import json
import os
import re
import urllib.request
import warnings
from typing import Optional, Union, Tuple, Dict

import ee
import pkg_resources

from ee_extra.STAC.utils import _get_platform_STAC
from ee_extra.utils import _load_JSON

def _get_expression_map(img: ee.Image, platformDict: dict) -> dict:
    """Gets the dictionary required for the map parameter i n ee.Image.expression() method.

    Args:
        img : Image to get the dictionary from.
        platformDict : Dictionary retrieved from the _get_STAC_platform() method.

    Returns:
        Map dictionary for the ee.Image.expression() method.
    """

    def lookupS2(img):
        return {
            "A": img.select("B1"),
            "B": img.select("B2"),
            "G": img.select("B3"),
            "R": img.select("B4"),
            "RE1": img.select("B5"),
            "RE2": img.select("B6"),
            "RE3": img.select("B7"),
            "N": img.select("B8"),
            "N2": img.select("B8A"),
            "WV": img.select("B9"),
            "S1": img.select("B11"),
            "S2": img.select("B12"),
            "lambdaG": 559.8,
            "lambdaR": 664.6,
            "lambdaN": 832.8,
        }

    def lookupL8(img):
        return {
            "A": img.select("B1"),
            "B": img.select("B2"),
            "G": img.select("B3"),
            "R": img.select("B4"),
            "N": img.select("B5"),
            "S1": img.select("B6"),
            "S2": img.select("B7"),
            "T1": img.select("B10"),
            "T2": img.select("B11"),
            "lambdaG": 560.0,
            "lambdaR": 655.0,
            "lambdaN": 865.0,
        }

    def lookupL8C2(img):
        return {
            "A": img.select("SR_B1"),
            "B": img.select("SR_B2"),
            "G": img.select("SR_B3"),
            "R": img.select("SR_B4"),
            "N": img.select("SR_B5"),
            "S1": img.select("SR_B6"),
            "S2": img.select("SR_B7"),
            "T1": img.select("ST_B10"),
            "lambdaG": 560.0,
            "lambdaR": 655.0,
            "lambdaN": 865.0,
        }

    def lookupL45(img):
        return {
            "B": img.select("B1"),
            "G": img.select("B2"),
            "R": img.select("B3"),
            "N": img.select("B4"),
            "S1": img.select("B5"),
            "T1": img.select("B6"),
            "S2": img.select("B7"),
            "lambdaG": 560.0,
            "lambdaR": 660.0,
            "lambdaN": 830.0,
        }

    def lookupL45C2(img):
        return {
            "B": img.select("SR_B1"),
            "G": img.select("SR_B2"),
            "R": img.select("SR_B3"),
            "N": img.select("SR_B4"),
            "S1": img.select("SR_B5"),
            "T1": img.select("ST_B6"),
            "S2": img.select("SR_B7"),
            "lambdaG": 560.0,
            "lambdaR": 660.0,
            "lambdaN": 830.0,
        }

    def lookupL7(img):
        return {
            "B": img.select("B1"),
            "G": img.select("B2"),
            "R": img.select("B3"),
            "N": img.select("B4"),
            "S1": img.select("B5"),
            "T1": img.select("B6"),
            "S2": img.select("B7"),
            "lambdaG": 560.0,
            "lambdaR": 660.0,
            "lambdaN": 835.0,
        }

    def lookupL7C2(img):
        return {
            "B": img.select("SR_B1"),
            "G": img.select("SR_B2"),
            "R": img.select("SR_B3"),
            "N": img.select("SR_B4"),
            "S1": img.select("SR_B5"),
            "T1": img.select("ST_B6"),
            "S2": img.select("SR_B7"),
            "lambdaG": 560.0,
            "lambdaR": 660.0,
            "lambdaN": 835.0,
        }

    def lookupMOD09GQ(img):
        return {
            "R": img.select("sur_refl_b01"),
            "N": img.select("sur_refl_b02"),
            "lambdaR": 645.0,
            "lambdaN": 858.5,
        }

    def lookupMOD09GA(img):
        return {
            "B": img.select("sur_refl_b03"),
            "G": img.select("sur_refl_b04"),
            "R": img.select("sur_refl_b01"),
            "N": img.select("sur_refl_b02"),
            "S1": img.select("sur_refl_b06"),
            "S2": img.select("sur_refl_b07"),
            "lambdaG": 555.0,
            "lambdaR": 645.0,
            "lambdaN": 858.5,
        }

    def lookupMCD43A4(img):
        return {
            "B": img.select("Nadir_Reflectance_Band3"),
            "G": img.select("Nadir_Reflectance_Band4"),
            "R": img.select("Nadir_Reflectance_Band1"),
            "N": img.select("Nadir_Reflectance_Band2"),
            "S1": img.select("Nadir_Reflectance_Band6"),
            "S2": img.select("Nadir_Reflectance_Band7"),
            "lambdaG": 555.0,
            "lambdaR": 645.0,
            "lambdaN": 858.5,
        }

    lookupPlatform = {
        "COPERNICUS/S2": lookupS2,
        "COPERNICUS/S2_SR": lookupS2,
        "LANDSAT/LC08/C01/T1_SR": lookupL8,
        "LANDSAT/LC08/C01/T2_SR": lookupL8,
        "LANDSAT/LC08/C02/T1_L2": lookupL8C2,
        "LANDSAT/LC08/C02/T2_L2": lookupL8C2,
        "LANDSAT/LC09/C02/T1_L2": lookupL8C2,
        "LANDSAT/LC09/C02/T2_L2": lookupL8C2,
        "LANDSAT/LE07/C01/T1_SR": lookupL7,
        "LANDSAT/LE07/C01/T2_SR": lookupL7,
        "LANDSAT/LE07/C02/T1_L2": lookupL7C2,
        "LANDSAT/LE07/C02/T2_L2": lookupL7C2,
        "LANDSAT/LT05/C01/T1_SR": lookupL45,
        "LANDSAT/LT05/C01/T2_SR": lookupL45,
        "LANDSAT/LT05/C02/T1_L2": lookupL45C2,
        "LANDSAT/LT05/C02/T2_L2": lookupL45C2,
        "LANDSAT/LT04/C01/T1_SR": lookupL45,
        "LANDSAT/LT04/C01/T2_SR": lookupL45,
        "LANDSAT/LT04/C02/T1_L2": lookupL45C2,
        "LANDSAT/LT04/C02/T2_L2": lookupL45C2,
        "MODIS/006/MOD09GQ": lookupMOD09GQ,
        "MODIS/006/MYD09GQ": lookupMOD09GQ,
        "MODIS/006/MOD09GA": lookupMOD09GA,
        "MODIS/006/MYD09GA": lookupMOD09GA,
        "MODIS/006/MOD09Q1": lookupMOD09GQ,
        "MODIS/006/MYD09Q1": lookupMOD09GQ,
        "MODIS/006/MOD09A1": lookupMOD09GA,
        "MODIS/006/MYD09A1": lookupMOD09GA,
        "MODIS/006/MCD43A4": lookupMCD43A4,
    }

    if platformDict["platform"] not in list(lookupPlatform.keys()):
        raise Exception(
            "Sorry, satellite platform not supported for index computation!"
        )

    return lookupPlatform[platformDict["platform"]](img)

def _get_indices(online: bool) -> dict:
    """Retrieves the dictionary of indices used for the index() method in ee.Image and ee.ImageCollection classes.

    Args:
        online : Wheter to retrieve the most recent list of indices directly from the GitHub repository and not from the local copy.

    Returns:
        Indices.
    """
    if online:
        with urllib.request.urlopen(
            "https://raw.githubusercontent.com/davemlz/awesome-ee-spectral-indices/main/output/spectral-indices-dict.json"
        ) as url:
            indices = json.loads(url.read().decode())
    else:
        indices = _load_JSON("spectral-indices-dict.json")

    return indices["SpectralIndices"]

def _get_kernel_image(
    img: ee.Image, lookup: dict, kernel: str, sigma: Union[str, float], a: str, b: str
) -> ee.Image:
    """Creates an ee.Image representing a kernel computed on bands [a] and [b].

    Args:
        img : Image to compute the kernel on.
        lookup : Dictionary retrieved from _get_expression_map().
        kernel : Kernel to use.
        sigma : Length-scale parameter. Used for kernel = 'RBF'.
        a : Key of the first band to use.
        b : Key of the second band to use.

    Returns:
        Kernel image.
    """
    if a not in list(lookup.keys()) or b not in list(lookup.keys()):
        return None
    else:
        lookupab = {"a": lookup[a], "b": lookup[b]}
        if isinstance(sigma, str):
            lookup = {**lookup, **lookupab, "sigma": img.expression(sigma, lookupab)}
        else:
            lookup = {**lookup, **lookupab, "sigma": sigma}
        kernels = {
            "linear": "a * b",
            "RBF": "exp((-1.0 * (a - b) ** 2.0)/(2.0 * sigma ** 2.0))",
            "poly": "((a * b) + c) ** p",
        }
        return img.expression(kernels[kernel], lookup)

def _remove_none_dict(dictionary: dict) -> dict:
    """Removes elements from a dictionary with None values.

    Args:
        dictionary : Dictionary to remove None values.

    Returns:
        Curated dictionary.
    """
    newDictionary = dict(dictionary)
    for key in dictionary.keys():
        if dictionary[key] is None:
            del newDictionary[key]
    return newDictionary

def _get_kernel_parameters(
    img: ee.Image, lookup: dict, kernel: str, sigma: Union[str, float]
) -> dict:
    """Gets the additional kernel parameters to compute kernel indices.

    Args:
        img : Image to compute the kernel parameters on.
        lookup : Dictionary retrieved from _get_expression_map().
        kernel : Kernel to use.
        sigma : Length-scale parameter. Used for kernel = 'RBF'.

    Returns:
        Kernel parameters.
    """
    kernelParameters = {
        "kNN": _get_kernel_image(img, lookup, kernel, sigma, "N", "N"),
        "kNR": _get_kernel_image(img, lookup, kernel, sigma, "N", "R"),
        "kNB": _get_kernel_image(img, lookup, kernel, sigma, "N", "B"),
        "kNL": _get_kernel_image(img, lookup, kernel, sigma, "N", "L"),
        "kGG": _get_kernel_image(img, lookup, kernel, sigma, "G", "G"),
        "kGR": _get_kernel_image(img, lookup, kernel, sigma, "G", "R"),
        "kGB": _get_kernel_image(img, lookup, kernel, sigma, "G", "B"),
        "kBB": _get_kernel_image(img, lookup, kernel, sigma, "B", "B"),
        "kBR": _get_kernel_image(img, lookup, kernel, sigma, "B", "R"),
        "kBL": _get_kernel_image(img, lookup, kernel, sigma, "B", "L"),
        "kRR": _get_kernel_image(img, lookup, kernel, sigma, "R", "R"),
        "kRB": _get_kernel_image(img, lookup, kernel, sigma, "R", "B"),
        "kRL": _get_kernel_image(img, lookup, kernel, sigma, "R", "L"),
        "kLL": _get_kernel_image(img, lookup, kernel, sigma, "L", "L"),
    }

    return kernelParameters

def _get_tc_coefficients(platform: str) -> dict:
    """Gets the platform-specific coefficient dictionary required for tasseled cap
    transformation.

    Platform matching is strict, meaning that data must be at the processing level
    specified by the reference literature that coefficients were sourced from, e.g.
    Landsat 8 SR cannot be transformed with Landsat 8 TOA coefficients.

    Args:
        platform : Platform name retrieved from the STAC.

    Returns:
        Map dictionary with band names and corresponding coefficients for brightness
        greenness, and wetness.

    Raises:
        Exception : If the platform has no supported coefficients.
    """

    SENTINEL2_1C = {
        "bands": (
            "B1",
            "B2",
            "B3",
            "B4",
            "B5",
            "B6",
            "B7",
            "B8",
            "B8A",
            "B9",
            "B10",
            "B11",
            "B12",
        ),
        "TCB": (
            0.2381,
            0.2569,
            0.2934,
            0.3020,
            0.3099,
            0.3740,
            0.4180,
            0.3580,
            0.3834,
            0.0103,
            0.0020,
            0.0896,
            0.0780,
        ),
        "TCG": (
            -0.2266,
            -0.2818,
            -0.3020,
            -0.4283,
            -0.2959,
            0.1602,
            0.3127,
            0.3138,
            0.4261,
            0.1454,
            -0.0017,
            -0.1341,
            -0.2538,
        ),
        "TCW": (
            0.1825,
            0.1763,
            0.1615,
            0.0486,
            0.0170,
            0.0223,
            0.0219,
            -0.0755,
            -0.0910,
            -0.1369,
            0.0003,
            -0.7701,
            -0.5293,
        ),
    }

    # Zhai et al. 2022 also provide coefficients with the blue band, but
    # recommend omitting it due to difficulties in atmospheric correction.
    LANDSAT8_SR = {
        "bands": ("SR_B3", "SR_B4", "SR_B5", "SR_B6", "SR_B7"),
        "TCB": (0.4596, 0.5046, 0.5458, 0.4114, 0.2589),
        "TCG": (-0.3374, -0.4901, 0.7909, 0.0177, -0.1416),
        "TCW": (0.2254, 0.3681, 0.2250, -0.6053, -0.6298)
    }

    # Zhai et al. 2022 coefficients were included for L8 TOA over the Baig
    # et al. 2014 coefficients for consistency with the L8 SR coefficients,
    # which were not calculated by Baig et al.
    LANDSAT8_TOA = {
        "bands": ("B3", "B4", "B5", "B6", "B7"),
        "TCB": (0.4321, 0.4971, 0.5695, 0.4192, 0.2569),
        "TCG": (-0.3318, -0.4844, 0.7856, -0.0331, -0.1923),
        "TCW": (0.2633, 0.3945, 0.1801, -0.6121, -0.6066)
    }

    # Coefficients for Landsat 8 OLI are usable for Landsat 9 OLI-2, per
    # Zhai et al. 2022
    LANDSAT9_SR = LANDSAT8_SR
    LANDSAT9_TOA = LANDSAT8_TOA

    LANDSAT7_TOA = {
        "bands": ("B1", "B2", "B3", "B4", "B5", "B7"),
        "TCB": (0.3561, 0.3972, 0.3904, 0.6966, 0.2286, 0.1596),
        "TCG": (-0.3344, -0.3544, -0.4556, 0.6966, -0.0242, -0.2630),
        "TCW": (0.2626, 0.2141, 0.0926, 0.0656, -0.7629, -0.5388),
    }

    LANDSAT4_DN = {
        "bands": ("B1", "B2", "B3", "B4", "B5", "B7"),
        "TCB": (0.3037, 0.2793, 0.4743, 0.5585, 0.5082, 0.1863),
        "TCG": (-0.2848, -0.2435, -0.5435, 0.7243, 0.0840, -0.1800),
        "TCW": (0.1509, 0.1973, 0.3279, 0.3406, -0.7112, -0.4572),
    }

    LANDSAT4_SR = {
        "bands": ("SR_B1", "SR_B2", "SR_B3", "SR_B4", "SR_B5", "SR_B7"),
        "TCB": (0.2043, 0.4158, 0.5524, 0.5741, 0.3124, 0.2303),
        "TCG": (-0.1603, -0.2819, -0.4934, 0.7940, -0.0002, -0.1446),
        "TCW": (0.0315, 0.2021, 0.3102, 0.1594, -0.6806, -0.6109),
    }

    LANDSAT5_DN = {
        "bands": ("B1", "B2", "B3", "B4", "B5", "B7"),
        "TCB": (0.2909, 0.2493, 0.4806, 0.5568, 0.4438, 0.1706),
        "TCG": (-0.2728, -0.2174, -0.5508, 0.7221, 0.0733, -0.1648),
        "TCW": (0.1446, 0.1761, 0.3322, 0.3396, -0.6210, -0.4186),
    }

    MODIS_NBAR = {
        "bands": (
            "Nadir_Reflectance_Band1",
            "Nadir_Reflectance_Band2",
            "Nadir_Reflectance_Band3",
            "Nadir_Reflectance_Band4",
            "Nadir_Reflectance_Band5",
            "Nadir_Reflectance_Band6",
            "Nadir_Reflectance_Band7",
        ),
        "TCB": (0.4395, 0.5945, 0.2460, 0.3918, 0.3506, 0.2136, 0.2678),
        "TCG": (-0.4064, 0.5129, -0.2744, -0.2893, 0.4882, -0.0036, -0.4169),
        "TCW": (0.1147, 0.2489, 0.2408, 0.3132, -0.3122, -0.6416, -0.5087),
    }

    platformCoeffs = {
        "COPERNICUS/S2": SENTINEL2_1C,
        "MODIS/006/MCD43A4": MODIS_NBAR,
        "LANDSAT/LC09/C02/T1_L2": LANDSAT9_SR,
        "LANDSAT/LC09/C02/T1_TOA": LANDSAT9_TOA,
        "LANDSAT/LC08/C02/T1_L2": LANDSAT8_SR,
        "LANDSAT/LC08/C02/T2_L2": LANDSAT8_SR,
        "LANDSAT/LC08/C01/T1_TOA": LANDSAT8_TOA,
        "LANDSAT/LC08/C01/T1_RT_TOA": LANDSAT8_TOA,
        "LANDSAT/LC08/C01/T2_TOA": LANDSAT8_TOA,
        "LANDSAT/LE07/C01/T1_TOA": LANDSAT7_TOA,
        "LANDSAT/LE07/C01/T1_RT_TOA": LANDSAT7_TOA,
        "LANDSAT/LE07/C01/T2_TOA": LANDSAT7_TOA,
        "LANDSAT/LT05/C01/T1": LANDSAT5_DN,
        "LANDSAT/LT05/C01/T2": LANDSAT5_DN,
        "LANDSAT/LT04/C02/T1_L2": LANDSAT4_SR,
        "LANDSAT/LT04/C02/T2_L2": LANDSAT4_SR,
        "LANDSAT/LT04/C01/T1": LANDSAT4_DN,
        "LANDSAT/LT04/C01/T2": LANDSAT4_DN,
    }

    if platform not in list(platformCoeffs.keys()):
        raise Exception(
            "Sorry, satellite platform not supported for tasseled cap transformation! Use one of "
            + str(list(platformCoeffs.keys()))
        )

    return platformCoeffs[platform]

def _match_histogram(
    source: ee.Image,
    target: ee.Image,
    bands: Optional[Dict[str, str]],
    geometry: Optional[ee.Geometry],
    maxBuckets: int,
) -> ee.Image:
    """Adjust the histogram of an image to match a target image.

    Args:
        source : Image to adjust.
        target : Image to use as the histogram reference.
        bands : An optional dictionary of band names to match, with source bands as keys
            and target bands as values. If none is provided, bands will be matched by name.
            Any bands not included here will be dropped.
        geometry : The optional region to match histograms in that overlaps both images.
            If none is provided, the geometry of the source image will be used. If the
            source image is unbounded and no geometry is provided, histogram matching will
            fail.
        maxBuckets : The maximum number of buckets to use when building histograms. More
            buckets will require more memory and time but will generate more accurate
            results. The number of buckets will be rounded to the nearest power of 2.

    Returns:
        The adjusted image containing the matched source bands.
    """

    def histogram_lookup(
        source_hist: ee.Array, target_hist: ee.Array
    ) -> Tuple[ee.List, ee.List]:
        """Build a list of target values with corresponding counts to source values from a source and target histogram.

        Args:
            source_hist : A histogram for a source image returned by ee.Reducer.autoHistogram
            target_hist : A histogram for a target image returned by ee.Reducer.autoHistogram

        Returns:
            Source histogram values and target histogram values with corresponding counts.
        """
        source_vals = source_hist.slice(1, 0, 1).project([0])
        source_counts = source_hist.slice(1, 1, 2).project([0])
        source_counts = source_counts.divide(source_counts.get([-1]))

        target_vals = target_hist.slice(1, 0, 1).project([0])
        target_counts = target_hist.slice(1, 1, 2).project([0])
        target_counts = target_counts.divide(target_counts.get([-1]))

        def lookup_value(n):
            """Find the first target value with at least n counts."""
            index = target_counts.gte(n).argmax()
            return target_vals.get(index)

        target_lookup_vals = source_counts.toList().map(lookup_value)

        return (source_vals.toList(), target_lookup_vals)

    geometry = ee.Element.geometry(source) if geometry is None else geometry

    source_bands = source.bandNames() if bands is None else list(bands.keys())
    target_bands = source.bandNames() if bands is None else list(bands.values())
    bands = ee.Dictionary.fromLists(source_bands, target_bands)

    source = source.select(source_bands)
    target = target.select(target_bands)

    source_histogram = source.reduceRegion(
        reducer=ee.Reducer.autoHistogram(maxBuckets=maxBuckets, cumulative=True),
        geometry=geometry,
        scale=30,
        maxPixels=1e13,
        bestEffort=True,
    )

    target_histogram = target.updateMask(source.mask()).reduceRegion(
        reducer=ee.Reducer.autoHistogram(maxBuckets=maxBuckets, cumulative=True),
        geometry=geometry,
        scale=30,
        maxPixels=1e13,
        bestEffort=True,
    )

    def match_bands(source_band: ee.String, target_band: ee.String) -> ee.Image:
        """Match the histogram of one source band to a target band.

        Args:
            source_band : The name of a band in the source image to adjust.
            target_band : The name of a corresponding band in the target image to match to.

        Returns:
            The source band image histogram-matched to the target band.
        """
        x, y = histogram_lookup(
            source_histogram.getArray(source_band),
            target_histogram.getArray(target_band),
        )
        matched = source.select([source_band]).interpolate(x, y)
        return matched

    matched = (
        ee.ImageCollection(bands.map(match_bands).values())
        .toBands()
        .rename(bands.keys())
    )

    # Preserve the metadata, band types, and band order of the source image.
    matched = ee.Image(matched.copyProperties(source, source.propertyNames()))
    matched = matched.cast(source.bandTypes(), source.bandNames())
    matched = matched.set("ee_extra:HISTOGRAM_TARGET", target)

    # If the source image was bounded, clip the matched output to its bounds. If the source
    # image doesn't have a `geometry` this will fail, but that seems exceptionally rare.
    matched = ee.Algorithms.If(
        source.geometry().isUnbounded(),
        matched,
        matched.clip(source.geometry().bounds()),
    )

    return ee.Image(matched)


# coding: utf-8
# Copyright (c) 2016, 2021, Oracle and/or its affiliates.  All rights reserved.
# This software is dual-licensed to you under the Universal Permissive License (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl or Apache License 2.0 as shown at http://www.apache.org/licenses/LICENSE-2.0. You may choose either license.

from .update_data_asset_details import UpdateDataAssetDetails
from oci.util import formatted_flat_dict, NONE_SENTINEL, value_allowed_none_or_none_sentinel  # noqa: F401
from oci.decorators import init_model_state_from_kwargs

@init_model_state_from_kwargs
class UpdateDataAssetFromAtp(UpdateDataAssetDetails):
    """
    Details for the Autonomous Transaction Processing data asset type.
    """

    def __init__(self, **kwargs):
        """
        Initializes a new UpdateDataAssetFromAtp object with values from keyword arguments. The default value of the :py:attr:`~oci.data_integration.models.UpdateDataAssetFromAtp.model_type` attribute
        of this class is ``ORACLE_ATP_DATA_ASSET`` and it should not be changed.
        The following keyword arguments are supported (corresponding to the getters/setters of this class):

        :param model_type:
            The value to assign to the model_type property of this UpdateDataAssetFromAtp.
            Allowed values for this property are: "ORACLE_DATA_ASSET", "ORACLE_OBJECT_STORAGE_DATA_ASSET", "ORACLE_ATP_DATA_ASSET", "ORACLE_ADWC_DATA_ASSET", "MYSQL_DATA_ASSET", "GENERIC_JDBC_DATA_ASSET", "FUSION_APP_DATA_ASSET", "AMAZON_S3_DATA_ASSET"
        :type model_type: str

        :param key:
            The value to assign to the key property of this UpdateDataAssetFromAtp.
        :type key: str

        :param model_version:
            The value to assign to the model_version property of this UpdateDataAssetFromAtp.
        :type model_version: str

        :param name:
            The value to assign to the name property of this UpdateDataAssetFromAtp.
        :type name: str

        :param description:
            The value to assign to the description property of this UpdateDataAssetFromAtp.
        :type description: str

        :param object_status:
            The value to assign to the object_status property of this UpdateDataAssetFromAtp.
        :type object_status: int

        :param object_version:
            The value to assign to the object_version property of this UpdateDataAssetFromAtp.
        :type object_version: int

        :param identifier:
            The value to assign to the identifier property of this UpdateDataAssetFromAtp.
        :type identifier: str

        :param external_key:
            The value to assign to the external_key property of this UpdateDataAssetFromAtp.
        :type external_key: str

        :param asset_properties:
            The value to assign to the asset_properties property of this UpdateDataAssetFromAtp.
        :type asset_properties: dict(str, str)

        :param registry_metadata:
            The value to assign to the registry_metadata property of this UpdateDataAssetFromAtp.
        :type registry_metadata: oci.data_integration.models.RegistryMetadata

        :param service_name:
            The value to assign to the service_name property of this UpdateDataAssetFromAtp.
        :type service_name: str

        :param driver_class:
            The value to assign to the driver_class property of this UpdateDataAssetFromAtp.
        :type driver_class: str

        :param credential_file_content:
            The value to assign to the credential_file_content property of this UpdateDataAssetFromAtp.
        :type credential_file_content: str

        :param wallet_secret:
            The value to assign to the wallet_secret property of this UpdateDataAssetFromAtp.
        :type wallet_secret: oci.data_integration.models.SensitiveAttribute

        :param wallet_password_secret:
            The value to assign to the wallet_password_secret property of this UpdateDataAssetFromAtp.
        :type wallet_password_secret: oci.data_integration.models.SensitiveAttribute

        :param region_id:
            The value to assign to the region_id property of this UpdateDataAssetFromAtp.
        :type region_id: str

        :param tenancy_id:
            The value to assign to the tenancy_id property of this UpdateDataAssetFromAtp.
        :type tenancy_id: str

        :param compartment_id:
            The value to assign to the compartment_id property of this UpdateDataAssetFromAtp.
        :type compartment_id: str

        :param autonomous_db_id:
            The value to assign to the autonomous_db_id property of this UpdateDataAssetFromAtp.
        :type autonomous_db_id: str

        :param default_connection:
            The value to assign to the default_connection property of this UpdateDataAssetFromAtp.
        :type default_connection: oci.data_integration.models.UpdateConnectionFromAtp

        """
        self.swagger_types = {
            'model_type': 'str',
            'key': 'str',
            'model_version': 'str',
            'name': 'str',
            'description': 'str',
            'object_status': 'int',
            'object_version': 'int',
            'identifier': 'str',
            'external_key': 'str',
            'asset_properties': 'dict(str, str)',
            'registry_metadata': 'RegistryMetadata',
            'service_name': 'str',
            'driver_class': 'str',
            'credential_file_content': 'str',
            'wallet_secret': 'SensitiveAttribute',
            'wallet_password_secret': 'SensitiveAttribute',
            'region_id': 'str',
            'tenancy_id': 'str',
            'compartment_id': 'str',
            'autonomous_db_id': 'str',
            'default_connection': 'UpdateConnectionFromAtp'
        }

        self.attribute_map = {
            'model_type': 'modelType',
            'key': 'key',
            'model_version': 'modelVersion',
            'name': 'name',
            'description': 'description',
            'object_status': 'objectStatus',
            'object_version': 'objectVersion',
            'identifier': 'identifier',
            'external_key': 'externalKey',
            'asset_properties': 'assetProperties',
            'registry_metadata': 'registryMetadata',
            'service_name': 'serviceName',
            'driver_class': 'driverClass',
            'credential_file_content': 'credentialFileContent',
            'wallet_secret': 'walletSecret',
            'wallet_password_secret': 'walletPasswordSecret',
            'region_id': 'regionId',
            'tenancy_id': 'tenancyId',
            'compartment_id': 'compartmentId',
            'autonomous_db_id': 'autonomousDbId',
            'default_connection': 'defaultConnection'
        }

        self._model_type = None
        self._key = None
        self._model_version = None
        self._name = None
        self._description = None
        self._object_status = None
        self._object_version = None
        self._identifier = None
        self._external_key = None
        self._asset_properties = None
        self._registry_metadata = None
        self._service_name = None
        self._driver_class = None
        self._credential_file_content = None
        self._wallet_secret = None
        self._wallet_password_secret = None
        self._region_id = None
        self._tenancy_id = None
        self._compartment_id = None
        self._autonomous_db_id = None
        self._default_connection = None
        self._model_type = 'ORACLE_ATP_DATA_ASSET'

    @property
    def service_name(self):
        """
        Gets the service_name of this UpdateDataAssetFromAtp.
        The Autonomous Transaction Processing instance service name.

        :return: The service_name of this UpdateDataAssetFromAtp.
        :rtype: str
        """
        return self._service_name

    @service_name.setter
    def service_name(self, service_name):
        """
        Sets the service_name of this UpdateDataAssetFromAtp.
        The Autonomous Transaction Processing instance service name.

        :param service_name: The service_name of this UpdateDataAssetFromAtp.
        :type: str
        """
        self._service_name = service_name

    @property
    def driver_class(self):
        """
        Gets the driver_class of this UpdateDataAssetFromAtp.
        The Autonomous Transaction Processing driver class

        :return: The driver_class of this UpdateDataAssetFromAtp.
        :rtype: str
        """
        return self._driver_class

    @driver_class.setter
    def driver_class(self, driver_class):
        """
        Sets the driver_class of this UpdateDataAssetFromAtp.
        The Autonomous Transaction Processing driver class

        :param driver_class: The driver_class of this UpdateDataAssetFromAtp.
        :type: str
        """
        self._driver_class = driver_class

    @property
    def credential_file_content(self):
        """
        Gets the credential_file_content of this UpdateDataAssetFromAtp.
        The credential file content from an Autonomous Transaction Processing wallet.

        :return: The credential_file_content of this UpdateDataAssetFromAtp.
        :rtype: str
        """
        return self._credential_file_content

    @credential_file_content.setter
    def credential_file_content(self, credential_file_content):
        """
        Sets the credential_file_content of this UpdateDataAssetFromAtp.
        The credential file content from an Autonomous Transaction Processing wallet.

        :param credential_file_content: The credential_file_content of this UpdateDataAssetFromAtp.
        :type: str
        """
        self._credential_file_content = credential_file_content

    @property
    def wallet_secret(self):
        """
        Gets the wallet_secret of this UpdateDataAssetFromAtp.

        :return: The wallet_secret of this UpdateDataAssetFromAtp.
        :rtype: oci.data_integration.models.SensitiveAttribute
        """
        return self._wallet_secret

    @wallet_secret.setter
    def wallet_secret(self, wallet_secret):
        """
        Sets the wallet_secret of this UpdateDataAssetFromAtp.

        :param wallet_secret: The wallet_secret of this UpdateDataAssetFromAtp.
        :type: oci.data_integration.models.SensitiveAttribute
        """
        self._wallet_secret = wallet_secret

    @property
    def wallet_password_secret(self):
        """
        Gets the wallet_password_secret of this UpdateDataAssetFromAtp.

        :return: The wallet_password_secret of this UpdateDataAssetFromAtp.
        :rtype: oci.data_integration.models.SensitiveAttribute
        """
        return self._wallet_password_secret

    @wallet_password_secret.setter
    def wallet_password_secret(self, wallet_password_secret):
        """
        Sets the wallet_password_secret of this UpdateDataAssetFromAtp.

        :param wallet_password_secret: The wallet_password_secret of this UpdateDataAssetFromAtp.
        :type: oci.data_integration.models.SensitiveAttribute
        """
        self._wallet_password_secret = wallet_password_secret

    @property
    def region_id(self):
        """
        Gets the region_id of this UpdateDataAssetFromAtp.
        The Autonomous Data Warehouse instance region Id.

        :return: The region_id of this UpdateDataAssetFromAtp.
        :rtype: str
        """
        return self._region_id

    @region_id.setter
    def region_id(self, region_id):
        """
        Sets the region_id of this UpdateDataAssetFromAtp.
        The Autonomous Data Warehouse instance region Id.

        :param region_id: The region_id of this UpdateDataAssetFromAtp.
        :type: str
        """
        self._region_id = region_id

    @property
    def tenancy_id(self):
        """
        Gets the tenancy_id of this UpdateDataAssetFromAtp.
        The Autonomous Data Warehouse instance tenancy Id.

        :return: The tenancy_id of this UpdateDataAssetFromAtp.
        :rtype: str
        """
        return self._tenancy_id

    @tenancy_id.setter
    def tenancy_id(self, tenancy_id):
        """
        Sets the tenancy_id of this UpdateDataAssetFromAtp.
        The Autonomous Data Warehouse instance tenancy Id.

        :param tenancy_id: The tenancy_id of this UpdateDataAssetFromAtp.
        :type: str
        """
        self._tenancy_id = tenancy_id

    @property
    def compartment_id(self):
        """
        Gets the compartment_id of this UpdateDataAssetFromAtp.
        The Autonomous Data Warehouse instance compartment Id.

        :return: The compartment_id of this UpdateDataAssetFromAtp.
        :rtype: str
        """
        return self._compartment_id

    @compartment_id.setter
    def compartment_id(self, compartment_id):
        """
        Sets the compartment_id of this UpdateDataAssetFromAtp.
        The Autonomous Data Warehouse instance compartment Id.

        :param compartment_id: The compartment_id of this UpdateDataAssetFromAtp.
        :type: str
        """
        self._compartment_id = compartment_id

    @property
    def autonomous_db_id(self):
        """
        Gets the autonomous_db_id of this UpdateDataAssetFromAtp.
        Tha Autonomous Database Id

        :return: The autonomous_db_id of this UpdateDataAssetFromAtp.
        :rtype: str
        """
        return self._autonomous_db_id

    @autonomous_db_id.setter
    def autonomous_db_id(self, autonomous_db_id):
        """
        Sets the autonomous_db_id of this UpdateDataAssetFromAtp.
        Tha Autonomous Database Id

        :param autonomous_db_id: The autonomous_db_id of this UpdateDataAssetFromAtp.
        :type: str
        """
        self._autonomous_db_id = autonomous_db_id

    @property
    def default_connection(self):
        """
        Gets the default_connection of this UpdateDataAssetFromAtp.

        :return: The default_connection of this UpdateDataAssetFromAtp.
        :rtype: oci.data_integration.models.UpdateConnectionFromAtp
        """
        return self._default_connection

    @default_connection.setter
    def default_connection(self, default_connection):
        """
        Sets the default_connection of this UpdateDataAssetFromAtp.

        :param default_connection: The default_connection of this UpdateDataAssetFromAtp.
        :type: oci.data_integration.models.UpdateConnectionFromAtp
        """
        self._default_connection = default_connection

    def __repr__(self):
        return formatted_flat_dict(self)

    def __eq__(self, other):
        if other is None:
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        return not self == other


#!/usr/bin/env python3

import argparse
import json
import sys
import traceback
import re

from sonic_py_common import device_info, logger
from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector, SonicDBConfig

INIT_CFG_FILE = '/etc/sonic/init_cfg.json'
SYSLOG_IDENTIFIER = 'db_migrator'

# Global logger instance
log = logger.Logger(SYSLOG_IDENTIFIER)

class DBMigrator():
    def __init__(self, namespace, socket=None):
        """
        Version string format:
           version_<major>_<minor>_<build>
              major: starting from 1, sequentially incrementing in master
                     branch.
              minor: in github branches, minor version stays in 0. This minor
                     version creates space for private branches derived from
                     github public branches. These private branches shall use
                     none-zero values.
              build: sequentially increase within a minor version domain.
        """
        self.CURRENT_VERSION = 'version_2_0_0'

        self.TABLE_NAME      = 'VERSIONS'
        self.TABLE_KEY       = 'DATABASE'
        self.TABLE_FIELD     = 'VERSION'

        db_kwargs = {}
        if socket:
            db_kwargs['unix_socket_path'] = socket

        if namespace is None:
            self.configDB = ConfigDBConnector(**db_kwargs)
        else:
            self.configDB = ConfigDBConnector(use_unix_socket_path=True, namespace=namespace, **db_kwargs)
        self.configDB.db_connect('CONFIG_DB')

        self.appDB = SonicV2Connector(host='127.0.0.1')
        if self.appDB is not None:
            self.appDB.connect(self.appDB.APPL_DB)

        self.stateDB = SonicV2Connector(host='127.0.0.1')
        if self.stateDB is not None:
            self.stateDB.connect(self.stateDB.STATE_DB)

        version_info = device_info.get_sonic_version_info()
        asic_type = version_info.get('asic_type')
        self.asic_type = asic_type

        if asic_type == "mellanox":
            from mellanox_buffer_migrator import MellanoxBufferMigrator
            self.mellanox_buffer_migrator = MellanoxBufferMigrator(self.configDB)

    def migrate_pfc_wd_table(self):
        '''
        Migrate all data entries from table PFC_WD_TABLE to PFC_WD
        '''
        data = self.configDB.get_table('PFC_WD_TABLE')
        for key in data:
            self.configDB.set_entry('PFC_WD', key, data[key])
        self.configDB.delete_table('PFC_WD_TABLE')

    def is_ip_prefix_in_key(self, key):
        '''
        Function to check if IP address is present in the key. If it
        is present, then the key would be a tuple or else, it shall be
        be string
        '''
        return (isinstance(key, tuple))

    def migrate_interface_table(self):
        '''
        Migrate all data from existing INTERFACE table with IP Prefix
        to have an additional ONE entry without IP Prefix. For. e.g, for an entry
        "Vlan1000|192.168.0.1/21": {}", this function shall add an entry without
        IP prefix as ""Vlan1000": {}". This is for VRF compatibility.
        '''
        if_db = []
        if_tables = {
                     'INTERFACE',
                     'PORTCHANNEL_INTERFACE',
                     'VLAN_INTERFACE',
                     'LOOPBACK_INTERFACE'
                    }
        for table in if_tables:
            data = self.configDB.get_table(table)
            for key in data:
                if not self.is_ip_prefix_in_key(key):
                    if_db.append(key)
                    continue

        for table in if_tables:
            data = self.configDB.get_table(table)
            for key in data:
                if not self.is_ip_prefix_in_key(key) or key[0] in if_db:
                    continue
                log.log_info('Migrating interface table for ' + key[0])
                self.configDB.set_entry(table, key[0], data[key])
                if_db.append(key[0])

    def migrate_intf_table(self):
        '''
        Migrate all data from existing INTF table in APP DB during warmboot with IP Prefix
        to have an additional ONE entry without IP Prefix. For. e.g, for an entry
        "Vlan1000:192.168.0.1/21": {}", this function shall add an entry without
        IP prefix as ""Vlan1000": {}". This also migrates 'lo' to 'Loopback0' interface
        '''
        if self.appDB is None:
            return

        data = self.appDB.keys(self.appDB.APPL_DB, "INTF_TABLE:*")

        if data is None:
            return

        if_db = []
        for key in data:
            if_name = key.split(":")[1]
            if if_name == "lo":
                self.appDB.delete(self.appDB.APPL_DB, key)
                key = key.replace(if_name, "Loopback0")
                log.log_info('Migrating lo entry to ' + key)
                self.appDB.set(self.appDB.APPL_DB, key, 'NULL', 'NULL')

            if '/' not in key:
                if_db.append(key.split(":")[1])
                continue

        data = self.appDB.keys(self.appDB.APPL_DB, "INTF_TABLE:*")
        for key in data:
            if_name = key.split(":")[1]
            if if_name in if_db:
                continue
            log.log_info('Migrating intf table for ' + if_name)
            table = "INTF_TABLE:" + if_name
            self.appDB.set(self.appDB.APPL_DB, table, 'NULL', 'NULL')
            if_db.append(if_name)

    def migrate_copp_table(self):
        '''
        Delete the existing COPP table
        '''
        if self.appDB is None:
            return

        keys = self.appDB.keys(self.appDB.APPL_DB, "COPP_TABLE:*")
        if keys is None:
            return
        for copp_key in keys:
            self.appDB.delete(self.appDB.APPL_DB, copp_key)

    def migrate_config_db_buffer_tables_for_dynamic_calculation(self, speed_list, cable_len_list, default_dynamic_th, default_lossless_profiles, abandon_method, append_item_method):
        '''
        Migrate buffer tables to dynamic calculation mode
        parameters
        @speed_list - list of speed supported
        @cable_len_list - list of cable length supported
        @default_dynamic_th - default dynamic th
        @default_lossless_profiles - default lossless profiles from the previous image
        @abandon_method - a function which is called to abandon the migration and keep the current configuration
                          if the current one doesn't match the default one
        @append_item_method - a function which is called to append an item to the list of pending commit items
                              any update to buffer configuration will be pended and won't be applied until
                              all configuration is checked and aligns with the default one

        1. Buffer profiles for lossless PGs in BUFFER_PROFILE table will be removed
           if their names have the convention of pg_lossless_<speed>_<cable_length>_profile
           where the speed and cable_length belongs speed_list and cable_len_list respectively
           and the dynamic_th is equal to default_dynamic_th
        2. Insert tables required for dynamic buffer calculation
           - DEFAULT_LOSSLESS_BUFFER_PARAMETER|AZURE: {'default_dynamic_th': default_dynamic_th}
           - LOSSLESS_TRAFFIC_PATTERN|AZURE: {'mtu': '1500', 'small_packet_percentage': '100'}
        3. For lossless dynamic PGs, remove the explicit referencing buffer profiles
           Before: BUFFER_PG|<port>|3-4: {'profile': 'BUFFER_PROFILE|pg_lossless_<speed>_<cable_length>_profile'}
           After:  BUFFER_PG|<port>|3-4: {'profile': 'NULL'}
        '''
        # Migrate BUFFER_PROFILEs, removing dynamically generated profiles
        dynamic_profile = self.configDB.get_table('BUFFER_PROFILE')
        profile_pattern = 'pg_lossless_([1-9][0-9]*000)_([1-9][0-9]*m)_profile'
        for name, info in dynamic_profile.items():
            m = re.search(profile_pattern, name)
            if not m:
                continue
            speed = m.group(1)
            cable_length = m.group(2)
            if speed in speed_list and cable_length in cable_len_list:
                log.log_info("current profile {} {}".format(name, info))
                log.log_info("default profile {} {}".format(name, default_lossless_profiles.get(name)))
                default_profile = default_lossless_profiles.get(name);
                if info.get("xon") == default_profile.get("xon") and info.get("size") == default_profile.get("size") and info.get('dynamic_th') == default_dynamic_th:
                    append_item_method(('BUFFER_PROFILE', name, None))
                    log.log_info("Lossless profile {} has been removed".format(name))
                else:
                    log.log_notice("Lossless profile {} doesn't match the default configuration, keep using traditional buffer calculation mode")
                    abandon_method()
                    return True

        # Migrate BUFFER_PGs, removing the explicit designated profiles
        buffer_pgs = self.configDB.get_table('BUFFER_PG')
        ports = self.configDB.get_table('PORT')
        all_cable_lengths = self.configDB.get_table('CABLE_LENGTH')
        if not buffer_pgs or not ports or not all_cable_lengths:
            log.log_notice("At lease one of tables BUFFER_PG, PORT and CABLE_LENGTH hasn't been defined, skip following migration")
            abandon_method()
            return True

        cable_lengths = all_cable_lengths[list(all_cable_lengths.keys())[0]]
        for name, profile in buffer_pgs.items():
            # do the db migration
            port, pg = name
            if pg != '3-4':
                continue
            try:
                profile_name = profile['profile'][1:-1].split('|')[1]
                m = re.search(profile_pattern, profile_name)
            except Exception:
                continue
            if not m:
                continue
            speed = m.group(1)
            cable_length = m.group(2)
            try:
                if speed == ports[port]['speed'] and cable_length == cable_lengths[port]:
                    append_item_method(('BUFFER_PG', name, {'profile': 'NULL'}))
                else:
                    log.log_notice("Lossless PG profile {} for port {} doesn't match its speed {} or cable length {}, keep using traditional buffer calculation mode".format(
                        profile_name, port, speed, cable_length))
                    abandon_method()
                    return True
            except Exception:
                continue

        # Insert other tables required for dynamic buffer calculation
        metadata = self.configDB.get_entry('DEVICE_METADATA', 'localhost')
        metadata['buffer_model'] = 'dynamic'
        append_item_method(('DEVICE_METADATA', 'localhost', metadata))
        append_item_method(('DEFAULT_LOSSLESS_BUFFER_PARAMETER', 'AZURE', {'default_dynamic_th': default_dynamic_th}))
        append_item_method(('LOSSLESS_TRAFFIC_PATTERN', 'AZURE', {'mtu': '1500', 'small_packet_percentage': '100'}))

        return True

    def prepare_dynamic_buffer_for_warm_reboot(self, buffer_pools = None, buffer_profiles = None, buffer_pgs = None):
        '''
        This is the very first warm reboot of buffermgrd (dynamic) if the system reboot from old image by warm-reboot
        In this case steps need to be taken to get buffermgrd prepared (for warm reboot)

        During warm reboot, buffer tables should be installed in the first place.
        However, it isn't able to achieve that when system is warm-rebooted from an old image
        without dynamic buffer supported, because the buffer info wasn't in the APPL_DB in the old image.
        The solution is to copy that info from CONFIG_DB into APPL_DB in db_migrator.
        During warm-reboot, db_migrator adjusts buffer info in CONFIG_DB by removing some fields
        according to requirement from dynamic buffer calculation.
        The buffer info before that adjustment needs to be copied to APPL_DB.

        1. set WARM_RESTART_TABLE|buffermgrd as {restore_count: 0}
        2. Copy the following tables from CONFIG_DB into APPL_DB in case of warm reboot
           The separator in fields that reference objects in other table needs to be updated from '|' to ':'
           - BUFFER_POOL
           - BUFFER_PROFILE, separator updated for field 'pool'
           - BUFFER_PG, separator updated for field 'profile'
           - BUFFER_QUEUE, separator updated for field 'profile
           - BUFFER_PORT_INGRESS_PROFILE_LIST, separator updated for field 'profile_list'
           - BUFFER_PORT_EGRESS_PROFILE_LIST, separator updated for field 'profile_list'

        '''
        warmreboot_state = self.stateDB.get(self.stateDB.STATE_DB, 'WARM_RESTART_ENABLE_TABLE|system', 'enable')
        mmu_size = self.stateDB.get(self.stateDB.STATE_DB, 'BUFFER_MAX_PARAM_TABLE|global', 'mmu_size')
        if warmreboot_state == 'true' and not mmu_size:
            log.log_notice("This is the very first run of buffermgrd (dynamic), prepare info required from warm reboot")
        else:
            return True

        buffer_table_list = [
            ('BUFFER_POOL', buffer_pools, None),
            ('BUFFER_PROFILE', buffer_profiles, 'pool'),
            ('BUFFER_PG', buffer_pgs, 'profile'),
            ('BUFFER_QUEUE', None, 'profile'),
            ('BUFFER_PORT_INGRESS_PROFILE_LIST', None, 'profile_list'),
            ('BUFFER_PORT_EGRESS_PROFILE_LIST', None, 'profile_list')
        ]

        for pair in buffer_table_list:
            keys_copied = []
            keys_ignored = []
            table_name, entries, reference_field_name = pair
            app_table_name = table_name + "_TABLE"
            if not entries:
                entries = self.configDB.get_table(table_name)
            for key, items in entries.items():
                # copy items to appl db
                if reference_field_name:
                    confdb_ref = items.get(reference_field_name)
                    if not confdb_ref or confdb_ref == "NULL":
                        keys_ignored.append(key)
                        continue
                    items_referenced = confdb_ref.split(',')
                    appdb_ref = ""
                    first_item = True
                    for item in items_referenced:
                        if first_item:
                            first_item = False
                        else:
                            appdb_ref += ','
                        subitems = item.split('|')
                        first_key = True
                        for subitem in subitems:
                            if first_key:
                                appdb_ref += subitem + '_TABLE'
                                first_key = False
                            else:
                                appdb_ref += ':' + subitem

                    items[reference_field_name] = appdb_ref
                keys_copied.append(key)
                if type(key) is tuple:
                    appl_db_key = app_table_name + ':' + ':'.join(key)
                else:
                    appl_db_key = app_table_name + ':' + key
                for field, data in items.items():
                    self.appDB.set(self.appDB.APPL_DB, appl_db_key, field, data)

            if keys_copied:
                log.log_info("The following items in table {} in CONFIG_DB have been copied to APPL_DB: {}".format(table_name, keys_copied))
            if keys_ignored:
                log.log_info("The following items in table {} in CONFIG_DB have been ignored: {}".format(table_name, keys_copied))

        return True

    def version_unknown(self):
        """
        version_unknown tracks all SONiC versions that doesn't have a version
        string defined in config_DB.
        Nothing can be assumped when migrating from this version to the next
        version.
        Any migration operation needs to test if the DB is in expected format
        before migrating date to the next version.
        """

        log.log_info('Handling version_unknown')

        # NOTE: Uncomment next 3 lines of code when the migration code is in
        #       place. Note that returning specific string is intentional,
        #       here we only intended to migrade to DB version 1.0.1.
        #       If new DB version is added in the future, the incremental
        #       upgrade will take care of the subsequent migrations.
        self.migrate_pfc_wd_table()
        self.migrate_interface_table()
        self.migrate_intf_table()
        self.set_version('version_1_0_2')
        return 'version_1_0_2'

    def version_1_0_1(self):
        """
        Version 1_0_1.
        """
        log.log_info('Handling version_1_0_1')

        self.migrate_interface_table()
        self.migrate_intf_table()
        self.set_version('version_1_0_2')
        return 'version_1_0_2'

    def version_1_0_2(self):
        """
        Version 1_0_2.
        """
        log.log_info('Handling version_1_0_2')
        # Check ASIC type, if Mellanox platform then need DB migration
        if self.asic_type == "mellanox":
            if self.mellanox_buffer_migrator.mlnx_migrate_buffer_pool_size('version_1_0_2', 'version_1_0_3') \
               and self.mellanox_buffer_migrator.mlnx_flush_new_buffer_configuration():
                self.set_version('version_1_0_3')
        else:
            self.set_version('version_1_0_3')
        return 'version_1_0_3'

    def version_1_0_3(self):
        """
        Version 1_0_3.
        """
        log.log_info('Handling version_1_0_3')

        # Check ASIC type, if Mellanox platform then need DB migration
        if self.asic_type == "mellanox":
            if self.mellanox_buffer_migrator.mlnx_migrate_buffer_pool_size('version_1_0_3', 'version_1_0_4') \
               and self.mellanox_buffer_migrator.mlnx_migrate_buffer_profile('version_1_0_3', 'version_1_0_4') \
               and self.mellanox_buffer_migrator.mlnx_flush_new_buffer_configuration():
                self.set_version('version_1_0_4')
        else:
            self.set_version('version_1_0_4')

        return 'version_1_0_4'

    def version_1_0_4(self):
        """
        Current latest version. Nothing to do here.
        """
        log.log_info('Handling version_1_0_4')

        # Check ASIC type, if Mellanox platform then need DB migration
        if self.asic_type == "mellanox":
            speed_list = self.mellanox_buffer_migrator.default_speed_list
            cable_len_list = self.mellanox_buffer_migrator.default_cable_len_list
            buffer_pools = self.configDB.get_table('BUFFER_POOL')
            buffer_profiles = self.configDB.get_table('BUFFER_PROFILE')
            buffer_pgs = self.configDB.get_table('BUFFER_PG')
            default_lossless_profiles = self.mellanox_buffer_migrator.mlnx_get_default_lossless_profile('version_1_0_4')
            abandon_method = self.mellanox_buffer_migrator.mlnx_abandon_pending_buffer_configuration
            append_method = self.mellanox_buffer_migrator.mlnx_append_item_on_pending_configuration_list

            if self.mellanox_buffer_migrator.mlnx_migrate_buffer_pool_size('version_1_0_4', 'version_2_0_0') \
               and self.mellanox_buffer_migrator.mlnx_migrate_buffer_profile('version_1_0_4', 'version_2_0_0') \
               and self.migrate_config_db_buffer_tables_for_dynamic_calculation(speed_list, cable_len_list, '0', default_lossless_profiles,
                                                                                abandon_method, append_method) \
               and self.mellanox_buffer_migrator.mlnx_flush_new_buffer_configuration() \
               and self.prepare_dynamic_buffer_for_warm_reboot(buffer_pools, buffer_profiles, buffer_pgs):
                metadata = self.configDB.get_entry('DEVICE_METADATA', 'localhost')
                if not metadata.get('buffer_model'):
                    metadata['buffer_model'] = 'traditional'
                    self.configDB.set_entry('DEVICE_METADATA', 'localhost', metadata)
                    log.log_notice('Setting buffer_model to traditional')
                else:
                    log.log_notice('Got buffer_model {}'.format(metadata.get('buffer_model')))

                self.set_version('version_2_0_0')
        else:
            self.prepare_dynamic_buffer_for_warm_reboot()

            metadata = self.configDB.get_entry('DEVICE_METADATA', 'localhost')
            metadata['buffer_model'] = 'traditional'
            self.configDB.set_entry('DEVICE_METADATA', 'localhost', metadata)
            log.log_notice('Setting buffer_model to traditional')

            self.set_version('version_2_0_0')

        return 'version_2_0_0'

    def version_2_0_0(self):
        """
        Current latest version. Nothing to do here.
        """
        log.log_info('Handling version_2_0_0')

        return None

    def get_version(self):
        version = self.configDB.get_entry(self.TABLE_NAME, self.TABLE_KEY)
        if version and version[self.TABLE_FIELD]:
            return version[self.TABLE_FIELD]

        return 'version_unknown'

    def set_version(self, version=None):
        if not version:
            version = self.CURRENT_VERSION
        log.log_info('Setting version to ' + version)
        entry = { self.TABLE_FIELD : version }
        self.configDB.set_entry(self.TABLE_NAME, self.TABLE_KEY, entry)

    def common_migration_ops(self):
        try:
            with open(INIT_CFG_FILE) as f:
                init_db = json.load(f)
        except Exception as e:
            raise Exception(str(e))

        for init_cfg_table, table_val in init_db.items():
            data = self.configDB.get_table(init_cfg_table)
            if data:
                # Ignore overriding the values that pre-exist in configDB
                continue
            log.log_info("Migrating table {} from INIT_CFG to config_db".format(init_cfg_table))
            # Update all tables that do not exist in configDB but are present in INIT_CFG
            for init_table_key, init_table_val in table_val.items():
                self.configDB.set_entry(init_cfg_table, init_table_key, init_table_val)

        self.migrate_copp_table()

    def migrate(self):
        version = self.get_version()
        log.log_info('Upgrading from version ' + version)
        while version:
            next_version = getattr(self, version)()
            if next_version == version:
                raise Exception('Version migrate from %s stuck in same version' % version)
            version = next_version
        # Perform common migration ops
        self.common_migration_ops()

def main():
    try:
        parser = argparse.ArgumentParser()

        parser.add_argument('-o',
                            dest='operation',
                            metavar='operation (migrate, set_version, get_version)',
                            type = str,
                            required = False,
                            choices=['migrate', 'set_version', 'get_version'],
                            help = 'operation to perform [default: get_version]',
                            default='get_version')
        parser.add_argument('-s',
                        dest='socket',
                        metavar='unix socket',
                        type = str,
                        required = False,
                        help = 'the unix socket that the desired database listens on',
                        default = None )
        parser.add_argument('-n',
                        dest='namespace',
                        metavar='asic namespace',
                        type = str,
                        required = False,
                        help = 'The asic namespace whose DB instance we need to connect',
                        default = None )
        args = parser.parse_args()
        operation = args.operation
        socket_path = args.socket
        namespace = args.namespace

        if args.namespace is not None:
            SonicDBConfig.load_sonic_global_db_config(namespace=args.namespace)

        if socket_path:
            dbmgtr = DBMigrator(namespace, socket=socket_path)
        else:
            dbmgtr = DBMigrator(namespace)

        result = getattr(dbmgtr, operation)()
        if result:
            print(str(result))

    except Exception as e:
        log.log_error('Caught exception: ' + str(e))
        traceback.print_exc()
        print(str(e))
        parser.print_help()
        sys.exit(1)

if __name__ == "__main__":
    main()


# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Downloads and prepares TriviaQA dataset."""
from unittest import mock

from absl import app
from absl import flags
from absl import logging
import apache_beam as beam
import tensorflow_datasets as tfds

from official.projects.triviaqa import dataset  # pylint: disable=unused-import

flags.DEFINE_integer('sequence_length', 4096, 'Max number of tokens.')

flags.DEFINE_integer(
    'global_sequence_length', None,
    'Max number of question tokens plus sentences. If not set, defaults to '
    'sequence_length // 16 + 64.')

flags.DEFINE_integer(
    'stride', 3072,
    'For documents longer than `sequence_length`, where to split them.')

flags.DEFINE_string(
    'sentencepiece_model_path', None,
    'SentencePiece model to use for tokenization.')

flags.DEFINE_string('data_dir', None, 'Data directory for TFDS.')

flags.DEFINE_string('runner', 'DirectRunner', 'Beam runner to use.')

FLAGS = flags.FLAGS

def main(argv):
  if len(argv) > 1:
    raise app.UsageError('Too many command-line arguments.')
  builder = tfds.builder(
      'bigbird_trivia_qa/rc_wiki.preprocessed',
      data_dir=FLAGS.data_dir,
      sentencepiece_model_path=FLAGS.sentencepiece_model_path,
      sequence_length=FLAGS.sequence_length,
      global_sequence_length=FLAGS.global_sequence_length,
      stride=FLAGS.stride)
  download_config = tfds.download.DownloadConfig(
      beam_options=beam.options.pipeline_options.PipelineOptions(flags=[
          f'--runner={FLAGS.runner}',
          '--direct_num_workers=8',
          '--direct_running_mode=multi_processing',
      ]))
  with mock.patch('tensorflow_datasets.core.download.extractor._normpath',
                  new=lambda x: x):
    builder.download_and_prepare(download_config=download_config)
  logging.info(builder.info.splits)

if __name__ == '__main__':
  flags.mark_flag_as_required('sentencepiece_model_path')
  app.run(main)


import os
import click
import shutil
import subprocess
import pkg_resources
import sys
import errno
import traceback

from monitor.logs import init_logging, logger

class ValidationExceptionBinaryNotFound(Exception):
    pass

class NotRunningRoot(Exception):
    pass

@click.group()
def cli():
    click.echo("FileWave Monitor v13 configuration.")

delay_30m = 60 * 30

def run_root_command(cmd_array, **kwargs):
    try:
        os.rename('/etc/foo', '/etc/bar')
    except IOError as e:
        if (e == errno.EPERM):
            return False

    proc = subprocess.Popen(cmd_array, stdout=subprocess.PIPE, **kwargs)
    return proc.communicate()[0].decode('utf-8')

def run_root_commands(commands):
    for c in commands:
        run_root_command(c, shell=True)

def running_on_a_fwxserver_host(exist_func=os.path.exists):
    '''
    Check directories exist to see if we are running on a FileWave server host installation
    This should return True if we are, regardless of being Mac/Linux/Docker etc.
    '''
    dirs_that_must_exist = ["bin", "certs",
                            "django", "log"]
    main_filewave_dir = os.path.join("/usr/local", "filewave")
    if not exist_func(main_filewave_dir):
        return False
    for f in [os.path.join(main_filewave_dir, d) for d in dirs_that_must_exist]:
        if not exist_func(f):
            return False
    return True

@cli.command('integrate', help="Integrates the module assuming you are running this on the FileWave Server")
def install_into_environment():
    init_logging()

    if running_on_a_fwxserver_host():
        if run_root_command(["ls", "-l"]) is False:
            logger.info(
                "provisioning is requested - but I've detected you are not running as root - aborting")
            raise NotRunningRoot(
                "provisioning is requested - but I've detected you are not running as root - aborting")

        try:
            provision_postgres_wal_interval()
            provision_apache_mod_status()
            provision_prometheus_binary()
            provision_mtail_binary()
            provision_exporters()
            provision_supervisord_conf()
            validate_provisioning()

            logger.info("Looks like everything is configured, please restart the server now, then validate installation.")

        except Exception as e:
            logger.error("Error during provisioning, are you using sudo?")
            logger.error(e)
            traceback.print_exc(file=sys.stdout)
            return
    else:
        logger.info("Didn't detect a FileWave Server host - configuration aborted")

def provision_postgres_wal_interval():
    # /usr/local/filewave/fwxserver/DB/pg_data/postgresql.conf
    # log_min_duration_statement = 200
    #
    cmds = [
        "sed -i 's/log_min_duration_statement = 10000/log_min_duration_statement = 200/g' /usr/local/filewave/fwxserver/DB/pg_data/postgresql.conf"
    ]

    return run_root_commands(cmds)

def provision_prometheus_binary():
    cmds = [
        "wget https://github.com/prometheus/prometheus/releases/download/v2.19.2/prometheus-2.19.2.linux-amd64.tar.gz",
        "tar xzf prometheus-2.19.2.linux-amd64.tar.gz",
        "mkdir -p /usr/local/etc/filewave/prometheus",
        "mkdir -p /usr/local/etc/filewave/prometheus/conf.d/rules",
        "mkdir -p /usr/local/etc/filewave/prometheus/conf.d/alerts",
        "mkdir -p /usr/local/filewave/prometheus/",
        "mv prometheus-2.19.2.linux-amd64/prometheus /usr/local/sbin/",
        "mv prometheus-2.19.2.linux-amd64/promtool /usr/local/sbin/",
        "mv prometheus-2.19.2.linux-amd64/tsdb /usr/local/sbin/",
        "mv prometheus-2.19.2.linux-amd64/console_libraries /usr/local/filewave/prometheus/",
        "mv prometheus-2.19.2.linux-amd64/consoles /usr/local/filewave/prometheus/",
        "mkdir -p /usr/local/etc/filewave/prometheus/conf.d/jobs/http",
        "chown -R root:root /usr/local/filewave/prometheus/",
        "rm -rf prometheus-2.19.2.linux-amd64"
    ]

    run_root_commands(cmds)

    prom_file = "prometheus.yml"
    data = pkg_resources.resource_string("monitor.config", prom_file).decode('utf-8')
    provisioning_file = os.path.join("/usr/local/etc/filewave/prometheus", prom_file)
    with open(provisioning_file, 'w') as f:
        f.write(data)
    shutil.chown(provisioning_file, user="root", group="root")
    shutil.chown("/usr/local/filewave/prometheus", user="root", group="root")

def provision_apache_mod_status():
    '''
    #LoadModule status_module modules/mod_status.so

    # Uncomment following lines to enable mod status = and connect to https://localhost:20443/server-status?refresh=5 to see server status
    # Used by the prometheus apache_exporter. Works only on localhost (intentional to reduce security exposure).
    <IfModule status_module>
        <Location /server-status>
            SetHandler server-status
            Order Deny,Allow
            Deny from all
            Allow from 127.0.0.1 ::1
        </Location>
        ExtendedStatus On
    </IfModule>
    '''
    cmds = [
        "sed -i 's/#LoadModule status_module modules\/mod_status\.so/LoadModule status_module modules\/mod_status\.so/g' /usr/local/filewave/apache/conf/httpd.conf"
    ]

    run_root_commands(cmds)

def provision_mtail_binary():
    logger.info("downloading mtail...")

    # mtail binary: 15th Jul 2020
    # https://github.com/google/mtail/releases/download/v3.0.0-rc36/mtail_v3.0.0-rc36_linux_amd64
    cmds = [
        "mkdir -p /usr/local/etc/filewave/mtail/progs",
        "chown -R root:root /usr/local/etc/filewave/mtail",
        "wget https://github.com/google/mtail/releases/download/v3.0.0-rc36/mtail_v3.0.0-rc36_linux_amd64",
        "cp mtail_v3.0.0-rc36_linux_amd64 /usr/local/sbin/mtail",
        "chmod +x /usr/local/sbin/mtail",
        "firewall-cmd --zone=public --add-port=21090/tcp --permanent",
        "firewall-cmd --reload"
    ]

    run_root_commands(cmds)

    # write .mtail programs into /usr/local/etc/filewave/mtail/progs
    for mtail_file in pkg_resources.resource_listdir("monitor", "config"):
        if mtail_file.endswith(".mtail"):
            logger.info(f"writing with: {mtail_file}")
            data = pkg_resources.resource_string("monitor.config", mtail_file).decode('utf-8')
            provisioning_file = os.path.join("/usr/local/etc/filewave/mtail/progs", mtail_file)
            with open(provisioning_file, 'w') as f:
                f.write(data)
            shutil.chown(provisioning_file, user="root", group="root")

def provision_exporters():
    logger.info("downloading postgres exporter...")
    # from https://github.com/wrouesnel/postgres_exporter/releases/download/v0.8.0/postgres_exporter_v0.8.0_linux-amd64.tar.gz
    cmds = [
        "wget https://github.com/wrouesnel/postgres_exporter/releases/download/v0.8.0/postgres_exporter_v0.8.0_linux-amd64.tar.gz",
        "tar xzf postgres_exporter_v0.8.0_linux-amd64.tar.gz",
        "mv -f postgres_exporter_v0.8.0_linux-amd64/postgres_exporter /usr/local/sbin/ && rm -rf postgres_exporter_v0.8.0_linux-amd64"
    ]

    run_root_commands(cmds)

    logger.info("downloading apache exporter...")
    cmds = [
        "wget https://github.com/Lusitaniae/apache_exporter/releases/download/v0.8.0/apache_exporter-0.8.0.linux-amd64.tar.gz",
        "tar xzf apache_exporter-0.8.0.linux-amd64.tar.gz",
        "mv -f apache_exporter-0.8.0.linux-amd64/apache_exporter /usr/local/sbin/ && rm -rf apache_exporter-0.8.0.linux-amd64"
    ]

    run_root_commands(cmds)

    logger.info("downloading node_exporter")
    cmds = [
        "wget https://github.com/prometheus/node_exporter/releases/download/v1.0.1/node_exporter-1.0.1.linux-amd64.tar.gz",
        "tar xzf node_exporter-1.0.1.linux-amd64.tar.gz",
        "mv -f node_exporter-1.0.1.linux-amd64/node_exporter /usr/local/sbin/ && rm -rf node_exporter-1.0.1.linux-amd64"
    ]

    run_root_commands(cmds)

def provision_supervisord_conf():
    cmds = [
        "sed -i 's/\; port\=\*\:9001/port=127\.0\.0\.1\:9001/g' /usr/local/etc/filewave/supervisor/supervisord-server.conf",
        "sed -i 's/\; \[inet_http_server\]/\[inet_http_server\]/g' /usr/local/etc/filewave/supervisor/supervisord-server.conf",
        "sed -i 's/\;\[include\]/\[include\]/g' /usr/local/etc/filewave/supervisor/supervisord-server.conf",
        "sed -i 's/\;files = relative\/directory\/\*\.ini/files=extras\/\*\.conf/g' /usr/local/etc/filewave/supervisor/supervisord-server.conf",
    ]

    supervisord_dir = os.path.join("/usr/local/etc/filewave/supervisor/", "extras")
    if not os.path.exists(supervisord_dir):
        os.makedirs(supervisord_dir)

    data = pkg_resources.resource_string("monitor.config", "monitor-v13.conf").decode('utf-8')
    provisioning_file = os.path.join(supervisord_dir, "monitor-v13.conf")
    with open(provisioning_file, "w") as f:
        f.write(data)

    run_root_commands(cmds)

def validate_provisioning():
    binaries = [ "node_exporter", "apache_exporter", "mtail", "postgres_exporter", "prometheus", "promtool", "tsdb" ]
    for b in binaries:
        f = os.path.join("/usr/local/sbin", b)
        if not os.path.exists(f):
            raise ValidationExceptionBinaryNotFound(f"failed to find required binary: {f}")
        else:
            logger.info(f"OK: {f}")
            shutil.chown(f, user="root", group="root")


#Rebecca Schuetz
#May 25, 2016
#Homework 2

#1) Make a list of the following numbers: 22, 90, 0, -10, 3, 22, and 48
numbers = [22, 90, 0, -10, 3, 22, 48]
#1) Display the number of elements in the list
print(len(numbers))
#2) Display the 4th element of this list.
print(numbers[3])
#3) Display the sum of the 2nd and 4th element of the list.
print(numbers[1] + numbers[3])
#4) Display the 2nd-largest value in the list.
print(sorted(numbers)[-2])
#5) Display the last element of the original unsorted list
print(list(numbers)[-1])
#6) For each number, display a number: if your original number is less than 10,
#multiply it by thirty. If it's also even, add six.
#If it's greater than 50 subtract ten.
#If it's not negative ten, subtract one.
#(For example: 2 is less than 10, so 2 * 30 = 60, 2 is also even,
#so 60 + 6 = 66, 2 is not negative ten, so 66 - 1 = 65.)
#print('The answers I know are right')
#for number in numbers:
#    if number < 10:
#        number_less_than_10 = number * 30
#        if number % 2 == 0:
#            if number == -10:
#                print(number_less_than_10 + 6)
#            else:
#                print(number_less_than_10 + 5)
#        else:
#            print(number_less_than_10 - 1)
#    elif number > 50:
#        print(number - 11)
#    else:
#        print(number - 1)

#print('A way of doing it without the awkward minus ones')
for number in numbers:
    newnumber = number
    if number < 10:
        newnumber = number * 30
        if number % 2 == 0:
            newnumber = newnumber + 6
    elif number > 50:
        newnumber = number - 10
    if number == -10:
        print(newnumber)
    else:
        print(newnumber - 1)
#7) Sum the result of each of the numbers divided by two.
print(sum(numbers) / 2)
#DICTIONARIES

#8) Sometimes dictionaries are used to describe multiple aspects of a single object.
#Like, say, a movie. Define a dictionary called movie that works with the following code.
movie = {'title': 'The Life Aquatic', 'year': 2004, 'director': 'Wes Anderson',
'budget': 50000000, 'revenue': 34806726}

print("My favorite movie is", movie['title'], "which was released in", movie['year'],
"and was directed by", movie['director'])

#9) Add entries to the movie dictionary for budget and revenue
#(you'll use code like movie['budget'] = 1000), and print out the difference between the two.

#10) If the movie cost more to make than it made in theaters, print "It was a flop".
#If the film's revenue was more than five times the amount it cost to make, print "That was a good investment."
if movie['revenue'] < movie['budget']:
    print("It was a flop.")
if movie['revenue'] > (movie['budget'] * 5):
    print("That was a good investment.")
#11) Sometimes dictionaries are used to describe the same aspects of many different objects.
#Make ONE dictionary that describes the population of the boroughs of NYC.
#Manhattan has 1.6 million residents,
#Brooklyn has 2.6m,
#Bronx has 1.4m,
#Queens has 2.3m and
#Staten Island has 470,000.
#(Tip: keeping it all in either millions or thousands is a good idea)
population = {'Manhattan': 1.6, 'Brooklyn': 2.6, 'Bronx': 1.4, 'Queens': 2.3,
'Staten Island': .47 }

#12) Display the population of Brooklyn.
print("Brooklyn has", population['Brooklyn'], 'million people.')
#13) Display the combined population of all five boroughs.
print("All five buroughs have", round(sum(population.values()),2), 'million people.')
#14) Display what percent of NYC's population lives in Manhattan.
print(round(population['Manhattan'] / sum(population.values()) * 100,2), "percent of NYC's population lives in Manhattan.")


import sys
import csv
import get_info

def main(argv):
    skip = int(argv[1])
    with open('final_movie_upload_data.csv', mode='r') as csv_file:
        csv_reader = csv.DictReader(csv_file)
        for i in range(0, skip):
            next(csv_reader)
        count = 0
        new_data = []
        for row in csv_reader:
            idx = row["idx"]
            title = row["title"]
            #if line_count == 0:
            #    print(f'Column names are {", ".join(row)}')
            #    line_count += 1
            #    continue
            print(f'\t{idx} {title}')
            data = get_info.search(title)
            if "title" in data and "description" in data:
                data["idx"] = idx
                data["title"] = f"{data['title']}({title})"
                new_data.append(data)
            count += 1
            if count == 3 :
                break;
        print(f'Processed {count} lines.')
        append(new_data)

def append(data):
    field_names = ['idx','title','description']
    with open('movie_info.csv', 'a+', newline='') as write_obj:
            dict_writer = csv.DictWriter(write_obj, fieldnames=field_names)
            for row in data:
                dict_writer.writerow(row)

if __name__ == "__main__":
    main(sys.argv)


from tensorflow.keras.models import model_from_json
import numpy as np
import cv2
import math
import tensorflow as tf
from tensorflow.keras.preprocessing import image
facec = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')
from matplotlib import pyplot as plt
import os
import shutil
from skimage.measure import compare_ssim

with open("model.json", "r") as json_file:   #Loading the saved model
    loaded_model_json = json_file.read()
    loaded_model = model_from_json(loaded_model_json)

loaded_model.load_weights("model_weights.h5")
loaded_model._make_predict_function()
label_to_text = {0:'angry', 1:'disgust', 2:'fear', 3:'happy', 4: 'sad'}

def pred(img_path):
    label_to_text = {0:'angry', 1:'disgust', 2:'fear', 3:'happy', 4: 'sad'}
    img=cv2.imread(img_path)									#read Image
    gray_fr = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)				#covert image to grayscale
    faces_rects = facec.detectMultiScale(gray_fr, scaleFactor = 1.2, minNeighbors = 5)  #opencv's cascade classifier will be used for detecting the face
    if len(faces_rects)!=0:
        for (x, y, w, h) in faces_rects:
            fc = gray_fr[y:y+h, x:x+w]     #extracting only the face part
        roi = cv2.resize(fc, (48, 48))	#resizing it according to the image that are acceptable by our model
        img = image.img_to_array(roi)
        img = img/255
        img = np.expand_dims(img, axis=0)
        return label_to_text[np.argmax(loaded_model.predict(img))],img  #model.predict is used for predicting the emotion
    else:
        return 0,0  #return 0 if the face is not found

def removeout():
    shutil.rmtree('output/')  #remove output folder

def vidframe(vidname):
	if vidname==0:
		cap = cv2.VideoCapture(0)
		# Define the codec and create VideoWriter object
		fourcc = cv2.VideoWriter_fourcc(*'XVID')
		out = cv2.VideoWriter('output.mp4',fourcc, 20.0, (640,480))

		while(cap.isOpened()):
		    ret, frame = cap.read()
		    if ret==True:
		        out.write(frame)
		        cv2.imshow('frame',frame)
		        if cv2.waitKey(1) & 0xFF == ord('q'):
		            break
		    else:
		        break

		# Release everything if job is finished
		cap.release()
		out.release()
		cv2.destroyAllWindows()
		vidname="output.mp4"

	if os.path.exists('output'):      #if output folder is present then delete it
		removeout()						#create Output folder for storing frame
	os.mkdir('output')
	cap = cv2.VideoCapture(vidname)			#capture  video
	frameRate=cap.get(5)
	count = 0
	while(cap.isOpened()):					#store the frames in output folder
		frameId = cap.get(1)
		ret, frame = cap.read()
		if (ret != True):
			break
		if (frameId % math.floor(frameRate) == 0):
			filename ="output/frame%d.jpg" % count;count+=1
			cv2.imwrite(filename, frame)
	cap.release()
	result=[]							# used for storing emotion
	face=[]								#used for storing face images
	for filename in os.listdir("output"): #loop through each frame
		a,b = pred("output/"+filename)  #run pred function to get emotion and face images
		result.append(a)
		face.append(b)
	removeout()
	result=[x for x in result if x!=0]        #removing null prediction
	face=[x for x in face if len(str(x))>1]
	return result, face

def ssimscore1(im1,im2):
    im1=im1.reshape(48, 48, 1).astype('float32')   #reshaping the flattened image array
    im2=im2.reshape(48, 48, 1).astype('float32')
    (score, diff) = compare_ssim(im1, im2, full=True,multichannel=True) #comparing the image for finding difference using compare_ssim function
    return score


from operator import attrgetter
import pyangbind.lib.xpathhelper as xpathhelper
from pyangbind.lib.yangtypes import RestrictedPrecisionDecimalType, RestrictedClassType, TypedListType
from pyangbind.lib.yangtypes import YANGBool, YANGListType, YANGDynClass, ReferenceType
from pyangbind.lib.base import PybindBase
from decimal import Decimal
from bitarray import bitarray
import __builtin__
class cpu_info_state(PybindBase):
  """
  This class was auto-generated by the PythonClass plugin for PYANG
  from YANG module brocade-qos-operational - based on the path /cpu-info-state. Each member element of
  the container is represented as a class variable - with a specific
  YANG type.

  YANG Description: CPU EGID and Group ID mapping
  """
  __slots__ = ('_pybind_generated_by', '_path_helper', '_yang_name', '_rest_name', '_extmethods', '__name','__egid','__group_id','__description',)

  _yang_name = 'cpu-info-state'
  _rest_name = 'cpu-info-state'

  _pybind_generated_by = 'container'

  def __init__(self, *args, **kwargs):

    path_helper_ = kwargs.pop("path_helper", None)
    if path_helper_ is False:
      self._path_helper = False
    elif path_helper_ is not None and isinstance(path_helper_, xpathhelper.YANGPathHelper):
      self._path_helper = path_helper_
    elif hasattr(self, "_parent"):
      path_helper_ = getattr(self._parent, "_path_helper", False)
      self._path_helper = path_helper_
    else:
      self._path_helper = False

    extmethods = kwargs.pop("extmethods", None)
    if extmethods is False:
      self._extmethods = False
    elif extmethods is not None and isinstance(extmethods, dict):
      self._extmethods = extmethods
    elif hasattr(self, "_parent"):
      extmethods = getattr(self._parent, "_extmethods", None)
      self._extmethods = extmethods
    else:
      self._extmethods = False
    self.__group_id = YANGDynClass(base=RestrictedClassType(base_type=int, restriction_dict={'range': ['-128..127']}, int_size=8), is_leaf=True, yang_name="group-id", rest_name="group-id", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, is_keyval=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='int8', is_config=False)
    self.__egid = YANGDynClass(base=RestrictedClassType(base_type=long, restriction_dict={'range': ['0..4294967295']}, int_size=32), is_leaf=True, yang_name="egid", rest_name="egid", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='uint32', is_config=False)
    self.__name = YANGDynClass(base=unicode, is_leaf=True, yang_name="name", rest_name="name", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='string', is_config=False)
    self.__description = YANGDynClass(base=unicode, is_leaf=True, yang_name="description", rest_name="description", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='string', is_config=False)

    load = kwargs.pop("load", None)
    if args:
      if len(args) > 1:
        raise TypeError("cannot create a YANG container with >1 argument")
      all_attr = True
      for e in self._pyangbind_elements:
        if not hasattr(args[0], e):
          all_attr = False
          break
      if not all_attr:
        raise ValueError("Supplied object did not have the correct attributes")
      for e in self._pyangbind_elements:
        nobj = getattr(args[0], e)
        if nobj._changed() is False:
          continue
        setmethod = getattr(self, "_set_%s" % e)
        if load is None:
          setmethod(getattr(args[0], e))
        else:
          setmethod(getattr(args[0], e), load=load)

  def _path(self):
    if hasattr(self, "_parent"):
      return self._parent._path()+[self._yang_name]
    else:
      return [u'cpu-info-state']

  def _rest_path(self):
    if hasattr(self, "_parent"):
      if self._rest_name:
        return self._parent._rest_path()+[self._rest_name]
      else:
        return self._parent._rest_path()
    else:
      return [u'cpu-info-state']

  def _get_name(self):
    """
    Getter method for name, mapped from YANG variable /cpu_info_state/name (string)

    YANG Description: CPU EGID name
    """
    return self.__name

  def _set_name(self, v, load=False):
    """
    Setter method for name, mapped from YANG variable /cpu_info_state/name (string)
    If this variable is read-only (config: false) in the
    source YANG file, then _set_name is considered as a private
    method. Backends looking to populate this variable should
    do so via calling thisObj._set_name() directly.

    YANG Description: CPU EGID name
    """
    if hasattr(v, "_utype"):
      v = v._utype(v)
    try:
      t = YANGDynClass(v,base=unicode, is_leaf=True, yang_name="name", rest_name="name", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='string', is_config=False)
    except (TypeError, ValueError):
      raise ValueError({
          'error-string': """name must be of a type compatible with string""",
          'defined-type': "string",
          'generated-type': """YANGDynClass(base=unicode, is_leaf=True, yang_name="name", rest_name="name", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='string', is_config=False)""",
        })

    self.__name = t
    if hasattr(self, '_set'):
      self._set()

  def _unset_name(self):
    self.__name = YANGDynClass(base=unicode, is_leaf=True, yang_name="name", rest_name="name", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='string', is_config=False)

  def _get_egid(self):
    """
    Getter method for egid, mapped from YANG variable /cpu_info_state/egid (uint32)

    YANG Description: CPU EGID value
    """
    return self.__egid

  def _set_egid(self, v, load=False):
    """
    Setter method for egid, mapped from YANG variable /cpu_info_state/egid (uint32)
    If this variable is read-only (config: false) in the
    source YANG file, then _set_egid is considered as a private
    method. Backends looking to populate this variable should
    do so via calling thisObj._set_egid() directly.

    YANG Description: CPU EGID value
    """
    if hasattr(v, "_utype"):
      v = v._utype(v)
    try:
      t = YANGDynClass(v,base=RestrictedClassType(base_type=long, restriction_dict={'range': ['0..4294967295']}, int_size=32), is_leaf=True, yang_name="egid", rest_name="egid", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='uint32', is_config=False)
    except (TypeError, ValueError):
      raise ValueError({
          'error-string': """egid must be of a type compatible with uint32""",
          'defined-type': "uint32",
          'generated-type': """YANGDynClass(base=RestrictedClassType(base_type=long, restriction_dict={'range': ['0..4294967295']}, int_size=32), is_leaf=True, yang_name="egid", rest_name="egid", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='uint32', is_config=False)""",
        })

    self.__egid = t
    if hasattr(self, '_set'):
      self._set()

  def _unset_egid(self):
    self.__egid = YANGDynClass(base=RestrictedClassType(base_type=long, restriction_dict={'range': ['0..4294967295']}, int_size=32), is_leaf=True, yang_name="egid", rest_name="egid", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='uint32', is_config=False)

  def _get_group_id(self):
    """
    Getter method for group_id, mapped from YANG variable /cpu_info_state/group_id (int8)

    YANG Description: CPU Group ID
    """
    return self.__group_id

  def _set_group_id(self, v, load=False):
    """
    Setter method for group_id, mapped from YANG variable /cpu_info_state/group_id (int8)
    If this variable is read-only (config: false) in the
    source YANG file, then _set_group_id is considered as a private
    method. Backends looking to populate this variable should
    do so via calling thisObj._set_group_id() directly.

    YANG Description: CPU Group ID
    """
    parent = getattr(self, "_parent", None)
    if parent is not None and load is False:
      raise AttributeError("Cannot set keys directly when" +
                             " within an instantiated list")

    if hasattr(v, "_utype"):
      v = v._utype(v)
    try:
      t = YANGDynClass(v,base=RestrictedClassType(base_type=int, restriction_dict={'range': ['-128..127']}, int_size=8), is_leaf=True, yang_name="group-id", rest_name="group-id", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, is_keyval=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='int8', is_config=False)
    except (TypeError, ValueError):
      raise ValueError({
          'error-string': """group_id must be of a type compatible with int8""",
          'defined-type': "int8",
          'generated-type': """YANGDynClass(base=RestrictedClassType(base_type=int, restriction_dict={'range': ['-128..127']}, int_size=8), is_leaf=True, yang_name="group-id", rest_name="group-id", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, is_keyval=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='int8', is_config=False)""",
        })

    self.__group_id = t
    if hasattr(self, '_set'):
      self._set()

  def _unset_group_id(self):
    self.__group_id = YANGDynClass(base=RestrictedClassType(base_type=int, restriction_dict={'range': ['-128..127']}, int_size=8), is_leaf=True, yang_name="group-id", rest_name="group-id", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, is_keyval=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='int8', is_config=False)

  def _get_description(self):
    """
    Getter method for description, mapped from YANG variable /cpu_info_state/description (string)

    YANG Description: Description of CPU EGID
    """
    return self.__description

  def _set_description(self, v, load=False):
    """
    Setter method for description, mapped from YANG variable /cpu_info_state/description (string)
    If this variable is read-only (config: false) in the
    source YANG file, then _set_description is considered as a private
    method. Backends looking to populate this variable should
    do so via calling thisObj._set_description() directly.

    YANG Description: Description of CPU EGID
    """
    if hasattr(v, "_utype"):
      v = v._utype(v)
    try:
      t = YANGDynClass(v,base=unicode, is_leaf=True, yang_name="description", rest_name="description", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='string', is_config=False)
    except (TypeError, ValueError):
      raise ValueError({
          'error-string': """description must be of a type compatible with string""",
          'defined-type': "string",
          'generated-type': """YANGDynClass(base=unicode, is_leaf=True, yang_name="description", rest_name="description", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='string', is_config=False)""",
        })

    self.__description = t
    if hasattr(self, '_set'):
      self._set()

  def _unset_description(self):
    self.__description = YANGDynClass(base=unicode, is_leaf=True, yang_name="description", rest_name="description", parent=self, path_helper=self._path_helper, extmethods=self._extmethods, register_paths=True, namespace='urn:brocade.com:mgmt:brocade-qos-operational', defining_module='brocade-qos-operational', yang_type='string', is_config=False)

  name = __builtin__.property(_get_name)
  egid = __builtin__.property(_get_egid)
  group_id = __builtin__.property(_get_group_id)
  description = __builtin__.property(_get_description)

  _pyangbind_elements = {'name': name, 'egid': egid, 'group_id': group_id, 'description': description, }