/** svd.cc
    Jeremy Barnes, 16 December 2014
    Copyright (c) 2014 mldb.ai inc.  All rights reserved.

    This file is part of MLDB. Copyright 2015 mldb.ai inc. All rights reserved.

    Implementation of an SVD algorithm for embedding of a dataset.
*/

#include "svd.h"
#include "mldb/builtin/matrix.h"
#include "mldb/core/mldb_engine.h"
#include "mldb/core/dataset.h"
#include "mldb/utils/distribution.h"
#include <boost/multi_array.hpp>
#include "mldb/base/scope.h"
#include "mldb/base/parallel.h"
#include "mldb/utils/pair_utils.h"
#include "mldb/arch/timers.h"
#include "mldb/arch/simd_vector.h"
#include "mldb/builtin/intersection_utils.h"
#include "mldb/utils/vector_utils.h"
#include "mldb/ext/svdlibc/svdlib.h"
#include "mldb/types/basic_value_descriptions.h"
#include "mldb/types/distribution_description.h"
#include "mldb/types/optional_description.h"
#include "mldb/sql/sql_expression.h"
#include "mldb/builtin/sql_config_validator.h"
#include "mldb/vfs/fs_utils.h"
#include "mldb/types/map_description.h"
#include "mldb/types/any_impl.h"
#include "mldb/types/annotated_exception.h"
#include "mldb/types/hash_wrapper_description.h"
#include "mldb/vfs/filter_streams.h"
#include "mldb/utils/progress.h"
#include "mldb/utils/log.h"
#include <sstream>

using namespace std;

namespace MLDB {

// jsonDecode implementation for any type which:
// 1) has a default description;
template<typename T>
T jsonDecodeFile(const std::string & filename, T * = 0)
{
    T result;

    MLDB::filter_istream stream(filename);

    static auto desc = MLDB::getDefaultDescriptionSharedT<T>();
    MLDB::StreamingJsonParsingContext context(filename, stream);
    desc->parseJson(&result, context);
    return result;
}

DEFINE_STRUCTURE_DESCRIPTION(SvdConfig);

SvdConfigDescription::
SvdConfigDescription()
{
    Optional<PolyConfigT<Dataset> > optionalOutputDataset;
    optionalOutputDataset.emplace(PolyConfigT<Dataset>().
                                  withType(SvdConfig::defaultOutputDatasetType));

    addField("trainingData", &SvdConfig::trainingData,
             "Specification of the data for input to the SVD Procedure.  This should be "
             "organized as an embedding, with each selected row containing the same "
             "set of columns with numeric values to be used as coordinates.  The select statement "
             "does not support groupby and having clauses.");
    addField("columnOutputDataset", &SvdConfig::columnOutput,
             "Output dataset for embedding (column singular vectors go here)",
             optionalOutputDataset);
    addField("rowOutputDataset", &SvdConfig::rowOutput,
             "Output dataset for embedding (row singular vectors go here)",
             optionalOutputDataset);
    addField("numSingularValues", &SvdConfig::numSingularValues,
             "Maximum number of singular values to work with.  "
             "If there are not enough "
             "degrees of freedom in the dataset (it is rank-deficient), then "
             "less than this number may be used", 100);
    addField("numDenseBasisVectors", &SvdConfig::numDenseBasisVectors,
             "Maximum number of dense basis vectors to use for the SVD.  "
             "This parameter gives the number of dimensions into which the "
             "projection is made.  Higher values may allow the SVD to model "
             "slightly more diverse behaviour.  "
             "The runtime goes up with the square of this parameter, "
             "in other words 10 times as many is 100 times as long to run.",
             2000);
    addField("outputColumn", &SvdConfig::outputColumn,
             "Base name of the column that will be written by the SVD.  "
             "It will be an embedding with numSingularValues elements.",
             PathElement("embedding"));
    addField("modelFileUrl", &SvdConfig::modelFileUrl,
             "URL where the model file (with extension '.svd') should be saved. "
             "This file can be loaded by the ![](%%doclink svd.embedRow function). "
             "This parameter is optional unless the `functionName` parameter is used.");
    addField("functionName", &SvdConfig::functionName,
             "If specified, an instance of the ![](%%doclink svd.embedRow function) of this name will be created using "
             "the trained model. Note that to use this parameter, the `modelFileUrl` must "
             "also be provided.");
    addParent<ProcedureConfig>();

    onPostValidate = chain(validateQuery(&SvdConfig::trainingData,
                                         NoGroupByHaving(),
                                         MustContainFrom()),
                           validateFunction<SvdConfig>());
}

DEFINE_STRUCTURE_DESCRIPTION(SimpleIntersectionEntry);

SimpleIntersectionEntryDescription::
SimpleIntersectionEntryDescription()
{
    addParent<ColumnSpec>();
    addField("singularVector", &SimpleIntersectionEntry::singularVector,
             "Singular vector for this column");
}

DEFINE_STRUCTURE_DESCRIPTION(SvdColumnIndexEntry);

SvdColumnIndexEntryDescription::
SvdColumnIndexEntryDescription()
{
    addField("columnName", &SvdColumnIndexEntry::columnName,
             "Name of the column");
    addField("values", &SvdColumnIndexEntry::values,
             "Values of fields for this column");
}

/** Given the other column, project it onto the basis. */
distribution<float>
SvdBasis::
rightSingularVector(const ColumnIndexEntries & basisColumns,
                    const ColumnIndexEntry & column,
                    shared_ptr<spdlog::logger> logger) const
{
    // For each basis vector, calculate the overlap
    distribution<float> result(singularValues.size());

    for (unsigned i = 0;  i < columns.size();  ++i) {
        double overlap = column.correlation(basisColumns.at(i));
        if (overlap == 0.0)
            continue;
        result += overlap * columns[i].singularVector;
    }

    result /= singularValues * singularValues;

    if (logger->should_log(spdlog::level::trace) && (result == 0.0).all()) {
        logger->trace() <<  "all zero projection"
                        << "column " << column.columnName;

        for (unsigned i = 0;  i < columns.size();  ++i) {
            double overlap = column.correlation(basisColumns.at(i));
            logger->trace() << "overlap with " << columns[i].columnName
                            << " is " << overlap;
        }

    }

    return result;
}

distribution<float>
SvdBasis::
rightSingularVectorForColumn(ColumnHash col, const CellValue & value,
                             int maxValues,
                             bool acceptUnknownValues,
                             shared_ptr<spdlog::logger> logger) const
{
    if (maxValues < 0 || maxValues > singularValues.size())
        maxValues = singularValues.size();

    // 1.  Find the columns involved with the index
    auto it = columnIndex.find(col);
    if (it == columnIndex.end()) {
        TRACE_MSG(logger) << "column " << col << " not found in "
                          << columnIndex.size() << " entries";
        return distribution<float>();
    }

    // 2.  Look up the value of the cell
    const SvdColumnIndexEntry & columnEntry = it->second;

    auto it2 = columnEntry.values.find(value);
    if (it2 == columnEntry.values.end()) {
        // Extract the cell value...
        if (value.isNumeric()) {
            double d = value.toDouble();

            TRACE_MSG(logger) << "looking into " << columnEntry.values.size() << " values"
                              << " for column " << it->second.columnName;

            for (auto & e: columnEntry.values) {
                if (logger->should_log(spdlog::level::trace)) {
                    logger->trace() << "e.first = " << jsonEncodeStr(e.first);
                    logger->trace() << "e.first.cellType() = " << jsonEncodeStr(e.first.cellType());
                    logger->trace() << "e.second = " << e.second;
                }

                if (!e.first.empty() && e.first != value)
                    continue;
                int columnNum = e.second;
                auto & col = columns[columnNum];
                distribution<float> result = col.singularVector;
                result.resize(maxValues);

                double oldd = d;

                d += col.offset;
                d *= col.scale;

                TRACE_MSG(logger) << "value " << oldd << " transformed to " << d;

                result *= d;
                return result;
            }

            if (columnEntry.values.size() == 1) {
                // MLDB-687
                // We saw only a single value in training, which makes it
                // essentially a "has value" or "doesn't have value"
                // feature.  Take the output for the one and only value
                // seen in training.

                auto & col = columns[columnEntry.values.begin()->second];

                distribution<float> result = col.singularVector;
                result.resize(maxValues);

                double oldd = d;

                d += col.offset;
                d *= col.scale;

                TRACE_MSG(logger) << "value " << oldd << " transformed to " << d;

                result *= d;
                return result;
            }

        }

        if (acceptUnknownValues) {
            DEBUG_MSG(logger) << "Numeric value not found";
            return distribution<float>();
        }

        Json::Value details;
        details["columnName"] = jsonEncode(it->second.columnName);
        details["columnValueNotFound"] = jsonEncode(value);
        details["columnValueNotFoundType"] = jsonEncode(value.cellType());

        auto & vals = details["firstTenknownColumnValues"];

        bool expectedNumber = false;
        bool onlyNumber = true;
        for (auto & v: it->second.values) {
            if (vals.size() >= 10)
                break;

            auto & col = columns.at(v.second);

            Json::Value thisVal;
            if (col.op == COL_VALUE) {
                thisVal = "<<any number>>";
                expectedNumber = true;
            }
            else {
                thisVal = jsonEncode(v.first);
                onlyNumber = false;
            }

            vals.append(thisVal);
        }

        Utf8String message;
        if (value.isNumeric()) {
            message = "Column '" + it->second.columnName.toUtf8String()
                + "' was a string in training but has numeric value "
                + jsonEncodeUtf8(value) + " when passed to SVD";
        }
        else if (expectedNumber && onlyNumber) {
            message = "Column '" + it->second.columnName.toUtf8String()
                + "' passed as a string value " + jsonEncodeUtf8(value)
                + " but only numbers were seen in training";
        }
        else {
            message = "Column '" + it->second.columnName.toUtf8String()
                + "' passed a value " + jsonEncodeUtf8(value)
                + " that was never seen in training when passed to SVD";
        }

        DEBUG_MSG(logger) << details;

        throw AnnotatedException(400, message, details);
    }

    distribution<float> result = columns[it2->second].singularVector;
    result.resize(maxValues);
    return result;
}

std::pair<distribution<float>, Date>
SvdBasis::
leftSingularVector(const std::vector<std::tuple<ColumnPath, CellValue, Date> > & row,
                   int maxValues,
                   bool acceptUnknownValues,
                   shared_ptr<spdlog::logger> logger) const
{
    return doLeftSingularVector(row, maxValues, acceptUnknownValues, logger);
}

std::pair<distribution<float>, Date>
SvdBasis::
leftSingularVector(const std::vector<std::tuple<ColumnHash, CellValue, Date> > & row,
                   int maxValues,
                   bool acceptUnknownValues,
                   shared_ptr<spdlog::logger> logger) const
{
    return doLeftSingularVector(row, maxValues, acceptUnknownValues, logger);
}

template<typename Tuple>
std::pair<distribution<float>, Date>
SvdBasis::
doLeftSingularVector(const std::vector<Tuple> & row,
                     int maxValues,
                     bool acceptUnknownValues,
                     shared_ptr<spdlog::logger> logger) const
{
    if (maxValues < 0 || maxValues > singularValues.size())
        maxValues = singularValues.size();

    Date ts = modelTs;
    distribution<float> result(maxValues);

    for (auto & v: row) {
        ColumnHash column;
        CellValue value;
        Date columnTs;

        std::tie(column, value, columnTs) = v;

        const distribution<float> & rsv
            = rightSingularVectorForColumn(column, value, maxValues, acceptUnknownValues, logger);

        // If it was excluded, it will have an empty vector calculated
        if (rsv.empty())
            continue;
        result += rsv;
        ts.setMax(columnTs);
    }

    for (unsigned i = 0;  i < result.size();  ++i)
        result[i] /= singularValues[i];

    return make_pair(std::move(result), ts);
}

void
SvdBasis::
validate()
{
    for (unsigned i = 0;  i < columns.size();  ++i) {
        auto & c = columns[i];
        ExcAssert(columnIndex.count(c.columnName));
        ExcAssert(columnIndex[c.columnName].values.count(c.cellValue));
        ExcAssertEqual(columnIndex[c.columnName].columnName, c.columnName);
        ExcAssertEqual(columnIndex[c.columnName].values[c.cellValue], i);
        if (c.op == COL_VALUE)
            ExcAssertEqual(c.cellValue, CellValue());
    }
}

DEFINE_STRUCTURE_DESCRIPTION(SvdBasis);

SvdBasisDescription::
SvdBasisDescription()
{
    addField("columns", &SvdBasis::columns, "Columns of SVD");
    addField("singularValues", &SvdBasis::singularValues,
             "Singular values of SVD");
    addField("columnIndex", &SvdBasis::columnIndex, "Index of columns");
    addField("modelTs", &SvdBasis::modelTs, "Timestamp of latest information incorporated into model");
}

struct SvdTrainer {
    static SvdBasis calcSvdBasis(const ColumnCorrelations & correlations,
                                 int numSingularValues,
                                 shared_ptr<spdlog::logger> logger);

    static SvdBasis calcRightSingular(const ClassifiedColumns & columns,
                                      const ColumnIndexEntries & columnIndex,
                                      const SvdBasis & svd,
                                      shared_ptr<spdlog::logger> logger);
};

SvdBasis
SvdTrainer::
calcSvdBasis(const ColumnCorrelations & correlations,
             int numSingularValues,
             shared_ptr<spdlog::logger> logger)
{
#if 0
    static int n = 0;
    {
        INFO_MSG(logger) << "saving correlations " << n;
        filter_ostream stream(MLDB::format("correlations-%d.json", n++));
        stream << jsonEncode(correlations.columns);
        for (unsigned i = 0;  i < correlations.correlations.shape()[0];  ++i) {
            for (unsigned j = 0;  j < correlations.correlations.shape()[1];  ++j) {
                stream << i << " " << j << " " << correlations.correlations[i][j]
                       << endl;
            }
        }
        INFO_MSG(logger) << "done saving correlations ";
    }
#endif

    int ndims = correlations.columnCount();

    Timer timer;

    if (logger->should_log(spdlog::level::trace)) {
        for (unsigned i = 0;  i < ndims;  ++i) {
            logger->trace() << "correlation between " << 0 << " and "
                            << i << " is " << correlations.correlations[0][i];
        }
    }

    /**************************************************************
     * multiplication of matrix B by vector x, where B = A'A,     *
     * and A is nrow by ncol (nrow >> ncol). Hence, B is of order *
     * n = ncol (y stores product vector).		              *
     **************************************************************/

    auto opb_fn = [&] (const double * x, double * y)
    {
        for (unsigned i = 0; i != ndims; i++) {
            y[i] = MLDB::SIMD::vec_dotprod_dp(&correlations.correlations[i][0], x, ndims);
        }
    };

    SVDParams params;
    params.opb = opb_fn;
    params.ierr = 0;
    params.nrows = ndims;
    params.ncols = ndims;
    params.nvals = 0;
    params.doU = false;
    params.calcPrecision(params.ncols);

    svdrec * svdResult = svdLAS2A(numSingularValues, params);
    Scope_Exit(svdFreeSVDRec(svdResult));

    INFO_MSG(logger) << "done SVD " << timer.elapsed();

    // It doesn't clean up the ones that didn't converge properly... do it ourselves
    // We go until we get a NaN or one with too small a ratio.
    // Eg, seen in the wild:
    // svalues = { 3.06081 2.01797 1.91045 1.39165 1.20556 1.0859 1.01295 0.973041 0.96686 0.795663 0.787847 0.753074 0.663018 0.58732 0.566861 0.53674 0.507972 0.481893 0.476135 0.451054 0.434212 0.428739 0.406749 0.396502 0.388368 0.383147 0.381553 0.34724 0.322744 0.311273 0.297784 0.285271 0.275972 0.272025 0.271609 0.265779 0.254749 0.244108 0.234286 0.229235 0.21586 0.208849 0.207129 0.194427 0.186311 0.184302 0.18284 0.170876 0.1612 0.153722 0.145908 0.145039 0.139881 0.136478 0.134853 0.131319 0.124427 0.112027 0.0839514 0.0766772 0.0687135 0.0484199 0.0354719 0.034498 9.62614e-05 7.98612e-05 7.48308e-05 6.6479e-05 5.5881e-05 5.00391e-05 4.59796e-05 4.33525e-05 3.0214e-05 2.67698e-05 2.66379e-05 1.749e-05 1.64916e-05 1.20429e-05 5.02268e-08 -nan -nan -nan -nan 2.46486e-09 -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan -nan 1.61711e-08 }

    unsigned realD = 0;
    while (realD < svdResult->d
           && isfinite(svdResult->S[realD])
           && svdResult->S[realD] / svdResult->S[0] > 1e-9)
        ++realD;

    INFO_MSG(logger) << "skipped " << svdResult->d - realD << " bad singular values";
    ExcAssertLessEqual(realD, svdResult->d);
    ExcAssertLessEqual(realD, numSingularValues);
    svdResult->d = realD;

    INFO_MSG(logger) << "got " << svdResult->d << " singular values";

    numSingularValues = svdResult->d;

    TRACE_MSG(logger) << "Vt rows " << svdResult->Vt->rows;
    TRACE_MSG(logger) << "Vt cols " << svdResult->Vt->cols;

    SvdBasis result;
    result.modelTs = correlations.modelTs;
    result.singularValues.resize(numSingularValues);
    std::copy(svdResult->S, svdResult->S + numSingularValues,
              result.singularValues.begin());

    INFO_MSG(logger) << "svalues = " << result.singularValues;

    result.columns.resize(ndims);
    std::copy(correlations.columns.begin(), correlations.columns.end(),
              result.columns.begin());

#if 1
    // Extract the singular vectors for the dense behaviours
    for (unsigned i = 0;  i < ndims;  ++i) {

        distribution<float> & d = result.columns[i].singularVector;
        d.resize(numSingularValues);
        for (unsigned j = 0;  j < numSingularValues;  ++j)
            d[j] = svdResult->Vt->value[j][i];

        ColumnPath columnName = result.columns[i].columnName;
        CellValue cellValue = result.columns[i].cellValue;

        result.columnIndex[columnName].values[cellValue] = i;
        result.columnIndex[columnName].columnName = columnName;

        //std::copy(svdResult->Vt->value[i],
        //          svdResult->Vt->value[i] + numSingularValues,
        //          d.begin());
    }
#endif

    TRACE_MSG(logger) << "result.columnIndex.size() = " << result.columnIndex.size();
    TRACE_MSG(logger) << "ndims = " << ndims;

    for (auto & i: result.columnIndex)
        ExcAssertNotEqual(i.second.columnName, ColumnPath());

    if (logger->should_log(spdlog::level::trace)) {
        logger->trace() << "Testing the orthonormal-ness of the singular vectors";

        for (unsigned i = 0;  i < ndims;  ++i) {
            const auto & v = result.columns[i].singularVector;
              logger->trace() << "dim " << i << ": two_norm = " << v.two_norm() << " min " << v.min()
                              << " max " << v.max() << " one_norm " << v.total();
        }

        for (unsigned i = 0;  i < ndims;  ++i) {
              logger->trace() << "dim 0 with dim " << i << ": two_norm = "
                              << result.columns[i].singularVector.dotprod(result.columns[0].singularVector);
        }
    }

    return result;
}

SvdBasis
SvdTrainer::
calcRightSingular(const ClassifiedColumns & columns,
                  const ColumnIndexEntries & columnIndex,
                  const SvdBasis & svd,
                  shared_ptr<spdlog::logger> logger)

{
    DEBUG_MSG(logger) << "project extra " << columns.continuousColumns.size() - svd.columns.size()
                      << " continuous columns onto basis";
    DEBUG_MSG(logger) << "projecting " << columns.sparseColumns.size()
                      << " discrete columns onto basis";

    Timer timer;

    std::atomic<int> numDone(0);
    double lastSeconds = 0.0;

    size_t totalColumns = columns.continuousColumns.size() + columns.sparseColumns.size();

    SvdBasis result(svd);
    result.modelTs = svd.modelTs;
    result.columns.resize(totalColumns);

    auto outputFirstRightSingular = [&svd, logger](const ColumnIndexEntries & columnIndex) {
        stringstream output;

        // Calc the right singular of the first continuous column
        auto vec0 = svd.rightSingularVector(columnIndex, columnIndex[0], logger);
        auto svec = svd.columns[0].singularVector;

        vec0.resize(10);
        svec.resize(10);

        output << "vec0 = " << vec0 << " svec = " << svec;
        return output.str();
    };

    if (logger->should_log(spdlog::level::debug) && svd.columns.size() > 0)
        logger->debug() << outputFirstRightSingular(columnIndex);

    auto calcRightSingular = [&] (int i)
        {
            int done = numDone.fetch_add(1);

            // If we've already done it, then skip
            if (!result.columns[i].singularVector.empty())
                return;

            auto vec = svd.rightSingularVector(columnIndex, columnIndex[i], logger);

            SimpleIntersectionEntry column;
            column = columnIndex[i];
            column.singularVector = std::move(vec);

            result.columns[i] = std::move(column);

            if (done % 1000 == 0) {
                double seconds = timer.elapsed_wall();
                INFO_MSG(logger) << "done " << done << " of " << totalColumns
                     << " in " << timer.elapsed();
                INFO_MSG(logger) << "Average " << done / seconds << " per second; inst "
                     << 1000 / (seconds - lastSeconds) << " per second";
                lastSeconds = seconds;
            }
        };

    parallelMap(0, totalColumns, calcRightSingular);

    for (unsigned i = 0;  i < totalColumns;  ++i) {
        ColumnPath columnName = result.columns[i].columnName;
        CellValue cellValue = result.columns[i].cellValue;
        if (result.columns[i].op == COL_VALUE)
            ExcAssertEqual(cellValue, CellValue());

        TRACE_MSG(logger) << "column " << i << " name " << columnName
                          << " value " << jsonEncodeStr(cellValue) << " op "
                          << result.columns[i].op;

        result.columnIndex[columnName].values[cellValue] = i;
        result.columnIndex[columnName].columnName = columnName;
    }

    for (auto & i: result.columnIndex) {
        ExcAssertNotEqual(i.second.columnName, ColumnPath());
        for (auto & v: i.second.values) {
            auto col = result.columns.at(v.second);
            auto val = v.first;

            ExcAssertEqual(val, col.cellValue);
            ExcAssertEqual(i.second.columnName, col.columnName);

            if (col.op == COL_VALUE)
                ExcAssertEqual(val, CellValue());
        }
    }

    ExcAssertLessEqual(result.columnIndex.size(), result.columns.size());

    for (auto & c: result.columns) {
        ExcAssertNotEqual(c.columnName, ColumnPath());
    }

    return result;
}

/*****************************************************************************/
/* SVD PROCEDURE                                                              */
/*****************************************************************************/

SvdProcedure::
SvdProcedure(MldbEngine * owner,
            PolyConfig config,
            const std::function<bool (const Json::Value &)> & onProgress)
    : Procedure(owner)
{
    this->svdConfig = config.params.convert<SvdConfig>();
}

Any
SvdProcedure::
getStatus() const
{
    return Any();
}

RunOutput
SvdProcedure::
run(const ProcedureRunConfig & run,
      const std::function<bool (const Json::Value &)> & onProgress) const
{
    auto runProcConf = applyRunConfOverProcConf(svdConfig, run);

    if (runProcConf.outputColumn.null()) {
        throw AnnotatedException
            (400, "SVD training procedure requires a non-empty output column name",
             "config", runProcConf);
    }

    if (!runProcConf.modelFileUrl.empty()) {
        checkWritability(runProcConf.modelFileUrl.toDecodedString(), "modelFileUrl");
    }

    int numBasisVectors = runProcConf.numDenseBasisVectors;

    SqlExpressionMldbScope context(engine);

    ConvertProgressToJson convertProgressToJson(onProgress);
    auto dataset = runProcConf.trainingData.stm->from->bind(context, convertProgressToJson).dataset;

    Progress svdProgress;
    std::shared_ptr<Step> classificationStep = svdProgress.steps({
            make_pair("classifying columns", "percentile"),
            make_pair("extracting features", "percentile"),
            make_pair("inverting features", "percentile"),
            make_pair("calculating correlations", "percentile")
            });

    ClassifiedColumns columns = classifyColumns(runProcConf.trainingData.stm->select,
                                                *dataset,
                                                runProcConf.trainingData.stm->when,
                                                *runProcConf.trainingData.stm->where,
                                                runProcConf.trainingData.stm->orderBy,
                                                runProcConf.trainingData.stm->offset,
                                                runProcConf.trainingData.stm->limit,
                                                logger,
                                                convertProgressToJson);

    auto outputColumns = [](const ClassifiedColumns & columns) {
        stringstream output;
        output << "continuous columns are:" << endl;
        for (auto & c: columns.continuousColumns) {
            output << "name " << c.columnName << " val " << c.cellValue << " op " << c.op
                   << " rowCount " << c.rowCount << endl;;
        }
        output << "sparse columns are:" << endl;
        for (auto & c: columns.sparseColumns) {
            output << "name " << c.columnName << " val " << c.cellValue << " op " << c.op
                   << " rowCount " << c.rowCount << endl;
        }
        return output.str();
    };

    DEBUG_MSG(logger) <<  "classified columns: " << columns.continuousColumns.size()
                      << " continuous columns, " << columns.sparseColumns.size()
                      << " sparse columns";
    DEBUG_MSG(logger) << outputColumns(columns);

    auto extractionStep = classificationStep->nextStep(1);

    FeatureBuckets extractedFeatures = extractFeaturesFromRows(runProcConf.trainingData.stm->select,
                                                               *dataset,
                                                               runProcConf.trainingData.stm->when,
                                                               runProcConf.trainingData.stm->where,
                                                               runProcConf.trainingData.stm->orderBy,
                                                               runProcConf.trainingData.stm->offset,
                                                               runProcConf.trainingData.stm->limit,
                                                               columns,
                                                               logger,
                                                               convertProgressToJson);

    auto inversionStep = extractionStep->nextStep(1);

    ColumnIndexEntries columnIndex = invertFeatures(columns, extractedFeatures, logger, convertProgressToJson);

    ColumnCorrelations correlations = calculateCorrelations(columnIndex, numBasisVectors, logger);
    SvdBasis svd = SvdTrainer::calcSvdBasis(correlations,
                                            runProcConf.numSingularValues,
                                            logger);

    auto outputSvdColumns = [](const SvdBasis & basis) {
        stringstream output;
         for (auto & c: basis.columns) {
             output << "name " << c.columnName << " val " << c.cellValue << " op " << c.op << endl;
         }
        return output.str();
    };

    DEBUG_MSG(logger) << "----------- SVD columns";
    DEBUG_MSG(logger) << outputSvdColumns(svd);

    SvdBasis allSvd = SvdTrainer::calcRightSingular(columns, columnIndex, svd, logger);

    DEBUG_MSG(logger) << "----------- ALL SVD columns";
    DEBUG_MSG(logger) << outputSvdColumns(allSvd);

    if (!runProcConf.modelFileUrl.empty()) {
        makeUriDirectory(runProcConf.modelFileUrl.toDecodedString());
        filter_ostream stream(runProcConf.modelFileUrl);
        jsonEncodeToStream(allSvd, stream);
    }

    Date ts;  // TODO: actually fill this in...

    // Save the column embedding to a dataset if we ask for it
    if (runProcConf.columnOutput) {

        PolyConfigT<Dataset> columnOutput = *runProcConf.columnOutput;
        if (columnOutput.type.empty())
            columnOutput.type = SvdConfig::defaultOutputDatasetType;

        auto onProgress2 = [&] (const Json::Value & progress) {
            Json::Value value;
            value["dataset"] = progress;
            return onProgress(value);
        };

        auto output = createDataset(engine, columnOutput, onProgress2, true /*overwrite*/);

        auto doColumn = [&] (size_t i)
            {
                auto & col = allSvd.columns[i];

                if (i % 10000 == 0)
                    INFO_MSG(logger) << "saving column " << i << " of " << allSvd.columns.size();

                ColumnPath outputName = col.columnName;

                if (col.op == COL_EQUAL) {
                    if (col.cellValue.empty()) {
                        outputName = outputName + PathElement("isNull");
                    }
                    else if (col.cellValue.isNumber()) {
                        outputName = outputName + PathElement("numberEquals")
                            + col.cellValue.toUtf8String();
                    }
                    else if (col.cellValue.isString()) {
                        if (col.cellValue.toUtf8String().empty()) {
                            outputName = outputName + PathElement("stringEmpty");
                        }
                        else {
                            outputName = outputName + PathElement("stringEquals")
                                + col.cellValue.toUtf8String();
                        }
                    }
#if 0
                    else if (col.cellValue.isTimestamp()) {
                    }
                    else if (col.cellValue.isTimeinterval()) {
                    }
                    else if (col.cellValue.isBlob()) {
                    }
#endif
                    else {
                        throw AnnotatedException
                            (400,"Can't apply an SVD to a column that's not "
                             "numeric or categorical (string)",
                             "columnValue", col.cellValue,
                             "columnName", col.columnName);
                    }
                }
                else if (col.op == COL_VALUE) {
                    outputName = outputName + PathElement("numericValue");
                }
                try {
                    StructValue cols;
                    cols.emplace_back(runProcConf.outputColumn,
                                      ExpressionValue(col.singularVector, ts));
                    output->recordRowExpr(outputName, std::move(cols));
                } catch (const std::exception & exc) {
                    rethrowException(-1, "Error adding SVD column '" + outputName.toUtf8String() + "' to output: "
                                         + exc.what(),
                                         "columnName", outputName);
                }
            };

        parallelMap(0, allSvd.columns.size(), doColumn);

        output->commit();
    }

    // Save the row embedding to a dataset.  This is optional (it's not actually
    // needed to apply the SVD).

    if (runProcConf.rowOutput) {

        PolyConfigT<Dataset> rowOutput = *runProcConf.rowOutput;
        if (rowOutput.type.empty())
            rowOutput.type = SvdConfig::defaultOutputDatasetType;

        auto onProgress2 = [&] (const Json::Value & progress) {
            Json::Value value;
            value["dataset"] = progress;
            return onProgress(value);
        };
        auto output = createDataset(engine, rowOutput, onProgress2, true /*overwrite*/);

        // getRowPaths can return row names in an arbitrary order as long as it is deterministic.
        auto rows = dataset->getMatrixView()->getRowPaths(0, -1);

        DEBUG_MSG(logger) << "writing embeddings for " << rows.size() << " rows to dataset "
                          << runProcConf.rowOutput->id;

        int numSingularValues = allSvd.numSingularValues();

        auto doRow = [&] (int rowNum)
            {
                if (rowNum % 10000 == 0)
                    INFO_MSG(logger) << "saving row " << rowNum << " of " << rows.size();

                auto row = dataset->getMatrixView()->getRow(rows[rowNum]);

                distribution<float> embedding;
                Date ts;

                std::tie(embedding, ts)
                = allSvd.leftSingularVector(row.columns, numSingularValues,
                                            false /* acceptUnknownValues*/,
                                            logger);

                StructValue cols;
                cols.emplace_back(runProcConf.outputColumn,
                                  ExpressionValue(std::move(embedding), ts));
                output->recordRowExpr(row.rowName, std::move(cols));
            };

        parallelMap(0, rows.size(), doRow);

        output->commit();
    }

    if(!runProcConf.functionName.empty()) {
        PolyConfig svdFuncPC;
        svdFuncPC.type = "svd.embedRow";
        svdFuncPC.id = runProcConf.functionName;
        svdFuncPC.params = SvdEmbedConfig(runProcConf.modelFileUrl);

        createFunction(engine, svdFuncPC, onProgress, true);
    }

    return Any();
}

/*****************************************************************************/
/* SVD EMBED ROW                                                             */
/*****************************************************************************/

DEFINE_STRUCTURE_DESCRIPTION(SvdEmbedConfig);

SvdEmbedConfigDescription::
SvdEmbedConfigDescription()
{
    addField("modelFileUrl", &SvdEmbedConfig::modelFileUrl,
             "URL of the model file (with extension '.svd') to load. "
             "This file is created by the ![](%%doclink svd.train procedure).");
    addField("maxSingularValues", &SvdEmbedConfig::maxSingularValues,
             "Maximum number of singular values to use (-1 = all)", -1);
    addField("acceptUnknownValues", &SvdEmbedConfig::acceptUnknownValues,
             "This parameter (which defaults to false) tells us whether or "
             "not unknown values should be accepted by the SVD.  An unknown "
             "value occurs when a column that was always a number in training "
             "is presented with a string value, or vice versa, or when a "
             "string valued column is presented with a value unknown in "
             "training.  If its value is true, an unknown value will be "
             "silently ignored.  If its value is false, an unknown value "
             "will return an error when the function is applied.",
             false);
}

DEFINE_STRUCTURE_DESCRIPTION(SvdInput);

SvdInputDescription::
SvdInputDescription()
{
    addField("row", &SvdInput::row,
             "Row to apply SVD to.  The embedding will be the average of "
             "the SVD of each of the keys of the row.");
}

DEFINE_STRUCTURE_DESCRIPTION(SvdOutput);

SvdOutputDescription::
SvdOutputDescription()
{
    addField("embedding", &SvdOutput::embedding,
             "Embedding of the row into the vector space defined by the "
             "SVD.  There will be a number of coordinates equal to the "
             "`maxSingluarValues` value of the configuration, or if not "
             "set, the number of SVD coordinates available in the SVD model "
             "file.  PathElementinates will have simple numerical names.");
}

SvdEmbedRow::
SvdEmbedRow(MldbEngine * owner,
            PolyConfig config,
            const std::function<bool (const Json::Value &)> & onProgress)
    : BaseT(owner, config)
{
    functionConfig = config.params.convert<SvdEmbedConfig>();
    svd = jsonDecodeFile<SvdBasis>(functionConfig.modelFileUrl.toDecodedString());

    std::map<ColumnHash, SvdColumnIndexEntry> columnIndex2;

    // Deal with the older version of the file
    for (auto & c: svd.columnIndex) {
        columnIndex2[c.second.columnName] = std::move(c.second);
    }

    svd.columnIndex = std::move(columnIndex2);

    svd.validate();

    nsv = functionConfig.maxSingularValues;
    if (nsv < 0 || nsv > svd.numSingularValues())
        nsv = svd.numSingularValues();
}

SvdOutput
SvdEmbedRow::
call(SvdInput input) const
{
    RowValue row;
    input.row.mergeToRowDestructive(row);

    distribution<float> embedding;
    Date ts;

    std::tie(embedding, ts)
        = svd.leftSingularVector(row, nsv,
                                 functionConfig.acceptUnknownValues,
                                 logger);

    DEBUG_MSG(logger) << "nsv = " << nsv;
    DEBUG_MSG(logger) << "embedding = " << embedding;

    // TODO: what to do when no embedding is returned?
    if (embedding.empty())
        embedding.resize(nsv);

    SvdOutput result;
    result.embedding = ExpressionValue(std::move((vector<float> &)embedding), ts);

    return result;
}

namespace {

RegisterProcedureType<SvdProcedure, SvdConfig>
regSvd(builtinPackage(),
       "Train a SVD to convert rows or columns to embedding coordinates",
       "procedures/Svd.md.html");

RegisterFunctionType<SvdEmbedRow, SvdEmbedConfig>
regSvdEmbedRow(builtinPackage(),
               "svd.embedRow",
               "Apply a trained SVD to embed a row into a coordinate space",
               "functions/SvdEmbedRow.md.html");

struct InitSvdParallelMap {
    InitSvdParallelMap()
    {
        svdParallelMap = &MLDB::parallelMap;
    }

} initSvdParallelMap;

} // file scope

} // namespace MLDB


// Boost.Geometry

// Copyright (c) 2007-2012 Barend Gehrels, Amsterdam, the Netherlands.

// This file was modified by Oracle on 2015-2017.
// Modifications copyright (c) 2015-2017 Oracle and/or its affiliates.

// Contributed and/or modified by Adam Wulkiewicz, on behalf of Oracle

// Use, modification and distribution is subject to the Boost Software License,
// Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)

#ifndef BOOST_GEOMETRY_STRATEGIES_GEOGRAPHIC_THOMAS_HPP
#define BOOST_GEOMETRY_STRATEGIES_GEOGRAPHIC_THOMAS_HPP

#include <boost/geometry/strategies/geographic/distance.hpp>
#include <boost/geometry/strategies/geographic/parameters.hpp>

namespace boost { namespace geometry
{

namespace strategy { namespace distance
{

/*!
\brief The solution of the inverse problem of geodesics on latlong coordinates,
       Forsyth-Andoyer-Lambert type approximation with second order terms.
\ingroup distance
\tparam Spheroid The reference spheroid model
\tparam CalculationType \tparam_calculation
\author See
    - Technical Report: PAUL D. THOMAS, MATHEMATICAL MODELS FOR NAVIGATION SYSTEMS, 1965
      http://www.dtic.mil/docs/citations/AD0627893
    - Technical Report: PAUL D. THOMAS, SPHEROIDAL GEODESICS, REFERENCE SYSTEMS, AND LOCAL GEOMETRY, 1970
      http://www.dtic.mil/docs/citations/AD703541
*/
template
<
    typename Spheroid = srs::spheroid<double>,
    typename CalculationType = void
>
class thomas
    : public strategy::distance::geographic
        <
            strategy::thomas, Spheroid, CalculationType
        >
{
    typedef strategy::distance::geographic
        <
            strategy::thomas, Spheroid, CalculationType
        > base_type;

public :
    inline thomas()
        : base_type()
    {}

    explicit inline thomas(Spheroid const& spheroid)
        : base_type(spheroid)
    {}
};

#ifndef DOXYGEN_NO_STRATEGY_SPECIALIZATIONS
namespace services
{

template <typename Spheroid, typename CalculationType>
struct tag<thomas<Spheroid, CalculationType> >
{
    typedef strategy_tag_distance_point_point type;
};

template <typename Spheroid, typename CalculationType, typename P1, typename P2>
struct return_type<thomas<Spheroid, CalculationType>, P1, P2>
    : thomas<Spheroid, CalculationType>::template calculation_type<P1, P2>
{};

template <typename Spheroid, typename CalculationType>
struct comparable_type<thomas<Spheroid, CalculationType> >
{
    typedef thomas<Spheroid, CalculationType> type;
};

template <typename Spheroid, typename CalculationType>
struct get_comparable<thomas<Spheroid, CalculationType> >
{
    static inline thomas<Spheroid, CalculationType> apply(thomas<Spheroid, CalculationType> const& input)
    {
        return input;
    }
};

template <typename Spheroid, typename CalculationType, typename P1, typename P2>
struct result_from_distance<thomas<Spheroid, CalculationType>, P1, P2 >
{
    template <typename T>
    static inline typename return_type<thomas<Spheroid, CalculationType>, P1, P2>::type
        apply(thomas<Spheroid, CalculationType> const& , T const& value)
    {
        return value;
    }
};

} // namespace services
#endif // DOXYGEN_NO_STRATEGY_SPECIALIZATIONS

}} // namespace strategy::distance

}} // namespace boost::geometry

#endif // BOOST_GEOMETRY_STRATEGIES_GEOGRAPHIC_THOMAS_HPP


/*=============================================================================
Copyright 2018 Pranam Lashkari <plashkari628@gmail.com>

Distributed under the Boost Software License, Version 1.0. (See accompanying
file License.txt or copy at https://www.boost.org/LICENSE_1_0.txt)
=============================================================================*/

#ifndef BOOST_ASTRONOMY_COORDINATE_ALT_AZ_HPP
#define BOOST_ASTRONOMY_COORDINATE_ALT_AZ_HPP

#include <type_traits>
#include <tuple>

#include <boost/geometry/core/cs.hpp>
#include <boost/geometry/geometries/point.hpp>
#include <boost/geometry/algorithms/transform.hpp>
#include <boost/geometry/algorithms/equals.hpp>
#include <boost/units/quantity.hpp>
#include <boost/units/systems/si/base.hpp>
#include <boost/units/systems/si/pressure.hpp>
#include <boost/units/systems/temperature/celsius.hpp>
#include <boost/units/systems/angle/degrees.hpp>
#include <boost/units/systems/si/volume.hpp>
#include <boost/units/systems/si/mass.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <boost/static_assert.hpp>

#include <boost/astronomy/detail/is_base_template_of.hpp>
#include <boost/astronomy/coordinate/ref_frame/base_frame.hpp>
#include <boost/astronomy/coordinate/rep/representation.hpp>
#include <boost/astronomy/coordinate/diff/differential.hpp>

namespace boost { namespace astronomy { namespace coordinate {

template <typename Representation, typename Differential>
struct alt_az : public base_frame<Representation, Differential>
{
    ///@cond INTERNAL
    BOOST_STATIC_ASSERT_MSG((std::is_base_of
        <spherical_representation<typename Representation::type,
        typename Representation::quantity1, typename Representation::quantity2,
        typename Representation::quantity3>, Representation>::value),
        "argument type is expected to be a spherical_representation class");
    BOOST_STATIC_ASSERT_MSG((std::is_base_of
        <spherical_coslat_differential<typename Differential::type,
        typename Differential::quantity1, typename Differential::quantity2,
        typename Differential::quantity3>, Differential>::value),
        "argument type is expected to be a spherical_coslat_differential class");
    ///@endcond

protected:
    //frame parameters
    spherical_equatorial_representation
        <
            double,
            bu::quantity<bu::degree::plane_angle>,
            bu::quantity<bu::degree::plane_angle>
        > earth_location;

    bu::quantity<bu::si::pressure> pressure = 0.0 * bu::si::pascals;
    bu::quantity<bu::celsius::temperature> temperature = 0.0 * bu::celsius::degrees;
    boost::posix_time::ptime obs_time;
    bu::quantity<bu::si::dimensionless> relative_humidity = 0.0;

public:
    alt_az() {}

    template <typename OtherRepresentation>
    alt_az(OtherRepresentation const& representation_data)
    {
        BOOST_STATIC_ASSERT_MSG((boost::astronomy::detail::is_base_template_of
         <boost::astronomy::coordinate::base_representation, OtherRepresentation>::value),
         "Invalid representation class");

        auto temp = make_spherical_representation
            <
            typename Representation::type,
            typename Representation::quantity1,
            typename Representation::quantity2,
            typename Representation::quantity3,
            typename OtherRepresentation::type,
            typename OtherRepresentation::quantity1,
            typename OtherRepresentation::quantity2,
            typename OtherRepresentation::quantity3
            >(representation_data);

        this->data = temp;
    }

    alt_az
    (
        typename Representation::quantity1 const& alt,
        typename Representation::quantity2 const& az,
        typename Representation::quantity3 const& distance
    )
    {
        this->data.set_lat_lon_dist(alt, az, distance);
    }

    alt_az
    (
        typename Representation::quantity1 const& alt,
        typename Representation::quantity2 const& az,
        typename Representation::quantity3 const& distance,
        typename Differential::quantity1 const& pm_alt,
        typename Differential::quantity2 const& pm_az_cosalt,
        typename Differential::quantity3 const& radial_velocity) :
        alt_az(alt, az, distance)
    {
        this->motion.set_dlat_dlon_coslat_ddist(pm_alt, pm_az_cosalt, radial_velocity);
    }

    template <typename OtherRepresentation, typename OtherDifferential>
    alt_az
    (
        OtherRepresentation const& representation_data,
        OtherDifferential const& differential_data
    )
    {
        BOOST_STATIC_ASSERT_MSG((boost::astronomy::detail::is_base_template_of
         <boost::astronomy::coordinate::base_representation, OtherRepresentation>::value),
         "argument type is expected to be a representation class");

        BOOST_STATIC_ASSERT_MSG((boost::astronomy::detail::is_base_template_of
          <boost::astronomy::coordinate::base_differential, OtherDifferential>::value),
            "argument type is expected to be a differential class");

        auto rep_temp = make_spherical_representation
            <
            typename Representation::type,
            typename Representation::quantity1,
            typename Representation::quantity2,
            typename Representation::quantity3,
            typename OtherRepresentation::type,
            typename OtherRepresentation::quantity1,
            typename OtherRepresentation::quantity2,
            typename OtherRepresentation::quantity3
            >(representation_data);
        this->data = rep_temp;

        auto dif_temp = make_spherical_coslat_differential
            <
            typename Differential::type,
            typename Differential::quantity1,
            typename Differential::quantity2,
            typename Differential::quantity3,
            typename OtherDifferential::type,
            typename OtherDifferential::quantity1,
            typename OtherDifferential::quantity2,
            typename OtherDifferential::quantity3
            >(differential_data);
        this->motion = dif_temp;
    }

    alt_az(alt_az<Representation, Differential> const& other)
    {
        this->data = other.get_data();
        this->motion = other.get_differential();
        this->earth_location = other.get_location();
        this->obs_time = other.get_obs_time();
        this->pressure = other.get_pressure();
        this->temperature = other.get_temprature();
        this->relative_humidity = other.get_relative_humidity();
    }

    //!returns altitude component of the coordinate
    typename Representation::quantity1 get_alt() const
    {
        return this->data.get_lat();
    }

    //!returns azimuth component of the coordinate
    typename Representation::quantity2 get_az() const
    {
        return this->data.get_lon();
    }

    //!returns distance component of the coordinate
    typename Representation::quantity3 get_distance() const
    {
        return this->data.get_dist();
    }

    //!returns the (alt, az, dist) in the form of tuple
    std::tuple
    <
        typename Representation::quantity1,
        typename Representation::quantity2,
        typename Representation::quantity3
    > get_alt_az_dist() const
    {
        return this->data.get_lat_lon_dist();
    }

    //!returns proper motion in altitude
    typename Differential::quantity1 get_pm_alt() const
    {
        return this->motion.get_dlat();
    }

    //!returns proper motion in azimuth including cos(alt)
    typename Differential::quantity2 get_pm_az_cosalt() const
    {
        return this->motion.get_dlon_coslat();
    }

    //!returns radial_velocity
    typename Differential::quantity3 get_radial_velocity() const
    {
        return this->motion.get_ddist();
    }

    //!returns the proper motion in tuple form
    std::tuple
    <
        typename Differential::quantity1,
        typename Differential::quantity2,
        typename Differential::quantity3
    > get_pm_alt_az_radial() const
    {
        return this->motion.get_dlat_dlon_coslat_ddist();
    }

    //!sets value of altitude component of the coordinate
    void set_alt(typename Representation::quantity1 const& alt)
    {
        this->data.set_lat(alt);
    }

    //!sets value of azimuth component of the coordinate
    void set_az(typename Representation::quantity3 const& az)
    {
        this->data.set_lon(az);
    }

    //!sets value of distance component of the coordinate
    void set_distance(typename Representation::quantity3 const& distance)
    {
        this->data.set_dist(distance);
    }

    //!sets value of all component of the coordinate including cos(alt)
    void set_alt_az_dist
    (
        typename Representation::quantity1 const& alt,
        typename Representation::quantity2 const& az,
        typename Representation::quantity3 const& dist)
    {
        this->data.set_lat_lon_dist(alt, az, dist);
    }

    //!sets the proper motion in altitude
    void set_pm_alt(typename Differential::quantity1 const& pm_alt)
    {
        this->motion.set_dlat(pm_alt);
    }

    //!sets the proper motion in azimuth including cos(alt)
    void set_pm_az_cosalt(typename Differential::quantity2 const& pm_az_cosalt)
    {
        this->motion.set_dlon_coslat(pm_az_cosalt);
    }

    //!sets the radial_velocity
    void set_radial_velocity(typename Differential::quantity3 const& radial_velocity)
    {
        this->motion.set_ddist(radial_velocity);
    }

    //!set value of motion
    void set_pm_alt_az_radial
    (
        typename Differential::quantity1 const& pm_alt,
        typename Differential::quantity2 const& pm_az_cosalt,
        typename Differential::quantity3 const& radial_velocity
    )
    {
        this->motion.set_dlat_dlon_coslat_ddist(pm_alt, pm_az_cosalt, radial_velocity);
    }

    //!set all the perameters of current frame object
    void set_frame_parameters
    (
        spherical_equatorial_representation
        <
            double,
            bu::quantity<bu::degree::plane_angle>,
            bu::quantity<bu::degree::plane_angle>
        > const& location,
        bu::quantity<bu::si::pressure> const& pressure,
        bu::quantity<bu::celsius::temperature> const& temperature,
        boost::posix_time::ptime const& obs_time,
        bu::quantity<bu::si::dimensionless> const& relative_humidity
    )
    {
        this->earth_location = location;
        this->pressure = pressure;
        this->temperature = temperature;
        this->obs_time = obs_time;
        this->relative_humidity = relative_humidity;
    }

    //!get all the perameters of current frame object
    std::tuple<
        spherical_equatorial_representation
        <
            double,
            bu::quantity<bu::degree::plane_angle>,
            bu::quantity<bu::degree::plane_angle>
        >,
        bu::quantity<bu::si::pressure>,
        bu::quantity<bu::celsius::temperature>,
        boost::posix_time::ptime,
        bu::quantity<bu::si::dimensionless>
    > get_frame_parameters() const
    {
        return std::make_tuple
        (
            this->earth_location,
            this->pressure,
            this->temperature,
            this->obs_time,
            this->relative_humidity
        );
    }

    //!get earth location of the current object
    spherical_equatorial_representation
    <
        double,
        bu::quantity<bu::degree::plane_angle>,
        bu::quantity<bu::degree::plane_angle>
    > get_location() const
    {
        return this->earth_location;
    }

    //!set earth location of the current object
    void set_location
    (
        spherical_equatorial_representation
        <
            double,
            bu::quantity<bu::degree::plane_angle>,
            bu::quantity<bu::degree::plane_angle>
        > const& location
    )
    {
        this->earth_location = location;
    }

    //!get atmospheric pressure
    bu::quantity<bu::si::pressure> get_pressure() const
    {
        return this->pressure;
    }

    //!set atmospheric pressure
    void set_pressure(bu::quantity<bu::si::pressure> const& pressure)
    {
        this->pressure = pressure;
    }

    //!get temperature of the location
    bu::quantity<bu::celsius::temperature> get_temprature() const
    {
        return this->temperature;
    }

    //!set temperature of the location
    void set_temprature(bu::quantity<bu::celsius::temperature> const& temperature)
    {
        this->temperature = temperature;
    }

    //!get observation time
    boost::posix_time::ptime get_obs_time() const
    {
        return this->obs_time;
    }

    //!set observation time
    void set_obs_time(boost::posix_time::ptime const& time)
    {
        this->obs_time = time;
    }

    //!get relative humidity
    bu::quantity<bu::si::dimensionless> get_relative_humidity() const
    {
        return this->relative_humidity;
    }

    //!set relative humidity
    void set_relative_humidity(bu::quantity<bu::si::dimensionless> const& humidity)
    {
        this->relative_humidity = humidity;
    }
};
}}} //namespace boost::astronomy::cordinate
#endif // !BOOST_ASTRONOMY_COORDINATE_ALT_AZ_HPP


/* Copyright (c) 2016 - 2021, the adamantine authors.
 *
 * This file is subject to the Modified BSD License and may not be distributed
 * without copyright and license information. Please refer to the file LICENSE
 * for the text and further information on this license.
 */

#ifndef THERMAL_PHYSICS_TEMPLATES_HH
#define THERMAL_PHYSICS_TEMPLATES_HH

#include <ThermalOperator.hh>
#if defined(ADAMANTINE_HAVE_CUDA) && defined(__CUDACC__)
#include <ThermalOperatorDevice.hh>
#endif
#include <CubeHeatSource.hh>
#include <ElectronBeamHeatSource.hh>
#include <GoldakHeatSource.hh>
#include <ThermalPhysics.hh>

#include <deal.II/base/geometry_info.h>
#include <deal.II/distributed/cell_data_transfer.templates.h>
#include <deal.II/dofs/dof_tools.h>
#include <deal.II/fe/fe_nothing.h>
#include <deal.II/fe/fe_q.h>
#include <deal.II/grid/filtered_iterator.h>
#include <deal.II/hp/fe_values.h>
#include <deal.II/hp/q_collection.h>
#include <deal.II/lac/precondition.h>
#include <deal.II/lac/solver_gmres.h>

#ifdef ADAMANTINE_WITH_CALIPER
#include <caliper/cali.h>
#endif

#include <algorithm>
#include <execution>

namespace adamantine
{
namespace
{
template <int dim, int fe_degree, typename MemorySpaceType,
          std::enable_if_t<
              std::is_same<MemorySpaceType, dealii::MemorySpace::Host>::value,
              int> = 0>
dealii::LA::distributed::Vector<double, MemorySpaceType>
evaluate_thermal_physics_impl(
    std::shared_ptr<ThermalOperatorBase<dim, MemorySpaceType>> thermal_operator,
    double const t, double const current_source_height,
    dealii::LA::distributed::Vector<double, MemorySpaceType> const &y,
    std::vector<Timer> &timers)
{
  timers[evol_time_eval_mat_prop].start();
  thermal_operator->evaluate_material_properties(y);
  timers[evol_time_eval_mat_prop].stop();

  timers[evol_time_eval_th_ph].start();
  thermal_operator->set_time_and_source_height(t, current_source_height);

  dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host> value(
      y.get_partitioner());
  value = 0.;
  // Apply the Thermal Operator.
  thermal_operator->vmult_add(value, y);

  // Multiply by the inverse of the mass matrix.
  value.scale(*thermal_operator->get_inverse_mass_matrix());

  timers[evol_time_eval_th_ph].stop();

  return value;
}

template <int dim, int fe_degree, typename MemorySpaceType,
          std::enable_if_t<
              std::is_same<MemorySpaceType, dealii::MemorySpace::Host>::value,
              int> = 0>
void init_dof_vector(
    double const value,
    dealii::LinearAlgebra::distributed::Vector<double, MemorySpaceType> &vector)
{
  unsigned int const local_size = vector.locally_owned_size();
  for (unsigned int i = 0; i < local_size; ++i)
    vector.local_element(i) = value;
}

#ifdef ADAMANTINE_HAVE_CUDA
template <int dim, int fe_degree, typename MemorySpaceType,
          std::enable_if_t<
              std::is_same<MemorySpaceType, dealii::MemorySpace::CUDA>::value,
              int> = 0>
dealii::LA::distributed::Vector<double, MemorySpaceType> vmult_and_scale(
    std::shared_ptr<ThermalOperatorBase<dim, MemorySpaceType>> const
        &thermal_operator,
    dealii::LA::distributed::Vector<double, MemorySpaceType> const &y,
    dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host> &value,
    std::vector<Timer> &timers)
{
  dealii::LA::distributed::Vector<double, MemorySpaceType> value_dev(
      value.get_partitioner());
  value_dev.import(value, dealii::VectorOperation::insert);

  // Apply the Thermal Operator.
  thermal_operator->vmult_add(value_dev, y);

  // Multiply by the inverse of the mass matrix.
  value_dev.scale(*thermal_operator->get_inverse_mass_matrix());

  timers[evol_time_eval_th_ph].stop();

  return value_dev;
}

template <int dim, int fe_degree, typename MemorySpaceType,
          std::enable_if_t<
              std::is_same<MemorySpaceType, dealii::MemorySpace::CUDA>::value,
              int> = 0>
dealii::LA::distributed::Vector<double, MemorySpaceType>
evaluate_thermal_physics_impl(
    std::shared_ptr<ThermalOperatorBase<dim, MemorySpaceType>> const
        &thermal_operator,
    dealii::hp::FECollection<dim> const &fe_collection, double const t,
    dealii::DoFHandler<dim> const &dof_handler,
    std::vector<std::shared_ptr<HeatSource<dim>>> const &heat_sources,
    double current_source_height, BoundaryType boundary_type,
    std::shared_ptr<MaterialProperty<dim>> const &material_properties,
    dealii::AffineConstraints<double> const &affine_constraints,
    dealii::LA::distributed::Vector<double, MemorySpaceType> const &y,
    std::vector<Timer> &timers)
{
  timers[evol_time_eval_mat_prop].start();
  // TODO do this on the GPU
  dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host> y_host(
      y.get_partitioner());
  y_host.import(y, dealii::VectorOperation::insert);
  thermal_operator->evaluate_material_properties(y_host);
  timers[evol_time_eval_mat_prop].stop();

  timers[evol_time_eval_th_ph].start();
  dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host> source(
      y.get_partitioner());
  source = 0.;

  // Compute the source term.
  dealii::hp::QCollection<dim> source_q_collection;
  source_q_collection.push_back(dealii::QGauss<dim>(fe_degree + 1));
  source_q_collection.push_back(dealii::QGauss<dim>(1));
  dealii::hp::FEValues<dim> hp_fe_values(fe_collection, source_q_collection,
                                         dealii::update_quadrature_points |
                                             dealii::update_values |
                                             dealii::update_JxW_values);
  unsigned int const dofs_per_cell = fe_collection.max_dofs_per_cell();
  unsigned int const n_q_points = source_q_collection.max_n_quadrature_points();
  std::vector<dealii::types::global_dof_index> local_dof_indices(dofs_per_cell);
  dealii::QGauss<dim - 1> face_quadrature(fe_degree + 1);
  dealii::FEFaceValues<dim> fe_face_values(
      fe_collection[0], face_quadrature,
      dealii::update_values | dealii::update_quadrature_points |
          dealii::update_JxW_values);
  unsigned int const n_face_q_points = face_quadrature.size();
  dealii::Vector<double> cell_source(dofs_per_cell);

  // Loop over the locally owned cells with an active FE index of zero
  for (auto const &cell : dealii::filter_iterators(
           dof_handler.active_cell_iterators(),
           dealii::IteratorFilters::LocallyOwnedCell(),
           dealii::IteratorFilters::ActiveFEIndexEqualTo(0)))
  {
    cell_source = 0.;
    hp_fe_values.reinit(cell);
    dealii::FEValues<dim> const &fe_values =
        hp_fe_values.get_present_fe_values();

    for (unsigned int i = 0; i < dofs_per_cell; ++i)
    {
      for (unsigned int q = 0; q < n_q_points; ++q)
      {
        double const inv_rho_cp = thermal_operator->get_inv_rho_cp(cell, q);
        double quad_pt_source = 0.;
        dealii::Point<dim> const &q_point = fe_values.quadrature_point(q);
        for (auto &beam : heat_sources)
          quad_pt_source += beam->value(q_point, t, current_source_height);

        cell_source[i] += inv_rho_cp * quad_pt_source *
                          fe_values.shape_value(i, q) * fe_values.JxW(q);
      }
    }

    // If we don't have a adiabatic boundary conditions, we need to add boundary
    // conditions
    if (!(boundary_type & BoundaryType::adiabatic))
    {
      for (unsigned int f = 0; f < dealii::GeometryInfo<dim>::faces_per_cell;
           ++f)
      {
        // We need to add the boundary conditions on the faces on the boundary
        // but also on the faces at the interface with FE_Nothing
        auto const &face = cell->face(f);
        if ((face->at_boundary()) &&
            ((!face->at_boundary()) &&
             (cell->neighbor(f)->active_fe_index() != 0)))
        {
          double conv_temperature_infty = 0.;
          double conv_heat_transfer_coef = 0.;
          double rad_temperature_infty = 0.;
          double rad_heat_transfer_coef = 0.;
          if (boundary_type & BoundaryType::convective)
          {
            conv_temperature_infty = material_properties->get_cell_value(
                cell, Property::convection_temperature_infty);
            conv_heat_transfer_coef = material_properties->get_cell_value(
                cell, StateProperty::convection_heat_transfer_coef);
          }
          if (boundary_type & BoundaryType::radiative)
          {
            rad_temperature_infty = material_properties->get_cell_value(
                cell, Property::radiation_temperature_infty);
            rad_heat_transfer_coef = material_properties->get_cell_value(
                cell, StateProperty::radiation_heat_transfer_coef);
          }

          fe_face_values.reinit(cell, face);
          for (unsigned int i = 0; i < dofs_per_cell; ++i)
          {
            for (unsigned int q = 0; q < n_face_q_points; ++q)
            {
              double const inv_rho_cp =
                  thermal_operator->get_inv_rho_cp(cell, q);
              cell_source[i] +=
                  inv_rho_cp *
                  (conv_heat_transfer_coef * conv_temperature_infty +
                   rad_heat_transfer_coef * rad_temperature_infty) *
                  fe_face_values.shape_value(i, q) * fe_face_values.JxW(q);
            }
          }
        }
      }
    }
    cell->get_dof_indices(local_dof_indices);
    affine_constraints.distribute_local_to_global(cell_source,
                                                  local_dof_indices, source);
  }

  source.compress(dealii::VectorOperation::add);

  return vmult_and_scale<dim, fe_degree, MemorySpaceType>(thermal_operator, y,
                                                          source, timers);
}

template <int dim, int fe_degree, typename MemorySpaceType,
          std::enable_if_t<
              std::is_same<MemorySpaceType, dealii::MemorySpace::CUDA>::value,
              int> = 0>
void init_dof_vector(
    double const value,
    dealii::LinearAlgebra::distributed::Vector<double, MemorySpaceType> &vector)
{
  dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host>
      vector_host(vector.get_partitioner());
  unsigned int const local_size = vector_host.local_size();
  for (unsigned int i = 0; i < local_size; ++i)
    vector_host.local_element(i) = value;

  vector.import(vector_host, dealii::VectorOperation::insert);
}
#endif
} // namespace

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
ThermalPhysics<dim, fe_degree, MemorySpaceType, QuadratureType>::ThermalPhysics(
    MPI_Comm const &communicator, boost::property_tree::ptree const &database,
    Geometry<dim> &geometry)
    : _boundary_type(BoundaryType::invalid), _geometry(geometry),
      _dof_handler(_geometry.get_triangulation())
{
  // Create the FECollection
  _fe_collection.push_back(dealii::FE_Q<dim>(fe_degree));
  _fe_collection.push_back(dealii::FE_Nothing<dim>());

  // Create the QCollection
  _q_collection.push_back(QuadratureType(fe_degree + 1));
  _q_collection.push_back(QuadratureType(1));

  // Create the material properties
  boost::property_tree::ptree const &material_database =
      database.get_child("materials");
  _material_properties.reset(new MaterialProperty<dim>(
      communicator, _geometry.get_triangulation(), material_database));

  // Create the heat sources
  boost::property_tree::ptree const &source_database =
      database.get_child("sources");
  // PropertyTreeInput sources.n_beams
  unsigned int const n_beams = source_database.get<unsigned int>("n_beams");
  _heat_sources.resize(n_beams);
  for (unsigned int i = 0; i < n_beams; ++i)
  {
    // PropertyTreeInput sources.beam_X.type
    boost::property_tree::ptree const &beam_database =
        source_database.get_child("beam_" + std::to_string(i));
    std::string type = beam_database.get<std::string>("type");
    if (type == "goldak")
    {
      _heat_sources[i] = std::make_shared<GoldakHeatSource<dim>>(beam_database);
    }
    else if (type == "electron_beam")
    {
      _heat_sources[i] =
          std::make_shared<ElectronBeamHeatSource<dim>>(beam_database);
    }
    else if (type == "cube")
    {
      _heat_sources[i] = std::make_shared<CubeHeatSource<dim>>(beam_database);
    }
    else
    {
      ASSERT_THROW(false, "Error: Beam type '" +
                              beam_database.get<std::string>("type") +
                              "' not recognized.");
    }
  }

  // Create the boundary condition type
  // PropertyTreeInput boundary.type
  std::string boundary_type_str = database.get<std::string>("boundary.type");
  // Parse the string
  size_t pos_str = 0;
  std::string boundary;
  std::string delimiter = ",";
  auto parse_boundary_type = [&](std::string const &boundary) {
    if (boundary == "adiabatic")
    {
      ASSERT_THROW(_boundary_type == BoundaryType::invalid,
                   "Adiabatic condition cannot be combined with another type.");
      _boundary_type = BoundaryType::adiabatic;
    }
    else
    {
      ASSERT_THROW(_boundary_type != BoundaryType::adiabatic,
                   "Adiabatic condition cannot be combined with another type.");

      if (boundary == "radiative")
      {
        _boundary_type |= BoundaryType::radiative;
      }
      else if (boundary == "convective")
      {
        _boundary_type |= BoundaryType::convective;
      }
      else
      {
        ASSERT_THROW(false, "Unknown boundary type.");
      }
    }
  };
  while ((pos_str = boundary_type_str.find(delimiter)) != std::string::npos)
  {
    boundary = boundary_type_str.substr(0, pos_str);
    parse_boundary_type(boundary);
    boundary_type_str.erase(0, pos_str + delimiter.length());
  }
  parse_boundary_type(boundary_type_str);

  // Create the thermal operator
  if (std::is_same<MemorySpaceType, dealii::MemorySpace::Host>::value)
    _thermal_operator =
        std::make_shared<ThermalOperator<dim, fe_degree, MemorySpaceType>>(
            communicator, _boundary_type, _material_properties, _heat_sources);
#if defined(ADAMANTINE_HAVE_CUDA) && defined(__CUDACC__)
  else
    _thermal_operator = std::make_shared<
        ThermalOperatorDevice<dim, fe_degree, MemorySpaceType>>(
        communicator, _material_properties);
#endif

  // Create the time stepping scheme
  boost::property_tree::ptree const &time_stepping_database =
      database.get_child("time_stepping");
  // PropertyTreeInput time_stepping.method
  std::string method = time_stepping_database.get<std::string>("method");
  std::transform(method.begin(), method.end(), method.begin(),
                 [](unsigned char c) { return std::tolower(c); });
  if (method.compare("forward_euler") == 0)
    _time_stepping =
        std::make_unique<dealii::TimeStepping::ExplicitRungeKutta<LA_Vector>>(
            dealii::TimeStepping::FORWARD_EULER);
  else if (method.compare("rk_third_order") == 0)
    _time_stepping =
        std::make_unique<dealii::TimeStepping::ExplicitRungeKutta<LA_Vector>>(
            dealii::TimeStepping::RK_THIRD_ORDER);
  else if (method.compare("rk_fourth_order") == 0)
    _time_stepping =
        std::make_unique<dealii::TimeStepping::ExplicitRungeKutta<LA_Vector>>(
            dealii::TimeStepping::RK_CLASSIC_FOURTH_ORDER);
  else if (method.compare("heun_euler") == 0)
  {
    _time_stepping = std::make_unique<
        dealii::TimeStepping::EmbeddedExplicitRungeKutta<LA_Vector>>(
        dealii::TimeStepping::HEUN_EULER);
    _embedded_method = true;
  }
  else if (method.compare("bogacki_shampine") == 0)
  {
    _time_stepping = std::make_unique<
        dealii::TimeStepping::EmbeddedExplicitRungeKutta<LA_Vector>>(
        dealii::TimeStepping::BOGACKI_SHAMPINE);
    _embedded_method = true;
  }
  else if (method.compare("dopri") == 0)
  {
    _time_stepping = std::make_unique<
        dealii::TimeStepping::EmbeddedExplicitRungeKutta<LA_Vector>>(
        dealii::TimeStepping::DOPRI);
    _embedded_method = true;
  }
  else if (method.compare("fehlberg") == 0)
  {
    _time_stepping = std::make_unique<
        dealii::TimeStepping::EmbeddedExplicitRungeKutta<LA_Vector>>(
        dealii::TimeStepping::FEHLBERG);
    _embedded_method = true;
  }
  else if (method.compare("cash_karp") == 0)
  {
    _time_stepping = std::make_unique<
        dealii::TimeStepping::EmbeddedExplicitRungeKutta<LA_Vector>>(
        dealii::TimeStepping::CASH_KARP);
    _embedded_method = true;
  }
  else if (method.compare("backward_euler") == 0)
  {
    _time_stepping =
        std::make_unique<dealii::TimeStepping::ImplicitRungeKutta<LA_Vector>>(
            dealii::TimeStepping::BACKWARD_EULER);
    _implicit_method = true;
  }
  else if (method.compare("implicit_midpoint") == 0)
  {
    _time_stepping =
        std::make_unique<dealii::TimeStepping::ImplicitRungeKutta<LA_Vector>>(
            dealii::TimeStepping::IMPLICIT_MIDPOINT);
    _implicit_method = true;
  }
  else if (method.compare("crank_nicolson") == 0)
  {
    _time_stepping =
        std::make_unique<dealii::TimeStepping::ImplicitRungeKutta<LA_Vector>>(
            dealii::TimeStepping::CRANK_NICOLSON);
    _implicit_method = true;
  }
  else if (method.compare("sdirk2") == 0)
  {
    _time_stepping =
        std::make_unique<dealii::TimeStepping::ImplicitRungeKutta<LA_Vector>>(
            dealii::TimeStepping::SDIRK_TWO_STAGES);
    _implicit_method = true;
  }

  if (_embedded_method == true)
  {
    // PropertyTreeInput time_steppping.coarsening_parameter
    double coarsen_param =
        time_stepping_database.get("coarsening_parameter", 1.2);
    // PropertyTreeInput time_steppping.refining_parameter
    double refine_param = time_stepping_database.get("refining_parameter", 0.8);
    // PropertyTreeInput time_stepping.min_time_step
    double min_delta = time_stepping_database.get("min_time_step", 1e-14);
    // PropertyTreeInput time_stepping.max_time_step
    double max_delta = time_stepping_database.get("max_time_step", 1e100);
    // PropertyTreeInput time_stepping.refining_tolerance
    double refine_tol = time_stepping_database.get("refining_tolerance", 1e-8);
    // PropertyTreeInput time_stepping.coarsening_tolerance
    double coarsen_tol =
        time_stepping_database.get("coarsening_tolerance", 1e-12);
    dealii::TimeStepping::EmbeddedExplicitRungeKutta<LA_Vector> *embedded_rk =
        static_cast<
            dealii::TimeStepping::EmbeddedExplicitRungeKutta<LA_Vector> *>(
            _time_stepping.get());
    embedded_rk->set_time_adaptation_parameters(coarsen_param, refine_param,
                                                min_delta, max_delta,
                                                refine_tol, coarsen_tol);
  }

  // If the time stepping scheme is implicit, set the parameters for the solver
  // and create the implicit operator.
  if (_implicit_method == true)
  {
    // PropertyTreeInput time_stepping.max_iteration
    _max_iter = time_stepping_database.get("max_iteration", 1000);
    // PropertyTreeInput time_stepping.tolerance
    _tolerance = time_stepping_database.get("tolerance", 1e-12);
    // PropertyTreeInput time_stepping.right_preconditioning
    _right_preconditioning =
        time_stepping_database.get("right_preconditioning", false);
    // PropertyTreeInput time_stepping.n_tmp_vectors
    _max_n_tmp_vectors = time_stepping_database.get("n_tmp_vectors", 30);
    // PropertyTreeInput time_stepping.newton_max_iteration
    unsigned int newton_max_iter =
        time_stepping_database.get("newton_max_iteration", 100);
    // PropertyTreeInput time_stepping.newton_tolerance
    double newton_tolerance =
        time_stepping_database.get("newton_tolerance", 1e-6);
    dealii::TimeStepping::ImplicitRungeKutta<LA_Vector> *implicit_rk =
        static_cast<dealii::TimeStepping::ImplicitRungeKutta<LA_Vector> *>(
            _time_stepping.get());
    implicit_rk->set_newton_solver_parameters(newton_max_iter,
                                              newton_tolerance);

    // PropertyTreeInput time_stepping.jfnk
    bool jfnk = time_stepping_database.get("jfnk", false);
    _implicit_operator = std::make_unique<ImplicitOperator<MemorySpaceType>>(
        _thermal_operator, jfnk);
  }

  // Set material on part of the domain
  // PropertyTreeInput geometry.material_height
  double const material_height = database.get("geometry.material_height", 1e9);
  for (auto const &cell :
       dealii::filter_iterators(_dof_handler.active_cell_iterators(),
                                dealii::IteratorFilters::LocallyOwnedCell()))
  {
    // If the center of the cell is below material_height, it contains material
    // otherwise it does not.
    if (cell->center()[axis<dim>::z] < material_height)
      cell->set_active_fe_index(0);
    else
      cell->set_active_fe_index(1);
  }

  // Set the initial height of the heat source. Right now this is just the
  // maximum heat source height, which can lead to unexpected behavior for
  // different sources with different heights.
  double temp_height = std::numeric_limits<double>::lowest();
  for (auto const &source : _heat_sources)
  {
    temp_height = std::max(temp_height, source->get_current_height(0.0));
  }
  _current_source_height = temp_height;
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
void ThermalPhysics<dim, fe_degree, MemorySpaceType,
                    QuadratureType>::setup_dofs()
{
  _dof_handler.distribute_dofs(_fe_collection);
  dealii::IndexSet locally_relevant_dofs;
  dealii::DoFTools::extract_locally_relevant_dofs(_dof_handler,
                                                  locally_relevant_dofs);
  _affine_constraints.clear();
  _affine_constraints.reinit(locally_relevant_dofs);
  dealii::DoFTools::make_hanging_node_constraints(_dof_handler,
                                                  _affine_constraints);
  _affine_constraints.close();

  _thermal_operator->reinit(_dof_handler, _affine_constraints, _q_collection);
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
void ThermalPhysics<dim, fe_degree, MemorySpaceType,
                    QuadratureType>::compute_inverse_mass_matrix()
{
  _thermal_operator->compute_inverse_mass_matrix(
      _dof_handler, _affine_constraints, _fe_collection);
  if (_implicit_method == true)
    _implicit_operator->set_inverse_mass_matrix(
        _thermal_operator->get_inverse_mass_matrix());
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
void ThermalPhysics<dim, fe_degree, MemorySpaceType, QuadratureType>::
    add_material(
        std::vector<
            typename dealii::DoFHandler<dim>::active_cell_iterator> const
            &elements_to_activate,
        double new_material_temperature,
        dealii::LA::distributed::Vector<double, MemorySpaceType> &solution)
{
#ifdef ADAMANTINE_WITH_CALIPER
  CALI_CXX_MARK_FUNCTION;
#endif
  std::vector<dealii::Vector<double>> data_to_transfer;
  unsigned int const dofs_per_cell = _dof_handler.get_fe().n_dofs_per_cell();
  dealii::Vector<double> cell_solution(dofs_per_cell);
  dealii::Vector<double> dummy_cell_solution(dofs_per_cell);
  for (auto &val : dummy_cell_solution)
  {
    val = std::numeric_limits<double>::infinity();
  }
  solution.update_ghost_values();
  std::vector<dealii::types::global_dof_index> dof_indices(dofs_per_cell);
  for (auto const &cell : _dof_handler.active_cell_iterators())
  {
    if ((cell->is_locally_owned()) && (cell->active_fe_index() == 0))
    {
      cell->get_dof_values(solution, cell_solution);
      data_to_transfer.push_back(cell_solution);
    }
    else
    {
      data_to_transfer.push_back(dummy_cell_solution);
    }
  }

  // Activate elements by updating the fe_index
  for (auto const &cell : elements_to_activate)
  {
    cell->set_active_fe_index(0);
  }

  dealii::parallel::distributed::Triangulation<dim> &triangulation =
      dynamic_cast<dealii::parallel::distributed::Triangulation<dim> &>(
          const_cast<dealii::Triangulation<dim> &>(
              _dof_handler.get_triangulation()));
  triangulation.prepare_coarsening_and_refinement();
  dealii::parallel::distributed::CellDataTransfer<
      dim, dim, std::vector<dealii::Vector<double>>>
      cell_data_trans(triangulation);
  cell_data_trans.prepare_for_coarsening_and_refinement(data_to_transfer);
  triangulation.execute_coarsening_and_refinement();

  setup_dofs();

  // Update MaterialProperty DoFHandler and resize the state vectors
  _material_properties->reinit_dofs();

  // Recompute the inverse of the mass matrix
  compute_inverse_mass_matrix();

  initialize_dof_vector(new_material_temperature, solution);
  std::vector<dealii::Vector<double>> transferred_data(
      triangulation.n_active_cells(), dealii::Vector<double>(dofs_per_cell));
  cell_data_trans.unpack(transferred_data);

  unsigned int cell_i = 0;
  for (auto const &cell : _dof_handler.active_cell_iterators())
  {
    if ((cell->is_locally_owned()) && (transferred_data[cell_i][0] !=
                                       std::numeric_limits<double>::infinity()))
    {
      cell->set_dof_values(transferred_data[cell_i], solution);
    }
    ++cell_i;
  }

  // Communicate the results.
  solution.compress(dealii::VectorOperation::min);

  // Set the value to the newly create DoFs. Here we need to be careful with the
  // hanging nodes. When there is a hanging node, the dofs at the vertices are
  // "doubled": there is a dof associated to the coarse cell and a dof
  // associated to the fine cell. The final value is decided by
  // AffineConstraints. Thus, we need to make sure that the newly activated
  // cells are at the same level than their neighbors.
  std::for_each(std::execution::par_unseq, solution.begin(), solution.end(),
                [&](double &val) {
                  if (val == std::numeric_limits<double>::infinity())
                  {
                    val = new_material_temperature;
                  }
                });
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
double ThermalPhysics<dim, fe_degree, MemorySpaceType, QuadratureType>::
    evolve_one_time_step(
        double t, double delta_t,
        dealii::LA::distributed::Vector<double, MemorySpaceType> &solution,
        std::vector<Timer> &timers)
{

  // Update the height of the heat source. Right now this is just the
  // maximum heat source height, which can lead to unexpected behavior for
  // different sources with different heights.
  double temp_height = std::numeric_limits<double>::lowest();
  for (auto const &source : _heat_sources)
  {
    temp_height = std::max(temp_height, source->get_current_height(t));
  }
  _current_source_height = temp_height;

  auto eval = [&](double const t, LA_Vector const &y) {
    return evaluate_thermal_physics(t, y, timers);
  };
  auto id_m_Jinv = [&](double const t, double const tau, LA_Vector const &y) {
    return id_minus_tau_J_inverse(t, tau, y, timers);
  };

  double time = _time_stepping->evolve_one_time_step(eval, id_m_Jinv, t,
                                                     delta_t, solution);

  // If the method is embedded, get the next time step. Otherwise, just use the
  // current time step.
  if (_embedded_method == false)
    _delta_t_guess = delta_t;
  else
  {
    dealii::TimeStepping::EmbeddedExplicitRungeKutta<LA_Vector> *embedded_rk =
        static_cast<
            dealii::TimeStepping::EmbeddedExplicitRungeKutta<LA_Vector> *>(
            _time_stepping.get());
    _delta_t_guess = embedded_rk->get_status().delta_t_guess;
  }

  // Return the time at the end of the time step. This may be different than
  // t+delta_t for embedded methods.
  return time;
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
void ThermalPhysics<dim, fe_degree, MemorySpaceType, QuadratureType>::
    initialize_dof_vector(
        dealii::LA::distributed::Vector<double, MemorySpaceType> &vector) const
{
  _thermal_operator->initialize_dof_vector(vector);
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
void ThermalPhysics<dim, fe_degree, MemorySpaceType, QuadratureType>::
    initialize_dof_vector(
        double const value,
        dealii::LA::distributed::Vector<double, MemorySpaceType> &vector) const
{
  // Resize the vector
  _thermal_operator->initialize_dof_vector(vector);

  init_dof_vector<dim, fe_degree, MemorySpaceType>(value, vector);
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
void ThermalPhysics<dim, fe_degree, MemorySpaceType,
                    QuadratureType>::get_state_from_material_properties()
{
  _thermal_operator->get_state_from_material_properties();
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
void ThermalPhysics<dim, fe_degree, MemorySpaceType,
                    QuadratureType>::set_state_to_material_properties()
{
  _thermal_operator->set_state_to_material_properties();
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
dealii::LA::distributed::Vector<double, MemorySpaceType>
ThermalPhysics<dim, fe_degree, MemorySpaceType, QuadratureType>::
    evaluate_thermal_physics(
        double const t,
        dealii::LA::distributed::Vector<double, MemorySpaceType> const &y,
        std::vector<Timer> &timers) const
{
#ifdef ADAMANTINE_WITH_CALIPER
  CALI_CXX_MARK_FUNCTION;
#endif
  if constexpr (std::is_same<MemorySpaceType, dealii::MemorySpace::Host>::value)
  {
    _thermal_operator->evaluate_material_properties(y);
    return evaluate_thermal_physics_impl<dim, fe_degree, MemorySpaceType>(
        _thermal_operator, t, _current_source_height, y, timers);
  }
  else
  {
    // TODO do this on the GPU
    dealii::LA::distributed::Vector<double, dealii::MemorySpace::Host> y_host(
        y.get_partitioner());
    y_host.import(y, dealii::VectorOperation::insert);
    _thermal_operator->evaluate_material_properties(y_host);
    return evaluate_thermal_physics_impl<dim, fe_degree, MemorySpaceType>(
        _thermal_operator, _fe_collection, t, _dof_handler, _heat_sources,
        _current_source_height, _boundary_type, _material_properties,
        _affine_constraints, y, timers);
  }

  // Dummy to silence warning
  return dealii::LA::distributed::Vector<double, MemorySpaceType>();
}

template <int dim, int fe_degree, typename MemorySpaceType,
          typename QuadratureType>
dealii::LA::distributed::Vector<double, MemorySpaceType>
ThermalPhysics<dim, fe_degree, MemorySpaceType, QuadratureType>::
    id_minus_tau_J_inverse(
        double const /*t*/, double const tau,
        dealii::LA::distributed::Vector<double, MemorySpaceType> const &y,
        std::vector<Timer> &timers) const
{
  timers[evol_time_J_inv].start();
  _implicit_operator->set_tau(tau);
  dealii::LA::distributed::Vector<double, MemorySpaceType> solution(
      y.get_partitioner());

  // TODO Add a geometric multigrid preconditioner.
  dealii::PreconditionIdentity preconditioner;

  dealii::SolverControl solver_control(_max_iter, _tolerance * y.l2_norm());
  // We need to inverse (I - tau M^{-1} J). While M^{-1} and J are SPD,
  // (I - tau M^{-1} J) is symmetric indefinite in the general case.
  typename dealii::SolverGMRES<
      dealii::LA::distributed::Vector<double, MemorySpaceType>>::AdditionalData
      additional_data(_max_n_tmp_vectors, _right_preconditioning);
  dealii::SolverGMRES<dealii::LA::distributed::Vector<double, MemorySpaceType>>
      solver(solver_control, additional_data);
  solver.solve(*_implicit_operator, solution, y, preconditioner);

  timers[evol_time_J_inv].stop();

  return solution;
}
} // namespace adamantine

#endif