Repository: IAMconsortium/pyam
Pull request #165 (2018-12-19 21:35:28)

Update README.md
# Please confirm that this PR has done the following:

- [x] Tests Added (N/A)
- [x] Documentation Added (N/A)
- [x] Description in RELEASE_NOTES.md Added (N/A)

## Adding to RELEASE_NOTES.md (remove section after adding to RELEASE_NOTES.md)

Please add a single line in the release notes similar to the following:

```
- (#XX)[http://link-to-pr.com] Added feature which does something
```

# Description of PR

Fixes syntax in README to make header appear as header (I'd forgotten the file was markdown not rst...)


coveralls: 
[![Coverage Status](https://coveralls.io/builds/20726699/badge)](https://coveralls.io/builds/20726699)

Coverage remained the same at 83.257% when pulling **0958bf8f207c758258d2aa72eddb0b3b8832ef0b on fix-readme-header** into **df399d6309aceb42363f3134a726b1b97c82892a on master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/20726699/badge)](https://coveralls.io/builds/20726699)

Coverage remained the same at 83.257% when pulling **0958bf8f207c758258d2aa72eddb0b3b8832ef0b on fix-readme-header** into **df399d6309aceb42363f3134a726b1b97c82892a on master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/20726699/badge)](https://coveralls.io/builds/20726699)

Coverage remained the same at 83.257% when pulling **0958bf8f207c758258d2aa72eddb0b3b8832ef0b on fix-readme-header** into **df399d6309aceb42363f3134a726b1b97c82892a on master**.


Patch:
diff --git a/.stickler.yml b/.stickler.yml
index ccca240..88fb889 100644
--- a/.stickler.yml
+++ b/.stickler.yml
@@ -2,7 +2,7 @@ linters:
   flake8:
     max-line-length: 79
     fixer: false
-    ignore: I002, F403
+    ignore: I002, F403, E402
     # stickler doesn't support 'exclude' for flake8 properly, so we disable it
     # below with files.ignore:
     # https://github.com/markstory/lint-review/issues/184
diff --git a/README.md b/README.md
index 8e4b1bb..38bdf72 100644
--- a/README.md
+++ b/README.md
@@ -73,15 +73,14 @@ To install from source after cloning this repository, simply run
 pip install -e .
 ```
 
-Development
-+++++++++++
+### Development
 
 To setup a development environment,
 
 ```
 # pyam can be replaced with any other name
 # you don't have to specify your python version if you don't want
-conda create --name pip pyam python=X.Y.Z
+conda create --name pyam pip python=X.Y.Z
 conda activate pyam  # may be source activate pyam or just activate pyam
 pip install -e .[tests,docs,deploy]
 # install other required packages (e.g. on a Unix like system)
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 9a12caa..6b161fc 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -1,10 +1,12 @@
 
 # Next Release
 
+- [#171](https://github.com/IAMconsortium/pyam/pull/171) Fix a bug when reading from an `ixmp.TimeSeries` object, refactor to mitigate circular dependency
 - [#162](https://github.com/IAMconsortium/pyam/pull/162) Add a function to sum and append timeseries components to an aggregate variable
 - [#152](https://github.com/IAMconsortium/pyam/pull/152) Fix bug where scatter plots did not work with property metadata when using two variables (#136, #152)
 - [#151](https://github.com/IAMconsortium/pyam/pull/151) Fix bug where excel files were not being written on Windows and MacOSX (#149)
 - [#145](https://github.com/IAMconsortium/pyam/pull/145) Support full semantic and VCS-style versioning with `versioneer`
+- [#132](https://github.com/IAMconsortium/pyam/pull/132) support time columns using the `datetime` format and additional `str` columns in `data`
 
 # Release v0.1.2
 
diff --git a/pyam/__init__.py b/pyam/__init__.py
index c9a6965..9251bb4 100644
--- a/pyam/__init__.py
+++ b/pyam/__init__.py
@@ -2,6 +2,7 @@ from pyam.core import *
 from pyam.utils import *
 from pyam.statistics import *
 from pyam.timeseries import *
+from pyam.read_ixmp import *
 from pyam.logger import *
 from pyam.run_control import *
 from pyam.iiasa import read_iiasa, read_iiasa_iamc15  # noqa: F401
diff --git a/pyam/core.py b/pyam/core.py
index 3940a08..c21a4b1 100644
--- a/pyam/core.py
+++ b/pyam/core.py
@@ -20,15 +20,17 @@ from pyam.logger import logger
 from pyam.run_control import run_control
 from pyam.utils import (
     write_sheet,
-    read_ix,
     read_files,
     read_pandas,
     format_data,
     pattern_match,
     years_match,
+    month_match,
+    hour_match,
+    day_match,
+    datetime_match,
     isstr,
     islistable,
-    cast_years_to_int,
     META_IDX,
     YEAR_IDX,
     REGION_IDX,
@@ -37,6 +39,7 @@ from pyam.utils import (
     LONG_IDX,
     GROUP_IDX
 )
+from pyam.read_ixmp import read_ix
 from pyam.timeseries import fill_series
 
 
@@ -64,15 +67,13 @@ class IamDataFrame(object):
         """
         # import data from pd.DataFrame or read from source
         if isinstance(data, pd.DataFrame) or isinstance(data, pd.Series):
-            self.data = format_data(data.copy())
+            _data = format_data(data.copy())
         elif has_ix and isinstance(data, ixmp.TimeSeries):
-            self.data = read_ix(data, **kwargs)
+            _data = read_ix(data, **kwargs)
         else:
-            self.data = read_files(data, **kwargs)
-
-        # cast year column to `int` if necessary
-        if not self.data.year.dtype == 'int64':
-            self.data.year = cast_years_to_int(self.data.year)
+            _data = read_files(data, **kwargs)
+        self.data, self.time_col, self.extra_cols = _data
+        self._LONG_IDX = IAMC_IDX + [self.time_col] + self.extra_cols
 
         # define a dataframe for categorization and other metadata indicators
         self.meta = self.data[META_IDX].drop_duplicates().set_index(META_IDX)
@@ -166,12 +167,15 @@ class IamDataFrame(object):
         inplace : bool, default False
             If True, do operation inplace and return None
         """
-        ret = copy.deepcopy(self) if not inplace else self
-
         if not isinstance(other, IamDataFrame):
             other = IamDataFrame(other, **kwargs)
             ignore_meta_conflict = True
 
+        if self.time_col is not other.time_col:
+            raise ValueError('incompatible time format (years vs. datetime)!')
+
+        ret = copy.deepcopy(self) if not inplace else self
+
         diff = other.meta.index.difference(ret.meta.index)
         intersect = other.meta.index.intersection(ret.meta.index)
 
@@ -205,11 +209,16 @@ class IamDataFrame(object):
             ret.meta = ret.meta.append(other.meta.loc[diff, :], **sort_kwarg)
 
         # append other.data (verify integrity for no duplicates)
-        ret.data.set_index(LONG_IDX, inplace=True)
-        other.data.set_index(LONG_IDX, inplace=True)
-        ret.data = ret.data.append(other.data, verify_integrity=True)\
+        ret.data.set_index(ret._LONG_IDX, inplace=True)
+        _other = other.data.set_index(other._LONG_IDX)
+        ret.data = ret.data.append(_other, verify_integrity=True)\
             .reset_index(drop=False)
 
+        # merge extra columns in `data` and set `LONG_IDX`
+        ret.extra_cols += [i for i in other.extra_cols
+                           if i not in ret.extra_cols]
+        ret._LONG_IDX = IAMC_IDX + [ret.time_col] + ret.extra_cols
+
         if not inplace:
             return ret
 
@@ -294,15 +303,26 @@ class IamDataFrame(object):
                   )
         return df
 
-    def timeseries(self):
-        """Returns a dataframe in the standard IAMC format
+    def timeseries(self, iamc_index=False):
+        """Returns a pd.DataFrame in wide format (years or timedate as columns)
+
+        Parameters
+        ----------
+        iamc_index: bool, default False
+            if True, use `['model', 'scenario', 'region', 'variable', 'unit']`;
+            else, use all `data` columns
         """
-        return (
+        index = IAMC_IDX if iamc_index else IAMC_IDX + self.extra_cols
+        df = (
             self.data
-            .pivot_table(index=IAMC_IDX, columns='year')
+            .pivot_table(index=index, columns=self.time_col)
             .value  # column name
             .rename_axis(None, axis=1)
         )
+        if df.index.has_duplicates:
+            raise ValueError('timeseries object has duplicates in index ',
+                             'use `iamc_index=False`')
+        return df
 
     def reset_exclude(self):
         """Reset exclusion assignment for all scenarios to `exclude: False`"""
@@ -322,8 +342,12 @@ class IamDataFrame(object):
         index: pyam.IamDataFrame, pd.DataFrame or pd.MultiIndex, optional
             index to be used for setting meta column (`['model', 'scenario']`)
         """
+        # check that name is valid and doesn't conflict with data columns
         if (name or (hasattr(meta, 'name') and meta.name)) in [None, False]:
             raise ValueError('Must pass a name or use a named pd.Series')
+        name = name or meta.name
+        if name in self.data.columns:
+            raise ValueError('`{}` already exists in `data`!'.format(name))
 
         # check if meta has a valid index and use it for further workflow
         if hasattr(meta, 'index') and hasattr(meta.index, 'names') \
@@ -350,7 +374,6 @@ class IamDataFrame(object):
 
         # create pd.Series from meta, index and name if provided
         meta = pd.Series(data=meta, index=index, name=name)
-        meta.name = name = name or meta.name
 
         # reduce index dimensions to model-scenario only
         meta = (
@@ -429,9 +452,9 @@ class IamDataFrame(object):
             required variable
         unit: str, default None
             name of unit (optional)
-        years: int or list, default None
+        year: int or list, default None
             years (optional)
-        exclude: bool, default False
+        exclude_on_fail: bool, default False
             flag scenarios missing the required variables as `exclude: True`
         """
         criteria = {'variable': variable}
@@ -440,7 +463,7 @@ class IamDataFrame(object):
         if year:
             criteria.update({'year': year})
 
-        keep = _apply_filters(self.data, self.meta, criteria)
+        keep = self._apply_filters(criteria)
         idx = self.meta.index.difference(_meta_idx(self.data[keep]))
 
         n = len(idx)
@@ -509,7 +532,7 @@ class IamDataFrame(object):
                 raise ValueError('Renaming by {} not supported!'.format(col))
             ret.data.loc[:, col] = ret.data.loc[:, col].replace(_mapping)
 
-        ret.data = ret.data.groupby(LONG_IDX).sum().reset_index()
+        ret.data = ret.data.groupby(self._LONG_IDX).sum().reset_index()
         if not inplace:
             return ret
 
@@ -675,6 +698,7 @@ class IamDataFrame(object):
                 df_var_to_add = self.filter(
                     region=region, variable=var_to_add
                 ).data.groupby(REGION_IDX + ['unit']).sum()['value']
+
                 df_var_to_add.index = df_var_to_add.index.droplevel('variable')
 
                 if len(df_var_to_add):
@@ -749,22 +773,25 @@ class IamDataFrame(object):
             if True, do operation inplace and return None
         filters by kwargs or dict (deprecated):
             The following columns are available for filtering:
-             - metadata columns: filter by category assignment in metadata
+             - metadata columns: filter by category assignment
              - 'model', 'scenario', 'region', 'variable', 'unit':
-               string or list of strings, where ``*`` can be used as a wildcard
+               string or list of strings, where `*` can be used as a wildcard
              - 'level': the maximum "depth" of IAM variables (number of '|')
                (exluding the strings given in the 'variable' argument)
              - 'year': takes an integer, a list of integers or a range
-                note that the last year of a range is not included,
-                so ``range(2010,2015)`` is interpreted as ``[2010, ..., 2014]``
-            - 'regexp=True' overrides pseudo-regexp syntax in `pattern_match()`
+               note that the last year of a range is not included,
+               so `range(2010, 2015)` is interpreted as `[2010, ..., 2014]`
+             - arguments for filtering by `datetime.datetime`
+               ('month', 'hour', 'time')
+             - 'regexp=True' disables pseudo-regexp syntax in `pattern_match()`
         """
         if filters is not None:
-            warnings.warn(
-                '`filters` keyword argument in filters() is deprecated and will be removed in the next release')
+            msg = '`filters` keyword argument in `filter()` is deprecated ' + \
+                'and will be removed in the next release'
+            warnings.warn(msg)
             kwargs.update(filters)
 
-        _keep = _apply_filters(self.data, self.meta, kwargs)
+        _keep = self._apply_filters(kwargs)
         _keep = _keep if keep else ~_keep
         ret = copy.deepcopy(self) if not inplace else self
         ret.data = ret.data[_keep]
@@ -780,6 +807,90 @@ class IamDataFrame(object):
         if not inplace:
             return ret
 
+    def _apply_filters(self, filters):
+        """Determine rows to keep in data for given set of filters
+
+        Parameters
+        ----------
+        filters: dict
+            dictionary of filters ({col: values}}); uses a pseudo-regexp syntax
+            by default, but accepts `regexp: True` to use regexp directly
+        """
+        regexp = filters.pop('regexp', False)
+        keep = np.array([True] * len(self.data))
+
+        # filter by columns and list of values
+        for col, values in filters.items():
+            if col in self.meta.columns:
+                matches = pattern_match(self.meta[col], values, regexp=regexp)
+                cat_idx = self.meta[matches].index
+                keep_col = (self.data[META_IDX].set_index(META_IDX)
+                                .index.isin(cat_idx))
+
+            elif col == 'variable':
+                level = filters['level'] if 'level' in filters else None
+                keep_col = pattern_match(self.data[col], values, level, regexp)
+
+            elif col == 'year':
+                if self.time_col is 'time':
+                    keep_col = years_match(self.data['time']
+                                               .apply(lambda x: x.year),
+                                           values)
+                else:
+                    keep_col = years_match(self.data[col], values)
+
+            elif col == 'month':
+                if self.time_col is not 'time':
+                    _raise_filter_error(col)
+                keep_col = month_match(self.data['time']
+                                           .apply(lambda x: x.month),
+                                       values)
+
+            elif col == 'day':
+                if self.time_col is not 'time':
+                    _raise_filter_error(col)
+                if isinstance(values, str):
+                    wday = True
+                elif isinstance(values, list) and isinstance(values[0], str):
+                    wday = True
+                else:
+                    wday = False
+
+                if wday:
+                    days = self.data['time'].apply(lambda x: x.weekday())
+                else:  # ints or list of ints
+                    days = self.data['time'].apply(lambda x: x.day)
+
+                keep_col = day_match(days, values)
+
+            elif col == 'hour':
+                if self.time_col is not 'time':
+                    _raise_filter_error(col)
+                keep_col = hour_match(self.data['time']
+                                          .apply(lambda x: x.hour),
+                                      values)
+
+            elif col == 'time':
+                keep_col = datetime_match(self.data[col], values)
+
+            elif col == 'level':
+                if 'variable' not in filters.keys():
+                    keep_col = pattern_match(self.data['variable'],
+                                             '*', values, regexp=regexp)
+                else:
+                    continue
+
+            elif col in self.data.columns:
+                keep_col = pattern_match(self.data[col], values, regexp=regexp)
+
+            else:
+                _raise_filter_error(col)
+
+            keep &= keep_col
+
+        return keep
+
+
     def col_apply(self, col, func, *args, **kwargs):
         """Apply a function to a column
 
@@ -795,46 +906,47 @@ class IamDataFrame(object):
         else:
             self.meta[col] = self.meta[col].apply(func, *args, **kwargs)
 
-    def _to_file_format(self):
+    def _to_file_format(self, iamc_index):
         """Return a dataframe suitable for writing to a file"""
-        df = self.timeseries().reset_index()
+        df = self.timeseries(iamc_index=iamc_index).reset_index()
         df = df.rename(columns={c: str(c).title() for c in df.columns})
         return df
 
-    def to_csv(self, path, index=False, **kwargs):
-        """Write data to a csv file
+    def to_csv(self, path, iamc_index=False, **kwargs):
+        """Write timeseries data to a csv file
 
         Parameters
         ----------
-        index: boolean, default False
-            write row names (index)
+        path: string
+            file path
+        iamc_index: bool, default False
+            if True, use `['model', 'scenario', 'region', 'variable', 'unit']`;
+            else, use all `data` columns
         """
-        self._to_file_format().to_csv(path, index=False, **kwargs)
+        self._to_file_format(iamc_index).to_csv(path, index=False, **kwargs)
 
-    def to_excel(self, path=None, writer=None, sheet_name='data', index=False,
-                 **kwargs):
-        """Write timeseries data to Excel using the IAMC template convention
-        (wrapper for `pd.DataFrame.to_excel()`)
+    def to_excel(self, excel_writer, sheet_name='data',
+                 iamc_index=False, **kwargs):
+        """Write timeseries data to Excel format
 
         Parameters
         ----------
         excel_writer: string or ExcelWriter object
-             file path or existing ExcelWriter
+            file path or existing ExcelWriter
         sheet_name: string, default 'data'
-            name of the sheet that will contain the (filtered) IamDataFrame
-        index: boolean, default False
-            write row names (index)
+            name of sheet which will contain `IamDataFrame.timeseries()` data
+        iamc_index: bool, default False
+            if True, use `['model', 'scenario', 'region', 'variable', 'unit']`;
+            else, use all `data` columns
         """
-        if (path is None and writer is None) or \
-           (path is not None and writer is not None):
-            raise ValueError('Only one of path and writer must have a value')
-        close = writer is None
-        if writer is None:
-            writer = pd.ExcelWriter(path)
-        self._to_file_format().to_excel(writer, sheet_name=sheet_name,
-                                        index=index, **kwargs)
+        if not isinstance(excel_writer, pd.ExcelWriter):
+            close = True
+            excel_writer = pd.ExcelWriter(excel_writer)
+        self._to_file_format(iamc_index)\
+            .to_excel(excel_writer, sheet_name=sheet_name, index=False,
+                      **kwargs)
         if close:
-            writer.close()
+            excel_writer.close()
 
     def export_metadata(self, path):
         """Export metadata to Excel
@@ -1072,7 +1184,7 @@ class IamDataFrame(object):
 
         # perform aggregations
         if agg == 'sum':
-            df = df.groupby(LONG_IDX).sum().reset_index()
+            df = df.groupby(self._LONG_IDX).sum().reset_index()
 
         ret.data = (df
                     .reindex(columns=columns_orderd)
@@ -1118,51 +1230,8 @@ def _aggregate_by_regions(df, regions, units=None):
     return df.groupby(REGION_IDX + ['unit']).sum()['value']
 
 
-def _apply_filters(data, meta, filters):
-    """Applies filters to the data and meta tables of an IamDataFrame.
-
-    Parametersp
-    ----------
-    data: pd.DataFrame
-        data table of an IamDataFrame
-    meta: pd.DataFrame
-        meta table of an IamDataFrame
-    filters: dict
-        dictionary of filters ({col: values}}); uses a pseudo-regexp syntax by
-        default, but accepts `regexp: True` to use direct regexp
-    """
-    regexp = filters.pop('regexp', False)
-    keep = np.array([True] * len(data))
-
-    # filter by columns and list of values
-    for col, values in filters.items():
-        if col in meta.columns:
-            matches = pattern_match(meta[col], values, regexp=regexp)
-            cat_idx = meta[matches].index
-            keep_col = data[META_IDX].set_index(META_IDX).index.isin(cat_idx)
-
-        elif col in ['model', 'scenario', 'region', 'unit']:
-            keep_col = pattern_match(data[col], values, regexp=regexp)
-
-        elif col == 'variable':
-            level = filters['level'] if 'level' in filters else None
-            keep_col = pattern_match(data[col], values, level, regexp)
-
-        elif col == 'year':
-            keep_col = years_match(data[col], values)
-
-        elif col == 'level':
-            if 'variable' not in filters.keys():
-                keep_col = pattern_match(data['variable'], '*', values,
-                                         regexp=regexp)
-            else:
-                continue
-        else:
-            raise ValueError(
-                'filter by column ' + col + ' not supported')
-        keep &= keep_col
-
-    return keep
+def _raise_filter_error(col):
+    raise ValueError('filter by `{}` not supported'.format(col))
 
 
 def _check_rows(rows, check, in_range=True, return_test='any'):
@@ -1174,7 +1243,7 @@ def _check_rows(rows, check, in_range=True, return_test='any'):
     rows: pd.DataFrame
         data rows
     check: dict
-        dictionary with possible values of "up", "lo", and "year"
+        dictionary with possible values of 'up', 'lo', and 'year'
     in_range: bool, optional
         check if values are inside or outside of provided range
     return_test: str, optional
diff --git a/pyam/iiasa.py b/pyam/iiasa.py
index c1e8b09..1d5bb0a 100644
--- a/pyam/iiasa.py
+++ b/pyam/iiasa.py
@@ -12,7 +12,7 @@ except ImportError:
 
 from pyam.core import IamDataFrame
 from pyam.logger import logger
-from pyam.utils import LONG_IDX, isstr, pattern_match
+from pyam.utils import META_IDX, isstr, pattern_match
 
 # quiet this fool
 logging.getLogger('requests').setLevel(logging.WARNING)
@@ -176,7 +176,26 @@ class Connection(object):
         data = json.dumps(self._query_post_data(**kwargs))
         url = self.base_url + 'runs/bulk/ts'
         r = requests.post(url, headers=headers, data=data)
-        return pd.read_json(r.content, orient='records')
+        # refactor returned json object to be castable to an IamDataFrame
+        df = (
+            pd.read_json(r.content, orient='records')
+            .drop(columns='runId')
+            .rename(columns={'time': 'subannual'})
+        )
+        # check if returned dataframe has subannual disaggregation, drop if not
+        if pd.Series([i in [-1, 'year'] for i in df.subannual]).all():
+            df.drop(columns='subannual', inplace=True)
+        # check if there are multiple version for any model/scenario
+        lst = (
+            df[META_IDX + ['version']].drop_duplicates()
+            .groupby(META_IDX).count().version
+        )
+        if max(lst) > 1:
+            raise ValueError('multiple versions for {}'.format(
+                lst[lst > 1].index.to_list()))
+        df.drop(columns='version', inplace=True)
+
+        return df
 
 
 def read_iiasa(name, **kwargs):
@@ -185,7 +204,7 @@ def read_iiasa(name, **kwargs):
     """
     conn = Connection(name)
     df = conn.query(**kwargs)
-    return IamDataFrame(df[LONG_IDX + ['value']])
+    return IamDataFrame(df)
 
 
 def read_iiasa_iamc15(**kwargs):
diff --git a/pyam/read_ixmp.py b/pyam/read_ixmp.py
new file mode 100644
index 0000000..a4b3e65
--- /dev/null
+++ b/pyam/read_ixmp.py
@@ -0,0 +1,24 @@
+# -*- coding: utf-8 -*-
+try:
+    import ixmp
+except ImportError:
+    pass
+
+
+def read_ix(ix, **kwargs):
+    """Read timeseries data from an ixmp object
+
+    Parameters
+    ----------
+    ix: ixmp.TimeSeries or ixmp.Scenario
+        this option requires the ixmp package as a dependency
+    kwargs: arguments passed to ixmp.TimeSeries.timeseries()
+    """
+    if not isinstance(ix, ixmp.TimeSeries):
+        error = 'not recognized as valid ixmp class: {}'.format(ix)
+        raise ValueError(error)
+
+    df = ix.timeseries(iamc=False, **kwargs)
+    df['model'] = ix.model
+    df['scenario'] = ix.scenario
+    return df, 'year', []
diff --git a/pyam/utils.py b/pyam/utils.py
index ee87e63..ae14262 100644
--- a/pyam/utils.py
+++ b/pyam/utils.py
@@ -5,15 +5,12 @@ import six
 import re
 import glob
 import collections
+import datetime
+import time
 
 import numpy as np
 import pandas as pd
 
-try:
-    import ixmp
-except ImportError:
-    pass
-
 try:
     import seaborn as sns
 except ImportError:
@@ -100,26 +97,6 @@ def write_sheet(writer, name, df, index=False):
         worksheet.set_column(xls_col, width)
 
 
-def read_ix(ix, **kwargs):
-    """Read timeseries data from an ix object
-
-    Parameters
-    ----------
-    ix: ixmp.TimeSeries or ixmp.Scenario
-        this option requires the ixmp package as a dependency
-    regions: list
-        list of regions to be loaded from the database snapshot
-    """
-    if not isinstance(ix, ixmp.TimeSeries):
-        error = 'not recognized as valid ixmp class: {}'.format(ix)
-        raise ValueError(error)
-
-    df = ix.timeseries(iamc=False, **kwargs)
-    df['model'] = ix.model
-    df['scenario'] = ix.scenario
-    return df
-
-
 def read_pandas(fname, *args, **kwargs):
     """Read a file and return a pd.DataFrame"""
     if not os.path.exists(fname):
@@ -138,18 +115,11 @@ def read_files(fnames, *args, **kwargs):
     """Read data from a snapshot file saved in the standard IAMC format
     or a table with year/value columns
     """
-    if isstr(fnames):
-        fnames = [fnames]
-
-    fnames = itertools.chain(*[glob.glob(f) for f in fnames])
-    dfs = []
-    for fname in fnames:
-        logger().info('Reading `{}`'.format(fname))
-        df = read_pandas(fname, *args, **kwargs)
-        df = format_data(df)
-        dfs.append(df)
-
-    return pd.concat(dfs)
+    if not isstr(fnames):
+        raise ValueError('reading multiple files not supported, '
+                         'please use `pyam.IamDataFrame.append()`')
+    logger().info('Reading `{}`'.format(fnames))
+    return format_data(read_pandas(fnames, *args, **kwargs))
 
 
 def format_data(df):
@@ -158,7 +128,8 @@ def format_data(df):
         df = df.to_frame()
 
     # all lower case
-    df.rename(columns={c: str(c).lower() for c in df.columns}, inplace=True)
+    str_cols = [c for c in df.columns if isstr(c)]
+    df.rename(columns={c: str(c).lower() for c in str_cols}, inplace=True)
 
     if 'notes' in df.columns:  # this came from the database
         logger().info('Ignoring notes column in dataframe')
@@ -181,23 +152,57 @@ def format_data(df):
         missing = list(set(IAMC_IDX) - set(df.columns))
         raise ValueError("missing required columns `{}`!".format(missing))
 
-    # check whether data in IAMC style or year/value layout
-    if 'value' not in df.columns:
-        numcols = sorted(set(df.columns) - set(IAMC_IDX))
-        df = pd.melt(df, id_vars=IAMC_IDX, var_name='year',
-                     value_vars=numcols, value_name='value')
-
-    # cast year and value columns to numeric
-    df['year'] = pd.to_numeric(df['year'])
+    # check whether data in wide format (IAMC) or long format (`value` column)
+    if 'value' in df.columns:
+        # check if time column is given as `year` (int) or `time` (datetime)
+        cols = df.columns
+        if 'year' in cols and 'time' not in cols:
+            time_col = 'year'
+        elif 'time' in cols and 'year' not in cols:
+            time_col = 'time'
+        else:
+            msg = 'invalid time format, must have either `year` or `time`!'
+            raise ValueError(msg)
+        extra_cols = list(set(cols) - set(IAMC_IDX + [time_col, 'value']))
+    else:
+        # if in wide format, check if columns are years (int) or datetime
+        cols = set(df.columns) - set(IAMC_IDX)
+        year_cols, time_cols, extra_cols = [], [], []
+        for i in cols:
+            try:
+                year_cols.append(i) if int(i) else None
+            except (ValueError, TypeError):
+                try:
+                    pd.to_datetime([i])
+                    time_cols.append(i)
+                except ValueError:
+                    extra_cols.append(i)
+        if year_cols and not time_cols:
+            time_col = 'year'
+            melt_cols = year_cols
+        elif not year_cols and time_cols:
+            time_col = 'time'
+            melt_cols = time_cols
+        else:
+            msg = 'invalid column format, must be either years or `datetime`!'
+            raise ValueError(msg)
+        df = pd.melt(df, id_vars=IAMC_IDX + extra_cols, var_name=time_col,
+                     value_vars=sorted(melt_cols), value_name='value')
+
+    # cast time_col to correct format
+    if time_col == 'year':
+        if not df.year.dtype == 'int64':
+            df['year'] = cast_years_to_int(pd.to_numeric(df['year']))
+    if time_col == 'time':
+        df['time'] = pd.to_datetime(df['time'])
+
+    # cast value columns to numeric, drop NaN's, sort data
     df['value'] = df['value'].astype('float64')
-
-    # drop NaN's
     df.dropna(inplace=True)
+    df.sort_values(META_IDX + ['variable', time_col, 'region'] + extra_cols,
+                   inplace=True)
 
-    # sort data
-    df.sort_values(SORT_IDX, inplace=True)
-
-    return df
+    return df, time_col, extra_cols
 
 
 def style_df(df, style='heatmap'):
@@ -268,9 +273,90 @@ def years_match(data, years):
     matching of year columns for data filtering
     """
     years = [years] if isinstance(years, int) else years
+    dt = datetime.datetime
+    if isinstance(years, dt) or isinstance(years[0], dt):
+        error_msg = "`year` can only be filtered with ints or lists of ints"
+        raise TypeError(error_msg)
     return data.isin(years)
 
 
+def month_match(data, months):
+    """
+    matching of months in time columns for data filtering
+    """
+    return time_match(data, months, ['%b', '%B'], "tm_mon", "months")
+
+
+def day_match(data, days):
+    """
+    matching of days in time columns for data filtering
+    """
+    return time_match(data, days, ['%a', '%A'], "tm_wday", "days")
+
+
+def hour_match(data, hours):
+    """
+    matching of days in time columns for data filtering
+    """
+    hours = [hours] if isinstance(hours, int) else hours
+    return data.isin(hours)
+
+
+def time_match(data, times, conv_codes, strptime_attr, name):
+    def conv_strs(strs_to_convert, conv_codes, name):
+        for conv_code in conv_codes:
+            try:
+                res = [getattr(time.strptime(t, conv_code), strptime_attr)
+                       for t in strs_to_convert]
+                break
+            except ValueError:
+                continue
+
+        try:
+            return res
+        except NameError:
+            raise ValueError("Could not convert {} to integer".format(name))
+
+    times = [times] if isinstance(times, (int, str)) else times
+    if isinstance(times[0], str):
+        to_delete = []
+        to_append = []
+        for i, timeset in enumerate(times):
+            if "-" in timeset:
+                ints = conv_strs(timeset.split("-"), conv_codes, name)
+                if ints[0] > ints[1]:
+                    error_msg = (
+                        "string ranges must lead to increasing integer ranges,"
+                        " {} becomes {}".format(timeset, ints)
+                    )
+                    raise ValueError(error_msg)
+
+                # + 1 to include last month
+                to_append += [j for j in range(ints[0], ints[1] + 1)]
+                to_delete.append(i)
+
+        for i in to_delete:
+            del times[i]
+
+        times = conv_strs(times, conv_codes, name)
+        times += to_append
+
+    return data.isin(times)
+
+
+def datetime_match(data, dts):
+    """
+    matching of datetimes in time columns for data filtering
+    """
+    dts = [dts] if isinstance(dts, datetime.datetime) else dts
+    if isinstance(dts, int) or isinstance(dts[0], int):
+        error_msg = (
+            "`time` can only be filtered with datetimes or lists of datetimes"
+        )
+        raise TypeError(error_msg)
+    return data.isin(dts)
+
+
 def cast_years_to_int(x, index=False):
     """Formatting series or timeseries columns to int and checking validity.
     If `index=False`, the function works on the `pd.Series x`; else,


Test patch:
diff --git a/tests/conftest.py b/tests/conftest.py
index 149d67d..c98772f 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1,13 +1,16 @@
+# has to go first for environment setup reasons
 import matplotlib
 matplotlib.use('agg')
 
 import os
 import pytest
-
 import pandas as pd
 
+
+from datetime import datetime
 from pyam import IamDataFrame
 
+
 here = os.path.dirname(os.path.realpath(__file__))
 IMAGE_BASELINE_DIR = os.path.join(here, 'expected_figs')
 TEST_DATA_DIR = os.path.join(here, 'data')
@@ -147,8 +150,25 @@ CHECK_AGG_REGIONAL_DF = pd.DataFrame([
 )
 
 
+TIME_AXES = [
+    [2005, 2010],
+    [datetime(2005, 6, 17), datetime(2010, 7, 21)],
+    ['2005-06-17', '2010-07-21'],
+    ['2005-06-17 00:00:00', '2010-07-21 12:00:00']
+]
+
+
+@pytest.fixture(scope="function", params=TIME_AXES)
+def test_df(request):
+    tdf = TEST_DF.iloc[:2]
+    tdf = tdf.rename({2005: request.param[0], 2010: request.param[1]},
+                     axis="columns")
+    df = IamDataFrame(data=tdf)
+    yield df
+
+
 @pytest.fixture(scope="function")
-def test_df():
+def test_df_year():
     df = IamDataFrame(data=TEST_DF.iloc[:2])
     yield df
 
diff --git a/tests/test_core.py b/tests/test_core.py
index 7d6f5dd..a8eaaba 100644
--- a/tests/test_core.py
+++ b/tests/test_core.py
@@ -1,6 +1,8 @@
 import os
 import copy
 import pytest
+import re
+import datetime
 
 import numpy as np
 import pandas as pd
@@ -49,6 +51,21 @@ def test_init_df_from_timeseries(test_df):
     pd.testing.assert_frame_equal(df.timeseries(), test_df.timeseries())
 
 
+def test_init_df_with_extra_col(test_pd_df):
+    tdf = test_pd_df.copy()
+
+    extra_col = "climate model"
+    extra_value = "scm_model"
+    tdf[extra_col] = extra_value
+
+    df = IamDataFrame(tdf)
+
+    assert df.extra_cols == [extra_col]
+    pd.testing.assert_frame_equal(df.timeseries().reset_index(),
+                                  tdf, check_like=True)
+
+
+
 def test_to_excel(test_df):
     fname = 'foo_testing.xlsx'
     test_df.to_excel(fname)
@@ -145,6 +162,221 @@ def test_filter_error(test_df):
     pytest.raises(ValueError, test_df.filter, foo='foo')
 
 
+def test_filter_year(test_df):
+    obs = test_df.filter(year=2005)
+    if "year" in test_df.data.columns:
+        npt.assert_equal(obs['year'].unique(), 2005)
+    else:
+        expected = np.array(pd.to_datetime('2005-06-17T00:00:00.0'),
+                            dtype=np.datetime64)
+        unique_time = obs['time'].unique()
+        assert len(unique_time) == 1
+        assert unique_time[0] == expected
+
+
+@pytest.mark.parametrize("test_month",
+                         [6, "June", "Jun", "jun", ["Jun", "jun"]])
+def test_filter_month(test_df, test_month):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape("filter by `month` not supported")
+        with pytest.raises(ValueError, match=error_msg):
+            obs = test_df.filter(month=test_month)
+    else:
+        obs = test_df.filter(month=test_month)
+        expected = np.array(pd.to_datetime('2005-06-17T00:00:00.0'),
+                            dtype=np.datetime64)
+        unique_time = obs['time'].unique()
+        assert len(unique_time) == 1
+        assert unique_time[0] == expected
+
+
+@pytest.mark.parametrize("test_month", [6, "Jun", "jun", ["Jun", "jun"]])
+def test_filter_year_month(test_df, test_month):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape("filter by `month` not supported")
+        with pytest.raises(ValueError, match=error_msg):
+            obs = test_df.filter(year=2005, month=test_month)
+    else:
+        obs = test_df.filter(year=2005, month=test_month)
+        expected = np.array(pd.to_datetime('2005-06-17T00:00:00.0'),
+                            dtype=np.datetime64)
+        unique_time = obs['time'].unique()
+        assert len(unique_time) == 1
+        assert unique_time[0] == expected
+
+
+@pytest.mark.parametrize("test_day",
+                         [17, "Fri", "Friday", "friday", ["Fri", "fri"]])
+def test_filter_day(test_df, test_day):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape("filter by `day` not supported")
+        with pytest.raises(ValueError, match=error_msg):
+            obs = test_df.filter(day=test_day)
+    else:
+        obs = test_df.filter(day=test_day)
+        expected = np.array(pd.to_datetime('2005-06-17T00:00:00.0'),
+                            dtype=np.datetime64)
+        unique_time = obs['time'].unique()
+        assert len(unique_time) == 1
+        assert unique_time[0] == expected
+
+
+@pytest.mark.parametrize("test_hour", [0, 12, [12, 13]])
+def test_filter_hour(test_df, test_hour):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape("filter by `hour` not supported")
+        with pytest.raises(ValueError, match=error_msg):
+            obs = test_df.filter(hour=test_hour)
+    else:
+        obs = test_df.filter(hour=test_hour)
+        test_hour = [test_hour] if isinstance(test_hour, int) else test_hour
+        expected_rows = (test_df.data["time"]
+                         .apply(lambda x: x.hour).isin(test_hour))
+        expected = test_df.data["time"].loc[expected_rows].unique()
+
+        unique_time = obs['time'].unique()
+        npt.assert_array_equal(unique_time, expected)
+
+
+def test_filter_time_exact_match(test_df):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape(
+            "`year` can only be filtered with ints or lists of ints"
+        )
+        with pytest.raises(TypeError, match=error_msg):
+            test_df.filter(year=datetime.datetime(2005, 6, 17))
+    else:
+        obs = test_df.filter(time=datetime.datetime(2005, 6, 17))
+        expected = np.array(pd.to_datetime('2005-06-17T00:00:00.0'),
+                            dtype=np.datetime64)
+        unique_time = obs['time'].unique()
+        assert len(unique_time) == 1
+        assert unique_time[0] == expected
+
+
+def test_filter_time_range(test_df):
+    error_msg = r".*datetime.datetime.*"
+    with pytest.raises(TypeError, match=error_msg):
+        test_df.filter(year=range(
+            datetime.datetime(2000, 6, 17),
+            datetime.datetime(2009, 6, 17)
+        ))
+
+
+def test_filter_time_range_year(test_df):
+    obs = test_df.filter(year=range(2000, 2008))
+
+    if "year" in test_df.data.columns:
+        unique_time = obs['year'].unique()
+        expected = np.array([2005])
+    else:
+        unique_time = obs['time'].unique()
+        expected = np.array(pd.to_datetime('2005-06-17T00:00:00.0'),
+                            dtype=np.datetime64)
+
+    assert len(unique_time) == 1
+    assert unique_time[0] == expected
+
+
+@pytest.mark.parametrize("month_range", [range(1, 7), "Mar-Jun"])
+def test_filter_time_range_month(test_df, month_range):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape("filter by `month` not supported")
+        with pytest.raises(ValueError, match=error_msg):
+            obs = test_df.filter(month=month_range)
+    else:
+        obs = test_df.filter(month=month_range)
+        expected = np.array(pd.to_datetime('2005-06-17T00:00:00.0'),
+                            dtype=np.datetime64)
+
+        unique_time = obs['time'].unique()
+        assert len(unique_time) == 1
+        assert unique_time[0] == expected
+
+
+@pytest.mark.parametrize("month_range", [["Mar-Jun", "Nov-Feb"]])
+def test_filter_time_range_round_the_clock_error(test_df, month_range):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape("filter by `month` not supported")
+        with pytest.raises(ValueError, match=error_msg):
+            test_df.filter(month=month_range)
+    else:
+        error_msg = re.escape(
+            "string ranges must lead to increasing integer ranges, "
+            "Nov-Feb becomes [11, 2]"
+        )
+        with pytest.raises(ValueError, match=error_msg):
+            test_df.filter(month=month_range)
+
+
+@pytest.mark.parametrize("day_range", [range(14, 20), "Thu-Sat"])
+def test_filter_time_range_day(test_df, day_range):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape("filter by `day` not supported")
+        with pytest.raises(ValueError, match=error_msg):
+            obs = test_df.filter(day=day_range)
+    else:
+        obs = test_df.filter(day=day_range)
+        expected = np.array(pd.to_datetime('2005-06-17T00:00:00.0'),
+                            dtype=np.datetime64)
+        unique_time = obs['time'].unique()
+        assert len(unique_time) == 1
+        assert unique_time[0] == expected
+
+
+@pytest.mark.parametrize("hour_range", [range(10, 14)])
+def test_filter_time_range_hour(test_df, hour_range):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape("filter by `hour` not supported")
+        with pytest.raises(ValueError, match=error_msg):
+            obs = test_df.filter(hour=hour_range)
+    else:
+        obs = test_df.filter(hour=hour_range)
+
+        expected_rows = (test_df.data["time"]
+                         .apply(lambda x: x.hour).isin(hour_range))
+        expected = test_df.data["time"].loc[expected_rows].unique()
+
+        unique_time = obs['time'].unique()
+        npt.assert_array_equal(unique_time, expected)
+
+
+def test_filter_time_no_match(test_df):
+    if "year" in test_df.data.columns:
+        error_msg = re.escape(
+            "`year` can only be filtered with ints or lists of ints"
+        )
+        with pytest.raises(TypeError, match=error_msg):
+            test_df.filter(year=datetime.datetime(2004, 6, 18))
+    else:
+        obs = test_df.filter(time=datetime.datetime(2004, 6, 18))
+        assert obs.data.empty
+
+
+def test_filter_time_not_datetime_error(test_df):
+    if "year" in test_df.data.columns:
+        with pytest.raises(KeyError, match=re.escape("'time")):
+            test_df.filter(time=datetime.datetime(2004, 6, 18))
+    else:
+        error_msg = re.escape(
+            "`time` can only be filtered with datetimes or lists of datetimes"
+        )
+        with pytest.raises(TypeError, match=error_msg):
+            test_df.filter(time=2005)
+
+
+def test_filter_time_not_datetime_range_error(test_df):
+    if "year" in test_df.data.columns:
+        with pytest.raises(KeyError, match=re.escape("'time")):
+            test_df.filter(time=range(2000, 2008))
+    else:
+        error_msg = re.escape(
+            "`time` can only be filtered with datetimes or lists of datetimes"
+        )
+        with pytest.raises(TypeError, match=error_msg):
+            test_df.filter(time=range(2000, 2008))
+
+
 def test_filter_as_kwarg(meta_df):
     obs = list(meta_df.filter(variable='Primary Energy|Coal').scenarios())
     assert obs == ['a_scenario']
@@ -313,18 +545,18 @@ def test_load_metadata(meta_df):
     pd.testing.assert_series_equal(obs['category'], exp['category'])
 
 
-def test_load_SSP_database_downloaded_file(test_df):
+def test_load_SSP_database_downloaded_file(test_df_year):
     obs_df = IamDataFrame(os.path.join(
         TEST_DATA_DIR, 'test_SSP_database_raw_download.xlsx')
     )
-    pd.testing.assert_frame_equal(obs_df.as_pandas(), test_df.as_pandas())
+    pd.testing.assert_frame_equal(obs_df.as_pandas(), test_df_year.as_pandas())
 
 
-def test_load_RCP_database_downloaded_file(test_df):
+def test_load_RCP_database_downloaded_file(test_df_year):
     obs_df = IamDataFrame(os.path.join(
         TEST_DATA_DIR, 'test_RCP_database_raw_download.xlsx')
     )
-    pd.testing.assert_frame_equal(obs_df.as_pandas(), test_df.as_pandas())
+    pd.testing.assert_frame_equal(obs_df.as_pandas(), test_df_year.as_pandas())
 
 
 def test_append_other_scenario(meta_df):
@@ -388,24 +620,24 @@ def test_append_same_scenario(meta_df):
     npt.assert_array_equal(ts.iloc[2], ts.iloc[3])
 
 
-def test_append_duplicates(test_df):
-    other = copy.deepcopy(test_df)
-    pytest.raises(ValueError, test_df.append, other=other)
+def test_append_duplicates(test_df_year):
+    other = copy.deepcopy(test_df_year)
+    pytest.raises(ValueError, test_df_year.append, other=other)
 
 
-def test_interpolate(test_df):
-    test_df.interpolate(2007)
+def test_interpolate(test_df_year):
+    test_df_year.interpolate(2007)
     dct = {'model': ['a_model'] * 3, 'scenario': ['a_scenario'] * 3,
            'years': [2005, 2007, 2010], 'value': [1, 3, 6]}
     exp = pd.DataFrame(dct).pivot_table(index=['model', 'scenario'],
                                         columns=['years'], values='value')
     variable = {'variable': 'Primary Energy'}
-    obs = test_df.filter(**variable).timeseries()
+    obs = test_df_year.filter(**variable).timeseries()
     npt.assert_array_equal(obs, exp)
 
     # redo the inpolation and check that no duplicates are added
-    test_df.interpolate(2007)
-    assert not test_df.filter(**variable).data.duplicated().any()
+    test_df_year.interpolate(2007)
+    assert not test_df_year.filter(**variable).data.duplicated().any()
 
 
 def test_set_meta_no_name(meta_df):
@@ -547,8 +779,8 @@ def test_filter_by_bool(meta_df):
 
 
 def test_filter_by_int(meta_df):
-    meta_df.set_meta([1, 2], name='value')
-    obs = meta_df.filter(value=[1, 3])
+    meta_df.set_meta([1, 2], name='test')
+    obs = meta_df.filter(test=[1, 3])
     assert obs['scenario'].unique() == 'a_scenario'
 
 
diff --git a/tests/test_feature_aggregate.py b/tests/test_feature_aggregate.py
index fb017b0..74d6df8 100644
--- a/tests/test_feature_aggregate.py
+++ b/tests/test_feature_aggregate.py
@@ -83,7 +83,6 @@ def run_check_agg_fail(pyam_df, tweak_dict, test_type):
     elif 'region' in test_type:
         expected_index['region'] = 'World'
 
-    # units get dropped during aggregation and the index is a list
     expected_index = [v for k, v in expected_index.items()]
 
     for variable in pyam_df.variables():