Repository: pyjanitor-devs/pyjanitor

Issue:
[bug] pivot_longer fails when number of entries differ for '.value'
# Brief Description

When pivoting to long form, with `.value`, it is expected that the sub structures will be dataframes. This is not the case all the time, and raises an error.

# System Information

- Operating system: Linux
- OS details (optional):  Linux
- Python version (required): 3.9

# Minimally Reproducible Code

```python

df = [{'x1': 4, 'y1': 7}, {'x1': 5, 'y1': 8}, {'x1': 6, 'y1': 9}]

df

   x1  y1
0   4   7
1   5   8
2   6   9


df.pivot_longer(names_to=".value", names_pattern = r"(.).")
```

<!-- If you provide minimal code that reproduces the problem, this makes it easier for us to debug what's going on.

Minimal code should be trivially copy/pastable into a Python interpreter in its entirety. Be sure to include imports.
-->

# Error Messages

<!-- If you get an error message, please paste it between the backticks here. -->

```
AttributeError                            Traceback (most recent call last)
<ipython-input-147-7afe0417624a> in <module>
----> 1 df.pivot_longer(names_to=".value", names_pattern = r"(.).")

~/miniforge3/envs/datatable/lib/python3.9/site-packages/pandas_flavor/register.py in __call__(self, *args, **kwargs)
     27             @wraps(method)
     28             def __call__(self, *args, **kwargs):
---> 29                 return method(self._obj, *args, **kwargs)
     30 
     31         register_dataframe_accessor(method.__name__)(AccessorMethod)

~/miniforge3/envs/datatable/lib/python3.9/site-packages/janitor/functions.py in pivot_longer(df, index, column_names, names_to, values_to, column_level, names_sep, names_pattern, sort_by_appearance, ignore_index)
   5744     )
   5745 
-> 5746     df = _computations_pivot_longer(
   5747         df,
   5748         index,

~/miniforge3/envs/datatable/lib/python3.9/site-packages/janitor/utils.py in _computations_pivot_longer(df, index, column_names, names_to, values_to, column_level, names_sep, names_pattern, sort_by_appearance, ignore_index)
   1522     else:  # .value
   1523         if df_columns.nlevels == 1:
-> 1524             df = [
   1525                 df[name].melt(ignore_index=False)
   1526                 # changing the name in `rename`

~/miniforge3/envs/datatable/lib/python3.9/site-packages/janitor/utils.py in <listcomp>(.0)
   1523         if df_columns.nlevels == 1:
   1524             df = [
-> 1525                 df[name].melt(ignore_index=False)
   1526                 # changing the name in `rename`
   1527                 # instead of passing it to `melt`

~/miniforge3/envs/datatable/lib/python3.9/site-packages/pandas/core/generic.py in __getattr__(self, name)
   5463             if self._info_axis._can_hold_identifiers_and_holds_name(name):
   5464                 return self[name]
-> 5465             return object.__getattribute__(self, name)
   5466 
   5467     def __setattr__(self, name: str, value) -> None:

AttributeError: 'Series' object has no attribute 'melt'

```



Discussion:

Resolving patch:
diff --git a/docs/conf.py b/docs/conf.py
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -11,6 +11,7 @@
 # -- Path setup --------------------------------------------------------------
 
 import datetime
+
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
@@ -45,7 +46,7 @@
 now = datetime.datetime.now()
 CurrentYear = str(now.year)
 copyright = CurrentYear + ", PyJanitor devs"
-author = "Eric J. Ma"
+author = "pyjanitor devs"
 
 # The short X.Y version
 version = "0.1.0"
@@ -97,7 +98,13 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path .
-exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "**.ipynb_checkpoints"]
+exclude_patterns = [
+    "_build",
+    "Thumbs.db",
+    ".DS_Store",
+    "**.ipynb_checkpoints",
+    "inflating_converting_currency*",  # not working on PR #828.
+]
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = "sphinx"
@@ -215,3 +222,4 @@
 # -- Other options -----------------------------------------------------------
 
 autosummary_generate = True  # Make _autosummary files and include them
+nbsphinx_execute = "never"
diff --git a/janitor/finance.py b/janitor/finance.py
--- a/janitor/finance.py
+++ b/janitor/finance.py
@@ -3,15 +3,15 @@
 """
 
 import json
-from datetime import date, datetime
+from datetime import date
 from functools import lru_cache
-from typing import Optional
 
 import pandas as pd
 import pandas_flavor as pf
 import requests
 
 from janitor import check
+from janitor.errors import JanitorError
 
 from .utils import deprecated_alias
 
@@ -321,6 +321,7 @@
 
 
 def _check_currency(currency: str):
+    """Check that currency is in supported set."""
     if currency not in currency_set:
         raise ValueError(
             f"currency {currency} not in supported currency set, "
@@ -329,6 +330,7 @@ def _check_currency(currency: str):
 
 
 def _check_wb_country(country: str):
+    """Check that world bank country is in supported set."""
     if (country not in wb_country_dict.keys()) & (
         country not in wb_country_dict.values()  # noqa: PD011
     ):
@@ -339,156 +341,194 @@ def _check_wb_country(country: str):
 
 
 def _check_wb_years(year: int):
+    """Check that year is in world bank dataset years."""
     if year < 1960:
         raise ValueError("year value must be 1960 or later")
 
 
-@lru_cache(maxsize=32)
-def _convert_currency(
-    from_currency: str = None,
-    to_currency: str = None,
-    historical_date: Optional[date] = None,
-) -> float:
-    """
-    Currency conversion for Pandas DataFrame column.
-
-    Helper function for `convert_currency` method.
-
-    The API used is: https://exchangeratesapi.io/
-    """
-
-    url = "https://api.exchangeratesapi.io"
-
-    if historical_date:
-        check("historical_date", historical_date, [datetime, date])
-        if isinstance(historical_date, datetime):
-            if historical_date < datetime(1999, 1, 4):
-                raise ValueError(
-                    "historical_date:datetime must be later than 1999-01-04!"
-                )
-            string_date = str(historical_date)[:10]
-        else:
-            if historical_date < date(1999, 1, 4):
-                raise ValueError(
-                    "historical_date:date must be later than 1999-01-04!"
-                )
-            string_date = str(historical_date)
-        url = url + "/%s" % string_date
-    else:
-        url = url + "/latest"
-
-    _check_currency(from_currency)
-    _check_currency(to_currency)
-
-    payload = {"base": from_currency, "symbols": to_currency}
-
-    result = requests.get(url, params=payload)
-
-    if result.status_code != 200:
-        raise ConnectionError(
-            "Exchange Rate API failed to receive a 200 "
-            "response from the server. "
-            "Please try again later."
-        )
-
-    currency_dict = json.loads(result.text)
-    rate = currency_dict["rates"][to_currency]
-    return rate
+# @lru_cache(maxsize=32)
+# def _convert_currency(
+#     api_key: str,
+#     from_currency: str = None,
+#     to_currency: str = None,
+#     historical_date: Optional[date] = None,
+# ) -> float:
+#     """
+#     Currency conversion for Pandas DataFrame column.
+
+#     Helper function for `convert_currency` method.
+
+#     The API used is https://exchangeratesapi.io/.
+#     """
+
+#     url = "http://api.exchangeratesapi.io"
+
+#     if historical_date:
+#         check("historical_date", historical_date, [datetime, date])
+#         if isinstance(historical_date, datetime):
+#             if historical_date < datetime(1999, 1, 4):
+#                 raise ValueError(
+#                     "historical_date:datetime must be later than 1999-01-04!"
+#                 )
+#             string_date = str(historical_date)[:10]
+#         else:
+#             if historical_date < date(1999, 1, 4):
+#                 raise ValueError(
+#                     "historical_date:date must be later than 1999-01-04!"
+#                 )
+#             string_date = str(historical_date)
+#         url = url + "/%s" % string_date
+#     else:
+#         url = url + "/latest"
+
+#     _check_currency(from_currency)
+#     _check_currency(to_currency)
+
+#     payload = {
+#         # "base": from_currency,
+#         "symbols": to_currency,
+#         "access_key": api_key,
+#     }
+
+#     result = requests.get(url, params=payload)
+
+#     if result.status_code != 200:
+#         raise ConnectionError(
+#             "Exchange Rate API failed to receive a 200 "
+#             "response from the server. "
+#             "Please try again later."
+#         )
+
+#     currency_dict = json.loads(result.text)
+#     rate = currency_dict["rates"][to_currency]
+#     return rate
 
 
 @pf.register_dataframe_method
 @deprecated_alias(colname="column_name")
 def convert_currency(
     df: pd.DataFrame,
+    api_key: str,
     column_name: str = None,
     from_currency: str = None,
     to_currency: str = None,
     historical_date: date = None,
     make_new_column: bool = False,
 ) -> pd.DataFrame:
-    """
-    Converts a column from one currency to another, with an option to
-    convert based on historical exchange values.
-
-    This method mutates the original DataFrame.
-
-    :param df: A pandas dataframe.
-    :param column_name: Name of the new column. Should be a string, in order
-        for the column name to be compatible with the Feather binary
-        format (this is a useful thing to have).
-    :param from_currency: The base currency to convert from.
-        May be any of: currency_set = {"AUD", "BGN", "BRL", "CAD", "CHF",
-        "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HRK", "HUF", "IDR",
-        "ILS", "INR", "ISK", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD",
-        "PHP", "PLN", "RON", "RUB", "SEK", "SGD", "THB", "TRY", "USD",
-        "ZAR"}
-    :param to_currency: The target currency to convert to.
-        May be any of: currency_set = {"AUD", "BGN", "BRL", "CAD", "CHF",
-        "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HRK", "HUF", "IDR",
-        "ILS", "INR", "ISK", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD",
-        "PHP", "PLN", "RON", "RUB", "SEK", "SGD", "THB", "TRY", "USD",
-        "ZAR"}
-    :param historical_date: If supplied, get exchange rate on a certain\
-        date. If not supplied, get the latest exchange rate. The exchange\
-        rates go back to Jan. 4, 1999.
-    :param make_new_column: Generates new column for converted currency if
-        True, otherwise, converts currency in place.
-    :returns: The dataframe with converted currency column.
-
-    :Setup:
-
-    .. code-block:: python
-
-        import pandas as pd
-        import janitor
-        from datetime import date
-
-        data_dict = {
-            "a": [1.23452345, 2.456234, 3.2346125] * 3,
-            "Bell__Chart": [1/3, 2/7, 3/2] * 3,
-            "decorated-elephant": [1/234, 2/13, 3/167] * 3,
-            "animals": ["rabbit", "leopard", "lion"] * 3,
-            "cities": ["Cambridge", "Shanghai", "Basel"] * 3,
-        }
-
-        example_dataframe = pd.DataFrame(data_dict)
-
-    :Example: Converting a column from one currency to another using rates
-
-    from 01/01/2018:
-
-    .. code-block:: python
-
-        example_dataframe.convert_currency('a', from_currency='USD',
-        to_currency='EUR', historical_date=date(2018,1,1))
-
-    :Output:
-
-    .. code-block:: python
-
-                    a  Bell__Chart  decorated-elephant  animals     cities
-        0  1.029370     0.333333            0.004274   rabbit  Cambridge
-        1  2.048056     0.285714            0.153846  leopard   Shanghai
-        2  2.697084     1.500000            0.017964     lion      Basel
-        3  1.029370     0.333333            0.004274   rabbit  Cambridge
-        4  2.048056     0.285714            0.153846  leopard   Shanghai
-        5  2.697084     1.500000            0.017964     lion      Basel
-        6  1.029370     0.333333            0.004274   rabbit  Cambridge
-        7  2.048056     0.285714            0.153846  leopard   Shanghai
-        8  2.697084     1.500000            0.017964     lion      Basel
-
-    """
-
-    rate = _convert_currency(from_currency, to_currency, historical_date)
-
-    if make_new_column:
-        new_column_name = column_name + "_" + to_currency
-        df[new_column_name] = df[column_name] * rate
+    """Deprecated function."""
+    raise JanitorError(
+        "The `convert_currency` function has been temporarily disabled due to "
+        "exchangeratesapi.io disallowing free pinging of its API. "
+        "(Our tests started to fail due to this issue.) "
+        "There is no easy way around this problem "
+        "except to find a new API to call on."
+        "Please comment on issue #829 "
+        "(https://github.com/pyjanitor-devs/pyjanitor/issues/829) "
+        "if you know of an alternative API that we can call on, "
+        "otherwise the function will be removed in pyjanitor's 1.0 release."
+    )
 
-    else:
-        df[column_name] = df[column_name] * rate
 
-    return df
+# @pf.register_dataframe_method
+# @deprecated_alias(colname="column_name")
+# def convert_currency(
+#     df: pd.DataFrame,
+#     api_key: str,
+#     column_name: str = None,
+#     from_currency: str = None,
+#     to_currency: str = None,
+#     historical_date: date = None,
+#     make_new_column: bool = False,
+# ) -> pd.DataFrame:
+#     """
+#     Converts a column from one currency to another, with an option to
+#     convert based on historical exchange values.
+
+#     On April 10 2021,
+#     we discovered that there was no more free API available.
+#     Thus, an API key is required to perform currency conversion.
+#     API keys should be set as an environment variable,
+#     for example, ``EXCHANGE_RATE_API_KEY``,
+#     and then passed into the function
+#     by calling on ``os.getenv("EXCHANGE_RATE_APIKEY")``.
+
+#     :param df: A pandas dataframe.
+#     :param api_key: exchangeratesapi.io API key.
+#     :param column_name: Name of the new column. Should be a string, in order
+#         for the column name to be compatible with the Feather binary
+#         format (this is a useful thing to have).
+#     :param from_currency: The base currency to convert from.
+#         May be any of: currency_set = {"AUD", "BGN", "BRL", "CAD", "CHF",
+#         "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HRK", "HUF", "IDR",
+#         "ILS", "INR", "ISK", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD",
+#         "PHP", "PLN", "RON", "RUB", "SEK", "SGD", "THB", "TRY", "USD",
+#         "ZAR"}
+#     :param to_currency: The target currency to convert to.
+#         May be any of: currency_set = {"AUD", "BGN", "BRL", "CAD", "CHF",
+#         "CNY", "CZK", "DKK", "EUR", "GBP", "HKD", "HRK", "HUF", "IDR",
+#         "ILS", "INR", "ISK", "JPY", "KRW", "MXN", "MYR", "NOK", "NZD",
+#         "PHP", "PLN", "RON", "RUB", "SEK", "SGD", "THB", "TRY", "USD",
+#         "ZAR"}
+#     :param historical_date: If supplied,
+#         get exchange rate on a certain date.
+#         If not supplied, get the latest exchange rate.
+#         The exchange rates go back to Jan. 4, 1999.
+#     :param make_new_column: Generates new column
+#         for converted currency if True,
+#         otherwise, converts currency in place.
+#     :returns: The dataframe with converted currency column.
+
+#     .. code-block:: python
+
+#         import pandas as pd
+#         import janitor
+#         from datetime import date
+
+#         data_dict = {
+#             "a": [1.23452345, 2.456234, 3.2346125] * 3,
+#             "Bell__Chart": [1/3, 2/7, 3/2] * 3,
+#             "decorated-elephant": [1/234, 2/13, 3/167] * 3,
+#             "animals": ["rabbit", "leopard", "lion"] * 3,
+#             "cities": ["Cambridge", "Shanghai", "Basel"] * 3,
+#         }
+
+#         example_dataframe = pd.DataFrame(data_dict)
+
+#     Example: Converting a column from one currency to another
+#     using rates from 01/01/2018.
+
+#     .. code-block:: python
+
+#         example_dataframe.convert_currency('a', from_currency='USD',
+#         to_currency='EUR', historical_date=date(2018,1,1))
+
+#     Output:
+
+#     .. code-block:: python
+
+#                     a  Bell__Chart  decorated-elephant  animals     cities
+#         0  1.029370     0.333333            0.004274   rabbit  Cambridge
+#         1  2.048056     0.285714            0.153846  leopard   Shanghai
+#         2  2.697084     1.500000            0.017964     lion      Basel
+#         3  1.029370     0.333333            0.004274   rabbit  Cambridge
+#         4  2.048056     0.285714            0.153846  leopard   Shanghai
+#         5  2.697084     1.500000            0.017964     lion      Basel
+#         6  1.029370     0.333333            0.004274   rabbit  Cambridge
+#         7  2.048056     0.285714            0.153846  leopard   Shanghai
+#         8  2.697084     1.500000            0.017964     lion      Basel
+#     """
+
+#     rate = _convert_currency(
+#         api_key, from_currency, to_currency, historical_date
+#     )
+
+#     if make_new_column:
+#         # new_column_name = column_name + "_" + to_currency
+#         column_name = column_name + "_" + to_currency
+
+#     df = df.assign(column_name=df[column_name] * rate)
+
+#     return df
 
 
 @lru_cache(maxsize=32)
diff --git a/janitor/utils.py b/janitor/utils.py
--- a/janitor/utils.py
+++ b/janitor/utils.py
@@ -1342,6 +1342,14 @@ def _pivot_longer_extractions(
                 """
             )
 
+    outcome = None
+    single_index_mapping = not isinstance(mapping, pd.MultiIndex)
+    if single_index_mapping:
+        outcome = pd.Series(mapping)
+        outcome = outcome.groupby(outcome).cumcount()
+        mapping = pd.MultiIndex.from_arrays([mapping, outcome])
+        outcome = None
+
     df.columns = mapping
 
     dot_value = any(
@@ -1354,68 +1362,56 @@ def _pivot_longer_extractions(
     dtypes = None
     cumcount = None
     if dot_value:
-        if not isinstance(mapping, pd.MultiIndex):
-            dtypes = CategoricalDtype(
-                categories=mapping.drop_duplicates(), ordered=True
-            )
-            df.columns = df.columns.astype(dtypes)
-
-        else:
-            if not mapping.is_unique:
-                cumcount = pd.factorize(mapping)[0]
-                cumcount = pd.Series(cumcount).groupby(cumcount).cumcount()
-            cumcount_check = cumcount is not None
-            mapping_names = mapping.names
-            mapping = [
-                mapping.get_level_values(name) for name in mapping_names
-            ]
-            dtypes = [
-                CategoricalDtype(
-                    categories=entry.drop_duplicates(), ordered=True
-                )
-                for entry in mapping
-            ]
-            mapping = [
-                entry.astype(dtype) for entry, dtype in zip(mapping, dtypes)
-            ]
+        if not mapping.is_unique:
+            cumcount = pd.factorize(mapping)[0]
+            cumcount = pd.Series(cumcount).groupby(cumcount).cumcount()
+        cumcount_check = cumcount is not None
+        mapping_names = mapping.names
+        mapping = [mapping.get_level_values(name) for name in mapping_names]
+        dtypes = [
+            CategoricalDtype(categories=entry.unique(), ordered=True)
+            for entry in mapping
+        ]
+        mapping = [
+            entry.astype(dtype) for entry, dtype in zip(mapping, dtypes)
+        ]
 
-            if cumcount_check:
-                mapping.append(cumcount)
-            mapping = pd.MultiIndex.from_arrays(mapping)
-            df.columns = mapping
-
-            mapping = df.columns
-            if cumcount_check:
-                mapping = mapping.droplevel(-1)
-
-            # test if all combinations are present
-            first = mapping.get_level_values(".value")
-            last = mapping.droplevel(".value")
-            outcome = first.groupby(last)
-            outcome = (value for _, value in outcome.items())
-            outcome = combinations(outcome, 2)
-            outcome = (
-                left.symmetric_difference(right).empty
-                for left, right in outcome
-            )
+        if cumcount_check:
+            mapping.append(cumcount)
+        mapping = pd.MultiIndex.from_arrays(mapping)
+        df.columns = mapping
+
+        mapping = df.columns
+        if cumcount_check:
+            mapping = mapping.droplevel(-1)
+
+        # test if all combinations are present
+        first = mapping.get_level_values(".value")
+        last = mapping.droplevel(".value")
+        outcome = first.groupby(last)
+        outcome = (value for _, value in outcome.items())
+        outcome = combinations(outcome, 2)
+        outcome = (
+            left.symmetric_difference(right).empty for left, right in outcome
+        )
 
-            # include all combinations into the columns
-            if not all(outcome):
-                if isinstance(last, pd.MultiIndex):
-                    indexer = (first.drop_duplicates(), last.drop_duplicates())
-                    complete_index = _complete_indexer_expand_grid(indexer)
-                    complete_index = complete_index.reorder_levels(
-                        [*mapping.names]
-                    )
+        # include all combinations into the columns
+        if not all(outcome):
+            if isinstance(last, pd.MultiIndex):
+                indexer = (first.drop_duplicates(), last.drop_duplicates())
+                complete_index = _complete_indexer_expand_grid(indexer)
+                complete_index = complete_index.reorder_levels(
+                    [*mapping.names]
+                )
 
-                else:
-                    complete_index = pd.MultiIndex.from_product(mapping.levels)
+            else:
+                complete_index = pd.MultiIndex.from_product(mapping.levels)
 
-                df = df.reindex(columns=complete_index)
-            if cumcount_check:
-                df = df.droplevel(-1, axis="columns")
+            df = df.reindex(columns=complete_index)
+        if cumcount_check:
+            df = df.droplevel(-1, axis=1)
 
-    return df
+    return df, single_index_mapping
 
 
 def _computations_pivot_longer(
@@ -1503,7 +1499,7 @@ def _computations_pivot_longer(
 
         return df
 
-    df = _pivot_longer_extractions(
+    df, single_index_mapping = _pivot_longer_extractions(
         df=df,
         index=index,
         column_names=column_names,
@@ -1512,53 +1508,57 @@ def _computations_pivot_longer(
         names_pattern=names_pattern,
     )
 
-    df_columns = df.columns
+    # df_columns = df.columns
+    unique_names = None
+    drop_column = None
+    dot_value = ".value" in df.columns.names
 
-    if ".value" not in df_columns.names:
+    if not dot_value:
+        if single_index_mapping:
+            unique_names = df.columns.names[0]
+            drop_column = "_".join([unique_names, values_to])
+            df.columns.names = [unique_names, drop_column]
         df = pd.melt(
             df, id_vars=None, value_name=values_to, ignore_index=False
         )
 
-    else:  # .value
-        if df_columns.nlevels == 1:
-            df = [
-                df[name].melt(ignore_index=False)
-                # changing the name in `rename`
-                # instead of passing it to `melt`
-                # avoids name collision,
-                # especially if `name` exists in `df`
-                # during melt
-                .iloc[:, -1].rename(name)
-                for name in df_columns.unique()
-            ]
-            df = pd.concat(df, axis=1)
-
-        else:
-            # ensures that the correct values are aligned,
-            # in preparation for the recombination
-            # of the columns downstream
-            df = df.sort_index(axis=1)
-            df = [
-                df.xs(key=name, level=".value", axis=1).melt(
-                    ignore_index=False, value_name=name
-                )
-                for name in df_columns.get_level_values(".value").unique()
-            ]
+    else:
+        unique_names = df.columns.get_level_values(".value").categories
+        if single_index_mapping:
+            # passing `drop_column` to `melt` downstream
+            # avoids any name conflict with `var_name`,
+            # especially if var_name exists in the names
+            # associated with .value.
+            drop_column = "_".join(unique_names)
+        # ensures that the correct values are aligned,
+        # in preparation for the recombination
+        # of the columns downstream
+        df = df.sort_index(axis=1)
+
+        df = [
+            df.xs(key=name, level=".value", axis=1).melt(
+                ignore_index=False, var_name=drop_column, value_name=name
+            )
+            for name in unique_names
+        ]
 
-            first, *rest = df
-
-            # `first` has all the required columns;
-            # as such, there is no need to keep these columns in
-            # the other dataframes in `rest`;
-            # plus, we avoid duplicate columns during concatenation
-            # the only column we need is the last column,
-            # from each dataframe in `rest`
-            # uniformity in the data is already assured
-            # with the categorical dtype creation,
-            # followed by the sorting on the columns earlier.
-            rest = [frame.iloc[:, -1] for frame in rest]
-            # df = first.join(rest, how = 'outer', sort = False)
-            df = pd.concat([first, *rest], axis=1)
+        first, *rest = df
+
+        # `first` has all the required columns;
+        # as such, there is no need to keep these columns in
+        # the other dataframes in `rest`;
+        # plus, we avoid duplicate columns during concatenation
+        # the only column we need is the last column,
+        # from each dataframe in `rest`
+        # uniformity in the data is already assured
+        # with the categorical dtype creation,
+        # followed by the sorting on the columns earlier.
+        rest = [frame.iloc[:, -1] for frame in rest]
+        # df = first.join(rest, how = 'outer', sort = False)
+        df = pd.concat([first, *rest], axis=1)
+
+    if single_index_mapping:
+        df = df.drop(columns=drop_column)
 
     if index:
         df = df.reset_index(level=index)


Test patch:
diff --git a/tests/finance/test_convert_currency.py b/tests/finance/test_convert_currency.py
--- a/tests/finance/test_convert_currency.py
+++ b/tests/finance/test_convert_currency.py
@@ -8,7 +8,7 @@
 
 
 @pytest.mark.finance
-@pytest.mark.xfail
+@pytest.mark.xfail(reason="changes made to web API prevent this from running")
 def test_make_currency_api_request():
     """
     Test for currency API request.
@@ -25,6 +25,7 @@ def test_make_currency_api_request():
     assert r.status_code == 200
 
 
+@pytest.mark.xfail(reason="changes made to web API prevent this from running")
 @pytest.mark.finance
 def test_make_new_currency_col(dataframe):
     """Test converting to same currency equals original currency column."""
@@ -33,6 +34,7 @@ def test_make_new_currency_col(dataframe):
 
 
 @pytest.mark.finance
+@pytest.mark.xfail(reason="changes made to web API prevent this from running")
 def test_historical_datetime(dataframe):
     """Test conversion raises exception for datetime outside API range."""
     with pytest.raises(ValueError):
@@ -46,6 +48,7 @@ def test_historical_datetime(dataframe):
 
 
 @pytest.mark.finance
+@pytest.mark.xfail(reason="changes made to web API prevent this from running")
 def test_historical_date(dataframe):
     """Test conversion raises exception for date outside API range."""
     with pytest.raises(ValueError):
@@ -59,6 +62,7 @@ def test_historical_date(dataframe):
 
 
 @pytest.mark.finance
+@pytest.mark.xfail(reason="changes made to web API prevent this from running")
 def test_currency_check(dataframe):
     """Test conversion raises exception for invalid currency."""
     with pytest.raises(ValueError):
diff --git a/tests/spark/conftest.py b/tests/spark/conftest.py
--- a/tests/spark/conftest.py
+++ b/tests/spark/conftest.py
@@ -1,3 +1,5 @@
+"""Spark fixtures."""
+
 import pytest
 
 try:
@@ -13,8 +15,9 @@
     pass
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture  # (scope="session")
 def spark():
+    """Create spark session."""
     spark = SparkSession.builder.getOrCreate()
     yield spark
     spark.stop()
@@ -22,6 +25,7 @@ def spark():
 
 @pytest.fixture
 def spark_df(spark):
+    """Create spark dataframe."""
     schema = StructType(
         [
             StructField("a", IntegerType(), True),
@@ -36,4 +40,5 @@ def spark_df(spark):
 
 @pytest.fixture
 def spark_dataframe(spark, dataframe):
+    """Another function to create spark dataframe."""
     return spark.createDataFrame(dataframe)