Repository: openclimatefix/Elexonpy (language: python)

Issue:
 Implement `datasets.py` module with basic functionality
<!--- Provide a general summary of the issue in the Title above -->

## Description
<!--- Provide a detailed description of the change or addition you are proposing -->

`datasets.py`  module will handle fetching data from the Elexon API for specific routes. The initial implementation will focus on key routes, such as `/datasets/ABUC` and `/datasets/AGPT`, to establish the pattern for future route implementations.


Discussion / hints:

Fix patch:
diff --git a/pyelexon/constants.py b/pyelexon/constants.py
new file mode 100644
index 0000000..d7d7d93
--- /dev/null
+++ b/pyelexon/constants.py
@@ -0,0 +1,3 @@
+BASE_URL = "https://data.elexon.co.uk/bmrs/api/v1"
+
+
diff --git a/pyelexon/datasets.py b/pyelexon/datasets.py
index c30b526..ba9e149 100644
--- a/pyelexon/datasets.py
+++ b/pyelexon/datasets.py
@@ -1,7 +1,89 @@
-""" Functions for /datasets """
+import requests
+import pandas as pd
+from pyelexon.constants import BASE_URL
 
-""" 
- See https://developer.data.elexon.co.uk/api-details#api=prod-insol-insights-api for more information 
+class Datasets:
+
+    def fetch_data(self, endpoint, params=None, convert_to_dataframe=False):
+        """
+        Parameters:
+        - endpoint (str): The endpoint path to fetch data from.
+        - params (dict): Optional query parameters.
+        - convert_to_dataframe (bool): Flag to convert response to DataFrame (default: False).
+
+        Returns:
+        - dict or DataFrame: JSON response from the API or DataFrame if convert_to_dataframe is True.
+        """
+        url = f"{BASE_URL}{endpoint}"
+
+        try:
+            response = requests.get(url, params=params)
+            response.raise_for_status()  # Raise an exception for errors
+
+            if convert_to_dataframe:
+                # Check if response is a list of dictionaries
+                if isinstance(response.json(), list):
+                    df = pd.DataFrame(response.json())
+                else:
+                    df = pd.DataFrame(response.json()['data'])
+                return df
+            else:
+                return response.json()
+
+        except requests.exceptions.HTTPError as http_err:
+            print(f"HTTP error: {http_err}")
+        except Exception as err:
+            print(f"Other error: {err}")
+
+        return None
+
+    def fetch_abuc(self, publish_date_time_from, publish_date_time_to, convert_to_dataframe=True):
+        """
+        Parameters:
+        - publish_date_time_from (str): The start datetime for fetching data.
+        - publish_date_time_to (str): The end datetime for fetching data.
+        - convert_to_dataframe (bool): Convert the response to a DataFrame (default: True).
+
+        Returns:
+        - dict or DataFrame: JSON response from the ABUC dataset endpoint or DataFrame if convert_to_dataframe is True.
+
+        API Documentation:
+        For more details on this API endpoint, refer to the [ABUC Dataset API Documentation](https://developer.data.elexon.co.uk/api-details#api=prod-insol-insights-api&operation=get-datasets-abuc-publishdatetimefrom-publishdatetimefrom-publishdatetimeto).
+        """
+        endpoint = '/datasets/ABUC'
+        params = {
+            'publishDateTimeFrom': publish_date_time_from,
+            'publishDateTimeTo': publish_date_time_to,
+            'format': 'json'  # Optional
+        }
+
+        return self.fetch_data(endpoint, params, convert_to_dataframe)
+
+    def fetch_abuc_stream(self, publish_date_time_from, publish_date_time_to, convert_to_dataframe=True):
+        """
+        Parameters:
+        - publish_date_time_from (str): The start datetime for fetching data.
+        - publish_date_time_to (str): The end datetime for fetching data.
+        - convert_to_dataframe (bool): Convert the response to a DataFrame (default: True).
+
+        Returns:
+        - dict or DataFrame: JSON response from the ABUC stream dataset endpoint or DataFrame if convert_to_dataframe is True.
+
+        API Documentation:
+        For more details on this API endpoint, refer to the [ABUC stream Dataset API Documentation](https://developer.data.elexon.co.uk/api-details#api=prod-insol-insights-api&operation=get-datasets-abuc-stream-publishdatetimefrom-publishdatetimefrom-publishdate)
+        """
+        endpoint = '/datasets/ABUC/stream'
+        params = {
+            'publishDateTimeFrom': publish_date_time_from,
+            'publishDateTimeTo': publish_date_time_to,
+            'format': 'json'  # Optional
+        }
+
+        return self.fetch_data(endpoint, params, convert_to_dataframe)
+
+
+"""
+See https://developer.data.elexon.co.uk/api-details#api=prod-insol-insights-api for more information
 This file should contain the following routes:
 """
 
@@ -158,4 +240,4 @@
 # Path: /datasets/YAFM/stream
 # Path: /datasets/YATL
 # Path: /datasets/YATL/stream
-# Path: /datasets/metadata/latest
+# Path: /datasets/metadata/latest
\ No newline at end of file


Test patch:
diff --git a/pyelexon/tests/test_datasets.py b/pyelexon/tests/test_datasets.py
new file mode 100644
index 0000000..58373cb
--- /dev/null
+++ b/pyelexon/tests/test_datasets.py
@@ -0,0 +1,143 @@
+import pytest
+import pandas as pd
+from unittest.mock import patch, MagicMock
+from pyelexon.datasets import Datasets
+import requests
+
+@pytest.fixture
+def mock_requests_get(mocker):
+    return mocker.patch('requests.get')
+
+def test_fetch_data_dict_response(mock_requests_get):
+    # Mock response data for dict response
+    mock_response = {
+        'dataset': 'ABUC',
+        'documentId': 'NGET-EMFIP-ABUC-00689635',
+        'documentRevisionNumber': 1,
+        'publishTime': '2024-06-25T08:29:53Z',
+        'businessType': 'Replacement reserve',
+        'psrType': 'Load',
+        'marketAgreementType': 'Daily',
+        'flowDirection': 'Down',
+        'settlementDate': '2024-06-26',
+        'quantity': 73.0
+    }
+    mock_requests_get.return_value.json.return_value = mock_response
+    mock_requests_get.return_value.status_code = 200
+
+    datasets = Datasets()
+    data_dict = datasets.fetch_data('/datasets/ABUC', convert_to_dataframe=False)
+
+    assert isinstance(data_dict, dict)
+    assert data_dict == mock_response
+
+def test_fetch_data_list_response(mock_requests_get):
+    # Mock response data for list response
+    mock_response = [
+        {
+            'dataset': 'ABUC',
+            'documentId': 'NGET-EMFIP-ABUC-00689635',
+            'documentRevisionNumber': 1,
+            'publishTime': '2024-06-25T08:29:53Z',
+            'businessType': 'Replacement reserve',
+            'psrType': 'Load',
+            'marketAgreementType': 'Daily',
+            'flowDirection': 'Down',
+            'settlementDate': '2024-06-26',
+            'quantity': 73.0
+        },
+        {
+            'dataset': 'ABUC',
+            'documentId': 'NGET-EMFIP-ABUC-00689636',
+            'documentRevisionNumber': 1,
+            'publishTime': '2024-06-25T08:30:00Z',
+            'businessType': 'Replacement reserve',
+            'psrType': 'Generation',
+            'marketAgreementType': 'Daily',
+            'flowDirection': 'Up',
+            'settlementDate': '2024-06-26',
+            'quantity': 1136.0
+        }
+    ]
+    mock_requests_get.return_value.json.return_value = mock_response
+    mock_requests_get.return_value.status_code = 200
+
+    datasets = Datasets()
+    data_df = datasets.fetch_data('/datasets/ABUC', convert_to_dataframe=True)
+
+    assert isinstance(data_df, pd.DataFrame)
+    assert data_df.equals(pd.DataFrame(mock_response))
+
+def test_fetch_abuc_dataframe(mock_requests_get):
+    # Mock response data for fetch_abuc
+    mock_response = [
+        {
+            'dataset': 'ABUC',
+            'documentId': 'NGET-EMFIP-ABUC-00689635',
+            'documentRevisionNumber': 1,
+            'publishTime': '2024-06-25T08:29:53Z',
+            'businessType': 'Replacement reserve',
+            'psrType': 'Load',
+            'marketAgreementType': 'Daily',
+            'flowDirection': 'Down',
+            'settlementDate': '2024-06-26',
+            'quantity': 73.0
+        },
+        {
+            'dataset': 'ABUC',
+            'documentId': 'NGET-EMFIP-ABUC-00689636',
+            'documentRevisionNumber': 1,
+            'publishTime': '2024-06-25T08:30:00Z',
+            'businessType': 'Replacement reserve',
+            'psrType': 'Generation',
+            'marketAgreementType': 'Daily',
+            'flowDirection': 'Up',
+            'settlementDate': '2024-06-26',
+            'quantity': 1136.0
+        }
+    ]
+    mock_requests_get.return_value.json.return_value = mock_response
+    mock_requests_get.return_value.status_code = 200
+
+    datasets = Datasets()
+    abuc_data_df = datasets.fetch_abuc('2024-06-25T00:00:00Z', '2024-06-26T00:00:00Z', convert_to_dataframe=True)
+
+    assert isinstance(abuc_data_df, pd.DataFrame)
+    assert abuc_data_df.equals(pd.DataFrame(mock_response))
+
+def test_fetch_abuc_stream_dataframe(mock_requests_get):
+    # Mock response data for fetch_abuc_stream
+    mock_response = [
+        {
+            'dataset': 'ABUC',
+            'documentId': 'NGET-EMFIP-ABUC-00689635',
+            'documentRevisionNumber': 1,
+            'publishTime': '2024-06-25T08:29:53Z',
+            'businessType': 'Replacement reserve',
+            'psrType': 'Load',
+            'marketAgreementType': 'Daily',
+            'flowDirection': 'Down',
+            'settlementDate': '2024-06-26',
+            'quantity': 73.0
+        },
+        {
+            'dataset': 'ABUC',
+            'documentId': 'NGET-EMFIP-ABUC-00689636',
+            'documentRevisionNumber': 1,
+            'publishTime': '2024-06-25T08:30:00Z',
+            'businessType': 'Replacement reserve',
+            'psrType': 'Generation',
+            'marketAgreementType': 'Daily',
+            'flowDirection': 'Up',
+            'settlementDate': '2024-06-26',
+            'quantity': 1136.0
+        }
+    ]
+    mock_requests_get.return_value.json.return_value = mock_response
+    mock_requests_get.return_value.status_code = 200
+
+    datasets = Datasets()
+    abuc_stream_data_df = datasets.fetch_abuc_stream('2024-06-25T00:00:00Z', '2024-06-26T00:00:00Z', convert_to_dataframe=True)
+
+    assert isinstance(abuc_stream_data_df, pd.DataFrame)
+    assert abuc_stream_data_df.equals(pd.DataFrame(mock_response))
\ No newline at end of file