Repository: RobMor/NetZero

Issue:
Best practices for configuration files
Standard practices for configuration files would suggest that JSON is not the best option. While it is an excellent transmission schema, industry standard seems to lean towards YAML or other property files.

Reasons:

* Low Signal to Noise ratio
	* There is a lot of unnecessary punctuation, redundant quotations and useless curly brackets for configuration management.
	* JSON also does not support trailing commas
* Long Strings
	* No support for multi-line strings
	* Strings that carries over onto another line of the file are not possible
* Numbers
	* As defined in the JSON spec, numbers are arbitrary precision finite floating point numbers in decimal notation. If you need to use hexadecimal notation or represent values like infinity or NaN, you might be out of luck

Suggestions: `YAML`, `TOML`, `Dhall` or any custom format implementation

Sources: 
[https://revelry.co/json-configuration-file-format/](https://revelry.co/json-configuration-file-format/)
[https://softwareengineering.stackexchange.com/questions/351126/how-bad-of-an-idea-is-it-to-use-python-files-as-configuration-files](https://softwareengineering.stackexchange.com/questions/351126/how-bad-of-an-idea-is-it-to-use-python-files-as-configuration-files)
[https://arp242.net/json-config.html](https://arp242.net/json-config.html)
[https://www.lucidchart.com/techblog/2018/07/16/why-json-isnt-a-good-configuration-language/](https://www.lucidchart.com/techblog/2018/07/16/why-json-isnt-a-good-configuration-language/)

Discussion:

Patch:
diff --git a/.gitignore b/.gitignore
index 107125a..ffcd445 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,7 +5,7 @@ venv/
 *.egg-info
 
 # Ignore personal configurations
-options.json
+options.ini
 
 # Ignore database files
 *.db
diff --git a/README.md b/README.md
index 3898bf4..b04162a 100644
--- a/README.md
+++ b/README.md
@@ -28,7 +28,7 @@ $ pip install git+https://github.com/RobMor/NetZero.git
 These commands will install all dependencies and install `netzero` on your
 machine.
 
-In order to actually use the tool you'll need to set up an `options.json` file.
+In order to actually use the tool you'll need to set up an `config.ini` file.
 A template with instructions is included in this repository.
 
 ## Usage
@@ -55,7 +55,7 @@ available commands:
 
 Say you wanted to collect Pepco, SolarEdge, and Weather data since August 2019.
 First you need to ensure that your configuration file is set up. In this example
-we will call it `config.json`. Also since we don't already have a database for 
+we will call it `config.ini`. Also since we don't already have a database for 
 our data, we will tell `netzero` to create one called `netzero.db`. Okay, to 
 collect our data we can run this command:
 
@@ -73,7 +73,7 @@ sources to collect from (You can get a full list of sources from `netzero collec
 * `--start=2019-08-01`: tells `netzero` to only collect data after August 1st 2019.
 The date must follow the format `YYYY-MM-DD`.
 
-* `-c config.json`: specifies the file name of our configuration file.
+* `-c config.ini`: specifies the file name of our configuration file.
 
 * `netzero.db`: gives the filename of our database.
 
diff --git a/config.ini b/config.ini
new file mode 100644
index 0000000..7d19d66
--- /dev/null
+++ b/config.ini
@@ -0,0 +1,38 @@
+# To learn more about this configuration file format you can read:
+# https://docs.python.org/3/library/configparser.html
+
+# **Quotes are required around entries in lists**
+
+[pepco]
+# This is a list of files containing your Pepco green button data files.
+# These files can be obtained in XML format from Pepco's website
+files = ["file1.xml", "file2.xml"]
+
+[solar]
+# This is the SolarEdge API key obtained from SolarEdge.
+# In order to obtain an api key, talk to SolarEdge customer support.
+api_key = 00000000000000000000000000000000
+# This is the SolarEdge site ID for your solar panels
+# Again, talk to SolarEdge customer support to obtain this
+site_id = 000000
+
+[weather]
+# This is the API key for NOAA's NCDC API service.
+# This key can be obtained from the API's token request page:
+# https://www.ncdc.noaa.gov/cdo-web/token
+api_key = 00000000000000000000000000000000
+# These are the local GHCND stations you wish to collect data from.
+# This tool requires that these stations provide daily average temperature.
+# You can find your local stations using this tool provided by NCDC
+# https://www.ncdc.noaa.gov/cdo-web/datatools/findstation
+# Be sure to limit your search to the 'Daily Summaries' dataset and 
+# 'Air Temperature' data categories.
+# This tool averages the results from each station so supply as many stations
+# as you want.
+stations = ["GHCND:USX00000000", "GHCND:USY00000000"]
+
+[gshp]
+# Your username and password for your Symphony Water Furnace control panel.
+# https://symphony.mywaterfurnace.com/
+username = email@example.com
+password = password
\ No newline at end of file
diff --git a/netzero/__main__.py b/netzero/__main__.py
index 1eda76c..fd2cdaf 100644
--- a/netzero/__main__.py
+++ b/netzero/__main__.py
@@ -3,13 +3,16 @@
 Provides the command line interface for collecting data from the various soures.
 """
 
+import sqlite3
+import datetime
+import configparser
+
 import netzero.sources
-import sqlite3, datetime, json
 
 
 def collect_data(arguments):
     if arguments.config:
-        config = json.load(arguments.config)
+        config = configparser.read_file(arguments.config)
     else:
         config = None
 
diff --git a/netzero/sources/pepco.py b/netzero/sources/pepco.py
index f7ff44d..14d200e 100644
--- a/netzero/sources/pepco.py
+++ b/netzero/sources/pepco.py
@@ -36,6 +36,7 @@ Green Button XML Format for Documentation Purposes:
 </feed>
 """
 
+import json
 import sqlite3
 import datetime
 import itertools
@@ -63,7 +64,7 @@ class Pepco(DataSource):
     def __init__(self, config, conn):
         super().validate_config(config, entry="pepco", fields=["files"])
 
-        self.files = config["pepco"]["files"]
+        self.files = json.loads(config["pepco"]["files"])
 
         self.conn = conn
 
diff --git a/netzero/sources/weather.py b/netzero/sources/weather.py
index 9cee024..e9c2fa0 100644
--- a/netzero/sources/weather.py
+++ b/netzero/sources/weather.py
@@ -1,6 +1,7 @@
 import sqlite3
 import datetime
 import requests
+import json
 
 from netzero.sources.base import DataSource
 from netzero.sources import util
@@ -16,7 +17,7 @@ class Weather(DataSource):
                                 fields=["api_key", "stations"])
 
         self.api_key = config["weather"]["api_key"]
-        self.stations = config["weather"]["stations"]
+        self.stations = json.loads(config["weather"]["stations"])
 
         self.conn = conn
 


Test patch:
diff --git a/tests/test_weather.py b/tests/test_weather.py
index fe8e4fb..af36a7f 100644
--- a/tests/test_weather.py
+++ b/tests/test_weather.py
@@ -3,6 +3,7 @@ import unittest
 import unittest.mock
 import datetime
 import itertools
+import configparser
 
 from netzero.sources import Weather
 
@@ -24,12 +25,13 @@ fake_results = {
 
 class TestWeatherSource(unittest.TestCase):
     def setUp(self):
-        self.config = {
-            "weather": {
-                "api_key": "fake",
-                "stations": ["fake1", "fake2"]
-            }
+        self.config = configparser.ConfigParser()
+        self.config["weather"] = {
+            "api_key": "fake",
+            "stations": '["fake1", "fake2"]'
         }
+        print(self.config)
+        print(self.config["weather"])
         self.conn = sqlite3.connect(":memory:",
                                     detect_types=sqlite3.PARSE_DECLTYPES)
 
@@ -88,9 +90,9 @@ class TestWeatherSource(unittest.TestCase):
                                 "query_api",
                                 return_value=fake_results)
     def test_weather_collect_max_range(self, mock_query):
-        # Three sources to get 1000 results in 333.33 days, which is less than
+        # Three stations to get 1000 results in 333.33 days, which is less than
         # a year
-        self.config["sources"] = ["fake1", "fake2", "fake3"]
+        self.config["weather"]["stations"] = '["fake1", "fake2", "fake3"]'
 
         start_date = datetime.date(2016, 1, 1)
         end_date = start_date + datetime.timedelta(days=333)
@@ -110,7 +112,7 @@ class TestWeatherSource(unittest.TestCase):
                                 "query_api",
                                 return_value=fake_results)
     def test_weather_collect_multiple_intervals(self, mock_query):
-        self.config["source"] = ["fake1"]
+        self.config["weather"]["stations"] = '["fake1"]'
 
         start_date = datetime.date(2016, 1, 1)
         end_date = start_date + datetime.timedelta(