Repository: gslab-econ/gslab_python (language: python)

Issue:
SaveData for Python
In this issue we'll bring in the Python implementation for SaveData that was created in another project, extend its functionality, and write unit tests. 
SaveData for Python
In this issue we'll bring in the Python implementation for SaveData that was created in another project, extend its functionality, and write unit tests. 


Discussion / hints:

Fix patch:
diff --git a/gslab_misc/SaveData/SaveData.py b/gslab_misc/SaveData/SaveData.py
new file mode 100644
index 0000000..644fcbc
--- /dev/null
+++ b/gslab_misc/SaveData/SaveData.py
@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Sat Dec 25 05:44:33 2021
+
+@author: user
+"""
+
+import pandas as pd
+import hashlib
+import re
+
+def SaveData(df, keys, out_file, log_file = '', append = False, sortbykey = True):
+    CheckExtension(out_file)
+    CheckKeys(df, keys)
+    df_hash = hashlib.md5(pd.util.hash_pandas_object(df).values).hexdigest() 
+    summary_stats = GetSummaryStats(df)
+    SaveDf(df, keys, out_file, sortbykey)
+    SaveLog(df_hash, keys, summary_stats, out_file, append, log_file)
+    
+def CheckExtension(out_file):
+    extension = re.findall(r'\.[a-z]+$', out_file)
+    if not extension[0] in ['.csv', '.dta']:
+        raise ValueError("File extension should be one of .csv or .dta.")
+    
+def CheckKeys(df, keys):
+    if not isinstance(keys, list):
+        raise TypeError("Keys must be specified as a list.")
+        
+    for key in keys:
+        if not key in df.columns:
+            print('%s is not a column name.' % (key))
+            raise ValueError('One of the keys you specified is not among the columns.')
+    
+    df_keys = df[keys]
+    
+    keys_with_missing = df_keys.columns[df_keys.isnull().any()]
+    if keys_with_missing.any():
+        missings_string = ', '.join(keys_with_missing)
+        print('The following keys are missing in some rows: %s .' % (missings_string))
+        raise ValueError('You have rows with missing keys.')
+    
+    
+    if not all(df.groupby(keys).size() == 1):
+        raise ValueError("Keys do not uniquely identify the observations.")
+        
+def GetSummaryStats(df):
+    var_types = df.dtypes
+    var_stats = df.describe().transpose()
+    summary_stats = pd.DataFrame({'type': var_types}).\
+        merge(var_stats, how = 'left', left_index = True, right_index = True)
+    summary_stats = summary_stats.round(4)
+    return(summary_stats)    
+
+def SaveDf(df, keys, out_file, sortbykey):
+    if sortbykey:
+        df.sort_values(keys)
+    
+    if re.search('.csv', out_file):
+        df.to_csv(out_file, index = False)
+    if re.search('.dta', out_file):
+        df.to_stata(out_file, write_index = False)
+    
+def SaveLog(df_hash, keys, summary_stats, out_file, append, log_file):
+    if log_file: 
+        if append:
+            with open(log_file, 'a') as f:
+                f.write('\n\n')
+                f.write('File: %s\n\n' % (out_file))
+                f.write('MD5 hash: %s\n\n' % (df_hash))
+                f.write('Keys: ')
+                for item in keys:
+                    f.write('%s ' % (item))
+                f.write('\n\n')
+                f.write(summary_stats.to_string(header = True, index = True))
+                f.write("\n\n")
+        else:
+            with open(log_file, 'w') as f:
+                f.write('File: %s\n\n' % (out_file))
+                f.write('MD5 hash: %s\n\n' % (df_hash))
+                f.write('Keys: ')
+                for item in keys:
+                    f.write('%s ' % (item))
+                f.write('\n\n')
+                f.write(summary_stats.to_string(header = True, index = True))
+                f.write("\n\n")
+        
+        f.close()    
+    else:
+        pass
+    
+
diff --git a/gslab_misc/SaveData/__init__.py b/gslab_misc/SaveData/__init__.py
new file mode 100644
index 0000000..b956cd8
--- /dev/null
+++ b/gslab_misc/SaveData/__init__.py
@@ -0,0 +1,7 @@
+'''
+SaveData: a Python library for saving data in a normalized format.
+=====================================================
+
+'''
+
+from SaveData import SaveData
diff --git a/gslab_misc/__init__.py b/gslab_misc/__init__.py
index ffbfc08..16e8ecc 100644
--- a/gslab_misc/__init__.py
+++ b/gslab_misc/__init__.py
@@ -3,6 +3,7 @@
 ============================================================================================
 Current tools: 
 		gencat: a Python library for concatenating text files. 
+        SaveData: a Python library for saving data in a normalized format.
 '''
 
 from .gencat.gencat import gencat
diff --git a/manifest.log b/manifest.log
new file mode 100644
index 0000000..bcca893
--- /dev/null
+++ b/manifest.log
@@ -0,0 +1,10 @@
+File: df_sample.dta
+
+MD5 hash: 8de2b44fd5bd9f42d138501aa48031b6
+
+Keys: a 
+
+    type  count  mean     std  min   25%  50%   75%  max
+a  int64    2.0   2.5  0.7071  2.0  2.25  2.5  2.75  3.0
+b  int64    2.0   4.5  0.7071  4.0  4.25  4.5  4.75  5.0
+


Test patch:
diff --git a/gslab_misc/SaveData/tests/__init__.py b/gslab_misc/SaveData/tests/__init__.py
new file mode 100644
index 0000000..a1e0e96
--- /dev/null
+++ b/gslab_misc/SaveData/tests/__init__.py
@@ -0,0 +1,7 @@
+'''
+This directory contains the `SaveData` library's unit tests. These tests can be run using
+`python -m unittest discover`
+from `SaveData/` or from `SaveData/tests/`. To run the tests with logging, use:
+`python run_all_tests.py`
+from `SaveData/tests/`.
+'''
diff --git a/gslab_misc/SaveData/tests/test_main.py b/gslab_misc/SaveData/tests/test_main.py
new file mode 100644
index 0000000..71c5610
--- /dev/null
+++ b/gslab_misc/SaveData/tests/test_main.py
@@ -0,0 +1,66 @@
+import unittest
+import sys
+import pandas as pd
+import numpy as np
+import os
+
+sys.path.append('..')
+
+from SaveData import SaveData
+
+class TestSaveData(unittest.TestCase):
+    
+    def test_wrong_extension(self):
+        df = pd.DataFrame({'a' : [4,3,6], 'b': [4,5,6], 'c': [1,2,3]})
+        with self.assertRaises(ValueError):
+            SaveData(df, ['a'], 'dfs.pdf')
+            
+    def test_wrong_keytype(self):
+        df = pd.DataFrame({'a' : [4,3,6], 'b': [4,5,6], 'c': [1,2,3]})
+        with self.assertRaises(TypeError):
+            SaveData(df, 'a', 'dfs.csv')        
+            
+    def test_missingkeys(self):
+        df = pd.DataFrame({'a' : [4,3,6], 'b': [4,np.nan,6], 'c': [1,2,3]})
+        with self.assertRaises(ValueError):
+            SaveData(df, ['b'], 'dfs.csv')
+
+    def test_duplicate_keys(self):
+        df = pd.DataFrame({'a' : [3,3,6], 'b': [4,5,6], 'c': [1,2,3]})
+        with self.assertRaises(ValueError):
+            SaveData(df, ['a'], 'dfs.csv')
+
+    def test_saves_desired_file(self):
+        df = pd.DataFrame({'a' : [3,3,6], 'b': [4,5,6], 'c': [1,2,3]})
+        SaveData(df, ['b'], 'df.csv')
+        exists = os.path.isfile('df.csv')
+        self.assertEqual(exists, True)
+        os.remove('df.csv')
+                
+    def test_saves_with_log(self):    
+        df = pd.DataFrame({'a' : [3,3,6], 'b': [4,5,6], 'c': [1,2,3]})
+        SaveData(df, ['b'], 'df.csv', 'df.log')
+        exists = os.path.isfile('df.log')
+        self.assertEqual(exists, True)
+        os.remove('df.log')
+        os.remove('df.csv')
+        
+    def test_saves_when_append_given(self):    
+        df = pd.DataFrame({'a' : [3,3,6], 'b': [4,5,6], 'c': [1,2,3]})
+        SaveData(df, ['b'], 'df.csv', 'df.log')
+        SaveData(df, ['c'], 'df.csv', 'df.log', append = True)
+        exists = os.path.isfile('df.log')
+        self.assertEqual(exists, True)
+        os.remove('df.log')
+        os.remove('df.csv')
+        
+    def test_saves_when_sort_is_false(self):    
+        df = pd.DataFrame({'a' : [3,3,6], 'b': [4,5,6], 'c': [1,2,3]})
+        SaveData(df, ['c'], 'df.csv', 'df.log', sortbykey = False)
+        exists = os.path.isfile('df.csv')
+        self.assertEqual(exists, True)
+        os.remove('df.log')
+        os.remove('df.csv')    
+        
+if __name__ == '__main__':
+    unittest.main()