Repository: Social-Media-Public-Analysis/dozent (language: python)

Issue:
Remove Murpheus as a dependancy?
During testing of the usage of Dozent to download some tweets, I realized that a lot of packages were being installed with the intention of being used for Murpheus. 

AFAIK, there isn't a ton of functionality that comes from Murpheus and we want to ensure that Dozent is as lightweight as possible. 

Because of this, I think it'll be a good idea to remove Murpheus. 

I'll make a relevant branch that can do that without any problems and remove all functionality to simplify Dozent


Discussion / hints:

Fix patch:
diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml
new file mode 100644
index 0000000..28af2e3
--- /dev/null
+++ b/.github/workflows/build-workflow.yml
@@ -0,0 +1,31 @@
+name: Build
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install flake8 pytest
+        pip install -r requirements.txt
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Test with pytest
+      run: |
+        pytest
diff --git a/.github/workflows/workflow-code-cov.yaml b/.github/workflows/workflow-code-cov.yaml
new file mode 100644
index 0000000..70b5990
--- /dev/null
+++ b/.github/workflows/workflow-code-cov.yaml
@@ -0,0 +1,31 @@
+name: Python Code Cov Workflow
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+  codecov:
+    name: Codecov Workflow
+    runs-on: ubuntu-18.04
+    
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up Python
+        uses: actions/setup-python@master
+        with:
+          python-version: 3.8
+      - name: Installing Dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          pip install -r requirements.txt
+      - name: Generate coverage report
+        run: |
+          pip install pytest
+          pip install pytest-cov
+          pytest --cov=./ --cov-config=.coveragerc --cov-report=xml
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v1
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.xml
+          flags: unittests
diff --git a/dozent/__init__.py b/dozent/__init__.py
index 1246cd2..db56ba7 100644
--- a/dozent/__init__.py
+++ b/dozent/__init__.py
@@ -1,8 +1,7 @@
 from dozent.dozent import Dozent
 
 from dozent.catch_thread_exceptions import install as _catch_thread_exceptions
-from dozent.preprocess import Preprocess
 
 _catch_thread_exceptions()
 
-__all__ = ['Dozent', 'Preprocess']
+__all__ = ['Dozent']
diff --git a/dozent/preprocess.py b/dozent/preprocess.py
deleted file mode 100644
index d7307f1..0000000
--- a/dozent/preprocess.py
+++ /dev/null
@@ -1,134 +0,0 @@
-import os
-import shutil
-import tarfile
-import zipfile
-
-from murpheus.data_loading import DataLoading
-from pathlib import Path
-from typing import Union, List
-
-
-class Preprocess:
-    __instance__ = None
-    _output_formats = ['csv', 'json', 'parquet', 'sql']
-    _supported_formats = ['csv', 'json']
-    _unsupported_formats = ['parquet', 'sql']
-
-    def __init__(self):
-        if Preprocess.__instance__ is None:
-            Preprocess.__instance__ = self
-        else:
-            raise RuntimeError(f"Singleton {self.__class__.__name__} class is created more than once!")
-
-    @staticmethod
-    def __untar_file(file_path: Union[str, Path], destination: Union[str, Path], verbose: bool): # skip_tests
-        """
-        Untars a single tar file into target directory
-        :return: None
-        """
-        if verbose:
-            print(f"Extracting {file_path}")
-        tar_file = tarfile.open(file_path)
-        try:
-            tar_file.extractall(destination)
-        except:
-            tar_file.close()
-        tar_file.close()
-
-    @staticmethod
-    def __unzip_file(file_path: Union[str, Path], destination: Union[str, Path], verbose: bool): # skip_tests
-        """
-        Unzips a single zip file into target directory
-        :return: None
-        """
-        if verbose:
-            print(f"Extracting {file_path}")
-        with zipfile.ZipFile(file_path, 'r') as zip_ref:
-            try:
-                zip_ref.extractall(destination)
-            except:
-                zip_ref.close()
-            zip_ref.close()
-
-    @staticmethod
-    def extract_directory(directory_path: Union[str, Path], destination: Union[str, Path], verbose: bool = True,
-                          delete_archive: bool = False):
-        """
-        Extracts all files in a directory into target directory
-        :param delete_archive:
-        :param directory_path: path of the directory
-        :param destination: where the files will be stored
-        :param verbose: show output?
-        :return: None
-        """
-
-        if not os.path.isdir(destination):
-            os.mkdir(destination)
-
-        files = [f for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f))]
-        for file in files:
-            file_extension = file[-3:]
-            if file_extension == 'tar':
-                if verbose:
-                    print(f"\nUntaring: {file}")
-                Preprocess.__untar_file(f"{directory_path}/{file}", destination, verbose)
-            elif file_extension == 'zip':
-                if verbose:
-                    print(f"\nUnzipping: {file}")
-                Preprocess.__unzip_file(f"{directory_path}/{file}", destination, verbose)
-            else:
-                raise RuntimeError(f"File extension .{file_extension} not recognized.")
-        if verbose:
-            print(f"\nSuccessfully extracted all files in {directory_path} to {destination}")
-
-        if delete_archive:
-            shutil.rmtree(directory_path)
-
-    @staticmethod
-    def store_tweets_to_file_format(directory_path: Union[str, Path], destination: Union[str, Path],
-                                    suffix: str = '*.json*',
-                                    recursive: bool = True,
-                                    output_format='csv') -> List[str]:
-        """
-        Extracting all tweets and storing them as csv.
-        This function recursively searches the `directory_path` for files that match the given `suffix`
-
-        :param directory_path: path of the directory where the data is stored.
-        :param suffix: suffix: suffix to search for files with
-        :param destination: where the output needs to be stored
-        :param recursive: recursively search for files?
-        :param output_format: the format that the data should be stored in.
-                              While the possible values are:  ['csv', 'json', 'parquet', 'sql'], we currently test for
-                              only ['csv', 'json']
-        :return: returns the list of all of the output files
-        """
-        if output_format not in Preprocess._supported_formats:
-            raise ValueError(f'The given format: {output_format} is not in {Preprocess._supported_formats}')
-
-        files_list = DataLoading.get_files_list(pathname=directory_path, suffix=suffix, recursive=recursive)
-        data = DataLoading.get_twitter_data_from_file_list(file_lst=files_list, remove_deleted_tweets=True)
-        data = data.to_dataframe()
-
-        if output_format == 'csv':
-            return data.to_csv(destination)
-
-        elif output_format == 'json':
-            return data.to_json(destination)
-
-        elif output_format == 'parquet':  # skip_tests
-            raise NotImplementedError('Parquet has not been yet supported because of the added dependency of '
-                                      'fastparque/pyarrow, which we might look into later'
-                                      'If you would like this feature, please make an issue here: '
-                                      'https://github.com/Twitter-Public-Analysis/Twitter-Public-Analysis/issues')
-
-        elif output_format == 'sql':  # skip_tests
-            raise NotImplementedError('SQL has not been yet implemented. '
-                                      'If you would like this feature, please make an issue here: '
-                                      'https://github.com/Twitter-Public-Analysis/Twitter-Public-Analysis/issues')
-
-        else:
-            raise ValueError(f'The given format: {output_format} is not recognized')
-
-
-if __name__ == "__main__":
-    pass
diff --git a/requirements.txt b/requirements.txt
index 2ab04e8..cd778f6 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,4 +2,4 @@ pySmartDL
 aria2p
 pytest
 pyfiglet
-murpheus==0.6
\ No newline at end of file
+numpy
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 420c969..6177cda 100644
--- a/setup.py
+++ b/setup.py
@@ -10,12 +10,12 @@
   name = 'dozent',
   packages = ['dozent'],
   version = '0.7',
-  license='MIT',
+  license='GNU-3',
   description = 'Dozent is a powerful downloader that is used to download a ton of twitter data from the internet archive.',
   long_description=long_description,
   long_description_content_type='text/markdown',
   author = 'Ali Abbas, Eric Burt, Keelin Becker-Wheeler',
-  author_email = 'eric.burt@protonmail.com',
+  author_email = 'socialmediapublicanalysis@gmail.com',
   url = 'https://github.com/Social-Media-Public-Analysis/dozent',
   package_data={
         '': ['*.json'],
@@ -26,8 +26,7 @@
             'pySmartDL',
             'aria2p',
             'pytest',
-            'pyfiglet',
-            'murpheus',
+            'pyfiglet'
       ],
   classifiers=[
     'Development Status :: 3 - Alpha',


Test patch:
diff --git a/tests/test_preprocess.py b/tests/test_preprocess.py
deleted file mode 100644
index 884bb18..0000000
--- a/tests/test_preprocess.py
+++ /dev/null
@@ -1,75 +0,0 @@
-import unittest
-from dozent.preprocess import Preprocess
-from os import mkdir
-from shutil import rmtree
-from pathlib import Path
-from tests import CommonTestSetup
-import os
-
-CURRENT_FILE_PATH = Path(__file__)
-TEST_EXTRACTOR_FILE_PATH = CURRENT_FILE_PATH.parent / 'test_extract_file_path'
-
-
-class PreprocessTestCase(unittest.TestCase):
-    def setUp(self) -> None:
-        self.data_path, self.path_prefix = CommonTestSetup.set_data_dir_path()
-        self.raw_file_path = Path(__file__).parent / 'compressed_test_files/raw'
-        self.compressed_files_directory = Path(__file__).parent / 'compressed_test_files/'
-        try:
-            mkdir(TEST_EXTRACTOR_FILE_PATH)
-        except FileExistsError:
-            rmtree(TEST_EXTRACTOR_FILE_PATH)
-            mkdir(TEST_EXTRACTOR_FILE_PATH)
-
-        try:
-            mkdir(self.raw_file_path)
-        except FileExistsError:
-            rmtree(self.raw_file_path)
-            mkdir(self.raw_file_path)
-
-    def tearDown(self) -> None:
-        rmtree(TEST_EXTRACTOR_FILE_PATH)
-        rmtree(self.raw_file_path)
-
-    def test_extract_directory(self):
-        Preprocess.extract_directory(self.compressed_files_directory, self.raw_file_path,
-                                     verbose=False)
-
-        self.assertTrue(os.path.isfile(os.path.join(self.raw_file_path, 'test_preprocess_file_1.txt')))
-        self.assertTrue(os.path.isfile(os.path.join(self.raw_file_path, 'test_preprocess_file_2.txt')))
-        self.assertTrue(os.path.isfile(os.path.join(self.raw_file_path, 'test_preprocess_file_3.txt')))
-        self.assertTrue(os.path.isfile(os.path.join(self.raw_file_path, 'test_preprocess_file_4.txt')))
-        self.assertTrue(os.path.isfile(os.path.join(self.raw_file_path, 'test_preprocess_file_5.txt')))
-        self.assertTrue(os.path.isfile(os.path.join(self.raw_file_path, 'test_preprocess_file_6.txt')))
-
-    def test_store_tweets_to_supported_file_formats(self):
-        for output_format in Preprocess._output_formats:
-            output_file_format = f'*.{output_format}'
-            if output_format in Preprocess._unsupported_formats:
-                try:
-                    Preprocess.store_tweets_to_file_format(
-                        directory_path=self.data_path,
-                        destination=TEST_EXTRACTOR_FILE_PATH / output_file_format,
-                        suffix='*.json.bz2',
-                        recursive=True,
-                        output_format=output_format
-                    )
-                    self.assertFalse(False)
-                except (NotImplementedError, ValueError):
-                    self.assertTrue(True)
-            else:
-                file_list = Preprocess.store_tweets_to_file_format(
-                    directory_path=self.data_path,
-                    destination=TEST_EXTRACTOR_FILE_PATH / output_file_format,
-                    suffix='*.json.bz2',
-                    recursive=True,
-                    output_format=output_format
-                )
-
-                file_list = [Path(x) for x in file_list]
-                test_file_list = [x for x in TEST_EXTRACTOR_FILE_PATH.glob(f'*.{output_format}')]
-
-                file_list.sort()
-                test_file_list.sort()
-
-                self.assertListEqual(file_list, test_file_list)