Repository: databricks-industry-solutions/auto-data-linkage (language: python)

Issue:
Set up CICD for ARC module.



Discussion / hints:

Fix patch:
diff --git a/.github/actions/python_build/action.yml b/.github/actions/python_build/action.yml
new file mode 100644
index 0000000..71d2dd3
--- /dev/null
+++ b/.github/actions/python_build/action.yml
@@ -0,0 +1,24 @@
+name: build arc python
+description: build arc python
+runs:
+  using: "composite"
+  steps:
+    - name: Configure python interpreter
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python }}
+    - name: Install python dependencies
+      shell: bash
+      run: |
+        cd python
+        pip install build wheel pyspark==${{ matrix.spark }}
+        pip install .
+    - name: Test and build python package
+      shell: bash
+      run: |
+        cd python
+        python -m unittest
+        python -m build
+    - name: Copy python artifacts to GH Actions run
+      shell: bash
+      run: cp python/dist/*.whl staging
\ No newline at end of file
diff --git a/.github/actions/upload_artefacts/action.yml b/.github/actions/upload_artefacts/action.yml
new file mode 100644
index 0000000..c153c44
--- /dev/null
+++ b/.github/actions/upload_artefacts/action.yml
@@ -0,0 +1,10 @@
+name: upload arc artefacts
+description: upload arc artefacts
+runs:
+  using: "composite"
+  steps:
+    - name: upload artefacts
+      uses: actions/upload-artifact@v3
+      with:
+        name: artefacts
+        path: staging/*
diff --git a/.github/workflows/build_main.yml b/.github/workflows/build_main.yml
new file mode 100644
index 0000000..1fd7a2c
--- /dev/null
+++ b/.github/workflows/build_main.yml
@@ -0,0 +1,25 @@
+name: build main
+on:
+  push:
+    branches-ignore:
+      - "python/*"
+      - "scala/*"
+  pull_request:
+    branches:
+      - '**'
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        python: [ 3.8.10 ]
+        spark: [ 3.2.1 ]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build python
+        uses: ./.github/actions/python_build
+      - name: upload artefacts
+        uses: ./.github/actions/upload_artefacts
\ No newline at end of file
diff --git a/.github/workflows/build_python.yml b/.github/workflows/build_python.yml
new file mode 100644
index 0000000..267b303
--- /dev/null
+++ b/.github/workflows/build_python.yml
@@ -0,0 +1,25 @@
+name: build_python
+# build for python
+on:
+  push:
+    branches:
+      - "python/*"
+  pull_request:
+    branches:
+      - "python/*"
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    env:
+      GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
+    strategy:
+      matrix:
+        python: [ 3.8.10 ]
+        spark: [ 3.2.1 ]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build python
+        uses: ./.github/actions/python_build
+      - name: upload artefacts
+        uses: ./.github/actions/upload_artefacts
\ No newline at end of file
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
new file mode 100644
index 0000000..d3c3180
--- /dev/null
+++ b/.github/workflows/codeql-analysis.yml
@@ -0,0 +1,61 @@
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main" ]
+  schedule:
+    - cron: '18 16 * * 0'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        
+        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+        
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #   echo "Run, Build Application using script"
+    #   ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
new file mode 100644
index 0000000..2e2e46b
--- /dev/null
+++ b/.github/workflows/docs.yml
@@ -0,0 +1,26 @@
+name: docs
+on:
+  push:
+    branches:
+    - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - name: install pandoc
+      run: sudo apt-get install pandoc
+    - uses: actions/setup-python@v2
+    - uses: actions/checkout@v2
+      with:
+        ref: main
+        fetch-depth: 0 # otherwise, you will failed to push refs to dest repo
+    - name: Build and Commit
+      uses: sphinx-notes/pages@v2
+      with:
+        documentation_path: docs/source
+        requirements_path: docs/docs-requirements.txt
+    - name: Push changes
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        branch: gh-pages
diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml
new file mode 100644
index 0000000..1efc059
--- /dev/null
+++ b/.github/workflows/pypi-release.yml
@@ -0,0 +1,25 @@
+name: publish python package to test-pypi
+on:
+  release:
+    types: [published]
+
+jobs:
+  build-n-publish:
+    name: Build project and publish to TestPyPI
+    runs-on: ubuntu-20.04
+    strategy:
+      matrix:
+        python: [3.8.10]
+        spark: [3.2.1]
+    steps:
+      - name: checkout code
+        uses: actions/checkout@v2
+      - name: build python
+        uses: ./.github/actions/python_build
+      - name: Publish a Python distribution to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          user: __token__
+          password: ${{ secrets.LABS_PYPI_TOKEN }}
+          packages_dir: python/dist/
+
diff --git a/docs/Makefile b/docs/Makefile
new file mode 100644
index 0000000..37b2bda
--- /dev/null
+++ b/docs/Makefile
@@ -0,0 +1,21 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
diff --git a/docs/docs-requirements.txt b/docs/docs-requirements.txt
new file mode 100644
index 0000000..7e676ff
--- /dev/null
+++ b/docs/docs-requirements.txt
@@ -0,0 +1,9 @@
+Sphinx==4.4.0
+sphinx-material==0.0.35
+nbsphinx==0.8.8
+pandoc==2.0.1
+ipython==8.0.1
+sphinxcontrib-fulltoc==1.2.0
+livereload==2.6.3
+autodocsumm==0.2.7
+sphinx-tabs==3.2.0
\ No newline at end of file
diff --git a/docs/source/conf.py b/docs/source/conf.py
new file mode 100644
index 0000000..fcb45e3
--- /dev/null
+++ b/docs/source/conf.py
@@ -0,0 +1,102 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+
+sys.path.insert(0, os.path.abspath('../../python'))
+
+# -- Project information -----------------------------------------------------
+
+project = 'ARC'
+copyright = '2022, Databricks Inc'
+author = 'Robert Whiffin, Marcell Ferencz, Milos Colic, Databricks Inc'
+
+# The full version, including alpha/beta/rc tags
+release = "v0.0.1"
+
+# -- General configuration ---------------------------------------------------
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    "sphinx_material",
+    "nbsphinx",
+    "sphinx_tabs.tabs",
+    "sphinx.ext.githubpages",
+    "sphinx.ext.autosectionlabel",
+    "sphinx.ext.todo"
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', ".env"]
+source_suffix = [".rst", ".md"]
+
+pygments_style = 'sphinx'
+nbsphinx_execute = 'never'
+napoleon_use_admonition_for_notes = True
+sphinx_tabs_disable_tab_closing = True
+todo_include_todos = True
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_material'
+
+# Material theme options (see theme.conf for more information)
+html_theme_options = {
+
+    # Set the name of the project to appear in the navigation.
+    'nav_title': f'ARC {release}',
+
+    # Specify a base_url used to generate sitemap.xml. If not
+    # specified, then no sitemap will be built.
+    # 'base_url': 'https://project.github.io/project',
+
+    # Set the color and the accent color
+    'color_primary': 'green',
+    'color_accent': 'green',
+
+    # Set the repo location to get a badge with stats
+    'repo_url': 'https://github.com/databricks-industry-solutions/splink-public-sector-hackathon',
+    'repo_name': 'ARC',
+
+    'globaltoc_depth': 3,
+    'globaltoc_collapse': False,
+    'globaltoc_includehidden': True,
+    'heroes': {'index': 'Simple, automated and unsupervised data linking solution on Databricks.',
+               'examples/index': 'examples and tutorials to get started with '
+                                 'ARC'},
+    "version_dropdown": True,
+    # "version_json": "../versions-v2.json",
+
+}
+html_title = project
+html_short_title = project
+# html_logo = 'images/logo.svg'
+html_sidebars = {
+    "**": ["logo-text.html", "globaltoc.html", "localtoc.html", "searchbox.html"]
+}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+html_css_files = ["css/custom.css"]
diff --git a/docs/source/index.rst b/docs/source/index.rst
new file mode 100644
index 0000000..9595131
--- /dev/null
+++ b/docs/source/index.rst
@@ -0,0 +1,33 @@
+.. ARC documentation master file, created by
+   sphinx-quickstart on Wed Feb  2 11:01:42 2022.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to ARC's documentation!
+
+Documentation
+=============
+
+.. toctree::
+   :glob:
+   :titlesonly:
+   :maxdepth: 2
+   :caption: Contents:
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`search`
+
+
+.. * :ref:`modindex`
+
+
+Project Support
+===============
+
+Please note that all projects in the ``industry-solutions`` github space are provided for your exploration only, and are not formally supported by Databricks with Service Level Agreements (SLAs). They are provided AS-IS and we do not make any guarantees of any kind. Please do not submit a support ticket relating to any issues arising from the use of these projects.
+
+Any issues discovered through the use of this project should be filed as GitHub Issues on the Repo. They will be reviewed as time permits, but there are no formal SLAs for support.
diff --git a/docs/source/reload.py b/docs/source/reload.py
new file mode 100644
index 0000000..1e811e6
--- /dev/null
+++ b/docs/source/reload.py
@@ -0,0 +1,13 @@
+from livereload import Server, shell
+
+if __name__ == '__main__':
+    server = Server()
+    server.watch('*.rst', shell('make -C .. html'), delay=1)
+    server.watch('*.md', shell('make -C .. html'), delay=1)
+    server.watch('*.py', shell('make -C .. html'), delay=1)
+    server.watch('api/*', shell('make -C .. html'), delay=1)
+    server.watch('images/*', shell('make -C .. html'), delay=1)
+    server.watch('usage/*', shell('make -C .. html'), delay=1)
+    server.watch('_static/*', shell('make -C .. html'), delay=1)
+    server.watch('_templates/*', shell('make -C .. html'), delay=1)
+    server.serve(root='../_build/html')
\ No newline at end of file
diff --git a/python/README.md b/python/README.md
new file mode 100644
index 0000000..6e0ad3c
--- /dev/null
+++ b/python/README.md
@@ -0,0 +1,1 @@
+# ARC Readme
\ No newline at end of file
diff --git a/python/arc/__init__.py b/python/arc/__init__.py
new file mode 100644
index 0000000..94d55ee
--- /dev/null
+++ b/python/arc/__init__.py
@@ -0,0 +1,3 @@
+
+
+__version__ = "0.0.1"
diff --git a/python/setup.cfg b/python/setup.cfg
new file mode 100644
index 0000000..a56af76
--- /dev/null
+++ b/python/setup.cfg
@@ -0,0 +1,52 @@
+[metadata]
+name = databricks-arc
+author = Databricks
+version = attr: arc.__version__
+description = ARC: data linking solution for Databricks with Splink
+url = https://github.com/databricks-industry-solutions/splink-public-sector-hackathon
+long_description = file: README.md
+long_description_content_type = text/markdown
+license = Databricks License
+classifiers =
+    License :: Other/Proprietary License
+    Topic :: Entity Resolution :: Data Linking
+    Programming Language :: Python
+    Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.7
+
+[options]
+packages = find:
+python_requires = >=3.7.0
+install_requires =
+    ipython==7.22.0
+
+[options.package_data]
+arc =
+    lib/*.jar
+    resources/*.png
+
+[options.extras_require]
+dev =
+    isort
+    black
+    build
+
+[isort]
+multi_line_output=3
+include_trailing_comma=True
+force_grid_wrap=0
+use_parentheses=True
+line_length=88
+known_first_party=arc
+default_section=THIRDPARTY
+
+[flake8]
+ignore = E203, E266, E501, W503
+max-line-length = 88
+max-complexity = 20
+select = B,C,E,F,W,T4
+exclude = .git,__pycache__,docs/conf.py,dist
+
+[mypy]
+files=arc
+ignore_missing_imports=true
diff --git a/python/setup.py b/python/setup.py
new file mode 100644
index 0000000..a39aee5
--- /dev/null
+++ b/python/setup.py
@@ -0,0 +1,6 @@
+try:
+    from setuptools import setup
+except ImportError:
+    from distutils.core import setup
+
+setup()


Test patch:
diff --git a/python/test/__init__.py b/python/test/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/python/test/arc_test.py b/python/test/arc_test.py
new file mode 100644
index 0000000..b43b7aa
--- /dev/null
+++ b/python/test/arc_test.py
@@ -0,0 +1,17 @@
+import unittest
+from importlib.metadata import version
+
+import arc
+
+
+class ArcTestCase(unittest.TestCase):
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        # setup testing if needed
+        super().setUpClass()
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        # teardown testing if needed
+        super().tearDownClass()
\ No newline at end of file
diff --git a/python/test/dummy_test.py b/python/test/dummy_test.py
new file mode 100644
index 0000000..09bd851
--- /dev/null
+++ b/python/test/dummy_test.py
@@ -0,0 +1,7 @@
+from python.test.arc_test import ArcTestCase
+
+
+class DummyTest(ArcTestCase):
+
+    def test_invoke_function(self):
+        self.assertEqual("HelloWorld", "HelloWorld")