Repository: LSSTDESC/gcr-catalogs

Issue:
Add configs for catalogs that are part of DC2 public release
(Edited by @yymao Dec 22)

Catalogs in the public release will have different paths than those for the DESC internal use. Hence we will need to add new catalog configs for them. We need to add three configs:

- [x] DPDD-only object catalog
- [x] Truth-match catalog (with only unique truth entries)
- [x] Composite catalog of Object + Truth-match (as an add-on)

The DPDD-only object catalog catalog will use [dc2_object.DC2ObjectParquetCatalog](https://github.com/LSSTDESC/gcr-catalogs/blob/v1.0.4/GCRCatalogs/dc2_object.py#L656) as its reader with the reader option `is_dpdd` set to true to disable translation. 

The paths for these catalogs will be the ones we use for globus integration. 

Discussion:

Patch:
diff --git a/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_object.yaml b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_object.yaml
new file mode 100644
index 0000000..0aad93b
--- /dev/null
+++ b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_object.yaml
@@ -0,0 +1,3 @@
+alias: desc_dc2_run2.2i_dr6_v1_object
+include_in_default_catalog_list: true
+public_release: v1
diff --git a/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_object_with_truth_match.yaml b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_object_with_truth_match.yaml
new file mode 100644
index 0000000..a795ca9
--- /dev/null
+++ b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_object_with_truth_match.yaml
@@ -0,0 +1,3 @@
+alias: desc_dc2_run2.2i_dr6_v1_object_with_truth_match
+include_in_default_catalog_list: true
+public_release: v1
diff --git a/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_truth.yaml b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_truth.yaml
new file mode 100644
index 0000000..b6d1d51
--- /dev/null
+++ b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_truth.yaml
@@ -0,0 +1,3 @@
+alias: desc_dc2_run2.2i_dr6_v1_truth
+include_in_default_catalog_list: true
+public_release: v1
diff --git a/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_v1_object.yaml b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_v1_object.yaml
new file mode 100644
index 0000000..463a062
--- /dev/null
+++ b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_v1_object.yaml
@@ -0,0 +1,8 @@
+subclass_name: dc2_object.DC2ObjectParquetCatalog
+base_dir: ^/lsstdesc-public/dc2/run2.2i-dr6-v1/object_dpdd
+filename_pattern: object_dpdd_tract\d+.parquet$
+is_dpdd: true
+
+description: DESC DC2 (Run2.2i) DR6 Object Table for the Public Release v1
+creators: Rubin LSST DESC DC2 Team
+public_release: v1
diff --git a/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_v1_object_with_truth_match.yaml b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_v1_object_with_truth_match.yaml
new file mode 100644
index 0000000..6bd698c
--- /dev/null
+++ b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_v1_object_with_truth_match.yaml
@@ -0,0 +1,17 @@
+subclass_name: composite.CompositeReader
+catalogs:
+  - catalog_name: desc_dc2_run2.2i_dr6_v1_object
+    is_main: true
+    include_native_quantities: false
+  - catalog_name: desc_dc2_run2.2i_dr6_v1_truth
+    as_object_addon: true
+    as_truth_table: false
+    matching_partition: true
+    matching_row_order: true
+    overwrite_quantities: false
+    overwrite_attributes: false
+    include_native_quantities: false
+
+description: DESC DC2 DR6 Object + Truth-match Table for the Public Release v1
+creators: Rubin LSST DESC DC2 Team
+public_release: v1
diff --git a/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_v1_truth.yaml b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_v1_truth.yaml
new file mode 100644
index 0000000..69a3b2d
--- /dev/null
+++ b/GCRCatalogs/catalog_configs/desc_dc2_run2.2i_dr6_v1_truth.yaml
@@ -0,0 +1,8 @@
+subclass_name: dc2_truth_match.DC2TruthMatchCatalog
+base_dir: ^/lsstdesc-public/dc2/run2.2i-dr6-v1/truth_match
+filename_pattern: truth_tract\d+.parquet$
+as_truth_table: true
+
+description: DESC DC2 (Run2.2i) DR6 Truth Table for the Public Release v1
+creators: Rubin LSST DESC DC2 Team
+public_release: v1


Test patch:
diff --git a/.github/workflows/desc-env-test.yml b/.github/workflows/desc-env-test.yml
index 0041647..6aaaa80 100644
--- a/.github/workflows/desc-env-test.yml
+++ b/.github/workflows/desc-env-test.yml
@@ -1,32 +1,35 @@
-name: DESC Env Docker Build Test
+name: desc-env-test
 
 on:
+  # Trigger the workflow on push, but only for the master branch
   push:
-    branches: [ master ]
+    branches:
+      - master
 
 jobs:
-  docker-build:
+  build:
+    name: Build on Ubuntu
     runs-on: ubuntu-18.04
-    strategy:
-      matrix:
-       include:
-         - docker_name: updateGCRpy
-           docker_image: lsstdesc/desc-python:latest
-           docker_image_updated: lsstdesc/desc-python:test
-           conda_activate: "conda activate desc;"
-           setup_script: /usr/local/py3.7/etc/profile.d/conda.sh
-         ## Disable stack build for now
-         #- docker_name: updateGCRstack
-         #  docker_image: lsstdesc/stack-jupyter:prod
-         #  docker_image_updated: lsstdesc/stack-jupyter:test
-         #  conda_activate: ""
-         #  setup_script: /opt/lsst/software/stack/loadLSST.bash
-
     steps:
-      - name: docker login
-        run: echo '${{ secrets.DOCKERHUB_ACCESSTOK }}' | docker login --username ${{ secrets.DOCKERHUB_USERNAME }} --password-stdin
-      - run: docker pull ${{ matrix.docker_image }}
-      - name: Install master branch of ${{ github.repository }}
-        run: docker run --name=${{ matrix.docker_name }} ${{ matrix.docker_image }} /bin/bash -c "source ${{ matrix.setup_script }}; ${{ matrix.conda_activate }} pip install https://github.com/${{ github.repository }}/archive/master.zip"
-      - run: docker commit -m="Installed ${{ github.repository }} master" ${{ matrix.docker_name }} ${{ matrix.docker_image_updated }}
-      - run: docker push ${{ matrix.docker_image_updated }}
+      - name: Docker login
+        run: echo '${{ secrets.DOCKERHUB_ACCESSTOK }}' | docker login --username heather999 --password-stdin
+      - name: checkout gcr-catalogs
+        uses: actions/checkout@v1
+        with:
+          fetch-depth: 1
+  #    - name: pull desc-stack
+  #      run: docker pull lsstdesc/stack-jupyter:prod 
+      - name: pull desc-python - using latest for now
+        run: docker pull lsstdesc/desc-python:latest
+      - name: Install gcr master in desc-python
+        run: docker run --name="updateGCRpy" lsstdesc/desc-python:latest /bin/bash -c "source /usr/local/py3.7/etc/profile.d/conda.sh; conda activate desc; pip install https://github.com/LSSTDESC/gcr-catalogs/archive/master.zip"
+   #   - name: Install gcr master in desc-stack:test
+   #     run: docker run --name="updateGCRstack" lsstdesc/stack-jupyter:prod /bin/bash -c "source /opt/lsst/software/stack/loadLSST.bash; pip install https://github.com/LSSTDESC/gcr-catalogs/archive/master.zip"
+      - name: docker commit desc-python:test
+        run: docker commit -m="Installed latest GCR master" updateGCRpy lsstdesc/desc-python:test
+    #  - name: docker commit desc-stack:test
+    #    run: docker commit -m="Installed latest GCR master" updateGCRstack lsstdesc/stack-jupyter:test
+      - name: push desc-python
+        run: docker push lsstdesc/desc-python:test
+    #  - name: push desc-stack
+    #   run: docker push lsstdesc/stack-jupyter:test
diff --git a/.github/workflows/python-ci-test.yml b/.github/workflows/pythonpackage.yml
similarity index 88%
rename from .github/workflows/python-ci-test.yml
rename to .github/workflows/pythonpackage.yml
index 4939653..d1239b8 100644
--- a/.github/workflows/python-ci-test.yml
+++ b/.github/workflows/pythonpackage.yml
@@ -1,17 +1,15 @@
-name: Python CI Test
+name: Python package
 
-on:
-  push:
-    branches: [ master ]
-  pull_request:
-    branches: [ master ]
+on: [push, pull_request]
 
 jobs:
   build:
+
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 4
       matrix:
-        python-version: [3.6, 3.7, 3.8]
+        python-version: [3.7]
 
     steps:
     - uses: actions/checkout@v2
@@ -23,13 +21,12 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-py${{ matrix.python-version }}-pip-
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install wheel
         pip install .[full]
     - name: Lint with flake8
       run: |