Repository: CybercentreCanada/assemblyline-service-server (language: python)

Issue:
Get the full safelist related to a specific tags



Discussion / hints:

Fix patch:
diff --git a/pipelines/azure-build.yaml b/pipelines/azure-build.yaml
index 6a162f5..9531b1a 100644
--- a/pipelines/azure-build.yaml
+++ b/pipelines/azure-build.yaml
@@ -6,164 +6,156 @@ trigger:
 pr: none
 
 pool:
-  vmImage: 'ubuntu-18.04'
+  vmImage: "ubuntu-20.04"
 
 resources:
   containers:
-  - container: redis
-    image: redis
-    ports:
-      - 6379:6379
-  - container: elasticsearch
-    image: cccs/elasticsearch:7.8.0
-    env:
-      ES_JAVA_OPTS: "-Xms256m -Xmx512m"
-      DISCOVERY_TYPE: 'single-node'
-    ports:
-      - 9200:9200
-  - container: minio
-    image: cccs/minio
-    env:
-      MINIO_ACCESS_KEY: al_storage_key
-      MINIO_SECRET_KEY: Ch@ngeTh!sPa33w0rd
-    ports:
-      - 9000:9000
-
-stages:
-- stage: build
-  jobs:
-  - job: build_package
-    displayName: Build Package
-    steps:
-    - task: UsePythonVersion@0
-      displayName: Set python version
-      inputs: {versionSpec: 3.8}
-    - script: |
-        export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
-        export COMMIT=`git rev-parse --verify ${BUILD_SOURCEBRANCH}`
-        export BRANCH=`git ls-remote --heads origin | grep $COMMIT | sed "s/.*\///"`
-        echo "Building $VERSION On branch $BRANCH"
-        if [[ "$VERSION" == *stable* ]] && [[ $BRANCH != "master" ]]; then exit 1; fi
-        if [[ "$VERSION" == *dev* ]] && [[ $BRANCH != "dev" ]]; then exit 1; fi
-        export VERSION=${VERSION/stable}
-        export VERSION=${VERSION/beta/b}
-        echo $VERSION > assemblyline_service_server/VERSION
-        sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U wheel cython pip
-        python setup.py bdist_wheel
-    - publish: $(System.DefaultWorkingDirectory)/dist/
-      artifact: dist
-    - publish: $(System.DefaultWorkingDirectory)/pipelines/
-      artifact: pipelines
-    - publish: $(System.DefaultWorkingDirectory)/test/
-      artifact: test
-- stage: test
-  jobs:
-  - job: run_test
-    strategy:
-      matrix:
-        python3_7:
-          python.version: '3.7'
-        Python3_8:
-          python.version: '3.8'
-    timeoutInMinutes: 10
-    services:
-      elasticsearch: elasticsearch
-      redis: redis
-      minio: minio
-    steps:
-    - checkout: none
-    - task: UsePythonVersion@0
-      displayName: Set python version
-      inputs:
-        versionSpec: '$(python.version)'
-    - download: current
-      artifact: dist
-    - download: current
-      artifact: pipelines
-    - download: current
-      artifact: test
-    - script: |
-        set -x  # Echo commands before they are run
-        sudo apt-get update
-        sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev
-        export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
-        if [[ "$VERSION" == *stable* ]]; then
-          sudo -E env "PATH=$PATH" python -m pip install "assemblyline" "assemblyline-core"
-        else
-          sudo -E env "PATH=$PATH" python -m pip install --pre "assemblyline" "assemblyline-core"
-        fi
-        export VERSION=${VERSION/stable}
-        export VERSION=${VERSION/beta/b}
-        sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -f $(Pipeline.Workspace)/dist/ "assemblyline-service-server==${VERSION}"
-        sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -r $(Pipeline.Workspace)/test/requirements.txt
-        sudo mkdir -p /etc/assemblyline/
-        sudo mkdir -p /var/cache/assemblyline/
-        sudo mkdir -p /var/lib/assemblyline/
-        sudo chmod a+rw /var/cache/assemblyline/
-        sudo chmod a+rw /var/lib/assemblyline/
-        sudo cp $(Pipeline.Workspace)/pipelines/config.yml /etc/assemblyline
-      displayName: Install package
-    - script: |
-        python -m assemblyline_service_server.app &
-        pytest -rsx -vv
-      workingDirectory: $(Pipeline.Workspace)/test
-      displayName: Test
-- stage: deploy
-  jobs:
-  - job: deploy
-    displayName: Deploy packages and containers
-    variables:
-      - group: deployment-information
-    steps:
-    - task: UsePythonVersion@0
-      displayName: Set python version
-      inputs: {versionSpec: 3.8}
-    - download: current
-      artifact: dist
-    - script: |
-        set -xv  # Echo commands before they are run
-        sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
-        ls dist
-        twine upload --skip-existing --repository-url $TEST_REPOSITORY_URL dist/*
-      workingDirectory: $(Pipeline.Workspace)
-      displayName: Deploy to Test PyPI
+    - container: redis
+      image: redis
+      ports:
+        - 6379:6379
+    - container: elasticsearch
+      image: cccs/elasticsearch:7.8.0
       env:
-        TWINE_USERNAME: $(twineUsername)
-        TWINE_PASSWORD: $(twinePassword)
-    - script: |
-        set -xv  # Echo commands before they are run
-        sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
-        ls dist
-        twine upload --skip-existing dist/*
-      workingDirectory: $(Pipeline.Workspace)
-      displayName: Deploy to PyPI
+        ES_JAVA_OPTS: "-Xms256m -Xmx512m"
+        DISCOVERY_TYPE: "single-node"
+      ports:
+        - 9200:9200
+    - container: minio
+      image: cccs/minio
       env:
-        TWINE_USERNAME: $(twineUsername)
-        TWINE_PASSWORD: $(twinePassword)
-    - task: Docker@2
-      displayName: Login to docker hub
-      inputs:
-        command: login
-        containerRegistry: dockerhub
-    - task: Docker@2
-      displayName: Login to github packages
-      inputs:
-        command: login
-        containerRegistry: github-packages-sa
-    - script: |
-        set -xv  # Echo commands before they are run
-        export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
-        if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
-        export VERSION=${TAG/stable}
-        export VERSION=${VERSION/beta/b}
-        export SERIES="`expr $TAG : '\([0-9]\+\.[0-9]\+\.\)'`${BUILD_TYPE}"
-        until sudo env "PATH=$PATH" python -m pip download "assemblyline-service-server==$VERSION" --pre --no-deps &> /dev/null; do sleep 2; done
+        MINIO_ACCESS_KEY: al_storage_key
+        MINIO_SECRET_KEY: Ch@ngeTh!sPa33w0rd
+      ports:
+        - 9000:9000
 
-        for IMAGE in "cccs/assemblyline-service-server" "docker.pkg.github.com/cybercentrecanada/assemblyline/assemblyline-service-server"
-        do
-          docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -t $IMAGE:$TAG -t $IMAGE:$BUILD_TYPE -t $IMAGE:$SERIES docker
-          docker push $IMAGE --all-tags
-        done
+stages:
+  - stage: build
+    jobs:
+      - job: build_package
+        displayName: Build Package
+        steps:
+          - task: UsePythonVersion@0
+            displayName: Set python version
+            inputs: { versionSpec: 3.9 }
+          - script: |
+              export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
+              export COMMIT=`git rev-parse --verify ${BUILD_SOURCEBRANCH}`
+              export BRANCH=`git ls-remote --heads origin | grep $COMMIT | sed "s/.*\///"`
+              echo "Building $VERSION On branch $BRANCH"
+              if [[ "$VERSION" == *stable* ]] && [[ $BRANCH != "master" ]]; then exit 1; fi
+              if [[ "$VERSION" == *dev* ]] && [[ $BRANCH != "dev" ]]; then exit 1; fi
+              export VERSION=${VERSION/stable}
+              export VERSION=${VERSION/beta/b}
+              echo $VERSION > assemblyline_service_server/VERSION
+              sudo env "PATH=$PATH" python -m pip install --no-cache-dir -U wheel cython pip
+              python setup.py bdist_wheel
+          - publish: $(System.DefaultWorkingDirectory)/dist/
+            artifact: dist
+          - publish: $(System.DefaultWorkingDirectory)/pipelines/
+            artifact: pipelines
+          - publish: $(System.DefaultWorkingDirectory)/test/
+            artifact: test
+  - stage: test
+    jobs:
+      - job: run_test
+        timeoutInMinutes: 10
+        services:
+          elasticsearch: elasticsearch
+          redis: redis
+          minio: minio
+        steps:
+          - checkout: none
+          - task: UsePythonVersion@0
+            displayName: Set python version
+            inputs: { versionSpec: 3.9 }
+          - download: current
+            artifact: dist
+          - download: current
+            artifact: pipelines
+          - download: current
+            artifact: test
+          - script: |
+              set -x  # Echo commands before they are run
+              sudo apt-get update
+              sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev
+              export VERSION=${BUILD_SOURCEBRANCH#"refs/tags/v"}
+              if [[ "$VERSION" == *stable* ]]; then
+                sudo -E env "PATH=$PATH" python -m pip install "assemblyline" "assemblyline-core"
+              else
+                sudo -E env "PATH=$PATH" python -m pip install --pre "assemblyline" "assemblyline-core"
+              fi
+              export VERSION=${VERSION/stable}
+              export VERSION=${VERSION/beta/b}
+              sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -f $(Pipeline.Workspace)/dist/ "assemblyline-service-server==${VERSION}"
+              sudo -E env "PATH=$PATH" python -m pip install --no-cache-dir -r $(Pipeline.Workspace)/test/requirements.txt
+              sudo mkdir -p /etc/assemblyline/
+              sudo mkdir -p /var/cache/assemblyline/
+              sudo mkdir -p /var/lib/assemblyline/
+              sudo chmod a+rw /var/cache/assemblyline/
+              sudo chmod a+rw /var/lib/assemblyline/
+              sudo cp $(Pipeline.Workspace)/pipelines/config.yml /etc/assemblyline
+            displayName: Install package
+          - script: |
+              python -m assemblyline_service_server.app &
+              pytest -rsx -vv
+            workingDirectory: $(Pipeline.Workspace)/test
+            displayName: Test
+  - stage: deploy
+    jobs:
+      - job: deploy
+        displayName: Deploy packages and containers
+        variables:
+          - group: deployment-information
+        steps:
+          - task: UsePythonVersion@0
+            displayName: Set python version
+            inputs: { versionSpec: 3.9 }
+          - download: current
+            artifact: dist
+          - script: |
+              set -xv  # Echo commands before they are run
+              sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
+              ls dist
+              twine upload --skip-existing --repository-url $TEST_REPOSITORY_URL dist/*
+            workingDirectory: $(Pipeline.Workspace)
+            displayName: Deploy to Test PyPI
+            env:
+              TWINE_USERNAME: $(twineUsername)
+              TWINE_PASSWORD: $(twinePassword)
+          - script: |
+              set -xv  # Echo commands before they are run
+              sudo env "PATH=$PATH" python -m pip install --no-cache-dir twine
+              ls dist
+              twine upload --skip-existing dist/*
+            workingDirectory: $(Pipeline.Workspace)
+            displayName: Deploy to PyPI
+            env:
+              TWINE_USERNAME: $(twineUsername)
+              TWINE_PASSWORD: $(twinePassword)
+          - task: Docker@2
+            displayName: Login to docker hub
+            inputs:
+              command: login
+              containerRegistry: dockerhub
+          - task: Docker@2
+            displayName: Login to github packages
+            inputs:
+              command: login
+              containerRegistry: github-packages-sa
+          - script: |
+              set -xv  # Echo commands before they are run
+              export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
+              if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
+              export VERSION=${TAG/stable}
+              export VERSION=${VERSION/beta/b}
+              export SERIES="`expr $TAG : '\([0-9]\+\.[0-9]\+\.\)'`${BUILD_TYPE}"
+              until sudo env "PATH=$PATH" python -m pip download "assemblyline-service-server==$VERSION" --pre --no-deps &> /dev/null; do sleep 2; done
 
-      displayName: Deploy to Docker Hub
+              for IMAGE in "cccs/assemblyline-service-server" "docker.pkg.github.com/cybercentrecanada/assemblyline/assemblyline-service-server"
+              do
+                docker build --build-arg version=$VERSION --build-arg branch=$BUILD_TYPE -t $IMAGE:$TAG -t $IMAGE:$BUILD_TYPE -t $IMAGE:$SERIES docker
+                docker push $IMAGE --all-tags
+              done
 
+            displayName: Deploy to Docker Hub


Test patch:
diff --git a/pipelines/azure-test.yaml b/pipelines/azure-test.yaml
index 41745c9..8ec7cfd 100644
--- a/pipelines/azure-test.yaml
+++ b/pipelines/azure-test.yaml
@@ -4,85 +4,85 @@ trigger: ["*"]
 pr: ["*"]
 
 pool:
-  vmImage: 'ubuntu-18.04'
+  vmImage: "ubuntu-20.04"
 
 resources:
   containers:
-  - container: redis
-    image: redis
-    ports:
-      - 6379:6379
-  - container: elasticsearch
-    image: cccs/elasticsearch:7.8.0
-    env:
-      ES_JAVA_OPTS: "-Xms256m -Xmx512m"
-      DISCOVERY_TYPE: 'single-node'
-    ports:
-      - 9200:9200
-  - container: minio
-    image: cccs/minio
-    env:
-      MINIO_ACCESS_KEY: al_storage_key
-      MINIO_SECRET_KEY: Ch@ngeTh!sPa33w0rd
-    ports:
-      - 9000:9000
+    - container: redis
+      image: redis
+      ports:
+        - 6379:6379
+    - container: elasticsearch
+      image: cccs/elasticsearch:7.8.0
+      env:
+        ES_JAVA_OPTS: "-Xms256m -Xmx512m"
+        DISCOVERY_TYPE: "single-node"
+      ports:
+        - 9200:9200
+    - container: minio
+      image: cccs/minio
+      env:
+        MINIO_ACCESS_KEY: al_storage_key
+        MINIO_SECRET_KEY: Ch@ngeTh!sPa33w0rd
+      ports:
+        - 9000:9000
 
   repositories:
-  - repository: assemblyline-base
-    type: github
-    endpoint: github
-    name: CybercentreCanada/assemblyline-base
+    - repository: assemblyline-base
+      type: github
+      endpoint: github
+      name: CybercentreCanada/assemblyline-base
 
 jobs:
-- job: run_test
-  strategy:
-    matrix:
-      python3_7:
-        python.version: '3.7'
-      Python3_8:
-        python.version: '3.8'
+  - job: run_test
+    strategy:
+      matrix:
+        python3_9:
+          python.version: "3.9"
+        # Python3_10:
+        #   python.version: '3.10'
 
-  timeoutInMinutes: 15
-  services:
-    elasticsearch: elasticsearch
-    redis: redis
-    minio: minio
+    timeoutInMinutes: 15
+    services:
+      elasticsearch: elasticsearch
+      redis: redis
+      minio: minio
 
-  steps:
-    - task: UsePythonVersion@0
-      displayName: Set python version
-      inputs:
-        versionSpec: '$(python.version)'
-    - checkout: self
-      displayName: Checkout assemblyline-service-server
-    - checkout: assemblyline-base
-      displayName: Checkout assemblyline-base
-    - script: |
-        sudo mkdir -p /etc/assemblyline/
-        sudo mkdir -p /var/cache/assemblyline/
-        sudo mkdir -p /var/lib/assemblyline/
-        sudo cp pipelines/config.yml /etc/assemblyline
-        sudo chmod a+rw /var/cache/assemblyline/
-        sudo chmod a+rw /var/lib/assemblyline/
-        sudo apt-get update
-        sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev
-      displayName: Setup Environment
-      workingDirectory: $(Pipeline.Workspace)/s/assemblyline-service-server
-    - script: |
-        # Try to checkout the matching branch, if the command fails, don't care.
-        export BRANCH_NAME=$(basename -- "$SYSTEM_PULLREQUEST_SOURCEBRANCH")
-        export BRANCH_NAME=${BRANCH_NAME:-"$BUILD_SOURCEBRANCHNAME"}
-        git checkout -b $BRANCH_NAME -t origin/$BRANCH_NAME || true
-        sudo env "PATH=$PATH" python -m pip install --no-cache-dir -e .
-      displayName: Install assemblyline
-      workingDirectory: $(Pipeline.Workspace)/s/assemblyline-base
-    - script: |
-        sudo env "PATH=$PATH" python -m pip install --no-cache-dir -r test/requirements.txt
-        sudo env "PATH=$PATH" python -m pip install --no-cache-dir -e .
-      displayName: Install assemblyline-service-server
-      workingDirectory: $(Pipeline.Workspace)/s/assemblyline-service-server
-    - script: |
-        python -m assemblyline_service_server.app &
-        pytest -rsx -vv
-      workingDirectory: $(Pipeline.Workspace)/s/assemblyline-service-server
-      displayName: Test
+    steps:
+      - task: UsePythonVersion@0
+        displayName: Set python version
+        inputs:
+          versionSpec: "$(python.version)"
+      - checkout: self
+        displayName: Checkout assemblyline-service-server
+      - checkout: assemblyline-base
+        displayName: Checkout assemblyline-base
+      - script: |
+          sudo mkdir -p /etc/assemblyline/
+          sudo mkdir -p /var/cache/assemblyline/
+          sudo mkdir -p /var/lib/assemblyline/
+          sudo cp pipelines/config.yml /etc/assemblyline
+          sudo chmod a+rw /var/cache/assemblyline/
+          sudo chmod a+rw /var/lib/assemblyline/
+          sudo apt-get update
+          sudo apt-get install -y build-essential libffi-dev libfuzzy-dev python3-dev
+        displayName: Setup Environment
+        workingDirectory: $(Pipeline.Workspace)/s/assemblyline-service-server
+      - script: |
+          # Try to checkout the matching branch, if the command fails, don't care.
+          export BRANCH_NAME=$(basename -- "$SYSTEM_PULLREQUEST_SOURCEBRANCH")
+          export BRANCH_NAME=${BRANCH_NAME:-"$BUILD_SOURCEBRANCHNAME"}
+          git checkout -b $BRANCH_NAME -t origin/$BRANCH_NAME || true
+          sudo env "PATH=$PATH" python -m pip install --no-cache-dir -e .
+        displayName: Install assemblyline
+        workingDirectory: $(Pipeline.Workspace)/s/assemblyline-base
+      - script: |
+          sudo env "PATH=$PATH" python -m pip install --no-cache-dir -r test/requirements.txt
+          sudo env "PATH=$PATH" python -m pip install --no-cache-dir -e .
+        displayName: Install assemblyline-service-server
+        workingDirectory: $(Pipeline.Workspace)/s/assemblyline-service-server
+      - script: |
+          python -m assemblyline_service_server.app &
+          pytest -rsx -vv
+        workingDirectory: $(Pipeline.Workspace)/s/assemblyline-service-server
+        displayName: Test