Repository: hyperskill/hyperstyle (language: python)

Issue:
Make tests compatible with production Docker environment
We run Hyperstyle in Docker (based on https://github.com/hyperskill/hyperstyle/blob/main/Dockerfile) on Stepik. Currently, tests do not pass within the production environment since you run tests in a different environment for GitHub Actions (https://github.com/hyperskill/hyperstyle/blob/main/.github/workflows/build.yml). 

Please do the following:
1. Use the same Docker environment to run tests.
2. Fix all failing test cases (some for eslint and flake8) in `main` and `main-upd` branches.
Change production configuration
We use `numpy` and `pandas` modules only in the `evaluation` module in the tests. We can ignore the installation of them in production if we don't run evaluation tests (see open PRs #44 & #46 for issue #43, alpine requires to add an additional step for it). Finally, instead of running all tests (`review/test/python`) we can run only `review/test/python/functional_tests` and `review/test/python/inspectors`. But `functional_tests` is used only for checking if the tool runs correctly for different CLI configurations. Tests affecting the tool correctness only are in `review/test/python/inspectors`.

Please, change the production configuration for tests running.


Discussion / hints:

Fix patch:
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 00000000..3f78c748
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,7 @@
+**/*.pyc
+**/.pytest_cache/
+**/__pycache__/
+.git
+.github
+.idea
+node_modules
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 507108db..c0fa5ce3 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -1,19 +1,22 @@
 name: Python build
 
-on: [push, pull_request]
+on: [ push, pull_request ]
 
 jobs:
 
   build:
     runs-on: ubuntu-latest
-    # Consistent with Version.md
-    container: nastyabirillo/hyperstyle:1.2.1
+    # Consistent with base image in Dockerfile
+    container: stepik/hyperstyle-base:py3.8.11-java11.0.11-node14.17.3
 
     steps:
-
       - name: Checkout
         uses: actions/checkout@v1
 
+      - name: Install requirements
+        run: |
+          pip install --no-cache-dir -r requirements-test.txt -r requirements.txt
+
       - name: Lint with flake8
         run: |
           # stop the build if there are Python syntax errors or undefined names
@@ -21,21 +24,24 @@ jobs:
           # TODO: change max-complexity into 10 after refactoring
           # TODO: remove R504, A003, E800, E402, WPS1, WPS2, WPS3, WPS4, WPS5, WPS6, H601
           flake8 . --count --max-complexity=11 --max-line-length=120 --max-doc-length=120 --ignore=R504,A003,E800,E402,W503,WPS1,WPS2,WPS3,WPS4,WPS5,WPS6,H601 --statistics --exclude=.git,__pycache__,docs/source/conf.py,old,build,dist,venv,test/resources,.eggs,review.egg-info,.pytest_cache,node_modules
-   
+
       - name: Set up Eslint
         run: |
-          npm install eslint --save-dev
-          ./node_modules/.bin/eslint --init
+          # Consistent with eslint version in Dockerfile
+          npm install eslint@7.5.0 -g && eslint --init
 
       - name: Test with pytest
         run: |
-            pytest
-            
-      # We should have only INFO errors      
+          pytest
+
       - name: Check installed module can run python linters
         run: |
-            python src/python/review/run_tool.py setup.py 
-             
+            python src/python/review/run_tool.py setup.py
+
       - name: Check installed module can run java linters
         run: |
             python src/python/review/run_tool.py test/resources/inspectors/java/test_algorithm_with_scanner.java
+
+      - name: Check installed module can run js linters
+        run: |
+          python src/python/review/run_tool.py test/resources/inspectors/js/case0_no_issues.js
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 41f14cdb..b33c5f72 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,35 +1,12 @@
-# This Dockerfile is used only for production
+FROM stepik/hyperstyle-base:py3.8.11-java11.0.11-node14.17.3
 
-FROM python:3.8.2-alpine3.11
-
-RUN apk --no-cache add openjdk11 --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community \
-    && apk add --update nodejs npm
-
-RUN npm i -g eslint@7.5.0
-
-RUN java -version
-RUN ls /usr/lib/jvm
-
-# Other dependencies
-RUN apk add bash
-
-# Set up Eslint
-RUN npm install eslint --save-dev && ./node_modules/.bin/eslint --init
-
-# Dependencies and package installation
-WORKDIR /
-
-COPY requirements-test.txt review/requirements-test.txt
-RUN pip3 install --no-cache-dir -r review/requirements-test.txt
-
-COPY requirements.txt review/requirements.txt
-RUN pip3 install --no-cache-dir -r review/requirements.txt
+RUN npm install eslint@7.5.0 -g \
+    && eslint --init
 
 COPY . review
-RUN pip3 install --no-cache-dir ./review
-
-# Container's enviroment variables
-ENV JAVA_HOME=/usr/lib/jvm/java-11-openjdk
-ENV PATH="$JAVA_HOME/bin:${PATH}"
+RUN pip install --no-cache-dir \
+    -r review/requirements-test.txt \
+    -r review/requirements.txt \
+    ./review
 
-CMD ["/bin/bash"]
\ No newline at end of file
+CMD ["/bin/bash"]
diff --git a/Dockerfile.base b/Dockerfile.base
new file mode 100644
index 00000000..c6869724
--- /dev/null
+++ b/Dockerfile.base
@@ -0,0 +1,170 @@
+FROM python:3.8.11-slim
+
+#########
+# Taken from https://github.com/docker-library/openjdk/blob/608f26c5ea63ca34070b439c904cb94a30f6b0c1/11/jdk/slim-buster/Dockerfile
+#########
+
+RUN set -eux; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+# utilities for keeping Debian and OpenJDK CA certificates in sync
+		ca-certificates p11-kit \
+	; \
+	rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME /usr/local/openjdk-11
+RUN { echo '#/bin/sh'; echo 'echo "$JAVA_HOME"'; } > /usr/local/bin/docker-java-home && chmod +x /usr/local/bin/docker-java-home && [ "$JAVA_HOME" = "$(docker-java-home)" ] # backwards compatibility
+ENV PATH $JAVA_HOME/bin:$PATH
+
+# Default to UTF-8 file.encoding
+ENV LANG C.UTF-8
+
+# https://adoptopenjdk.net/upstream.html
+# >
+# > What are these binaries?
+# >
+# > These binaries are built by Red Hat on their infrastructure on behalf of the OpenJDK jdk8u and jdk11u projects. The binaries are created from the unmodified source code at OpenJDK. Although no formal support agreement is provided, please report any bugs you may find to https://bugs.java.com/.
+# >
+ENV JAVA_VERSION 11.0.11+9
+# https://github.com/docker-library/openjdk/issues/320#issuecomment-494050246
+# >
+# > I am the OpenJDK 8 and 11 Updates OpenJDK project lead.
+# > ...
+# > While it is true that the OpenJDK Governing Board has not sanctioned those releases, they (or rather we, since I am a member) didn't sanction Oracle's OpenJDK releases either. As far as I am aware, the lead of an OpenJDK project is entitled to release binary builds, and there is clearly a need for them.
+# >
+
+RUN set -eux; \
+	\
+	arch="$(dpkg --print-architecture)"; \
+	case "$arch" in \
+		'amd64') \
+			downloadUrl='https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.11%2B9/OpenJDK11U-jdk_x64_linux_11.0.11_9.tar.gz'; \
+			;; \
+		'arm64') \
+			downloadUrl='https://github.com/AdoptOpenJDK/openjdk11-upstream-binaries/releases/download/jdk-11.0.11%2B9/OpenJDK11U-jdk_aarch64_linux_11.0.11_9.tar.gz'; \
+			;; \
+		*) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \
+	esac; \
+	\
+	savedAptMark="$(apt-mark showmanual)"; \
+	apt-get update; \
+	apt-get install -y --no-install-recommends \
+		dirmngr \
+		gnupg \
+		wget \
+	; \
+	rm -rf /var/lib/apt/lists/*; \
+	\
+	wget --progress=dot:giga -O openjdk.tgz "$downloadUrl"; \
+	wget --progress=dot:giga -O openjdk.tgz.asc "$downloadUrl.sign"; \
+	\
+	export GNUPGHOME="$(mktemp -d)"; \
+# pre-fetch Andrew Haley's (the OpenJDK 8 and 11 Updates OpenJDK project lead) key so we can verify that the OpenJDK key was signed by it
+# (https://github.com/docker-library/openjdk/pull/322#discussion_r286839190)
+# we pre-fetch this so that the signature it makes on the OpenJDK key can survive "import-clean" in gpg
+	gpg --batch --keyserver keyserver.ubuntu.com --recv-keys EAC843EBD3EFDB98CC772FADA5CD6035332FA671; \
+# TODO find a good link for users to verify this key is right (https://mail.openjdk.java.net/pipermail/jdk-updates-dev/2019-April/000951.html is one of the only mentions of it I can find); perhaps a note added to https://adoptopenjdk.net/upstream.html would make sense?
+# no-self-sigs-only: https://salsa.debian.org/debian/gnupg2/commit/c93ca04a53569916308b369c8b218dad5ae8fe07
+	gpg --batch --keyserver keyserver.ubuntu.com --keyserver-options no-self-sigs-only --recv-keys CA5F11C6CE22644D42C6AC4492EF8D39DC13168F; \
+	gpg --batch --list-sigs --keyid-format 0xLONG CA5F11C6CE22644D42C6AC4492EF8D39DC13168F \
+		| tee /dev/stderr \
+		| grep '0xA5CD6035332FA671' \
+		| grep 'Andrew Haley'; \
+	gpg --batch --verify openjdk.tgz.asc openjdk.tgz; \
+	gpgconf --kill all; \
+	rm -rf "$GNUPGHOME"; \
+	\
+	mkdir -p "$JAVA_HOME"; \
+	tar --extract \
+		--file openjdk.tgz \
+		--directory "$JAVA_HOME" \
+		--strip-components 1 \
+		--no-same-owner \
+	; \
+	rm openjdk.tgz*; \
+	\
+	apt-mark auto '.*' > /dev/null; \
+	[ -z "$savedAptMark" ] || apt-mark manual $savedAptMark > /dev/null; \
+	apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false; \
+	\
+# update "cacerts" bundle to use Debian's CA certificates (and make sure it stays up-to-date with changes to Debian's store)
+# see https://github.com/docker-library/openjdk/issues/327
+#     http://rabexc.org/posts/certificates-not-working-java#comment-4099504075
+#     https://salsa.debian.org/java-team/ca-certificates-java/blob/3e51a84e9104823319abeb31f880580e46f45a98/debian/jks-keystore.hook.in
+#     https://git.alpinelinux.org/aports/tree/community/java-cacerts/APKBUILD?id=761af65f38b4570093461e6546dcf6b179d2b624#n29
+	{ \
+		echo '#!/usr/bin/env bash'; \
+		echo 'set -Eeuo pipefail'; \
+		echo 'trust extract --overwrite --format=java-cacerts --filter=ca-anchors --purpose=server-auth "$JAVA_HOME/lib/security/cacerts"'; \
+	} > /etc/ca-certificates/update.d/docker-openjdk; \
+	chmod +x /etc/ca-certificates/update.d/docker-openjdk; \
+	/etc/ca-certificates/update.d/docker-openjdk; \
+	\
+# https://github.com/docker-library/openjdk/issues/331#issuecomment-498834472
+	find "$JAVA_HOME/lib" -name '*.so' -exec dirname '{}' ';' | sort -u > /etc/ld.so.conf.d/docker-openjdk.conf; \
+	ldconfig; \
+	\
+# https://github.com/docker-library/openjdk/issues/212#issuecomment-420979840
+# https://openjdk.java.net/jeps/341
+	java -Xshare:dump; \
+	\
+# basic smoke test
+	fileEncoding="$(echo 'System.out.println(System.getProperty("file.encoding"))' | jshell -s -)"; [ "$fileEncoding" = 'UTF-8' ]; rm -rf ~/.java; \
+	javac --version; \
+	java --version
+
+#########
+# Taken from  https://github.com/nodejs/docker-node/blob/fd130acf063b312355a5d88d51716db3ff34ae49/14/buster-slim/Dockerfile
+#########
+
+ENV NODE_VERSION 14.17.3
+
+RUN ARCH= && dpkgArch="$(dpkg --print-architecture)" \
+    && case "${dpkgArch##*-}" in \
+      amd64) ARCH='x64';; \
+      ppc64el) ARCH='ppc64le';; \
+      s390x) ARCH='s390x';; \
+      arm64) ARCH='arm64';; \
+      armhf) ARCH='armv7l';; \
+      i386) ARCH='x86';; \
+      *) echo "unsupported architecture"; exit 1 ;; \
+    esac \
+    && set -ex \
+    # libatomic1 for arm
+    && apt-get update && apt-get install -y ca-certificates curl wget gnupg dirmngr xz-utils libatomic1 --no-install-recommends \
+    && rm -rf /var/lib/apt/lists/* \
+    && for key in \
+      4ED778F539E3634C779C87C6D7062848A1AB005C \
+      94AE36675C464D64BAFA68DD7434390BDBE9B9C5 \
+      74F12602B6F1C4E913FAA37AD3A89613643B6201 \
+      71DCFD284A79C3B38668286BC97EC7A07EDE3FC1 \
+      8FCCA13FEF1D0C2E91008E09770F7A9A5AE15600 \
+      C4F0DFFF4E8C1A8236409D08E73BC641CC11F4C8 \
+      C82FA3AE1CBEDC6BE46B9360C43CEC45C17AB93C \
+      DD8F2338BAE7501E3DD5AC78C273792F7D83545D \
+      A48C2BEE680E841632CD4E44F07496B3EB3C1762 \
+      108F52B48DB57BB0CC439B2997B01419BD92F80A \
+      B9E2F5981AA6E0CD28160D9FF13993A75599653C \
+    ; do \
+      gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \
+      gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key" ; \
+    done \
+    && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.xz" \
+    && curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/SHASUMS256.txt.asc" \
+    && gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc \
+    && grep " node-v$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c - \
+    && tar -xJf "node-v$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner \
+    && rm "node-v$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt \
+    && apt-mark auto '.*' > /dev/null \
+    && find /usr/local -type f -executable -exec ldd '{}' ';' \
+      | awk '/=>/ { print $(NF-1) }' \
+      | sort -u \
+      | xargs -r dpkg-query --search \
+      | cut -d: -f1 \
+      | sort -u \
+      | xargs -r apt-mark manual \
+    && apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
+    && ln -s /usr/local/bin/node /usr/local/bin/nodejs \
+    # smoke tests
+    && node --version \
+    && npm --version
diff --git a/README.md b/README.md
index ca364037..bd656c79 100644
--- a/README.md
+++ b/README.md
@@ -201,7 +201,8 @@ __Note__: If you have `ModuleNotFoundError` while you try to run tests, please c
 
 __Note__: We use [eslint](https://eslint.org/) and [open-jdk 11](https://openjdk.java.net/projects/jdk/11/)
 in the tests. Please, set up the environment before running the tests. 
-You can see en example of the environment configuration in 
-the [Dockerfile](./docker/dev/Dockerfile) file.
+You can see en example of the environment configuration in
+the [Dockerfile](Dockerfile) file.
 
 Use `pytest` from the root directory to run __ALL__ tests.
+
diff --git a/requirements-evaluation.txt b/requirements-evaluation.txt
index 014f2f6c..046f8026 100644
--- a/requirements-evaluation.txt
+++ b/requirements-evaluation.txt
@@ -1,3 +1,1 @@
-openpyxl==3.0.7
-pandarallel
-plotly
+plotly==4.14.3
diff --git a/requirements-roberta.txt b/requirements-roberta.txt
index 287d83b5..15421cf1 100644
--- a/requirements-roberta.txt
+++ b/requirements-roberta.txt
@@ -1,5 +1,5 @@
 tqdm==4.49.0
-scikit-learn~=0.24.2
+scikit-learn==0.24.2
 transformers==4.6.1
 tokenizers==0.10.2
 torch==1.8.1
diff --git a/src/python/review/inspectors/detekt/detekt.py b/src/python/review/inspectors/detekt/detekt.py
index 8f5f0946..041c6b2d 100644
--- a/src/python/review/inspectors/detekt/detekt.py
+++ b/src/python/review/inspectors/detekt/detekt.py
@@ -41,7 +41,7 @@ def _create_command(cls, path: Path, output_path: Path):
             '--input', str(path),
         ]
 
-    def inspect(self, path: Path, config) -> List[BaseIssue]:
+    def inspect(self, path: Path, config: dict) -> List[BaseIssue]:
         with new_temp_dir() as temp_dir:
             output_path = temp_dir / 'output.xml'
             command = self._create_command(path, output_path)
diff --git a/src/python/review/inspectors/eslint/eslint.py b/src/python/review/inspectors/eslint/eslint.py
index 839a166c..c7d0b1b4 100644
--- a/src/python/review/inspectors/eslint/eslint.py
+++ b/src/python/review/inspectors/eslint/eslint.py
@@ -21,8 +21,9 @@ class ESLintInspector(BaseInspector):
     }
 
     @classmethod
-    def _create_command(cls, path: Path, output_path: Path, is_local: bool = False) -> List[str]:
-        eslint_command = 'eslint' if not is_local else './node_modules/.bin/eslint'
+    def _create_command(cls, path: Path, output_path: Path) -> List[str]:
+        local_path = 'node_modules/.bin/eslint'  # used only in local dev environment
+        eslint_command = local_path if Path(local_path).exists() else 'eslint'
         return [
             eslint_command,
             '-c', PATH_ESLINT_CONFIG,
@@ -31,10 +32,10 @@ def _create_command(cls, path: Path, output_path: Path, is_local: bool = False)
             path,
         ]
 
-    def inspect(self, path: Path, config: dict, is_local: bool = False) -> List[BaseIssue]:
+    def inspect(self, path: Path, config: dict) -> List[BaseIssue]:
         with new_temp_dir() as temp_dir:
             output_path = temp_dir / 'output.xml'
-            command = self._create_command(path, output_path, is_local)
+            command = self._create_command(path, output_path)
             run_in_subprocess(command)
 
             issues = parse_checkstyle_file_result(output_path,


Test patch:
diff --git a/requirements-test.txt b/requirements-test.txt
index cb0543a3..dc0f70dc 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,9 +1,11 @@
-pytest~=6.2.3
-pytest-runner
-pytest-subtests
-jsonschema~=3.2.0
-django~=3.2
-pylint~=2.7.4
-requests~=2.25.1
-setuptools~=56.0.0
-openpyxl==3.0.7
\ No newline at end of file
+pytest==6.2.3
+pytest-runner==5.2
+pytest-subtests==0.4.0
+jsonschema==3.2.0
+pandas==1.2.3
+django==3.2
+pylint==2.7.4
+requests==2.25.1
+setuptools==56.0.0
+openpyxl==3.0.7
+pandarallel==1.5.2
\ No newline at end of file
diff --git a/test/python/inspectors/test_eslint_inspector.py b/test/python/inspectors/test_eslint_inspector.py
index a69f30e5..d7aceeac 100644
--- a/test/python/inspectors/test_eslint_inspector.py
+++ b/test/python/inspectors/test_eslint_inspector.py
@@ -19,7 +19,7 @@ def test_file_with_issues(file_name: str, n_issues: int):
 
     path_to_file = JS_DATA_FOLDER / file_name
     with use_file_metadata(path_to_file) as file_metadata:
-        issues = inspector.inspect(file_metadata.path, {}, True)
+        issues = inspector.inspect(file_metadata.path, {})
         issues = filter_low_measure_issues(issues, Language.JS)
 
     assert len(issues) == n_issues