Repository: deepchecks/deepchecks (branch: main)
Commit date: 2023-11-22T17:09:33Z

Workflow file: .github/workflows/lint.yml (Code Style)
# ----------------------------------------------------------------------------
# Copyright (C) 2021-2023 Deepchecks (https://www.deepchecks.com)
#
# This file is part of Deepchecks.
# Deepchecks is distributed under the terms of the GNU Affero General
# Public License (version 3 or later).
# You should have received a copy of the GNU Affero General Public License
# along with Deepchecks.  If not, see <http://www.gnu.org/licenses/>.
# ----------------------------------------------------------------------------
#
name: Code Style

on:
  push:
    branches: [ main ]
  pull_request: 
    branches: [ main, nlp-base]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  DISABLE_DEEPCHECKS_ANONYMOUS_TELEMETRY: 'true'
  PIP_INDEX_URL: http://localhost:8629/2023-11-22
  UV_INDEX_URL: http://localhost:8629/2023-11-22

jobs:
  pylint:
    runs-on: ubuntu-latest
    services:
      pypi_wayback:
        image: ghcr.io/waleko/pypi-wayback
        ports:
          - 8629:8080


    steps:
    - uses: actions/checkout@v2
    - name: Set up Python 3.9
      uses: actions/setup-python@v2
      with:
        python-version: 3.9

    - name: Run Pylint
      run: make pylint
    - name: Check import order
      uses: jamescurtin/isort-action@master
      with:
        sortPaths: deepchecks/
  docstring:
    runs-on: ubuntu-latest
    services:
      pypi_wayback:
        image: ghcr.io/waleko/pypi-wayback
        ports:
          - 8629:8080
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python 3.9
      uses: actions/setup-python@v2
      with:
        python-version: 3.9
    - name: Run Pydocstyle
      run: make docstring

  license-check:
    runs-on: ubuntu-latest
    services:
      pypi_wayback:
        image: ghcr.io/waleko/pypi-wayback
        ports:
          - 8629:8080
    
    needs: docstring
    
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python 3.9
      uses: actions/setup-python@v2
      with:
        python-version: 3.9
    - name: Run License eye
      run: make license-check


Failed build logs:
step_name: pylint/4_Run Pylint.txt
log: ##[group]Run make pylint
[36;1mmake pylint[0m
shell: /usr/bin/bash -e {0}
env:
  DISABLE_DEEPCHECKS_ANONYMOUS_TELEMETRY: true
  pythonLocation: /opt/hostedtoolcache/Python/3.9.18/x64
  LD_LIBRARY_PATH: /opt/hostedtoolcache/Python/3.9.18/x64/lib
##[endgroup]
#### Creating Python Virtual Environment [ /home/runner/work/deepchecks/deepchecks/venv ] ####
external python_exe is /opt/hostedtoolcache/Python/3.9.18/x64/bin/python3
####  installing development dependencies, it could take some time, please wait! ####
  DEPRECATION: pathtools is being installed using the legacy 'setup.py install' method, because it does not have a 'pyproject.toml' and the 'wheel' package is not installed. pip 23.1 will enforce this behaviour change. A possible replacement is to enable the '--use-pep517' option. Discussion can be found at https://github.com/pypa/pip/issues/8559
  DEPRECATION: docopt is being installed using the legacy 'setup.py install' method, because it does not have a 'pyproject.toml' and the 'wheel' package is not installed. pip 23.1 will enforce this behaviour change. A possible replacement is to enable the '--use-pep517' option. Discussion can be found at https://github.com/pypa/pip/issues/8559
  DEPRECATION: promise is being installed using the legacy 'setup.py install' method, because it does not have a 'pyproject.toml' and the 'wheel' package is not installed. pip 23.1 will enforce this behaviour change. A possible replacement is to enable the '--use-pep517' option. Discussion can be found at https://github.com/pypa/pip/issues/8559
  DEPRECATION: gower is being installed using the legacy 'setup.py install' method, because it does not have a 'pyproject.toml' and the 'wheel' package is not installed. pip 23.1 will enforce this behaviour change. A possible replacement is to enable the '--use-pep517' option. Discussion can be found at https://github.com/pypa/pip/issues/8559
  DEPRECATION: sentence-transformers is being installed using the legacy 'setup.py install' method, because it does not have a 'pyproject.toml' and the 'wheel' package is not installed. pip 23.1 will enforce this behaviour change. A possible replacement is to enable the '--use-pep517' option. Discussion can be found at https://github.com/pypa/pip/issues/8559

[notice] A new release of pip is available: 23.0.1 -> 23.3.1
[notice] To update, run: python3 -m pip install --upgrade pip
/home/runner/work/deepchecks/deepchecks/venv/bin/pylint -j 0 deepchecks
************* Module deepchecks.nlp.utils.text_embeddings
deepchecks/nlp/utils/text_embeddings.py:229:0: C0304: Final newline missing (missing-final-newline)

------------------------------------
Your code has been rated at 10.00/10

make: *** [makefile:213: pylint] Error 16
##[error]Process completed with exit code 2.


Changed files:
deepchecks/nlp/utils/text_embeddings.py
deepchecks/nlp/utils/text_properties.py

Fix diff:
diff --git a/deepchecks/nlp/utils/text_embeddings.py b/deepchecks/nlp/utils/text_embeddings.py
index 53e8992ca..9db8ecf25 100644
--- a/deepchecks/nlp/utils/text_embeddings.py
+++ b/deepchecks/nlp/utils/text_embeddings.py
@@ -10,7 +10,6 @@
 #
 """Utils module for calculating embeddings for text."""
 import sys
-import re
 import warnings
 from itertools import islice
 from typing import Optional
@@ -23,10 +22,6 @@ EMBEDDING_DIM = 1536
 EMBEDDING_CTX_LENGTH = 8191
 EMBEDDING_ENCODING = 'cl100k_base'
 
-PATTERN_SPECIAL_CHARS = re.compile(r"[!@#$%^&*()_+{}|:\"<>?~`\-=\[\]\;',.\/]")
-PATTERN_SPACE_CHARS = re.compile(r'\s')
-PATTERN_BR_CHARS = re.compile(r'<br />')
-
 
 def batched(iterable, n):
     """Batch data into tuples of length n. The last batch may be shorter."""
@@ -209,21 +204,11 @@ def calculate_builtin_embeddings(text: np.array, model: str = 'miniLM',
         np.save(file_path, embeddings)
     return embeddings
 
-def _clean_special_chars(text: str) -> str:
-    """
-    Remove special characters, replaces space characters with space.
-
-    Parameters
-    ----------
-    text : str
-        The `text` parameter is a string that represents the input text that needs to be cleaned.
 
-    Returns
-    -------
-        text
-            Cleaned text string
-    """
-    text = PATTERN_SPECIAL_CHARS.sub('', text)
-    text = PATTERN_SPACE_CHARS.sub(' ', text)
-    text = PATTERN_BR_CHARS.sub(' ', text)
-    return text
\ No newline at end of file
+def _clean_special_chars(text):
+    special_chars = r'!@#$%^&*()_+{}|:"<>?~`-=[]\;\',./'
+    for char in special_chars:
+        text = text.replace(char, '')
+    text = text.replace('\n', ' ').replace('\r', ' ').replace('\t', ' ')
+    text = text.replace('<br />', ' ')
+    return text
diff --git a/deepchecks/nlp/utils/text_properties.py b/deepchecks/nlp/utils/text_properties.py
index bf690d839..c3c166964 100644
--- a/deepchecks/nlp/utils/text_properties.py
+++ b/deepchecks/nlp/utils/text_properties.py
@@ -152,6 +152,9 @@ def language(
     prediction = fasttext_model.predict(text.replace('\n', ' '), k=1, threshold=lang_certainty_threshold)[0]
     # label is empty for detection below threshold:
     language_code = prediction[0].replace('__label__', '') if prediction else None
+
+    if language_code == 'eng':  # both are english but different labels
+        return 'en'
     return language_code