Repository: fpgmaas/deptry (language: python)

Issue:
deptry hangs
**Describe the bug**

Using v0.13.0 there's no issue. Only when upgrading to v0.14.0a1 / v0.14.0.

As far as I start deptry with `deptry .` I see the output "Scanning 100 files..." where it hangs forever.

Adding `--verbose` to the command gives the following output:

```
No configuration for deptry was found in pyproject.toml.
Running with the following configuration:
root: (PosixPath('.'),)
config: pyproject.toml
no_ansi: False
per_rule_ignores: {}
ignore: ()
exclude: ('venv', '\\.venv', '\\.direnv', 'tests', '\\.git', 'setup\\.py')
extend_exclude: ()
using_default_exclude: True
ignore_notebooks: False
requirements_txt: ('requirements.txt',)
requirements_txt_dev: ('dev-requirements.txt', 'requirements-dev.txt')
known_first_party: ()
json_output: None
package_module_name_map: {}

pyproject.toml found!
pyproject.toml does not contain a [tool.poetry.dependencies] section, so Poetry is not used to specify the project's dependencies.
pyproject.toml does not contain a [tool.pdm.dev-dependencies] section, so PDM is not used to specify the project's dependencies.
pyproject.toml does not contain a [project] section, so PEP 621 is not used to specify the project's dependencies.
Dependency specification found in 'requirements.txt'. Will use this to determine the project's dependencies.

Scanning requirements.txt for dependencies
The project contains the following dependencies:
Dependency 'aws-lambda-powertools' with top-levels: {'aws_lambda_powertools'}.
Dependency 'jsonschema' with top-levels: {'jsonschema'}.
Dependency 'pydantic' with top-levels: {'pydantic'}.
Dependency 'pydantic' (optional) with top-levels: {'pydantic'}.
Dependency 'pysnow' with top-levels: {'pysnow'}.
Dependency 'requests' with top-levels: {'requests'}.
Dependency 'shortuuid' with top-levels: {'shortuuid'}.

Found files with development requirements! ['requirements-dev.txt']
Scanning requirements-dev.txt for dependencies
The project contains the following dev dependencies:
Dependency 'aws-lambda-powertools' with top-levels: {'aws_lambda_powertools'}.
Dependency 'jsonschema' with top-levels: {'jsonschema'}.
Dependency 'pydantic' with top-levels: {'pydantic'}.
Dependency 'pydantic' (optional) with top-levels: {'pydantic'}.
Dependency 'pysnow' with top-levels: {'pysnow'}.
Dependency 'requests' with top-levels: {'requests'}.
Dependency 'shortuuid' with top-levels: {'shortuuid'}.
Dependency 'boto3' with top-levels: {'boto3'}.
Dependency 'boto3-stubs' (optional) with top-levels: {'boto3-stubs'}.
Dependency 'codespell' with top-levels: {'codespell_lib'}.
Dependency 'freezegun' with top-levels: {'freezegun'}.
Dependency 'moto' with top-levels: {'moto'}.
Dependency 'pre-commit' with top-levels: {'pre_commit'}.
Dependency 'pytest' with top-levels: {'_pytest', 'py', 'pytest'}.
Dependency 'pytest-cov' with top-levels: {'pytest_cov'}.
Dependency 'pytest-html' with top-levels: {'pytest_html'}.
Dependency 'pytest-mock' with top-levels: {'pytest_mock'}.
Dependency 'pytest-pep8' with top-levels: {'pytest_pep8'}.
Dependency 'ruff' with top-levels: {'ruff'}.

Collecting Python files to scan...
Python files to scan for imports:
src/api/v1_/....py
<99 more .py files here>

Scanning 100 files...
```

**To Reproduce**

Steps to reproduce the behavior:

1. Upgrade from v0.13.0 to v0.14.0
2. `deptry .`


**Expected behavior**

Should run

**System [please complete the following information]:**

-   OS: Devcontainer (mcr.microsoft.com/vscode/devcontainers/python:3.12-bullseye) / Debian Bullseye
-   Language Version: Python 3.12
-   Poetry version: no poetry in use, I'm using uv


Discussion / hints:
Thanks for raising this issue! I did some local testing, and I think this is potentially related to https://github.com/fpgmaas/deptry/issues/611. We use `rayon` to parse the files in parallel, but whenever one of the files cannot be parsed it seems it can get stuck. Relevant code [here](https://github.com/fpgmaas/deptry/blob/a85339ecf5c2464853d633be1208b3cd2829c643/src/imports.rs#L22) and [here](https://github.com/fpgmaas/deptry/blob/a85339ecf5c2464853d633be1208b3cd2829c643/src/imports.rs#L83). I need some more time to figure out a solution though.

Fix patch:
diff --git a/src/imports.rs b/src/imports.rs
index 25db77c7..34404dd4 100644
--- a/src/imports.rs
+++ b/src/imports.rs
@@ -26,24 +26,40 @@ pub fn get_imports_from_py_files(py: Python, file_paths: Vec<&PyString>) -> PyRe
         .collect();
 
     // Process each file in parallel and collect results
-    let results: PyResult<Vec<_>> = rust_file_paths
+    let results: Vec<_> = rust_file_paths
         .par_iter()
-        .map(|path_str| _get_imports_from_py_file(path_str))
+        .map(|path_str| match _get_imports_from_py_file(path_str) {
+            Ok(result) => (path_str, Ok(result)),
+            Err(e) => (path_str, Err(e)),
+        })
         .collect();
 
-    let results = results?;
-
     // Merge results from each thread
     let mut all_imports = HashMap::new();
-    for file_result in results {
-        for (module, locations) in file_result {
-            all_imports
-                .entry(module)
-                .or_insert_with(Vec::new)
-                .extend(locations);
+    let mut errors = Vec::new();
+
+    for (path, file_result) in results {
+        match file_result {
+            Ok(file_result) => {
+                for (module, locations) in file_result {
+                    all_imports
+                        .entry(module)
+                        .or_insert_with(Vec::new)
+                        .extend(locations);
+                }
+            }
+            Err(e) => errors.push((path.to_string(), e)),
         }
     }
 
+    for (path, error) in errors {
+        log::warn!(
+            "Warning: Skipping processing of {} because of the following error: \"{}\".",
+            path,
+            error
+        );
+    }
+
     convert_to_python_dict(py, all_imports)
 }
 
@@ -60,19 +76,12 @@ pub fn get_imports_from_py_file(py: Python, file_path: &PyString) -> PyResult<Py
 /// Core helper function that extracts import statements and their locations from the content of a single Python file.
 /// Used internally by both parallel and single file processing functions.
 fn _get_imports_from_py_file(path_str: &str) -> PyResult<HashMap<String, Vec<Location>>> {
-    let file_content = match read_file(path_str) {
-        Ok(content) => content,
-        Err(_) => {
-            log::warn!("Warning: File {} could not be read. Skipping...", path_str);
-            return Ok(HashMap::new());
-        }
-    };
+    let file_content = read_file(path_str)?;
 
-    let ast = get_ast_from_file_content(&file_content, path_str)
-        .map_err(|e| PySyntaxError::new_err(format!("Error parsing file {}: {}", path_str, e)))?;
+    let ast = parse(&file_content, Mode::Module, path_str)
+        .map_err(|e| PySyntaxError::new_err(e.to_string()))?;
 
     let imported_modules = extract_imports_from_ast(ast);
-
     Ok(convert_imports_with_textranges_to_location_objects(
         imported_modules,
         path_str,
@@ -80,12 +89,6 @@ fn _get_imports_from_py_file(path_str: &str) -> PyResult<HashMap<String, Vec<Loc
     ))
 }
 
-/// Parses the content of a Python file into an abstract syntax tree (AST).
-pub fn get_ast_from_file_content(file_content: &str, file_path: &str) -> PyResult<Mod> {
-    parse(file_content, Mode::Module, file_path)
-        .map_err(|e| PySyntaxError::new_err(format!("Error parsing file {}: {}", file_path, e)))
-}
-
 /// Iterates through an AST to identify and collect import statements, and returns them together with their
 /// respective TextRange for each occurrence.
 fn extract_imports_from_ast(ast: Mod) -> HashMap<String, Vec<TextRange>> {


Test patch:
diff --git a/tests/unit/imports/test_extract.py b/tests/unit/imports/test_extract.py
index 05dd79a5..96da43b8 100644
--- a/tests/unit/imports/test_extract.py
+++ b/tests/unit/imports/test_extract.py
@@ -122,19 +122,33 @@ def test_import_parser_file_encodings_ipynb(code_cell_content: list[str], encodi
         assert get_imported_modules_from_list_of_files([random_file]) == {"foo": [Location(random_file, 1, 0)]}
 
 
-def test_import_parser_file_encodings_warning(tmp_path: Path, caplog: LogCaptureFixture) -> None:
-    file_path = Path("file1.py")
+def test_import_parser_errors(tmp_path: Path, caplog: LogCaptureFixture) -> None:
+    file_ok = Path("file_ok.py")
+    file_with_bad_encoding = Path("file_with_bad_encoding.py")
+    file_with_syntax_error = Path("file_with_syntax_error.py")
 
     with run_within_dir(tmp_path):
-        # The characters below are represented differently in ISO-8859-1 and UTF-8, so this should raise an error.
-        with file_path.open("w", encoding="ISO-8859-1") as f:
+        with file_ok.open("w") as f:
+            f.write("import black")
+
+        with file_with_bad_encoding.open("w", encoding="ISO-8859-1") as f:
             f.write("# -*- coding: utf-8 -*-\nprint('ÆØÅ')")
 
-        with caplog.at_level(logging.WARNING):
-            assert get_imported_modules_from_list_of_files([file_path]) == {}
+        with file_with_syntax_error.open("w") as f:
+            f.write("invalid_syntax:::")
 
-        # //TODO logging from Rust still includes it's own warning and file + line number. Can we get rid of that?
-        pattern = re.compile(
-            r"WARNING  deptry.imports:imports.rs:\d+ Warning: File file1.py could not be read. Skipping...\n"
+        with caplog.at_level(logging.WARNING):
+            assert get_imported_modules_from_list_of_files([
+                file_ok,
+                file_with_bad_encoding,
+                file_with_syntax_error,
+            ]) == {"black": [Location(file=Path("file_ok.py"), line=1, column=8)]}
+
+        assert re.search(
+            r"WARNING  deptry.imports:imports.rs:\d+ Warning: Skipping processing of file_with_bad_encoding.py because of the following error: \"OSError: Failed to decode file content with the detected encoding.\".",
+            caplog.text,
+        )
+        assert re.search(
+            r"WARNING  deptry.imports:imports.rs:\d+ Warning: Skipping processing of file_with_syntax_error.py because of the following error: \"SyntaxError: invalid syntax. Got unexpected token ':' at byte offset 15\".",
+            caplog.text,
         )
-        assert pattern.search(caplog.text) is not None