Repository: rucio/rucio

Issue:
Remove/change `bz2file` dependency
Motivation
----------
`bz2file` has not been updated since 2014. Switch to a different dependency if needed, or remove entirely from the codebase.


Discussion:
Would plain bz2 (currently mainteined) be OK? 

Motivation for using bz2file looks to be here.
https://github.com/rucio/rucio/blob/680b88876e5df65f87c7a3888020de28acb33c5e/lib/rucio/common/dumper/__init__.py#L38-L42

I assume that by dropping python2 the problem won't be there anymore. Remains python 3.0 to 3.2, but I would not care that much if its just for the multistream TBH.
And I assume that the transition to bz2 would be quite transparent.

Resolving patch:
diff --git a/lib/rucio/common/dumper/__init__.py b/lib/rucio/common/dumper/__init__.py
--- a/lib/rucio/common/dumper/__init__.py
+++ b/lib/rucio/common/dumper/__init__.py
@@ -1,4 +1,5 @@
-# Copyright 2012-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2015-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,13 +14,16 @@
 # limitations under the License.
 #
 # Authors:
-# - Fernando Lopez <felopez@cern.ch>, 2015
+# - Fernando López <felopez@cern.ch>, 2015-2016
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2016
-# - Mario Lassnig <mario.lassnig@cern.ch>, 2017-2018
-# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018
+# - Martin Barisits <martin.barisits@cern.ch>, 2017
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2017
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2017-2020
+# - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018-2021
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
-#
-# PY3K COMPATIBLE
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Fabio Luchetti <fabio.luchetti@cern.ch>, 2022
 
 import contextlib
 import datetime
@@ -34,12 +38,7 @@
 import gfal2
 import magic
 import requests
-
-# bz2 module doesn't support multi-stream files in Python < 3.3
-# https://bugs.python.org/issue1625
-# using bz2file (a backport from the lastest CPython bz2 implementation)
-# fixes this problem
-import bz2file
+import bz2
 
 from rucio.common import config
 
@@ -151,7 +150,7 @@ def smart_open(filename):
         if file_type.find('gzip') > -1:
             f = gzip.GzipFile(filename, 'rt')
         elif file_type.find('bzip2') > -1:
-            f = bz2file.open(filename, 'rt')
+            f = bz2.open(filename, 'rt')
         else:
             pass  # Not supported format
     return f
diff --git a/setuputil.py b/setuputil.py
--- a/setuputil.py
+++ b/setuputil.py
@@ -17,6 +17,7 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2021
 # - Martin Barisits <martin.barisits@cern.ch>, 2021-2022
 # - Joel Dierkes <joel.dierkes@cern.ch>, 2022
+# - Fabio Luchetti <fabio.luchetti@cern.ch>, 2022
 
 from __future__ import print_function
 
@@ -51,7 +52,6 @@
     'sftp': ['pysftp'],
     # 'gfal2': ['gfal2-python'],
     'dumper': [
-        'bz2file',
         # 'gfal2-python',
         'python-magic',
     ],
@@ -81,7 +81,6 @@
     'install_requires': clients_requirements_table['install_requires'] + [
         'argcomplete',
         'boto',
-        'bz2file',
         'python-magic',
         'paramiko',
         'boto3',


Test patch:
diff --git a/lib/rucio/tests/test_dumper.py b/lib/rucio/tests/test_dumper.py
--- a/lib/rucio/tests/test_dumper.py
+++ b/lib/rucio/tests/test_dumper.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2015-2020 CERN for the benefit of the ATLAS collaboration.
+# Copyright 2015-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,8 +19,7 @@
 # - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2019
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
-#
-# PY3K COMPATIBLE
+# - Fabio Luchetti <fabio.luchetti@cern.ch>, 2022
 
 import json
 import os
@@ -28,7 +27,7 @@
 import tempfile
 from datetime import datetime
 
-import bz2file
+import bz2
 import pytest
 import requests
 
@@ -105,11 +104,11 @@ def test_smart_open_for_text_file():
 
 def test_smart_open_for_bz2_file():
     fd, path = tempfile.mkstemp()
-    comp = bz2file.BZ2Compressor()
+    comp = bz2.BZ2Compressor()
     with os.fdopen(fd, 'wb') as f:
         f.write(comp.compress('abcdef'.encode()))
         f.write(comp.flush())
-    assert not isinstance(dumper.smart_open(path), bz2file.BZ2File)
+    assert not isinstance(dumper.smart_open(path), bz2.BZ2File)
     os.unlink(path)