|
import pkg_resources |
|
from pkg_resources import DistributionNotFound, VersionConflict |
|
|
|
from src.utils import remove |
|
from tests.utils import wrap_test_forked |
|
|
|
|
|
def get_all_requirements(): |
|
import glob |
|
requirements_all = [] |
|
reqs_http_all = [] |
|
for req_name in ['requirements.txt'] + glob.glob('reqs_optional/req*.txt'): |
|
requirements1, reqs_http1 = get_requirements(req_name) |
|
requirements_all.extend(requirements1) |
|
reqs_http_all.extend(reqs_http1) |
|
return requirements_all, reqs_http_all |
|
|
|
|
|
def get_requirements(req_file="requirements.txt"): |
|
req_tmp_file = req_file + '.tmp.txt' |
|
try: |
|
|
|
reqs_http = [] |
|
|
|
with open(req_file, 'rt') as f: |
|
contents = f.readlines() |
|
with open(req_tmp_file, 'wt') as g: |
|
for line in contents: |
|
if 'http://' not in line and 'https://' not in line: |
|
g.write(line) |
|
else: |
|
reqs_http.append(line.replace('\n', '')) |
|
reqs_http = [x for x in reqs_http if x] |
|
print('reqs_http: %s' % reqs_http, flush=True) |
|
|
|
with open(req_tmp_file, "rt") as f: |
|
requirements = pkg_resources.parse_requirements(f.read()) |
|
finally: |
|
remove(req_tmp_file) |
|
return requirements, reqs_http |
|
|
|
|
|
@wrap_test_forked |
|
def test_requirements(): |
|
"""Test that each required package is available.""" |
|
packages_all = [] |
|
packages_dist = [] |
|
packages_version = [] |
|
packages_unkn = [] |
|
|
|
requirements, reqs_http = get_all_requirements() |
|
|
|
for requirement in requirements: |
|
try: |
|
requirement = str(requirement) |
|
pkg_resources.require(requirement) |
|
except DistributionNotFound: |
|
packages_all.append(requirement) |
|
packages_dist.append(requirement) |
|
except VersionConflict: |
|
packages_all.append(requirement) |
|
packages_version.append(requirement) |
|
except pkg_resources.extern.packaging.requirements.InvalidRequirement: |
|
packages_all.append(requirement) |
|
packages_unkn.append(requirement) |
|
|
|
packages_all.extend(reqs_http) |
|
if packages_dist or packages_version: |
|
print('Missing packages: %s' % packages_dist, flush=True) |
|
print('Wrong version of packages: %s' % packages_version, flush=True) |
|
print("Can't determine (e.g. http) packages: %s" % packages_unkn, flush=True) |
|
print('\n\nRUN THIS:\n\n', flush=True) |
|
print( |
|
'pip uninstall peft transformers accelerate -y ; CUDA_HOME=/usr/local/cuda-11.7 pip install %s --upgrade' % str( |
|
' '.join(packages_all)), flush=True) |
|
print('\n\n', flush=True) |
|
|
|
raise ValueError(packages_all) |
|
|
|
|
|
import requests |
|
import json |
|
|
|
try: |
|
from packaging.version import parse |
|
except ImportError: |
|
from pip._vendor.packaging.version import parse |
|
|
|
URL_PATTERN = 'https://pypi.python.org/pypi/{package}/json' |
|
|
|
|
|
def get_version(package, url_pattern=URL_PATTERN): |
|
"""Return version of package on pypi.python.org using json.""" |
|
req = requests.get(url_pattern.format(package=package)) |
|
version = parse('0') |
|
if req.status_code == requests.codes.ok: |
|
j = json.loads(req.text.encode(req.encoding)) |
|
releases = j.get('releases', []) |
|
for release in releases: |
|
ver = parse(release) |
|
if not ver.is_prerelease: |
|
version = max(version, ver) |
|
return version |
|
|
|
|
|
@wrap_test_forked |
|
def test_what_latest_packages(): |
|
|
|
import requirements |
|
import glob |
|
for req_name in ['requirements.txt'] + glob.glob('reqs_optional/req*.txt'): |
|
print("\n File: %s" % req_name, flush=True) |
|
with open(req_name, 'rt') as fd: |
|
for req in requirements.parse(fd): |
|
from importlib.metadata import version |
|
try: |
|
current_version = version(req.name) |
|
latest_version = get_version(req.name) |
|
if str(current_version) != str(latest_version): |
|
print("%s: %s -> %s" % (req.name, current_version, latest_version), flush=True) |
|
except Exception as e: |
|
print("Exception: %s" % str(e), flush=True) |
|
|
