CVE-2020-7698: Improper Neutralization of Special Elements used in an OS Command ('OS Command Injection') (CWE-78)
value: This affects the package Gerapy from 0 and before 0.9.3. The input being passed to Popen, via the project_configure endpoint, isn’t being sanitized.
CWE description: The product constructs all or part of an OS command using externally-influenced input from an upstream component, but it does not neutralize or incorrectly neutralizes special elements that could modify the intended OS command when it is sent to a downstream component.
CVSS v3 base score: 9.8
Published: 2020-07-29T13:15Z
Repository: https://github.com/Gerapy/Gerapy
Language: Python
Files changed: README.md, gerapy/__version__.py, gerapy/server/core/views.py, requirements.txt
Commit date: 2020-07-06 22:57:10 +0800
Commit message: to b1

Diff: diff --git a/README.md b/README.md
index 8e8fae0..dc001fe 100755
--- a/README.md
+++ b/README.md
@@ -6,13 +6,10 @@
 [![GitHub stars](https://img.shields.io/github/stars/Gerapy/Gerapy)](https://github.com/Gerapy/Gerapy/stargazers)
 ![PyPI - Downloads](https://img.shields.io/pypi/dm/gerapy)
 ![Docker Pulls](https://img.shields.io/docker/pulls/germey/gerapy)
 ![PyPI - License](https://img.shields.io/pypi/l/gerapy)
 
-
-> 注：从 Gerapy 2.x 开始，其定位发生改变，不再支持 Scrapyd，转而支持 Docker、Kubernetes 的部署，另外开发还会迁移到 Scrapy 可视化配置和智能解析方面，敬请期待。
-
 Distributed Crawler Management Framework Based on Scrapy, Scrapyd, Scrapyd-Client, Scrapyd-API, Django and Vue.js.
 
 ## Documentation
 
 Documentation is available online at [https://docs.gerapy.com/](https://docs.gerapy.com/) and [https://github.com/Gerapy/Docs](https://github.com/Gerapy/Docs).
diff --git a/gerapy/__version__.py b/gerapy/__version__.py
index c620621..4fb2fa4 100755
--- a/gerapy/__version__.py
+++ b/gerapy/__version__.py
@@ -1,5 +1,5 @@
-VERSION = (0, 9, '3a3')
+VERSION = (0, 9, '3b1')
 
 __version__ = '.'.join(map(str, VERSION))
 
 version = lambda: __version__
diff --git a/gerapy/server/core/views.py b/gerapy/server/core/views.py
index e8a4ad3..bcbddea 100755
--- a/gerapy/server/core/views.py
+++ b/gerapy/server/core/views.py
@@ -1,5 +1,6 @@
+import re
 from pathlib import Path
 from urllib.parse import unquote
 import base64
 import json, os, requests, time, pytz, pymongo
 from shutil import rmtree
@@ -249,10 +250,12 @@ def project_configure(request, project_name):
         project = Project.objects.filter(name=project_name)
         data = json.loads(request.body)
         configuration = json.dumps(data.get('configuration'), ensure_ascii=False)
         project.update(**{'configuration': configuration})
         
+        # for safe protection
+        project_name = re.sub('[\!\@\#\$\;\&\*\~\"\'\{\}\]\[\-\+\%\^]+', '', project_name)
         # execute generate cmd
         cmd = ' '.join(['gerapy', 'generate', project_name])
         p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
         stdout, stderr = bytes2str(p.stdout.read()), bytes2str(p.stderr.read())
         
@@ -632,21 +635,19 @@ def job_list(request, client_id, project_name):
     :return: list of jobs
     """
     if request.method == 'GET':
         client = Client.objects.get(id=client_id)
         scrapyd = get_scrapyd(client)
-        try:
-            result = scrapyd.list_jobs(project_name)
-            jobs = []
-            statuses = ['pending', 'running', 'finished']
-            for status in statuses:
-                for job in result.get(status):
-                    job['status'] = status
-                    jobs.append(job)
-            return JsonResponse(jobs)
-        except ConnectionError:
-            return JsonResponse({'message': 'Connect Error'}, status=500)
+        result = scrapyd.list_jobs(project_name)
+        jobs = []
+        statuses = ['pending', 'running', 'finished']
+        for status in statuses:
+            for job in result.get(status):
+                job['status'] = status
+                jobs.append(job)
+        return JsonResponse(jobs)
+    
 
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticated])
 def job_log(request, client_id, project_name, spider_name, job_id):
@@ -661,25 +662,22 @@ def job_log(request, client_id, project_name, spider_name, job_id):
     """
     if request.method == 'GET':
         client = Client.objects.get(id=client_id)
         # get log url
         url = log_url(client.ip, client.port, project_name, spider_name, job_id)
-        try:
-            # get last 1000 bytes of log
-            response = requests.get(url, timeout=5, headers={
-                'Range': 'bytes=-1000'
-            }, auth=(client.username, client.password) if client.auth else None)
-            # Get encoding
-            encoding = response.apparent_encoding
-            # log not found
-            if response.status_code == 404:
-                return JsonResponse({'message': 'Log Not Found'}, status=404)
-            # bytes to string
-            text = response.content.decode(encoding, errors='replace')
-            return HttpResponse(text)
-        except requests.ConnectionError:
-            return JsonResponse({'message': 'Load Log Error'}, status=500)
+        # get last 1000 bytes of log
+        response = requests.get(url, timeout=5, headers={
+            'Range': 'bytes=-1000'
+        }, auth=(client.username, client.password) if client.auth else None)
+        # Get encoding
+        encoding = response.apparent_encoding
+        # log not found
+        if response.status_code == 404:
+            return JsonResponse({'message': 'Log Not Found'}, status=404)
+        # bytes to string
+        text = response.content.decode(encoding, errors='replace')
+        return HttpResponse(text)
 
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticated])
 def job_cancel(request, client_id, project_name, job_id):
@@ -691,42 +689,33 @@ def job_cancel(request, client_id, project_name, job_id):
     :param job_id: job id
     :return: json of cancel
     """
     if request.method == 'GET':
         client = Client.objects.get(id=client_id)
-        try:
-            scrapyd = get_scrapyd(client)
-            result = scrapyd.cancel(project_name, job_id)
-            return JsonResponse(result)
-        except ConnectionError:
-            return JsonResponse({'message': 'Connect Error'})
+        scrapyd = get_scrapyd(client)
+        result = scrapyd.cancel(project_name, job_id)
+        return JsonResponse(result)
 
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticated])
 def del_version(request, client_id, project, version):
     if request.method == 'GET':
         client = Client.objects.get(id=client_id)
-        try:
-            scrapyd = get_scrapyd(client)
-            result = scrapyd.delete_version(project=project, version=version)
-            return JsonResponse(result)
-        except ConnectionError:
-            return JsonResponse({'message': 'Connect Error'})
+        scrapyd = get_scrapyd(client)
+        result = scrapyd.delete_version(project=project, version=version)
+        return JsonResponse(result)
 
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticated])
 def del_project(request, client_id, project):
     if request.method == 'GET':
         client = Client.objects.get(id=client_id)
-        try:
-            scrapyd = get_scrapyd(client)
-            result = scrapyd.delete_project(project=project)
-            return JsonResponse(result)
-        except ConnectionError:
-            return JsonResponse({'message': 'Connect Error'})
+        scrapyd = get_scrapyd(client)
+        result = scrapyd.delete_project(project=project)
+        return JsonResponse(result)
 
 
 @api_view(['POST'])
 @permission_classes([IsAuthenticated])
 def monitor_db_list(request):
@@ -827,22 +816,20 @@ def task_remove(request, task_id):
     remove task by task_id
     :param request:
     :return:
     """
     if request.method == 'POST':
-        try:
-            # delete job from DjangoJob
-            task = Task.objects.get(id=task_id)
-            clients = clients_of_task(task)
-            for client in clients:
-                job_id = get_job_id(client, task)
-                DjangoJob.objects.filter(name=job_id).delete()
-            # delete task
-            Task.objects.filter(id=task_id).delete()
-            return JsonResponse({'result': '1'})
-        except:
-            return JsonResponse({'result': '0'})
+        # delete job from DjangoJob
+        task = Task.objects.get(id=task_id)
+        clients = clients_of_task(task)
+        for client in clients:
+            job_id = get_job_id(client, task)
+            DjangoJob.objects.filter(name=job_id).delete()
+        # delete task
+        Task.objects.filter(id=task_id).delete()
+        return JsonResponse({'result': '1'})
+    
 
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticated])
 def task_info(request, task_id):
@@ -913,12 +900,9 @@ def render_html(request):
     if request.method == 'GET':
         url = request.GET.get('url')
         url = unquote(base64.b64decode(url).decode('utf-8'))
         js = request.GET.get('js', 0)
         script = request.GET.get('script')
-        try:
-            response = requests.get(url, timeout=5)
-            response.encoding = response.apparent_encoding
-            html = process_html(response.text)
-            return HttpResponse(html)
-        except Exception as e:
-            return JsonResponse({'message': e.args}, status=500)
+        response = requests.get(url, timeout=5)
+        response.encoding = response.apparent_encoding
+        html = process_html(response.text)
+        return HttpResponse(html)
diff --git a/requirements.txt b/requirements.txt
index 068412e..e77eb14 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,16 +3,16 @@ cryptography==2.8
 django==1.11.29
 django-cors-headers==3.2.0
 django-apscheduler==0.3.0
 furl==2.1.0
 jinja2==2.10.1
-scrapy>=1.4.0
+scrapy==1.5.0
 scrapy-redis==0.6.8
 scrapy-splash==0.7.2
 python-scrapyd-api==2.1.2
 redis==2.10.5
-requests>=2.20.0
+requests==2.20.0
 pymongo==3.9.0
 pymysql==0.7.10
 pyquery==1.2.17
 beautifulsoup4==4.7.0
 djangorestframework==3.9.2

Vulnerable code: 
> 注：从 Gerapy 2.x 开始，其定位发生改变，不再支持 Scrapyd，转而支持 Docker、Kubernetes 的部署，另外开发还会迁移到 Scrapy 可视化配置和智能解析方面，敬请期待。

VERSION = (0, 9, '3a3')
        try:
            result = scrapyd.list_jobs(project_name)
            jobs = []
            statuses = ['pending', 'running', 'finished']
            for status in statuses:
                for job in result.get(status):
                    job['status'] = status
                    jobs.append(job)
            return JsonResponse(jobs)
        except ConnectionError:
            return JsonResponse({'message': 'Connect Error'}, status=500)
        try:
            # get last 1000 bytes of log
            response = requests.get(url, timeout=5, headers={
                'Range': 'bytes=-1000'
            }, auth=(client.username, client.password) if client.auth else None)
            # Get encoding
            encoding = response.apparent_encoding
            # log not found
            if response.status_code == 404:
                return JsonResponse({'message': 'Log Not Found'}, status=404)
            # bytes to string
            text = response.content.decode(encoding, errors='replace')
            return HttpResponse(text)
        except requests.ConnectionError:
            return JsonResponse({'message': 'Load Log Error'}, status=500)
        try:
            scrapyd = get_scrapyd(client)
            result = scrapyd.cancel(project_name, job_id)
            return JsonResponse(result)
        except ConnectionError:
            return JsonResponse({'message': 'Connect Error'})
        try:
            scrapyd = get_scrapyd(client)
            result = scrapyd.delete_version(project=project, version=version)
            return JsonResponse(result)
        except ConnectionError:
            return JsonResponse({'message': 'Connect Error'})
        try:
            scrapyd = get_scrapyd(client)
            result = scrapyd.delete_project(project=project)
            return JsonResponse(result)
        except ConnectionError:
            return JsonResponse({'message': 'Connect Error'})
        try:
            # delete job from DjangoJob
            task = Task.objects.get(id=task_id)
            clients = clients_of_task(task)
            for client in clients:
                job_id = get_job_id(client, task)
                DjangoJob.objects.filter(name=job_id).delete()
            # delete task
            Task.objects.filter(id=task_id).delete()
            return JsonResponse({'result': '1'})
        except:
            return JsonResponse({'result': '0'})
        try:
            response = requests.get(url, timeout=5)
            response.encoding = response.apparent_encoding
            html = process_html(response.text)
            return HttpResponse(html)
        except Exception as e:
            return JsonResponse({'message': e.args}, status=500)
scrapy>=1.4.0
requests>=2.20.0
Fixed code: VERSION = (0, 9, '3b1')
import re
        # for safe protection
        project_name = re.sub('[\!\@\#\$\;\&\*\~\"\'\{\}\]\[\-\+\%\^]+', '', project_name)
        result = scrapyd.list_jobs(project_name)
        jobs = []
        statuses = ['pending', 'running', 'finished']
        for status in statuses:
            for job in result.get(status):
                job['status'] = status
                jobs.append(job)
        return JsonResponse(jobs)
    
        # get last 1000 bytes of log
        response = requests.get(url, timeout=5, headers={
            'Range': 'bytes=-1000'
        }, auth=(client.username, client.password) if client.auth else None)
        # Get encoding
        encoding = response.apparent_encoding
        # log not found
        if response.status_code == 404:
            return JsonResponse({'message': 'Log Not Found'}, status=404)
        # bytes to string
        text = response.content.decode(encoding, errors='replace')
        return HttpResponse(text)
        scrapyd = get_scrapyd(client)
        result = scrapyd.cancel(project_name, job_id)
        return JsonResponse(result)
        scrapyd = get_scrapyd(client)
        result = scrapyd.delete_version(project=project, version=version)
        return JsonResponse(result)
        scrapyd = get_scrapyd(client)
        result = scrapyd.delete_project(project=project)
        return JsonResponse(result)
        # delete job from DjangoJob
        task = Task.objects.get(id=task_id)
        clients = clients_of_task(task)
        for client in clients:
            job_id = get_job_id(client, task)
            DjangoJob.objects.filter(name=job_id).delete()
        # delete task
        Task.objects.filter(id=task_id).delete()
        return JsonResponse({'result': '1'})
    
        response = requests.get(url, timeout=5)
        response.encoding = response.apparent_encoding
        html = process_html(response.text)
        return HttpResponse(html)
scrapy==1.5.0
requests==2.20.0