CVE-2021-43857: Insufficient Information (NVD-CWE-noinfo)
value: Gerapy is a distributed crawler management framework. Gerapy prior to version 0.9.8 is vulnerable to remote code execution, and this issue is patched in version 0.9.8.
CWE description: There is insufficient information about the issue to classify it; details are unkown or unspecified.
CVSS v3 base score: 8.8
Published: 2021-12-27T19:15Z
Repository: https://github.com/Gerapy/Gerapy
Language: Python
Files changed: gerapy/server/core/views.py
Commit date: 2021-12-26 17:50:00 +0800
Commit message: fix remote execute

Diff: diff --git a/gerapy/server/core/views.py b/gerapy/server/core/views.py
index bcbddea..3404f57 100755
--- a/gerapy/server/core/views.py
+++ b/gerapy/server/core/views.py
@@ -1,10 +1,15 @@
 import re
 from pathlib import Path
 from urllib.parse import unquote
 import base64
-import json, os, requests, time, pytz, pymongo
+import json
+import os
+import requests
+import time
+import pytz
+import pymongo
 from shutil import rmtree
 from requests.exceptions import ConnectionError
 from os.path import join, exists
 from django.shortcuts import render
 from django.core.serializers import serialize
@@ -171,11 +176,12 @@ def spider_list(request, client_id, project_name):
     """
     if request.method == 'GET':
         client = Client.objects.get(id=client_id)
         scrapyd = get_scrapyd(client)
         spiders = scrapyd.list_spiders(project_name)
-        spiders = [{'name': spider, 'id': index + 1} for index, spider in enumerate(spiders)]
+        spiders = [{'name': spider, 'id': index + 1}
+                   for index, spider in enumerate(spiders)]
         return JsonResponse(spiders)
 
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticated])
@@ -240,27 +246,29 @@ def project_configure(request, project_name):
     """
     # get configuration
     if request.method == 'GET':
         project = Project.objects.get(name=project_name)
         project = model_to_dict(project)
-        project['configuration'] = json.loads(project['configuration']) if project['configuration'] else None
+        project['configuration'] = json.loads(
+            project['configuration']) if project['configuration'] else None
         return JsonResponse(project)
-    
+
     # update configuration
     elif request.method == 'POST':
         project = Project.objects.filter(name=project_name)
         data = json.loads(request.body)
-        configuration = json.dumps(data.get('configuration'), ensure_ascii=False)
+        configuration = json.dumps(
+            data.get('configuration'), ensure_ascii=False)
         project.update(**{'configuration': configuration})
-        
         # for safe protection
-        project_name = re.sub('[\!\@\#\$\;\&\*\~\"\'\{\}\]\[\-\+\%\^]+', '', project_name)
+        project_name = re.sub(
+            '[\s\!\@\#\$\;\&\*\~\"\'\{\}\]\[\-\+\%\^]+', '', project_name)
         # execute generate cmd
-        cmd = ' '.join(['gerapy', 'generate', project_name])
-        p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        cmd = ['gerapy', 'generate', project_name]
+        p = Popen(cmd, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
         stdout, stderr = bytes2str(p.stdout.read()), bytes2str(p.stderr.read())
-        
+
         if not stderr:
             return JsonResponse({'status': '1'})
         else:
             return JsonResponse({'status': '0', 'message': stderr})
 
@@ -292,11 +300,12 @@ def project_create(request):
     if request.method == 'POST':
         data = json.loads(request.body)
         data['configurable'] = 1
         project, result = Project.objects.update_or_create(**data)
         # generate a single project folder
-        path = join(os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER)), data['name'])
+        path = join(os.path.abspath(
+            join(os.getcwd(), PROJECTS_FOLDER)), data['name'])
         os.mkdir(path)
         return JsonResponse(model_to_dict(project))
 
 
 @api_view(['POST'])
@@ -332,16 +341,17 @@ def project_clone(request):
         data = json.loads(request.body)
         address = data.get('address')
         if not address.startswith('http'):
             return JsonResponse({'status': False})
         address = address + '.git' if not address.endswith('.git') else address
-        cmd = 'git clone {address} {target}'.format(address=address, target=join(PROJECTS_FOLDER, Path(address).stem))
+        cmd = ['git', 'clone', 'address', join(PROJECTS_FOLDER, Path(address).stem)]
         logger.debug('clone cmd %s', cmd)
-        p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
+        p = Popen(cmd, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
         stdout, stderr = bytes2str(p.stdout.read()), bytes2str(p.stderr.read())
         logger.debug('clone run result %s', stdout)
-        if stderr: logger.error(stderr)
+        if stderr:
+            logger.error(stderr)
         return JsonResponse({'status': True}) if not stderr else JsonResponse({'status': False})
 
 
 @api_view(['POST'])
 @permission_classes([IsAuthenticated])
@@ -391,14 +401,16 @@ def project_version(request, client_id, project_name):
                 versions = scrapyd.list_versions(project_name)
             except ConnectionError:
                 return JsonResponse({'message': 'Connect Error'}, status=500)
             if len(versions) > 0:
                 version = versions[-1]
-                deployed_at = timezone.datetime.fromtimestamp(int(version), tz=pytz.timezone(TIME_ZONE))
+                deployed_at = timezone.datetime.fromtimestamp(
+                    int(version), tz=pytz.timezone(TIME_ZONE))
             else:
                 deployed_at = None
-            deploy, result = Deploy.objects.update_or_create(client=client, project=project, deployed_at=deployed_at)
+            deploy, result = Deploy.objects.update_or_create(
+                client=client, project=project, deployed_at=deployed_at)
         # return deploy json info
         return JsonResponse(model_to_dict(deploy))
 
 
 @api_view(['POST'])
@@ -444,11 +456,11 @@ def project_build(request, project_name):
     :return: json
     """
     # get project folder
     path = os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER))
     project_path = join(path, project_name)
-    
+
     # get build version
     if request.method == 'GET':
         egg = find_egg(project_path)
         # if built, save or update project to db
         if egg:
@@ -468,11 +480,11 @@ def project_build(request, project_name):
                 Project(name=project_name).save()
             model = Project.objects.get(name=project_name)
         # transfer model to dict then dumps it to json
         data = model_to_dict(model)
         return JsonResponse(data)
-    
+
     # build operation manually by clicking button
     elif request.method == 'POST':
         data = json.loads(request.body)
         description = data['description']
         build_project(project_name)
@@ -481,11 +493,12 @@ def project_build(request, project_name):
             return JsonResponse({'message': 'egg not found'}, status=500)
         # update built_at info
         built_at = timezone.now()
         # if project does not exists in db, create it
         if not Project.objects.filter(name=project_name):
-            Project(name=project_name, description=description, built_at=built_at, egg=egg).save()
+            Project(name=project_name, description=description,
+                    built_at=built_at, egg=egg).save()
             model = Project.objects.get(name=project_name)
         # if project exists, update egg, description, built_at info
         else:
             model = Project.objects.get(name=project_name)
             model.built_at = built_at
@@ -524,21 +537,20 @@ def project_parse(request, project_name):
         }
         # set request body
         body = data.get('body', '')
         if args.get('method').lower() != 'get':
             args['body'] = "'" + json.dumps(body, ensure_ascii=False) + "'"
-        
-        args_cmd = ' '.join(
-            ['--{arg} {value}'.format(arg=arg, value=value) for arg, value in args.items()])
-        logger.debug('args cmd %s', args_cmd)
-        cmd = 'gerapy parse {args_cmd} {project_path} {spider_name}'.format(
-            args_cmd=args_cmd,
-            project_path=project_path,
-            spider_name=spider_name
-        )
+
+        args_array = []
+        for arg, value in args.items():
+            args_array.append(f'--{arg}')
+            args_array.append(f'{value}')
+        cmd = ['gerapy', 'parse'] + args_array + [project_path] + [spider_name]
+        print('cmd', cmd)
         logger.debug('parse cmd %s', cmd)
-        p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
+        p = Popen(cmd, shell=False, stdin=PIPE,
+                         stdout=PIPE, stderr=PIPE, close_fds=True)
         stdout, stderr = bytes2str(p.stdout.read()), bytes2str(p.stderr.read())
         logger.debug('stdout %s, stderr %s', stdout, stderr)
         if not stderr:
             return JsonResponse({'status': True, 'result': json.loads(stdout)})
         else:
@@ -643,11 +655,10 @@ def job_list(request, client_id, project_name):
         for status in statuses:
             for job in result.get(status):
                 job['status'] = status
                 jobs.append(job)
         return JsonResponse(jobs)
-    
 
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticated])
 def job_log(request, client_id, project_name, spider_name, job_id):
@@ -661,11 +672,12 @@ def job_log(request, client_id, project_name, spider_name, job_id):
     :return: log of job
     """
     if request.method == 'GET':
         client = Client.objects.get(id=client_id)
         # get log url
-        url = log_url(client.ip, client.port, project_name, spider_name, job_id)
+        url = log_url(client.ip, client.port,
+                      project_name, spider_name, job_id)
         # get last 1000 bytes of log
         response = requests.get(url, timeout=5, headers={
             'Range': 'bytes=-1000'
         }, auth=(client.username, client.password) if client.auth else None)
         # Get encoding
@@ -763,11 +775,12 @@ def monitor_create(request):
     :return: json of create
     """
     if request.method == 'POST':
         data = json.loads(request.body)
         data = data['form']
-        data['configuration'] = json.dumps(data['configuration'], ensure_ascii=False)
+        data['configuration'] = json.dumps(
+            data['configuration'], ensure_ascii=False)
         monitor = Monitor.objects.create(**data)
         return JsonResponse(model_to_dict(monitor))
 
 
 @api_view(['POST'])
@@ -783,11 +796,12 @@ def task_create(request):
         task = Task.objects.create(clients=json.dumps(data.get('clients'), ensure_ascii=False),
                                    project=data.get('project'),
                                    name=data.get('name'),
                                    spider=data.get('spider'),
                                    trigger=data.get('trigger'),
-                                   configuration=json.dumps(data.get('configuration'), ensure_ascii=False),
+                                   configuration=json.dumps(
+                                       data.get('configuration'), ensure_ascii=False),
                                    modified=1)
         return JsonResponse({'result': '1', 'data': model_to_dict(task)})
 
 
 @api_view(['POST'])
@@ -801,11 +815,12 @@ def task_update(request, task_id):
     """
     if request.method == 'POST':
         task = Task.objects.filter(id=task_id)
         data = json.loads(request.body)
         data['clients'] = json.dumps(data.get('clients'), ensure_ascii=False)
-        data['configuration'] = json.dumps(data.get('configuration'), ensure_ascii=False)
+        data['configuration'] = json.dumps(
+            data.get('configuration'), ensure_ascii=False)
         data['modified'] = 1
         task.update(**data)
         return JsonResponse(model_to_dict(Task.objects.get(id=task_id)))
 
 
@@ -821,15 +836,14 @@ def task_remove(request, task_id):
         # delete job from DjangoJob
         task = Task.objects.get(id=task_id)
         clients = clients_of_task(task)
         for client in clients:
             job_id = get_job_id(client, task)
-            DjangoJob.objects.filter(name=job_id).delete()
+            DjangoJob.objects.filter(id=job_id).delete()
         # delete task
         Task.objects.filter(id=task_id).delete()
         return JsonResponse({'result': '1'})
-    
 
 
 @api_view(['GET'])
 @permission_classes([IsAuthenticated])
 def task_info(request, task_id):
@@ -873,16 +887,18 @@ def task_status(request, task_id):
         result = []
         task = Task.objects.get(id=task_id)
         clients = clients_of_task(task)
         for client in clients:
             job_id = get_job_id(client, task)
-            jobs = DjangoJob.objects.filter(name=job_id)
+            jobs = DjangoJob.objects.filter(id=job_id)
             logger.debug('jobs from djangojob %s', jobs)
             # if job does not exist, for date mode exceed time
-            if not jobs: continue
-            job = DjangoJob.objects.get(name=job_id)
-            executions = serialize('json', DjangoJobExecution.objects.filter(job=job))
+            if not jobs:
+                continue
+            job = DjangoJob.objects.get(id=job_id)
+            executions = serialize(
+                'json', DjangoJobExecution.objects.filter(job=job))
             result.append({
                 'client': model_to_dict(client),
                 'next': job.next_run_time,
                 'executions': json.loads(executions)
             })

Vulnerable code: import json, os, requests, time, pytz, pymongo
        spiders = [{'name': spider, 'id': index + 1} for index, spider in enumerate(spiders)]
        project['configuration'] = json.loads(project['configuration']) if project['configuration'] else None
    
        configuration = json.dumps(data.get('configuration'), ensure_ascii=False)
        
        project_name = re.sub('[\!\@\#\$\;\&\*\~\"\'\{\}\]\[\-\+\%\^]+', '', project_name)
        cmd = ' '.join(['gerapy', 'generate', project_name])
        p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        
        path = join(os.path.abspath(join(os.getcwd(), PROJECTS_FOLDER)), data['name'])
        cmd = 'git clone {address} {target}'.format(address=address, target=join(PROJECTS_FOLDER, Path(address).stem))
        p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        if stderr: logger.error(stderr)
                deployed_at = timezone.datetime.fromtimestamp(int(version), tz=pytz.timezone(TIME_ZONE))
            deploy, result = Deploy.objects.update_or_create(client=client, project=project, deployed_at=deployed_at)
    
    
            Project(name=project_name, description=description, built_at=built_at, egg=egg).save()
        
        args_cmd = ' '.join(
            ['--{arg} {value}'.format(arg=arg, value=value) for arg, value in args.items()])
        logger.debug('args cmd %s', args_cmd)
        cmd = 'gerapy parse {args_cmd} {project_path} {spider_name}'.format(
            args_cmd=args_cmd,
            project_path=project_path,
            spider_name=spider_name
        )
        p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True)
    
        url = log_url(client.ip, client.port, project_name, spider_name, job_id)
        data['configuration'] = json.dumps(data['configuration'], ensure_ascii=False)
                                   configuration=json.dumps(data.get('configuration'), ensure_ascii=False),
        data['configuration'] = json.dumps(data.get('configuration'), ensure_ascii=False)
            DjangoJob.objects.filter(name=job_id).delete()
    
            jobs = DjangoJob.objects.filter(name=job_id)
            if not jobs: continue
            job = DjangoJob.objects.get(name=job_id)
            executions = serialize('json', DjangoJobExecution.objects.filter(job=job))
Fixed code: import json
import os
import requests
import time
import pytz
import pymongo
        spiders = [{'name': spider, 'id': index + 1}
                   for index, spider in enumerate(spiders)]
        project['configuration'] = json.loads(
            project['configuration']) if project['configuration'] else None

        configuration = json.dumps(
            data.get('configuration'), ensure_ascii=False)
        project_name = re.sub(
            '[\s\!\@\#\$\;\&\*\~\"\'\{\}\]\[\-\+\%\^]+', '', project_name)
        cmd = ['gerapy', 'generate', project_name]
        p = Popen(cmd, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)

        path = join(os.path.abspath(
            join(os.getcwd(), PROJECTS_FOLDER)), data['name'])
        cmd = ['git', 'clone', 'address', join(PROJECTS_FOLDER, Path(address).stem)]
        p = Popen(cmd, shell=False, stdin=PIPE, stdout=PIPE, stderr=PIPE)
        if stderr:
            logger.error(stderr)
                deployed_at = timezone.datetime.fromtimestamp(
                    int(version), tz=pytz.timezone(TIME_ZONE))
            deploy, result = Deploy.objects.update_or_create(
                client=client, project=project, deployed_at=deployed_at)


            Project(name=project_name, description=description,
                    built_at=built_at, egg=egg).save()

        args_array = []
        for arg, value in args.items():
            args_array.append(f'--{arg}')
            args_array.append(f'{value}')
        cmd = ['gerapy', 'parse'] + args_array + [project_path] + [spider_name]
        print('cmd', cmd)
        p = Popen(cmd, shell=False, stdin=PIPE,
                         stdout=PIPE, stderr=PIPE, close_fds=True)
        url = log_url(client.ip, client.port,
                      project_name, spider_name, job_id)
        data['configuration'] = json.dumps(
            data['configuration'], ensure_ascii=False)
                                   configuration=json.dumps(
                                       data.get('configuration'), ensure_ascii=False),
        data['configuration'] = json.dumps(
            data.get('configuration'), ensure_ascii=False)
            DjangoJob.objects.filter(id=job_id).delete()
            jobs = DjangoJob.objects.filter(id=job_id)
            if not jobs:
                continue
            job = DjangoJob.objects.get(id=job_id)
            executions = serialize(
                'json', DjangoJobExecution.objects.filter(job=job))