Repository: scrapy/scrapy

Issue:
scrapy startproject alters executable permissions in local virtualenv
### Description

When running `scrapy startproject` after creating a virtualenv in .venv in the project directory, the executable files have their permissions changed.

### Steps to Reproduce (using Poetry)

1. Enter new project directory
2. `poetry init -n`
3. `poetry config virtualenvs.in-project true --local`
4. `poetry add scrapy`
5. `poetry shell`
6. `scrapy startproject scraper .`
7. `scrapy` <-- fails
8. `python` <-- fails

**Expected behavior:** [What you expect to happen]

I expect to still be able to execute `scrapy` or `python` commands.

**Actual behavior:** [What actually happens]

I am unable to execute `scrapy` or `python` commands because everything in .venv/bin has had its executable permissions removed including the python executable that is the target of the `python` symlink.

**Reproduces how often:** [What percentage of the time does it reproduce?]

100%

### Versions

Scrapy       : 2.2.0
lxml         : 4.5.1.0
libxml2      : 2.9.10
cssselect    : 1.1.0
parsel       : 1.6.0
w3lib        : 1.22.0
Twisted      : 20.3.0
Python       : 3.8.3 (default, Jun 27 2020, 12:47:10) - [Clang 11.0.0 (clang-1100.0.33.8)]
pyOpenSSL    : 19.1.0 (OpenSSL 1.1.1g  21 Apr 2020)
cryptography : 2.9.2
Platform     : macOS-10.15.5-x86_64-i386-64bit

Remove Command._copytree from startproject.py
Python 3.8 introduced `dirs_exist_ok`, which makes our method unnecessary.


Discussion:
Ouch, seems like the issue appears even earlier. I think this is related to #4604.


```
 eugenio  ~   mkdir asdf
 eugenio  ~   cd asdf
 eugenio  ~/asdf   python3.7 -m venv .venv
 eugenio  ~/asdf   source .venv/bin/activate
 eugenio  ⓔ .venv  ~/asdf   pip install scrapy
Collecting scrapy

(...)

Installing collected packages: itemadapter, lxml, zope.interface, six, w3lib, cssselect, parsel, protego, pycparser, cffi, cryptography, PyDispatcher, pyOpenSSL, attrs, pyasn1, pyasn1-modules, service-identity, Automat, constantly, idna, hyperlink, PyHamcrest, incremental, Twisted, queuelib, scrapy
  Running setup.py install for protego ... done
  Running setup.py install for PyDispatcher ... done
Successfully installed Automat-20.2.0 PyDispatcher-2.0.5 PyHamcrest-2.0.2 Twisted-20.3.0 attrs-19.3.0 cffi-1.14.0 constantly-15.1.0 cryptography-2.9.2 cssselect-1.1.0 hyperlink-19.0.0 idna-2.10 incremental-17.5.0 itemadapter-0.1.0 lxml-4.5.1 parsel-1.6.0 protego-0.1.16 pyOpenSSL-19.1.0 pyasn1-0.4.8 pyasn1-modules-0.2.8 pycparser-2.20 queuelib-1.5.0 scrapy-2.2.0 service-identity-18.1.0 six-1.15.0 w3lib-1.22.0 zope.interface-5.1.0
You are using pip version 19.0.3, however version 20.2b1 is available.
You should consider upgrading via the 'pip install --upgrade pip' command.
 eugenio  ⓔ .venv  ~/asdf   scrapy startproject scraper .
Traceback (most recent call last):
  File "/home/eugenio/asdf/.venv/bin/scrapy", line 10, in <module>
    sys.exit(execute())
  File "/home/eugenio/asdf/.venv/lib/python3.7/site-packages/scrapy/cmdline.py", line 143, in execute
    _run_print_help(parser, _run_command, cmd, args, opts)
  File "/home/eugenio/asdf/.venv/lib/python3.7/site-packages/scrapy/cmdline.py", line 98, in _run_print_help
    func(*a, **kw)
  File "/home/eugenio/asdf/.venv/lib/python3.7/site-packages/scrapy/cmdline.py", line 151, in _run_command
    cmd.run(args, opts)
  File "/home/eugenio/asdf/.venv/lib/python3.7/site-packages/scrapy/commands/startproject.py", line 125, in run
    self._copytree(self.templates_dir, abspath(project_dir))
  File "/home/eugenio/asdf/.venv/lib/python3.7/site-packages/scrapy/commands/startproject.py", line 82, in _copytree
    self._set_rw_permissions(dst)
  File "/home/eugenio/asdf/.venv/lib/python3.7/site-packages/scrapy/commands/startproject.py", line 104, in _set_rw_permissions
    os.chmod(join(root, file), mode_rw)
PermissionError: [Errno 1] Operation not permitted: '/home/eugenio/asdf/.venv/bin/python3.7'
 eugenio  ⓔ .venv  ~/asdf  126   scrapy startproject scraper .
bash: /home/eugenio/asdf/.venv/bin/scrapy: Permission denied
```


Resolving patch:
diff --git a/scrapy/commands/startproject.py b/scrapy/commands/startproject.py
--- a/scrapy/commands/startproject.py
+++ b/scrapy/commands/startproject.py
@@ -1,10 +1,10 @@
 import re
 import os
-import stat
 import string
 from importlib import import_module
 from os.path import join, exists, abspath
 from shutil import ignore_patterns, move, copy2, copystat
+from stat import S_IWUSR as OWNER_WRITE_PERMISSION
 
 import scrapy
 from scrapy.commands import ScrapyCommand
@@ -20,7 +20,12 @@
     ('${project_name}', 'middlewares.py.tmpl'),
 )
 
-IGNORE = ignore_patterns('*.pyc', '.svn')
+IGNORE = ignore_patterns('*.pyc', '__pycache__', '.svn')
+
+
+def _make_writable(path):
+    current_permissions = os.stat(path).st_mode
+    os.chmod(path, current_permissions | OWNER_WRITE_PERMISSION)
 
 
 class Command(ScrapyCommand):
@@ -78,30 +83,10 @@ def _copytree(self, src, dst):
                 self._copytree(srcname, dstname)
             else:
                 copy2(srcname, dstname)
-        copystat(src, dst)
-        self._set_rw_permissions(dst)
+                _make_writable(dstname)
 
-    def _set_rw_permissions(self, path):
-        """
-        Sets permissions of a directory tree to +rw and +rwx for folders.
-        This is necessary if the start template files come without write
-        permissions.
-        """
-        mode_rw = (stat.S_IRUSR
-                   | stat.S_IWUSR
-                   | stat.S_IRGRP
-                   | stat.S_IROTH)
-
-        mode_x = (stat.S_IXUSR
-                  | stat.S_IXGRP
-                  | stat.S_IXOTH)
-
-        os.chmod(path, mode_rw | mode_x)
-        for root, dirs, files in os.walk(path):
-            for dir in dirs:
-                os.chmod(join(root, dir), mode_rw | mode_x)
-            for file in files:
-                os.chmod(join(root, file), mode_rw)
+        copystat(src, dst)
+        _make_writable(dst)
 
     def run(self, args, opts):
         if len(args) not in (1, 2):


Test patch:
diff --git a/tests/test_commands.py b/tests/test_commands.py
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -2,11 +2,14 @@
 import json
 import optparse
 import os
+from stat import S_IWRITE as ANYONE_WRITE_PERMISSION
 import subprocess
 import sys
 import tempfile
 from contextlib import contextmanager
+from itertools import chain
 from os.path import exists, join, abspath
+from pathlib import Path
 from shutil import rmtree, copytree
 from tempfile import mkdtemp
 from threading import Timer
@@ -15,6 +18,7 @@
 
 import scrapy
 from scrapy.commands import ScrapyCommand
+from scrapy.commands.startproject import IGNORE
 from scrapy.settings import Settings
 from scrapy.utils.python import to_unicode
 from scrapy.utils.test import get_testenv
@@ -119,6 +123,29 @@ def test_startproject_with_project_dir(self):
         self.assertEqual(2, self.call('startproject', self.project_name, project_dir, 'another_params'))
 
 
+def get_permissions_dict(path, renamings=None, ignore=None):
+    renamings = renamings or tuple()
+    permissions_dict = {
+        '.': os.stat(path).st_mode,
+    }
+    for root, dirs, files in os.walk(path):
+        nodes = list(chain(dirs, files))
+        if ignore:
+            ignored_names = ignore(root, nodes)
+            nodes = [node for node in nodes if node not in ignored_names]
+        for node in nodes:
+            absolute_path = os.path.join(root, node)
+            relative_path = os.path.relpath(absolute_path, path)
+            for search_string, replacement in renamings:
+                relative_path = relative_path.replace(
+                    search_string,
+                    replacement
+                )
+            permissions = os.stat(absolute_path).st_mode
+            permissions_dict[relative_path] = permissions
+    return permissions_dict
+
+
 class StartprojectTemplatesTest(ProjectTest):
 
     def setUp(self):
@@ -139,6 +166,149 @@ def test_startproject_template_override(self):
         self.assertIn(self.tmpl_proj, out)
         assert exists(join(self.proj_path, 'root_template'))
 
+    def test_startproject_permissions_from_writable(self):
+        """Check that generated files have the right permissions when the
+        template folder has the same permissions as in the project, i.e.
+        everything is writable."""
+        scrapy_path = scrapy.__path__[0]
+        project_template = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject1'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            project_template,
+            renamings,
+            IGNORE,
+        )
+
+        destination = mkdtemp()
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+            ),
+            cwd=destination,
+            env=self.env,
+        )
+        process.wait()
+
+        project_dir = os.path.join(destination, project_name)
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
+    def test_startproject_permissions_from_read_only(self):
+        """Check that generated files have the right permissions when the
+        template folder has been made read-only, which is something that some
+        systems do.
+
+        See https://github.com/scrapy/scrapy/pull/4604
+        """
+        scrapy_path = scrapy.__path__[0]
+        templates_dir = os.path.join(scrapy_path, 'templates')
+        project_template = os.path.join(templates_dir, 'project')
+        project_name = 'startproject2'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            project_template,
+            renamings,
+            IGNORE,
+        )
+
+        def _make_read_only(path):
+            current_permissions = os.stat(path).st_mode
+            os.chmod(path, current_permissions & ~ANYONE_WRITE_PERMISSION)
+
+        read_only_templates_dir = str(Path(mkdtemp()) / 'templates')
+        copytree(templates_dir, read_only_templates_dir)
+
+        for root, dirs, files in os.walk(read_only_templates_dir):
+            for node in chain(dirs, files):
+                _make_read_only(os.path.join(root, node))
+
+        destination = mkdtemp()
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+                '--set',
+                'TEMPLATES_DIR={}'.format(read_only_templates_dir),
+            ),
+            cwd=destination,
+            env=self.env,
+        )
+        process.wait()
+
+        project_dir = os.path.join(destination, project_name)
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
+    def test_startproject_permissions_unchanged_in_destination(self):
+        """Check that pre-existing folders and files in the destination folder
+        do not see their permissions modified."""
+        scrapy_path = scrapy.__path__[0]
+        project_template = os.path.join(scrapy_path, 'templates', 'project')
+        project_name = 'startproject3'
+        renamings = (
+            ('module', project_name),
+            ('.tmpl', ''),
+        )
+        expected_permissions = get_permissions_dict(
+            project_template,
+            renamings,
+            IGNORE,
+        )
+
+        destination = mkdtemp()
+        project_dir = os.path.join(destination, project_name)
+
+        existing_nodes = {
+            oct(permissions)[2:] + extension: permissions
+            for extension in ('', '.d')
+            for permissions in (
+                0o444, 0o555, 0o644, 0o666, 0o755, 0o777,
+            )
+        }
+        os.mkdir(project_dir)
+        project_dir_path = Path(project_dir)
+        for node, permissions in existing_nodes.items():
+            path = project_dir_path / node
+            if node.endswith('.d'):
+                path.mkdir(mode=permissions)
+            else:
+                path.touch(mode=permissions)
+            expected_permissions[node] = path.stat().st_mode
+
+        process = subprocess.Popen(
+            (
+                sys.executable,
+                '-m',
+                'scrapy.cmdline',
+                'startproject',
+                project_name,
+                '.',
+            ),
+            cwd=project_dir,
+            env=self.env,
+        )
+        process.wait()
+
+        actual_permissions = get_permissions_dict(project_dir)
+
+        self.assertEqual(actual_permissions, expected_permissions)
+
 
 class CommandTest(ProjectTest):