Repository: GoogleCloudPlatform/django-cloud-deploy

Issue:
Update GCS bucket Cross-Origin Resource Sharing Policy after creation
Webbrowser deny to access font files in GCS bucket. Error message:
```
Access to font at 'https://storage.googleapis.com/django-447998/static/webfonts/fa-solid-900.woff2' from origin 'https://django-447998.appspot.com' has been blocked by CORS policy: No 'Access-Control-Allow-Origin' header is present on the requested resource.
```

How to set bucket CORS policy:
https://cloud.google.com/storage/docs/configuring-cors

Api information:
https://cloud.google.com/storage/docs/json_api/v1/buckets#resource



Discussion:

Patch:
diff --git a/django_cloud_deploy/.yapfignore b/django_cloud_deploy/.yapfignore
index fe867f6..db9c862 100644
--- a/django_cloud_deploy/.yapfignore
+++ b/django_cloud_deploy/.yapfignore
@@ -3,3 +3,6 @@
 
 # Django project source files used in test
 **/data
+
+# Automatically generated nox files
+.nox
\ No newline at end of file
diff --git a/django_cloud_deploy/cli/cloudify.py b/django_cloud_deploy/cli/cloudify.py
index 7ea83b1..e82657a 100644
--- a/django_cloud_deploy/cli/cloudify.py
+++ b/django_cloud_deploy/cli/cloudify.py
@@ -37,10 +37,21 @@ def add_arguments(parser):
 
     parser.add_argument(
         '--project-path',
-        dest='django_directory_path',
+        dest='django_directory_path_cloudify',
         help='The location where the generated Django project code should be '
         'stored.')
 
+    parser.add_argument(
+        '--django_requirements_path',
+        dest='django_requirements_path',
+        help='The location of the Django project requirements.txt.')
+
+    parser.add_argument(
+        '--billing_account_name',
+        dest='billing_account_name',
+        help='Name of the GCP Billing account name to be associated with the '
+        'project.')
+
     parser.add_argument(
         '--database-password',
         dest='database_password',
diff --git a/django_cloud_deploy/cli/new.py b/django_cloud_deploy/cli/new.py
index c9ff18a..6db6f70 100644
--- a/django_cloud_deploy/cli/new.py
+++ b/django_cloud_deploy/cli/new.py
@@ -40,6 +40,12 @@ def add_arguments(parser):
         help='The location where the generated Django project code should be '
         'stored.')
 
+    parser.add_argument(
+        '--billing_account_name',
+        dest='billing_account_name',
+        help='Name of the GCP Billing account name to be associated with the '
+        'project.')
+
     parser.add_argument(
         '--database-password',
         dest='database_password',
diff --git a/django_cloud_deploy/cloudlib/static_content_serve.py b/django_cloud_deploy/cloudlib/static_content_serve.py
index 8a31b2f..de47183 100644
--- a/django_cloud_deploy/cloudlib/static_content_serve.py
+++ b/django_cloud_deploy/cloudlib/static_content_serve.py
@@ -312,3 +312,28 @@ class StaticContentServeClient(object):
                 'Not able to collect static files.') from e
         finally:
             os.chdir(cwd)
+
+    def set_cors_policy(self, bucket_name: str, origin: str):
+        """Make the given bucket able to serve fonts to the given origins.
+
+        Args:
+            bucket_name: Name of a GCS bucket for static content serving.
+            origin: Url of the website which need fonts in the bucket.
+        """
+        request = self._storage_service.buckets().get(bucket=bucket_name)
+        bucket_body = request.execute(num_retries=5)
+        cors_policy = [{
+            'origin': [origin],
+            'method': ['GET'],
+            'responseHeader': ['Content-Type'],
+            'maxAgeSeconds': 3600,
+        }]
+        bucket_body['cors'] = cors_policy
+        try:
+            request = self._storage_service.buckets().patch(
+                bucket=bucket_name, body=bucket_body)
+            request.execute(num_retries=5)
+        except errors.HttpError as e:
+            raise StaticContentServeError(
+                'Fail to change CORS policy of bucket {}.'.format(
+                    bucket_name)) from e
diff --git a/django_cloud_deploy/skeleton/templates/admin_template/logger.py-tpl b/django_cloud_deploy/skeleton/templates/admin_template/logger.py-tpl
index 8fecc3c..e587a81 100644
--- a/django_cloud_deploy/skeleton/templates/admin_template/logger.py-tpl
+++ b/django_cloud_deploy/skeleton/templates/admin_template/logger.py-tpl
@@ -12,7 +12,7 @@ def generate_logging_client():
     try:  # Kubernetes Engine
         credentials = (
             service_account.Credentials.from_service_account_file(
-                '/secrets/django_app/logging-credentials.json',
+                '/secrets/django_app/django-app-credentials.json',
                 scopes=['https://www.googleapis.com/auth/logging.read']))
 
         return logging.Client(credentials=credentials)
diff --git a/django_cloud_deploy/skeleton/templates/app.yaml b/django_cloud_deploy/skeleton/templates/app.yaml
index 839ea38..7d8785b 100644
--- a/django_cloud_deploy/skeleton/templates/app.yaml
+++ b/django_cloud_deploy/skeleton/templates/app.yaml
@@ -1,4 +1,5 @@
 # [START django_app]
+instance_class: F2
 service: {{ service_name }}
 runtime: python37
 entrypoint: gunicorn -b :$PORT --access-logfile - --error-logfile - {{ project_name }}.wsgi
@@ -6,13 +7,7 @@ entrypoint: gunicorn -b :$PORT --access-logfile - --error-logfile - {{ project_n
 env_variables:
   DATABASE_USER: "postgres"
 
-
 handlers:
-# This configures Google App Engine to serve the files in the app's static
-# directory.
-- url: /static
-  static_dir: static/
-
 # This handler routes all requests not caught above to your main app. It is
 # required when static routes are defined, but can be omitted (along with
 # the entire handlers section) when there are no static files defined.
diff --git a/django_cloud_deploy/skeleton/templates/settings_template/cloud_settings.py-tpl b/django_cloud_deploy/skeleton/templates/settings_template/cloud_settings.py-tpl
index e5b370b..1c4a88f 100644
--- a/django_cloud_deploy/skeleton/templates/settings_template/cloud_settings.py-tpl
+++ b/django_cloud_deploy/skeleton/templates/settings_template/cloud_settings.py-tpl
@@ -1,5 +1,6 @@
 import json
 from google.cloud import storage
+from google.oauth2 import service_account
 
 from .google_settings import *
 
@@ -10,6 +11,23 @@ def get_database_password():
     creds = json.loads(blob.download_as_string())
     return creds['password']
 
+def get_gs_credentials():
+    if os.getenv('GAE_APPLICATION', None):
+        client = storage.Client()
+        bucket = client.get_bucket('secrets-{{ project_id }}')
+        blob = bucket.get_blob('secrets/django-app-credentials.json')
+        key_data = json.loads(blob.download_as_string().decode('utf-8'))
+        info = json.loads(key_data['django-app-credentials.json'])
+        return service_account.Credentials.from_service_account_info(info)
+
+    try:  # Kubernetes Engine
+        return service_account.Credentials.from_service_account_file(
+            '/secrets/django_app/django-app-credentials.json')
+    except:
+        # When generating the files we need to run the set up code, but
+        # this should not be reached when running on the platform
+        return None
+
 
 # SECURITY WARNING: If you deploy a Django app to production, make sure to set
 # an appropriate host here.
@@ -29,26 +47,27 @@ if os.getenv('GAE_APPLICATION', None):
     DATABASES = {
         'default': {
             'ENGINE': 'django.db.backends.postgresql',
-	        'NAME': '{{ database_name }}',
+            'NAME': '{{ database_name }}',
             'USER': os.environ['DATABASE_USER'],
             'PASSWORD': get_database_password(),
             'HOST': '/cloudsql/{{ cloud_sql_connection }}',
         }
     }
 else:
-	DATABASES = {
-	    'default': {
-	        'ENGINE': 'django.db.backends.postgresql',
-	        'NAME': '{{ database_name }}',
-	        'USER': os.getenv('DATABASE_USER'),
-	        'PASSWORD': os.getenv('DATABASE_PASSWORD'),
-	        'HOST': '127.0.0.1',
-	        'PORT': os.environ.get('CLOUD_SQL_PROXY_PORT') or '5432',
-	    }
-	}
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.postgresql',
+            'NAME': '{{ database_name }}',
+            'USER': os.getenv('DATABASE_USER'),
+            'PASSWORD': os.getenv('DATABASE_PASSWORD'),
+            'HOST': '127.0.0.1',
+            'PORT': os.environ.get('CLOUD_SQL_PROXY_PORT') or '5432',
+        }
+    }
 
 STATIC_URL = 'https://storage.googleapis.com/{{ bucket_name }}/static/'
 
 DEFAULT_FILE_STORAGE = 'storages.backends.gcloud.GoogleCloudStorage'
 GS_BUCKET_NAME = '{{ file_bucket_name }}'
-GS_FILE_OVERWRITE = False
\ No newline at end of file
+GS_FILE_OVERWRITE = False
+GS_CREDENTIALS = get_gs_credentials()
\ No newline at end of file
diff --git a/django_cloud_deploy/skeleton/utils.py b/django_cloud_deploy/skeleton/utils.py
index 022f238..cc18fd0 100644
--- a/django_cloud_deploy/skeleton/utils.py
+++ b/django_cloud_deploy/skeleton/utils.py
@@ -54,6 +54,20 @@ def parse_settings_module(file_path: str) -> Optional[str]:
         # Find strings between "" or ''
         raw_settings_module = re.findall(r'[\"\'][\w+\.]+[\"\']',
                                          settings_module_line.group(0))
+
+        # raw_settings_module should be like
+        # ['"DJANGO_SETTINGS_MODULE"', '"mysite.settings"']
+        # If it is not like this, then we are not able to parse the settings
+        # module.
+        if len(raw_settings_module) < 2:
+            return None
+
+        commas = ['\'', '\"']
+        for module in raw_settings_module:
+            # The settings module is not between " or '
+            if (len(module) < 2 or module[0] not in commas or
+                    module[-1] not in commas):
+                return None
         # Remove empty spaces and delete quotation marks at the start and end
         settings_module = raw_settings_module[-1].strip()[1:-1]
         return settings_module
diff --git a/django_cloud_deploy/workflow/__init__.py b/django_cloud_deploy/workflow/__init__.py
index b71d782..a394cc4 100644
--- a/django_cloud_deploy/workflow/__init__.py
+++ b/django_cloud_deploy/workflow/__init__.py
@@ -195,50 +195,50 @@ class WorkflowManager(object):
             self._billing_client.enable_project_billing(project_id,
                                                         billing_account_name)
 
-        self._console_io.tell('[3/{}]: Django Source Generation'.format(
-            self._TOTAL_NEW_STEPS))
-
-        # Source generation requires service account ids.
-        required_service_accounts = (
-            required_service_accounts or
-            self._service_account_workflow.load_service_accounts())
-        cloud_sql_secrets, django_secrets = self._load_secret_names(
-            required_service_accounts)
-        if deploy_existing_django_project:
-            self._source_generator.generate_from_existing(
-                project_id=project_id,
-                project_name=django_project_name,
-                project_dir=django_directory_path,
-                database_user=database_username,
-                database_password=database_password,
-                django_requirements_path=django_requirements_path,
-                django_settings_path=django_settings_path,
-                instance_name=database_instance_name,
-                database_name=database_name,
-                cloud_sql_proxy_port=cloud_sql_proxy_port,
-                cloud_storage_bucket_name=cloud_storage_bucket_name,
-                file_storage_bucket_name=file_storage_bucket_name,
-                cloudsql_secrets=cloud_sql_secrets,
-                django_secrets=django_secrets,
-                service_name=appengine_service_name,
-                image_tag=image_name)
-        else:
-            self._source_generator.generate_new(
-                project_id=project_id,
-                project_name=django_project_name,
-                app_name=django_app_name,
-                project_dir=django_directory_path,
-                database_user=database_username,
-                database_password=database_password,
-                instance_name=database_instance_name,
-                database_name=database_name,
-                cloud_sql_proxy_port=cloud_sql_proxy_port,
-                cloud_storage_bucket_name=cloud_storage_bucket_name,
-                file_storage_bucket_name=file_storage_bucket_name,
-                cloudsql_secrets=cloud_sql_secrets,
-                django_secrets=django_secrets,
-                service_name=appengine_service_name,
-                image_tag=image_name)
+        with self._console_io.progressbar(
+                60, '[3/{}]: Django Source Generation'.format(
+                    self._TOTAL_NEW_STEPS)):
+            # Source generation requires service account ids.
+            required_service_accounts = (
+                required_service_accounts or
+                self._service_account_workflow.load_service_accounts())
+            cloud_sql_secrets, django_secrets = self._load_secret_names(
+                required_service_accounts)
+            if deploy_existing_django_project:
+                self._source_generator.generate_from_existing(
+                    project_id=project_id,
+                    project_name=django_project_name,
+                    project_dir=django_directory_path,
+                    database_user=database_username,
+                    database_password=database_password,
+                    django_requirements_path=django_requirements_path,
+                    django_settings_path=django_settings_path,
+                    instance_name=database_instance_name,
+                    database_name=database_name,
+                    cloud_sql_proxy_port=cloud_sql_proxy_port,
+                    cloud_storage_bucket_name=cloud_storage_bucket_name,
+                    file_storage_bucket_name=file_storage_bucket_name,
+                    cloudsql_secrets=cloud_sql_secrets,
+                    django_secrets=django_secrets,
+                    service_name=appengine_service_name,
+                    image_tag=image_name)
+            else:
+                self._source_generator.generate_new(
+                    project_id=project_id,
+                    project_name=django_project_name,
+                    app_name=django_app_name,
+                    project_dir=django_directory_path,
+                    database_user=database_username,
+                    database_password=database_password,
+                    instance_name=database_instance_name,
+                    database_name=database_name,
+                    cloud_sql_proxy_port=cloud_sql_proxy_port,
+                    cloud_storage_bucket_name=cloud_storage_bucket_name,
+                    file_storage_bucket_name=file_storage_bucket_name,
+                    cloudsql_secrets=cloud_sql_secrets,
+                    django_secrets=django_secrets,
+                    service_name=appengine_service_name,
+                    image_tag=image_name)
 
         with self._console_io.progressbar(
                 300, '[4/{}]: Database Set Up'.format(self._TOTAL_NEW_STEPS)):
@@ -299,7 +299,8 @@ class WorkflowManager(object):
                     300, '[9/{}]: Deployment'.format(self._TOTAL_NEW_STEPS)):
                 app_url = self.deploy_workflow.deploy_gae_app(
                     project_id, django_directory_path, is_new=is_new)
-
+        self._static_content_workflow.set_cors_policy(cloud_storage_bucket_name,
+                                                      app_url)
         # Create configuration file to save information needed in "update"
         # command.
 
@@ -522,8 +523,6 @@ class WorkflowManager(object):
     def _create_files_for_secrets(path: str, secrets: Dict[str, Any]):
         """Create secret files for GAE that will be uploaded to GCS buckets.
 
-        Currently, only needed for database password.
-
         Generates JSON files that will be used by the django on GAE.
 
         Args:
@@ -531,10 +530,8 @@ class WorkflowManager(object):
               secrets: Contains the information regarding the credentials.
         """
         os.makedirs(path)
-        secret_name = 'cloudsql'
-        content = secrets['cloudsql']
-        filename = '{}.json'.format(secret_name)
-        file_path = os.path.join(path, filename)
-        with open(file_path, 'w') as file:
-            if secret_name == 'cloudsql':
+        for secret, content in secrets.items():
+            filename = '{}.json'.format(secret)
+            file_path = os.path.join(path, filename)
+            with open(file_path, 'w') as file:
                 json.dump(content, file)
diff --git a/django_cloud_deploy/workflow/_deploygae.py b/django_cloud_deploy/workflow/_deploygae.py
index 0d3ade2..ece59e4 100644
--- a/django_cloud_deploy/workflow/_deploygae.py
+++ b/django_cloud_deploy/workflow/_deploygae.py
@@ -17,9 +17,11 @@ import os
 import shutil
 import subprocess
 import time
-import yaml
+from typing import Any, Dict
 
+import backoff
 from googleapiclient import discovery
+import yaml
 
 from google.auth import credentials
 
@@ -58,6 +60,31 @@ class DeploygaeWorkflow(object):
                 break
             time.sleep(2)
 
+    @staticmethod
+    @backoff.on_predicate(
+        backoff.expo, lambda x: x.returncode != 0, max_tries=5)
+    def _app_deploy_with_retry(
+            gcloud_path: str, project: str, app_yaml_path: str,
+            env_vars: Dict[str, Any]) -> subprocess.CompletedProcess:
+        """Run 'gcloud app deploy' with retries.
+
+        Args:
+            gcloud_path: The path of your gcloud cli tool.
+            project: GCP project id.
+            app_yaml_path: Absolute path of your app.yaml.
+            env_vars: A dictionary of the environment variables.
+
+        Returns:
+            The result of the subprocess run.
+        """
+        gcloud_result = subprocess.run(
+            [gcloud_path, '-q', project, 'app', 'deploy', app_yaml_path],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+            universal_newlines=True,
+            env=env_vars)
+        return gcloud_result
+
     def deploy_gae_app(self,
                        project_id: str,
                        django_directory_path: str,
@@ -91,12 +118,8 @@ class DeploygaeWorkflow(object):
         # We need to grab all environment variables to pass to the subprocess
         env_vars = dict(os.environ)
         env_vars['CLOUDSDK_METRICS_ENVIRONMENT'] = 'django-cloud-deploy'
-        gcloud_result = subprocess.run(
-            [gcloud_path, '-q', project, 'app', 'deploy', app_yaml_path],
-            stdout=subprocess.DEVNULL,
-            stderr=subprocess.PIPE,
-            universal_newlines=True,
-            env=env_vars)
+        gcloud_result = self._app_deploy_with_retry(gcloud_path, project,
+                                                    app_yaml_path, env_vars)
         if gcloud_result.returncode != 0:
             raise DeployNewAppError(gcloud_result.stderr)
 
diff --git a/django_cloud_deploy/workflow/_static_content_serve.py b/django_cloud_deploy/workflow/_static_content_serve.py
index 0811b7c..17cbf54 100644
--- a/django_cloud_deploy/workflow/_static_content_serve.py
+++ b/django_cloud_deploy/workflow/_static_content_serve.py
@@ -45,6 +45,9 @@ class StaticContentServeWorkflow(object):
         self._static_content_serve_client.upload_content(
             bucket_name, static_content_dir)
 
+    def set_cors_policy(self, bucket_name: str, origin: str):
+        self._static_content_serve_client.set_cors_policy(bucket_name, origin)
+
     def serve_secret_content(self, project_id: str, bucket_name: str,
                              secrec_content_dir: str):
         """Do all the work for serving secret content of the provided project.
diff --git a/django_cloud_deploy/workflow/data/service_accounts.json b/django_cloud_deploy/workflow/data/service_accounts.json
index a36cae3..6d4c908 100644
--- a/django_cloud_deploy/workflow/data/service_accounts.json
+++ b/django_cloud_deploy/workflow/data/service_accounts.json
@@ -12,12 +12,14 @@
         }
     ],
     "django": [
+
         {
-            "id": "logging-credentials",
-            "name": "Logging Credentials",
-            "file_name": "logging-credentials.json",
+            "id": "django-app-credentials",
+            "name": "Django App Credentials",
+            "file_name": "django-app-credentials.json",
             "roles": [
-                "roles/logging.viewer"
+                "roles/logging.viewer",
+                "roles/storage.admin"
             ]
         }
     ]


Test patch:
diff --git a/django_cloud_deploy/tests/integration/cloudlib_test.py b/django_cloud_deploy/tests/integration/cloudlib_test.py
index d8d1d76..1754f36 100644
--- a/django_cloud_deploy/tests/integration/cloudlib_test.py
+++ b/django_cloud_deploy/tests/integration/cloudlib_test.py
@@ -20,6 +20,8 @@ from django_cloud_deploy.cloudlib import static_content_serve
 from django_cloud_deploy.tests.lib import test_base
 from django_cloud_deploy.tests.lib import utils
 
+from googleapiclient import discovery
+
 
 class StaticContentServeClientIntegrationTest(test_base.DjangoFileGeneratorTest,
                                               test_base.ResourceCleanUp):
@@ -38,6 +40,24 @@ class StaticContentServeClientIntegrationTest(test_base.DjangoFileGeneratorTest,
                 self._static_content_serve_client.create_bucket(
                     self.project_id, bucket_name)
 
+    def test_set_cors_policy(self):
+        bucket_name = utils.get_resource_name('bucket')
+        with self.clean_up_bucket(bucket_name):
+            self._static_content_serve_client.create_bucket(
+                self.project_id, bucket_name)
+            url = 'http://www.example.com'
+            self._static_content_serve_client.set_cors_policy(bucket_name, url)
+            client = discovery.build(
+                'storage',
+                'v1',
+                credentials=self.credentials,
+                cache_discovery=False)
+            request = client.buckets().get(bucket=bucket_name)
+            bucket_body = request.execute(num_retries=5)
+            cors_policy = bucket_body.get('cors')
+            self.assertNotEmpty(cors_policy)
+            self.assertIn(url, cors_policy[0].get('origin'))
+
 
 class ServiceAccountClientIntegrationTest(test_base.ResourceCleanUp):
     """Integration test for cloudlib.service_account."""
diff --git a/django_cloud_deploy/tests/unit/skeleton/utils_test.py b/django_cloud_deploy/tests/unit/skeleton/utils_test.py
index 2abe004..e8ebe3d 100644
--- a/django_cloud_deploy/tests/unit/skeleton/utils_test.py
+++ b/django_cloud_deploy/tests/unit/skeleton/utils_test.py
@@ -54,6 +54,21 @@ class GetDjangoProjectNameTest(unittest.TestCase):
             utils.get_django_project_name(project_dir)
         shutil.rmtree(project_dir)
 
+    def test_manage_py_uses_variable_for_settings_module(self):
+        """Test case for manage.py uses a variable for settings module."""
+        project_dir = tempfile.mkdtemp()
+        management.call_command('startproject', 'mysite', project_dir)
+        manage_py_path = os.path.join(project_dir, 'manage.py')
+        with open(manage_py_path) as f:
+            file_content = f.read()
+            file_content = file_content.replace('\'mysite.settings\'',
+                                                'module_variable')
+        with open(manage_py_path, 'wt') as f:
+            f.write(file_content)
+        with self.assertRaises(utils.ProjectContentError):
+            utils.get_django_project_name(project_dir)
+        shutil.rmtree(project_dir)
+
     def test_get_project_name_invalid_manage_py(self):
         # Create a temporary directory to put Django project files
         project_dir = tempfile.mkdtemp()