Repository: apache/airflow

Issue:
Integration list synchronization with airflow-site
Hello friends.

Today I am coming to you with another ticket. This time, while working on the website, I discovered that we have some information that is slightly outdated.

[`landing-pages/site/static/integrations.json`](https://github.com/apache/airflow-site/blob/master/landing-pages/site/static/integrations.json
) file have lists all the integrations that have been added to Airflow, but unfortunately it is updated manually.  Recently, new services were added there on Oct 2019

We can think of a tool that will update this list more automatically by reading the integration list from [provider.yaml](https://github.com/apache/airflow/find/master) files, and then creating an integration.json file that can be published to the airflow-site.

For now, we need a manual run script, but in the future, we can also run it on Github Action or during updates and documentation.

CC: @ryw 


Discussion:
I can work on this


Resolving patch:
diff --git a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
--- a/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
+++ b/scripts/ci/pre_commit/pre_commit_check_provider_yaml_files.py
@@ -271,6 +271,7 @@ def check_invalid_integration(yaml_files: Dict[str, Dict]):
 def check_doc_files(yaml_files: Dict[str, Dict]):
     print("Checking doc files")
     current_doc_urls = []
+    current_logo_urls = []
     for provider in yaml_files.values():
         if 'integrations' in provider:
             current_doc_urls.extend(
@@ -279,6 +280,9 @@ def check_doc_files(yaml_files: Dict[str, Dict]):
                 if 'how-to-guide' in guides
                 for guide in guides['how-to-guide']
             )
+            current_logo_urls.extend(
+                integration['logo'] for integration in provider['integrations'] if 'logo' in integration
+            )
         if 'transfers' in provider:
             current_doc_urls.extend(
                 op['how-to-guide'] for op in provider['transfers'] if 'how-to-guide' in op
@@ -293,9 +297,15 @@ def check_doc_files(yaml_files: Dict[str, Dict]):
         "/docs/" + os.path.relpath(f, start=DOCS_DIR)
         for f in glob(f"{DOCS_DIR}/apache-airflow-providers-*/operators.rst", recursive=True)
     }
+    expected_logo_urls = {
+        "/" + os.path.relpath(f, start=DOCS_DIR)
+        for f in glob(f"{DOCS_DIR}/integration-logos/**/*", recursive=True)
+        if os.path.isfile(f)
+    }
 
     try:
         assert_sets_equal(set(expected_doc_urls), set(current_doc_urls))
+        assert_sets_equal(set(expected_logo_urls), set(current_logo_urls))
     except AssertionError as ex:
         print(ex)
         sys.exit(1)
diff --git a/scripts/tools/generate-integrations-json.py b/scripts/tools/generate-integrations-json.py
new file mode 100755
--- /dev/null
+++ b/scripts/tools/generate-integrations-json.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import json
+import os
+import re
+import shutil
+
+# pylint: disable=no-name-in-module
+from docs.exts.provider_yaml_utils import load_package_data
+
+# pylint: enable=no-name-in-module
+
+AIRFLOW_SITE_DIR = os.environ.get('AIRFLOW_SITE_DIRECTORY')
+ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
+DOCS_DIR = os.path.join(ROOT_DIR, 'docs')
+
+if __name__ != "__main__":
+    raise SystemExit(
+        "This file is intended to be executed as an executable program. You cannot use it as a module."
+        "To run this script, run the ./generate-integrations-json.py command"
+    )
+
+if not (
+    AIRFLOW_SITE_DIR
+    and os.path.isdir(AIRFLOW_SITE_DIR)
+    and os.path.isdir(os.path.join(AIRFLOW_SITE_DIR, 'docs-archive'))
+):
+    raise SystemExit(
+        'Before using this script, set the environment variable AIRFLOW_SITE_DIRECTORY. This variable '
+        'should contain the path to the airflow-site repository directory. '
+        '${AIRFLOW_SITE_DIRECTORY}/docs-archive must exists.'
+    )
+
+ALL_PROVIDER_YAMLS = load_package_data()
+
+result_integrations = []
+for provider_info in ALL_PROVIDER_YAMLS:
+    for integration in provider_info.get('integrations', []):
+        doc_url = integration.get("how-to-guide")
+        if doc_url:
+            doc_url = doc_url[0].strip()
+            doc_url = re.sub(f'/{provider_info["package-name"]}/', r"\g<0>stable/", doc_url)
+            doc_url = re.sub(r'\.rst', '.html', doc_url)
+        else:
+            doc_url = f"/docs/{provider_info['package-name'].lower()}/stable/index.html"
+        logo = integration.get("logo")
+
+        result = {
+            'name': integration['integration-name'],
+            'url': doc_url,
+        }
+        if logo:
+            result['logo'] = logo
+        result_integrations.append(result)
+
+result_integrations = sorted(result_integrations, key=lambda x: x['name'])
+with open(os.path.join(AIRFLOW_SITE_DIR, 'landing-pages/site/static/integrations.json'), 'w') as f:
+    f.write(
+        json.dumps(
+            result_integrations,
+            indent=4,
+        )
+    )
+
+shutil.copytree(
+    src=os.path.join(DOCS_DIR, 'integration-logos'),
+    dst=os.path.join(AIRFLOW_SITE_DIR, 'landing-pages/site/static/integration-logos'),
+    dirs_exist_ok=True,
+)


Test patch: