Repository: apache/airflow

Issue:
Statsd (tbc. if more extras) tries to install provider package where it is missing
**Apache Airflow version**: 2.0.0b2

**What happened**:

Starting in 2.0.0b2, extras that aren't also providers like `statsd` prevents Airflow from being installed.

**How to reproduce it**:
```
$ docker run --rm -ti python:3.6 bash
# pip install apache-airflow[statsd]==2.0.0b2
Collecting apache-airflow[statsd]==2.0.0b2                 
  Downloading apache_airflow-2.0.0b2-py3-none-any.whl (4.5 MB)                    
...
Collecting statsd<4.0,>=3.3.0; extra == "statsd"          
  Downloading statsd-3.3.0-py2.py3-none-any.whl (11 kB)
ERROR: Could not find a version that satisfies the requirement apache-airflow-providers-statsd; extra == "statsd" (from apache-airflow[statsd]==2.0.0b2) (from versions: none)         
ERROR: No matching distribution found for apache-airflow-providers-statsd; extra == "statsd" (from apache-airflow[statsd]==2.0.0b2)
```

I believe this is from https://github.com/apache/airflow/pull/12233


Discussion:
I was quite sure I tested it :( 
I wiill double-check and fix it 
Just to clarify - Indeed I tested it. This is just a problem with "statsd" which I mistakenly marked as "provider" extra. @jedcunningham  - other extras like "kerberos" and "ldap" should work fine (as they are not mistakenly marked as having providers).  Could you please double check if those work ?
Yes, those do work (sorry, I generalized without checking others).
Cool ! :) No problem. I wil add automated check for that before I fix it :). And see if there are more cases. We have ~70 providers so maybe others have similar problem too :)

Resolving patch:
diff --git a/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py b/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py
new file mode 100755
--- /dev/null
+++ b/scripts/ci/pre_commit/pre_commit_check_extras_have_providers.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Check if the extras have providers defined.
+import os
+import sys
+from os.path import dirname
+from textwrap import wrap
+from typing import List
+
+AIRFLOW_SOURCES_DIR = os.path.abspath(os.path.join(dirname(__file__), os.pardir, os.pardir, os.pardir))
+
+sys.path.insert(0, AIRFLOW_SOURCES_DIR)
+# flake8: noqa: F401
+# pylint: disable=wrong-import-position
+from setup import EXTRAS_PROVIDERS_PACKAGES  # noqa
+
+sys.path.append(AIRFLOW_SOURCES_DIR)
+
+
+def get_provider_directory(provider: str):
+    return os.path.join(AIRFLOW_SOURCES_DIR, "airflow", "providers", *provider.split('.'))
+
+
+def check_all_providers() -> List[str]:
+    errors: List[str] = []
+    for extra, providers in EXTRAS_PROVIDERS_PACKAGES.items():
+        for provider in providers:
+            provider_directory = get_provider_directory(provider)
+            if not os.path.isdir(provider_directory):
+                errors.append(
+                    f"The {extra} has provider {provider} that has missing {provider_directory} directory"
+                )
+                continue
+            if not os.path.exists(os.path.join(provider_directory, "__init__.py")):
+                errors.append(
+                    f"The {extra} has provider {provider} that has"
+                    f" missing __init__.py in the {provider_directory} directory"
+                )
+            if not os.path.exists(os.path.join(provider_directory, "README.md")):
+                errors.append(
+                    f"The {extra} has provider {provider} that has"
+                    f" missing README.md in the {provider_directory} directory"
+                )
+    return errors
+
+
+if __name__ == '__main__':
+    errors = check_all_providers()
+    if errors:
+        for message in errors:
+            print(message, file=sys.stderr)
+        sys.exit(1)
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -569,7 +569,6 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     "apache.hive": hive,
     "apache.kylin": kylin,
     "apache.pinot": pinot,
-    "apache.presto": presto,
     "apache.spark": spark,
     "apache.webhdfs": webhdfs,
     'async': async_packages,
@@ -673,7 +672,6 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     "apache.hive": ["apache.hive"],
     "apache.kylin": ["apache.kylin"],
     "apache.pinot": ["apache.pinot"],
-    "apache.presto": ["apache.presto"],
     "apache.spark": ["apache.spark"],
     "apache.webhdfs": ["apache.hdfs"],
     'async': [],
@@ -685,7 +683,7 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'cgroups': [],
     'cloudant': ["cloudant"],
     'cncf.kubernetes': ["cncf.kubernetes"],
-    'dask': ["dask"],
+    'dask': [],
     'databricks': ["databricks"],
     'datadog': ["datadog"],
     'devel': ["cncf.kubernetes", "mysql"],
@@ -727,25 +725,26 @@ def write_version(filename: str = os.path.join(*[my_dir, "airflow", "git_version
     'presto': ["presto"],
     'qds': ["qubole"],  # TODO: remove this in Airflow 2.1
     'qubole': ["qubole"],
-    'rabbitmq': ["rabbitmq"],
+    'rabbitmq': [],
     'redis': ["redis"],
     'salesforce': ["salesforce"],
     'samba': ["samba"],
     'segment': ["segment"],
     'sendgrid': ["sendgrid"],
-    'sentry': ["sentry"],
+    'sentry': [],
     'singularity': ["singularity"],
     'slack': ["slack"],
     'snowflake': ["snowflake"],
-    'spark': ["spark"],
+    'spark': ["apache.spark"],
     'ssh': ["ssh"],
-    'statsd': ["statsd"],
-    'tableau': ["tableau"],
+    'statsd': [],
+    'tableau': [],
     'vertica': ["vertica"],
-    'virtualenv': ["virtualenv"],
+    'virtualenv': [],
     'webhdfs': ["apache.hdfs"],  # TODO: remove this in Airflow 2.1
     'winrm': ["microsoft.winrm"],  # TODO: remove this in Airflow 2.1
-    'yandexcloud': ["yandexcloud"],
+    'yandexcloud': ["yandex"],  # TODO: remove this in Airflow 2.1
+    'yandex': ["yandex"],
 }
 
 


Test patch: