Moves provider packages scripts to dev (#12082)

The change #10806 made airflow works with implicit packages when "airflow" got imported. This is a good change, however it has some unforeseen consequences. The 'provider_packages' script copy all the providers code for backports in order to refactor them to the empty "airflow" directory in provider_packages folder. The #10806 change turned that empty folder in 'airflow' package because it was in the same directory as the provider_packages scripts. Moving the scripts to dev solves this problem.
apache · Nov 9, 2020 · b2a28d1 · b2a28d1
1 parent eaac361
commit b2a28d1
Show file tree

Hide file tree

Showing 159 changed files with 533 additions and 9,319 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -378,10 +378,10 @@ jobs:
  runs-on: ubuntu-latest
  needs: [build-info, ci-images]
  env:
- INSTALL_AIRFLOW_VERSION: "2.0.0-dev" # Note that this causes local installation
+ INSTALL_AIRFLOW_VERSION: "wheel"
  PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
- VERSION_SUFFIX_FOR_PYPI: "a0"
- VERSION_SUFFIX_FOR_SVN: "a0"
+ VERSION_SUFFIX_FOR_PYPI: "a2"
+ VERSION_SUFFIX_FOR_SVN: "a2"
  if: needs.build-info.outputs.image-build == 'true'
  steps:
  - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"

diff --git a/.gitignore b/.gitignore
@@ -186,7 +186,10 @@ dmypy.json
 /.kube
 /.inputrc
 log.txt*
+
+# Provider-related ignores
 /provider_packages/CHANGELOG.txt
+/airflow/providers/__init__.py
 
 # Local .terraform directories
 **/.terraform/*

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -199,7 +199,7 @@ repos:
  hooks:
  - id: isort
  name: Run isort to sort imports
- types: [python]
+ files: \.py$
  # To keep consistent with the global isort skip config defined in setup.cfg
  exclude: ^build/.*$|^.tox/.*$|^venv/.*$
  - repo: https://github.com/pycqa/pydocstyle
@@ -335,7 +335,7 @@ repos:
  ^airflow/operators/.*$|
  ^airflow/sensors/.*$|
  ^airflow/providers/.*$|
- ^provider_packages/.*$
+ ^dev/provider_packages/.*$
  - id: base-operator
  language: pygrep
  name: Make sure BaseOperator[Link] is imported from airflow.models outside of core
@@ -438,6 +438,17 @@ repos:
  files: ^airflow/www/.*\.(css|scss|sass)$
  # Keep dependency versions in sync w/ airflow/www/package.json
  additional_dependencies: ['[email protected]', '[email protected]']
+ - id: version-sync
+ name: Version sync
+ files: ^airflow/version.py$|setup.py
+ entry: ./scripts/ci/pre_commit/pre_commit_sync_version.sh
+ language: system
+ - id: providers-init-file
+ name: Provider init file
+ pass_filenames: false
+ always_run: true
+ entry: ./scripts/ci/pre_commit/pre_commit_check_providers_init.sh
+ language: system
  - id: mermaid
  name: Generate mermaid images
  entry: ./scripts/ci/pre_commit/pre_commit_mermaid.sh

diff --git a/BREEZE.rst b/BREEZE.rst
@@ -1207,6 +1207,9 @@ This is the current syntax for `./breeze <./breeze>`_:
  image building time in production image and at container entering time for CI image. One of:
 
  1.10.12 1.10.11 1.10.10 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2
+ wheel
+
+ When 'wheel' is used, Airflow is installed from /dist/apache_airflow-*.whl file.
 
  -t, --install-airflow-reference INSTALL_AIRFLOW_REFERENCE
  If specified, installs Airflow directly from reference in GitHub. This happens at
@@ -2004,11 +2007,11 @@ This is the current syntax for `./breeze <./breeze>`_:
  dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 forbid-tabs
  helm-lint incorrect-use-of-LoggingMixin insert-license isort language-matters
  lint-dockerfile lint-openapi mermaid mixed-line-ending mypy mypy-helm
- no-relative-imports pre-commit-descriptions provide-create-sessions pydevd
- pydocstyle pylint pylint-tests python-no-log-warn pyupgrade restrict-start_date
- rst-backticks setup-order setup-installation shellcheck sort-in-the-wild stylelint
- trailing-whitespace update-breeze-file update-extras update-local-yml-file
- update-setup-cfg-file yamllint
+ no-relative-imports pre-commit-descriptions provide-create-sessions
+ providers-init-file pydevd pydocstyle pylint pylint-tests python-no-log-warn
+ pyupgrade restrict-start_date rst-backticks setup-order setup-installation
+ shellcheck sort-in-the-wild stylelint trailing-whitespace update-breeze-file
+ update-extras update-local-yml-file update-setup-cfg-file version-sync yamllint
 
  You can pass extra arguments including options to to the pre-commit framework as
  <EXTRA_ARGS> passed after --. For example:
@@ -2231,6 +2234,9 @@ This is the current syntax for `./breeze <./breeze>`_:
  image building time in production image and at container entering time for CI image. One of:
 
  1.10.12 1.10.11 1.10.10 1.10.9 1.10.8 1.10.7 1.10.6 1.10.5 1.10.4 1.10.3 1.10.2
+ wheel
+
+ When 'wheel' is used, Airflow is installed from /dist/apache_airflow-*.whl file.
 
  -t, --install-airflow-reference INSTALL_AIRFLOW_REFERENCE
  If specified, installs Airflow directly from reference in GitHub. This happens at

diff --git a/CI.rst b/CI.rst
@@ -203,7 +203,8 @@ You can use those variables when you try to reproduce the build locally.
 | Image variables |
 +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
 | ``INSTALL_AIRFLOW_VERSION`` | | | | Installs Airflow version from PyPI when |
-| | | | | building image. |
+| | | | | building image. Can be "wheel" to install from |
+| | | | | the wheel package instead. |
 +-----------------------------------------+-------------+-------------+------------+-------------------------------------------------+
 | ``INSTALL_AIRFLOW_REFERENCE`` | | | | Installs Airflow version from GitHub |
 | | | | | branch or tag. |

diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
@@ -126,6 +126,8 @@ require Breeze Docker images to be installed locally:
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``provide-create-sessions`` Make sure provide-session and create-session imports are OK.
 ----------------------------------- ---------------------------------------------------------------- ------------
+``providers-init-file`` Check that provider's __init__.py file is removed
+----------------------------------- ---------------------------------------------------------------- ------------
 ``pydevd`` Check for accidentally committed pydevd statements.
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``pydocstyle`` Runs pydocstyle.
@@ -156,10 +158,12 @@ require Breeze Docker images to be installed locally:
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``update-local-yml-file`` Updates mounts in local.yml file.
 ----------------------------------- ---------------------------------------------------------------- ------------
-``update-setup-cfg-file`` Update setup.cfg file with all licenses.
+``update-setup-cfg-file``  Update setup.cfg file with all licenses.
 ----------------------------------- ---------------------------------------------------------------- ------------
 ``update-extras`` Updates extras in the documentation.
 ----------------------------------- ---------------------------------------------------------------- ------------
+``version-sync`` Synchronizes versions setup.py <-> airflow/version.py.
+----------------------------------- ---------------------------------------------------------------- ------------
 ``yamllint`` Checks yaml files with yamllint.
 =================================== ================================================================ ============
 

diff --git a/airflow/api_connexion/exceptions.py b/airflow/api_connexion/exceptions.py
@@ -21,9 +21,10 @@
 
 from airflow import version
 
-doc_link = f'https://airflow.apache.org/docs/{version.version}/stable-rest-api-ref.html'
-if 'dev' in version.version:
+if any(suffix in version.version for suffix in ['dev', 'a', 'b']):
  doc_link = "https://airflow.readthedocs.io/en/latest/stable-rest-api-ref.html"
+else:
+ doc_link = f'https://airflow.apache.org/docs/{version.version}/stable-rest-api-ref.html'
 
 EXCEPTIONS_LINK_MAP = {
  400: f"{doc_link}#section/Errors/BadRequest",
@@ -38,6 +39,7 @@
 def common_error_handler(exception):
  """
  Used to capture connexion exceptions and add link to the type field
+
  :type exception: Exception
  """
  if isinstance(exception, ProblemException):

diff --git a/airflow/providers/amazon/backport_provider_setup.cfg b/airflow/providers/amazon/backport_provider_setup.cfg
diff --git a/airflow/providers/amazon/backport_provider_setup.py b/airflow/providers/amazon/backport_provider_setup.py
diff --git a/airflow/providers/apache/cassandra/backport_provider_setup.cfg b/airflow/providers/apache/cassandra/backport_provider_setup.cfg