Move plyvel to google provider extra (#15812)

Plyvel does not build on macOS without levelDB installed in system. Its better to make it an optional install.
apache · May 19, 2021 · 76a80bb · 76a80bb
1 parent 20f3639
commit 76a80bb
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 12 deletions.
diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst
@@ -583,11 +583,11 @@ apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery
 cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci,
 devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, facebook, ftp, gcp,
 gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
-jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo,
-mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot,
-plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid,
-sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino,
-vertica, virtualenv, webhdfs, winrm, yandex, zendesk
+jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql,
+microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill,
+password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
+segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau,
+telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
 
  .. END EXTRAS HERE
 

diff --git a/INSTALL b/INSTALL
@@ -95,11 +95,11 @@ apache.sqoop, apache.webhdfs, asana, async, atlas, aws, azure, cassandra, celery
 cncf.kubernetes, crypto, dask, databricks, datadog, deprecated_api, devel, devel_all, devel_ci,
 devel_hadoop, dingding, discord, doc, docker, druid, elasticsearch, exasol, facebook, ftp, gcp,
 gcp_api, github_enterprise, google, google_auth, grpc, hashicorp, hdfs, hive, http, imap, jdbc,
-jenkins, jira, kerberos, kubernetes, ldap, microsoft.azure, microsoft.mssql, microsoft.winrm, mongo,
-mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill, password, pinot,
-plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba, segment, sendgrid,
-sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau, telegram, trino,
-vertica, virtualenv, webhdfs, winrm, yandex, zendesk
+jenkins, jira, kerberos, kubernetes, ldap, leveldb, microsoft.azure, microsoft.mssql,
+microsoft.winrm, mongo, mssql, mysql, neo4j, odbc, openfaas, opsgenie, oracle, pagerduty, papermill,
+password, pinot, plexus, postgres, presto, qds, qubole, rabbitmq, redis, s3, salesforce, samba,
+segment, sendgrid, sentry, sftp, singularity, slack, snowflake, spark, sqlite, ssh, statsd, tableau,
+telegram, trino, vertica, virtualenv, webhdfs, winrm, yandex, zendesk
 
 # END EXTRAS HERE
 

diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml
@@ -748,3 +748,4 @@ extra-links:
 
 additional-extras:
  apache.beam: apache-beam[gcp]
+ leveldb: plyvel
diff --git a/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst b/docs/apache-airflow-providers-google/operators/leveldb/leveldb.rst
@@ -17,6 +17,8 @@
 
 
 
+.. _howto/operator:LevelDBOperator:
+
 Google LevelDB Operator
 ================================
 
@@ -27,7 +29,10 @@ an ordered mapping from string keys to string values.
  :depth: 1
  :local:
 
-.. _howto/operator:LevelDBOperator:
+.. note::
+
+ To use LevelDB hooks and operators you must requires installation of ``plyvel``. It will be
+ installed if you specify the extra ``apache-airflow-providers-google[leveldb]``.
 
 Put key
 ^^^^^^^^^^^^^^^

diff --git a/docs/apache-airflow/extra-packages-ref.rst b/docs/apache-airflow/extra-packages-ref.rst
@@ -60,6 +60,8 @@ python dependencies for the provided package.
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | ldap | ``pip install 'apache-airflow[ldap]'`` | LDAP authentication for users |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
+| leveldb | ``pip install 'apache-airflow[leveldb]'`` | Required for use leveldb extra in google provider |
++---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | password | ``pip install 'apache-airflow[password]'`` | Password authentication for users |
 +---------------------+-----------------------------------------------------+----------------------------------------------------------------------------+
 | rabbitmq | ``pip install 'apache-airflow[rabbitmq]'`` | RabbitMQ support as a Celery backend |

diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
@@ -1430,6 +1430,7 @@ utils
 uuid
 validator
 vals
+ve
 vendored
 venvs
 versionable

diff --git a/setup.py b/setup.py
@@ -326,7 +326,6 @@ def get_sphinx_theme_version() -> str:
  # pandas-gbq 0.15.0 release broke google provider's bigquery import
  # _check_google_client_version (airflow/providers/google/cloud/hooks/bigquery.py:49)
  'pandas-gbq<0.15.0',
- 'plyvel',
 ]
 grpc = [
  'google-auth>=1.0.0, <2.0.0dev',
@@ -381,6 +380,7 @@ def get_sphinx_theme_version() -> str:
  'ldap3>=2.5.1',
  'python-ldap',
 ]
+leveldb = ['plyvel']
 mongo = [
  'dnspython>=1.13.0,<2.0.0',
  'pymongo>=3.6.0',
@@ -636,6 +636,7 @@ def get_sphinx_theme_version() -> str:
  'google_auth': flask_oauth,
  'kerberos': kerberos,
  'ldap': ldap,
+ 'leveldb': leveldb,
  'password': password,
  'rabbitmq': rabbitmq,
  'sentry': sentry,