Add more SQL template fields renderers (#21237)

apache · Feb 4, 2022 · 39e395f · 39e395f
1 parent 648e980
commit 39e395f
Show file tree

Hide file tree

Showing 35 changed files with 60 additions and 12 deletions.
diff --git a/airflow/operators/sql.py b/airflow/operators/sql.py
@@ -126,6 +126,7 @@ class SQLCheckOperator(BaseSQLOperator):
  ".hql",
  ".sql",
  )
+ template_fields_renderers = {"sql": "sql"}
  ui_color = "#fff7e6"
 
  def __init__(
@@ -179,6 +180,7 @@ class SQLValueCheckOperator(BaseSQLOperator):
  ".hql",
  ".sql",
  )
+ template_fields_renderers = {"sql": "sql"}
  ui_color = "#fff7e6"
 
  def __init__(
@@ -403,6 +405,7 @@ class SQLThresholdCheckOperator(BaseSQLOperator):
  ".hql",
  ".sql",
  )
+ template_fields_renderers = {"sql": "sql"}
 
  def __init__(
  self,
@@ -481,6 +484,7 @@ class BranchSQLOperator(BaseSQLOperator, SkipMixin):
 
  template_fields: Sequence[str] = ("sql",)
  template_ext: Sequence[str] = (".sql",)
+ template_fields_renderers = {"sql": "sql"}
  ui_color = "#a22034"
  ui_fgcolor = "#F7F7F7"
 

diff --git a/airflow/providers/amazon/aws/operators/redshift_sql.py b/airflow/providers/amazon/aws/operators/redshift_sql.py
@@ -44,6 +44,7 @@ class RedshiftSQLOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql',)
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {"sql": "postgresql"}
 
  def __init__(
  self,

diff --git a/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py b/airflow/providers/amazon/aws/transfers/hive_to_dynamodb.py
@@ -50,6 +50,7 @@ class HiveToDynamoDBOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql',)
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {"sql": "hql"}
  ui_color = '#a0e08c'
 
  def __init__(

diff --git a/airflow/providers/apache/druid/transfers/hive_to_druid.py b/airflow/providers/apache/druid/transfers/hive_to_druid.py
@@ -66,6 +66,7 @@ class HiveToDruidOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql', 'intervals')
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'hql'}
 
  def __init__(
  self,

diff --git a/airflow/providers/apache/hive/operators/hive.py b/airflow/providers/apache/hive/operators/hive.py
@@ -69,6 +69,7 @@ class HiveOperator(BaseOperator):
  '.hql',
  '.sql',
  )
+ template_fields_renderers = {'sql': 'hql'}
  ui_color = '#f0e4ec'
 
  def __init__(

diff --git a/airflow/providers/apache/hive/transfers/hive_to_mysql.py b/airflow/providers/apache/hive/transfers/hive_to_mysql.py
@@ -57,6 +57,7 @@ class HiveToMySqlOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql', 'mysql_table', 'mysql_preoperator', 'mysql_postoperator')
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'hql', 'mysql_preoperator': 'mysql', 'mysql_postoperator': 'mysql'}
  ui_color = '#a0e08c'
 
  def __init__(

diff --git a/airflow/providers/apache/hive/transfers/hive_to_samba.py b/airflow/providers/apache/hive/transfers/hive_to_samba.py
@@ -47,6 +47,7 @@ class HiveToSambaOperator(BaseOperator):
  '.hql',
  '.sql',
  )
+ template_fields_renderers = {'hql': 'hql'}
 
  def __init__(
  self,

diff --git a/airflow/providers/apache/hive/transfers/mssql_to_hive.py b/airflow/providers/apache/hive/transfers/mssql_to_hive.py
@@ -65,6 +65,7 @@ class MsSqlToHiveOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql', 'partition', 'hive_table')
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'tsql'}
  ui_color = '#a0e08c'
 
  def __init__(

diff --git a/airflow/providers/apache/hive/transfers/mysql_to_hive.py b/airflow/providers/apache/hive/transfers/mysql_to_hive.py
@@ -70,6 +70,7 @@ class MySqlToHiveOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql', 'partition', 'hive_table')
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'mysql'}
  ui_color = '#a0e08c'
 
  def __init__(

diff --git a/airflow/providers/apache/hive/transfers/vertica_to_hive.py b/airflow/providers/apache/hive/transfers/vertica_to_hive.py
@@ -62,6 +62,7 @@ class VerticaToHiveOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql', 'partition', 'hive_table')
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = '#b4e0ff'
 
  def __init__(

diff --git a/airflow/providers/apache/spark/operators/spark_sql.py b/airflow/providers/apache/spark/operators/spark_sql.py
@@ -53,6 +53,7 @@ class SparkSqlOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('_sql',)
  template_ext: Sequence[str] = (".sql", ".hql")
+ template_fields_renderers = {'_sql': 'sql'}
 
  def __init__(
  self,

diff --git a/airflow/providers/apache/sqoop/operators/sqoop.py b/airflow/providers/apache/sqoop/operators/sqoop.py
@@ -110,6 +110,7 @@ class SqoopOperator(BaseOperator):
  'hcatalog_table',
  'schema',
  )
+ template_fields_renderers = {'query': 'sql'}
  ui_color = '#7D8CA4'
 
  def __init__(

diff --git a/airflow/providers/exasol/operators/exasol.py b/airflow/providers/exasol/operators/exasol.py
@@ -40,6 +40,7 @@ class ExasolOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql',)
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = '#ededed'
 
  def __init__(

diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py
@@ -550,6 +550,7 @@ class BigQueryExecuteQueryOperator(BaseOperator):
  'impersonation_chain',
  )
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = BigQueryUIColors.QUERY.value
 
  @property

diff --git a/airflow/providers/google/cloud/operators/cloud_sql.py b/airflow/providers/google/cloud/operators/cloud_sql.py
@@ -963,6 +963,7 @@ class CloudSQLExecuteQueryOperator(BaseOperator):
  # [START gcp_sql_query_template_fields]
  template_fields: Sequence[str] = ('sql', 'gcp_cloudsql_conn_id', 'gcp_conn_id')
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  # [END gcp_sql_query_template_fields]
 
  def __init__(

diff --git a/airflow/providers/google/cloud/operators/dataflow.py b/airflow/providers/google/cloud/operators/dataflow.py
@@ -822,6 +822,7 @@ class DataflowStartSqlJobOperator(BaseOperator):
  "project_id",
  "gcp_conn_id",
  )
+ template_fields_renderers = {"query": "sql"}
 
  def __init__(
  self,

diff --git a/airflow/providers/google/cloud/operators/dataproc.py b/airflow/providers/google/cloud/operators/dataproc.py
@@ -1231,6 +1231,7 @@ class DataprocSubmitSparkSqlJobOperator(DataprocJobBaseOperator):
  'impersonation_chain',
  )
  template_ext = ('.q',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = '#0273d4'
  job_type = 'spark_sql_job'
 

diff --git a/airflow/providers/google/cloud/operators/spanner.py b/airflow/providers/google/cloud/operators/spanner.py
@@ -221,6 +221,7 @@ class SpannerQueryDatabaseInstanceOperator(BaseOperator):
  'impersonation_chain',
  )
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'query': 'sql'}
  # [END gcp_spanner_query_template_fields]
 
  def __init__(
@@ -324,6 +325,7 @@ class SpannerDeployDatabaseInstanceOperator(BaseOperator):
  'impersonation_chain',
  )
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'ddl_statements': 'sql'}
  # [END gcp_spanner_database_deploy_template_fields]
 
  def __init__(
@@ -421,6 +423,7 @@ class SpannerUpdateDatabaseInstanceOperator(BaseOperator):
  'impersonation_chain',
  )
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'ddl_statements': 'sql'}
  # [END gcp_spanner_database_update_template_fields]
 
  def __init__(

diff --git a/airflow/providers/google/cloud/transfers/salesforce_to_gcs.py b/airflow/providers/google/cloud/transfers/salesforce_to_gcs.py
@@ -58,6 +58,7 @@ class SalesforceToGcsOperator(BaseOperator):
  'object_name',
  )
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
 
  def __init__(
  self,

diff --git a/airflow/providers/google/cloud/transfers/sql_to_gcs.py b/airflow/providers/google/cloud/transfers/sql_to_gcs.py
@@ -86,6 +86,7 @@ class BaseSQLToGCSOperator(BaseOperator):
  'impersonation_chain',
  )
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = '#a0e08c'
 
  def __init__(

diff --git a/airflow/providers/jdbc/operators/jdbc.py b/airflow/providers/jdbc/operators/jdbc.py
@@ -45,6 +45,7 @@ class JdbcOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql',)
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = '#ededed'
 
  def __init__(

diff --git a/airflow/providers/microsoft/mssql/operators/mssql.py b/airflow/providers/microsoft/mssql/operators/mssql.py
@@ -49,6 +49,7 @@ class MsSqlOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql',)
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'tsql'}
  ui_color = '#ededed'
 
  def __init__(

diff --git a/airflow/providers/mysql/operators/mysql.py b/airflow/providers/mysql/operators/mysql.py
@@ -47,7 +47,7 @@ class MySqlOperator(BaseOperator):
  """
 
  template_fields: Sequence[str] = ('sql', 'parameters')
- template_fields_renderers = {'sql': 'sql', 'parameters': 'json'}
+ template_fields_renderers = {'sql': 'mysql', 'parameters': 'json'}
  template_ext: Sequence[str] = ('.sql', '.json')
  ui_color = '#ededed'
 

diff --git a/airflow/providers/mysql/transfers/presto_to_mysql.py b/airflow/providers/mysql/transfers/presto_to_mysql.py
@@ -44,7 +44,7 @@ class PrestoToMySqlOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql', 'mysql_table', 'mysql_preoperator')
  template_ext: Sequence[str] = ('.sql',)
- template_fields_renderers = {"sql": "sql", "mysql_preoperator": "sql"}
+ template_fields_renderers = {"sql": "sql", "mysql_preoperator": "mysql"}
  ui_color = '#a0e08c'
 
  def __init__(

diff --git a/airflow/providers/mysql/transfers/trino_to_mysql.py b/airflow/providers/mysql/transfers/trino_to_mysql.py
@@ -44,7 +44,7 @@ class TrinoToMySqlOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql', 'mysql_table', 'mysql_preoperator')
  template_ext: Sequence[str] = ('.sql',)
- template_fields_renderers = {"sql": "sql", "mysql_preoperator": "sql"}
+ template_fields_renderers = {"sql": "sql", "mysql_preoperator": "mysql"}
  ui_color = '#a0e08c'
 
  def __init__(

diff --git a/airflow/providers/mysql/transfers/vertica_to_mysql.py b/airflow/providers/mysql/transfers/vertica_to_mysql.py
@@ -57,8 +57,8 @@ class VerticaToMySqlOperator(BaseOperator):
  template_ext: Sequence[str] = ('.sql',)
  template_fields_renderers = {
  "sql": "sql",
- "mysql_preoperator": "sql",
- "mysql_postoperator": "sql",
+ "mysql_preoperator": "mysql",
+ "mysql_postoperator": "mysql",
  }
  ui_color = '#a0e08c'
 

diff --git a/airflow/providers/oracle/operators/oracle.py b/airflow/providers/oracle/operators/oracle.py
@@ -41,6 +41,7 @@ class OracleOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql',)
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = '#ededed'
 
  def __init__(

diff --git a/airflow/providers/postgres/operators/postgres.py b/airflow/providers/postgres/operators/postgres.py
@@ -40,7 +40,7 @@ class PostgresOperator(BaseOperator):
  """
 
  template_fields: Sequence[str] = ('sql',)
- template_fields_renderers = {'sql': 'sql'}
+ template_fields_renderers = {'sql': 'postgresql'}
  template_ext: Sequence[str] = ('.sql',)
  ui_color = '#ededed'
 

diff --git a/airflow/providers/snowflake/operators/snowflake.py b/airflow/providers/snowflake/operators/snowflake.py
@@ -77,6 +77,7 @@ class SnowflakeOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql',)
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = '#ededed'
 
  def __init__(

diff --git a/airflow/providers/snowflake/transfers/snowflake_to_slack.py b/airflow/providers/snowflake/transfers/snowflake_to_slack.py
@@ -61,7 +61,7 @@ class SnowflakeToSlackOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql', 'slack_message')
  template_ext: Sequence[str] = ('.sql', '.jinja', '.j2')
- template_fields_renderers = {"slack_message": "jinja"}
+ template_fields_renderers = {"sql": "sql", "slack_message": "jinja"}
  times_rendered = 0
 
  def __init__(

diff --git a/airflow/providers/sqlite/operators/sqlite.py b/airflow/providers/sqlite/operators/sqlite.py
@@ -39,6 +39,7 @@ class SqliteOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql',)
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = '#cdaaed'
 
  def __init__(

diff --git a/airflow/providers/vertica/operators/vertica.py b/airflow/providers/vertica/operators/vertica.py
@@ -36,6 +36,7 @@ class VerticaOperator(BaseOperator):
 
  template_fields: Sequence[str] = ('sql',)
  template_ext: Sequence[str] = ('.sql',)
+ template_fields_renderers = {'sql': 'sql'}
  ui_color = '#b4e0ff'
 
  def __init__(

diff --git a/airflow/www/utils.py b/airflow/www/utils.py
@@ -414,21 +414,24 @@ def get_attr_renderer():
  return {
  'bash': lambda x: render(x, lexers.BashLexer),
  'bash_command': lambda x: render(x, lexers.BashLexer),
- 'hql': lambda x: render(x, lexers.SqlLexer),
- 'html': lambda x: render(x, lexers.HtmlLexer),
- 'sql': lambda x: render(x, lexers.SqlLexer),
  'doc': lambda x: render(x, lexers.TextLexer),
  'doc_json': lambda x: render(x, lexers.JsonLexer),
+ 'doc_md': wrapped_markdown,
  'doc_rst': lambda x: render(x, lexers.RstLexer),
  'doc_yaml': lambda x: render(x, lexers.YamlLexer),
- 'doc_md': wrapped_markdown,
+ 'hql': lambda x: render(x, lexers.SqlLexer),
+ 'html': lambda x: render(x, lexers.HtmlLexer),
  'jinja': lambda x: render(x, lexers.DjangoLexer),
  'json': lambda x: json_render(x, lexers.JsonLexer),
  'md': wrapped_markdown,
+ 'mysql': lambda x: render(x, lexers.MySqlLexer),
+ 'postgresql': lambda x: render(x, lexers.PostgresLexer),
  'powershell': lambda x: render(x, lexers.PowerShellLexer),
  'py': lambda x: render(get_python_source(x), lexers.PythonLexer),
  'python_callable': lambda x: render(get_python_source(x), lexers.PythonLexer),
  'rst': lambda x: render(x, lexers.RstLexer),
+ 'sql': lambda x: render(x, lexers.SqlLexer),
+ 'tsql': lambda x: render(x, lexers.TransactSqlLexer),
  'yaml': lambda x: render(x, lexers.YamlLexer),
  }
 

diff --git a/docs/apache-airflow/howto/custom-operator.rst b/docs/apache-airflow/howto/custom-operator.rst
@@ -243,19 +243,29 @@ configuration at ``query.sql`` to be rendered with the SQL lexer.
 Currently available lexers:
 
  - bash
+ - bash_command
  - doc
+ - doc_json
+ - doc_md
+ - doc_rst
+ - doc_yaml
+ - doc_md
  - hql
  - html
  - jinja
  - json
  - md
+ - mysql
+ - postgresql
  - powershell
  - py
+ - python_callable
  - rst
  - sql
+ - tsql
  - yaml
 
-If you use a non existing lexer then the value of the template field will be rendered as a pretty printed object.
+If you use a non-existing lexer then the value of the template field will be rendered as a pretty-printed object.
 
 Define an operator extra link
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt
@@ -456,6 +456,7 @@ apikey
 apis
 appbuilder
 approle
+aren
 arg
 args
 argv
@@ -1117,6 +1118,7 @@ pooler
 postMessage
 postfix
 postgres
+postgresql
 powershell
 pql
 pre
@@ -1264,6 +1266,7 @@ sharded
 shellcheck
 shellcmd
 shm
+shouldn
 sie
 sigv
 skipable
@@ -1402,6 +1405,7 @@ triggerers
 trino
 trojan
 truthy
+tsql
 tsv
 ttl
 tunables