Stop using start_date in default_args in example_dags (2) (#9985)

apache · Jul 25, 2020 · 7d24b08 · 7d24b08
1 parent 7cc1c8b
commit 7d24b08
Show file tree

Hide file tree

Showing 50 changed files with 74 additions and 85 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -326,7 +326,7 @@ repos:
  - id: restrict-start_date-in-default_args-in-example_dags
  language: pygrep
  name: "'start_date' should not be defined in default_args in example_dags"
- entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')"
+ entry: "default_args\\s*=\\s*{\\s*(\"|')start_date(\"|')|(\"|')start_date(\"|'):"
  files: \.*example_dags.*.py$
  pass_filenames: true
  - id: check-integrations

diff --git a/airflow/example_dags/example_bash_operator.py b/airflow/example_dags/example_bash_operator.py
@@ -27,13 +27,13 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': days_ago(2),
 }
 
 dag = DAG(
  dag_id='example_bash_operator',
  default_args=args,
  schedule_interval='0 0 * * *',
+ start_date=days_ago(2),
  dagrun_timeout=timedelta(minutes=60),
  tags=['example']
 )

diff --git a/airflow/example_dags/example_branch_operator.py b/airflow/example_dags/example_branch_operator.py
@@ -27,12 +27,12 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': days_ago(2),
 }
 
 dag = DAG(
  dag_id='example_branch_operator',
  default_args=args,
+ start_date=days_ago(2),
  schedule_interval="@daily",
  tags=['example']
 )

diff --git a/airflow/example_dags/example_branch_python_dop_operator_3.py b/airflow/example_dags/example_branch_python_dop_operator_3.py
@@ -28,13 +28,13 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': days_ago(2),
  'depends_on_past': True,
 }
 
 dag = DAG(
  dag_id='example_branch_dop_operator_v3',
  schedule_interval='*/1 * * * *',
+ start_date=days_ago(2),
  default_args=args,
  tags=['example']
 )

diff --git a/airflow/example_dags/example_kubernetes_executor.py b/airflow/example_dags/example_kubernetes_executor.py
@@ -27,13 +27,13 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': days_ago(2)
 }
 
 with DAG(
  dag_id='example_kubernetes_executor',
  default_args=args,
  schedule_interval=None,
+ start_date=days_ago(2),
  tags=['example'],
 ) as dag:
 

diff --git a/airflow/example_dags/example_kubernetes_executor_config.py b/airflow/example_dags/example_kubernetes_executor_config.py
@@ -27,13 +27,13 @@
 
 default_args = {
  'owner': 'airflow',
- 'start_date': days_ago(2)
 }
 
 with DAG(
  dag_id='example_kubernetes_executor_config',
  default_args=default_args,
  schedule_interval=None,
+ start_date=days_ago(2),
  tags=['example'],
 ) as dag:
 

diff --git a/airflow/example_dags/example_passing_params_via_test_command.py b/airflow/example_dags/example_passing_params_via_test_command.py
@@ -30,9 +30,9 @@
  "example_passing_params_via_test_command",
  default_args={
  "owner": "airflow",
- "start_date": days_ago(1),
  },
  schedule_interval='*/1 * * * *',
+ start_date=days_ago(1),
  dagrun_timeout=timedelta(minutes=4),
  tags=['example']
 )

diff --git a/airflow/example_dags/example_python_operator.py b/airflow/example_dags/example_python_operator.py
@@ -26,13 +26,13 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': days_ago(2),
 }
 
 dag = DAG(
  dag_id='example_python_operator',
  default_args=args,
  schedule_interval=None,
+ start_date=days_ago(2),
  tags=['example']
 )
 

diff --git a/airflow/example_dags/example_short_circuit_operator.py b/airflow/example_dags/example_short_circuit_operator.py
@@ -25,10 +25,14 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': dates.days_ago(2),
 }
 
-dag = DAG(dag_id='example_short_circuit_operator', default_args=args, tags=['example'])
+dag = DAG(
+ dag_id='example_short_circuit_operator',
+ default_args=args,
+ start_date=dates.days_ago(2),
+ tags=['example'],
+)
 
 cond_true = ShortCircuitOperator(
  task_id='condition_is_True',

diff --git a/airflow/example_dags/example_skip_dag.py b/airflow/example_dags/example_skip_dag.py
@@ -25,7 +25,6 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': days_ago(2),
 }
 
 
@@ -57,6 +56,6 @@ def create_test_pipeline(suffix, trigger_rule, dag_):
  join >> final
 
 
-dag = DAG(dag_id='example_skip_dag', default_args=args, tags=['example'])
+dag = DAG(dag_id='example_skip_dag', default_args=args, start_date=days_ago(2), tags=['example'])
 create_test_pipeline('1', 'all_success', dag)
 create_test_pipeline('2', 'one_success', dag)
diff --git a/airflow/example_dags/example_subdag_operator.py b/airflow/example_dags/example_subdag_operator.py
@@ -29,12 +29,12 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': days_ago(2),
 }
 
 dag = DAG(
  dag_id=DAG_NAME,
  default_args=args,
+ start_date=days_ago(2),
  schedule_interval="@once",
  tags=['example']
 )

diff --git a/airflow/example_dags/example_trigger_controller_dag.py b/airflow/example_dags/example_trigger_controller_dag.py
@@ -27,7 +27,8 @@
 
 dag = DAG(
  dag_id="example_trigger_controller_dag",
- default_args={"owner": "airflow", "start_date": days_ago(2)},
+ default_args={"owner": "airflow"},
+ start_date=days_ago(2),
  schedule_interval="@once",
  tags=['example']
 )

diff --git a/airflow/example_dags/example_xcom.py b/airflow/example_dags/example_xcom.py
@@ -21,12 +21,13 @@
 from airflow.operators.python import PythonOperator
 from airflow.utils.dates import days_ago
 
-args = {
- 'owner': 'airflow',
- 'start_date': days_ago(2),
-}
-
-dag = DAG('example_xcom', schedule_interval="@once", default_args=args, tags=['example'])
+dag = DAG(
+ 'example_xcom',
+ schedule_interval="@once",
+ start_date=days_ago(2),
+ default_args={'owner': 'airflow'},
+ tags=['example']
+)
 
 value_1 = [1, 2, 3]
 value_2 = {'a': 'b'}

diff --git a/airflow/example_dags/example_xcomargs.py b/airflow/example_dags/example_xcomargs.py
@@ -25,11 +25,6 @@
 
 log = logging.getLogger(__name__)
 
-args = {
- 'owner': 'airflow',
- 'start_date': days_ago(2),
-}
-
 
 def generate_value():
  """Dummy function"""
@@ -45,7 +40,8 @@ def print_value(value):
 
 with DAG(
  dag_id='example_xcom_args',
- default_args=args,
+ default_args={'owner': 'airflow'},
+ start_date=days_ago(2),
  schedule_interval=None,
  tags=['example']
 ) as dag:

diff --git a/airflow/example_dags/subdags/subdag.py b/airflow/example_dags/subdags/subdag.py
@@ -21,6 +21,7 @@
 # [START subdag]
 from airflow import DAG
 from airflow.operators.dummy_operator import DummyOperator
+from airflow.utils.dates import days_ago
 
 
 def subdag(parent_dag_name, child_dag_name, args):
@@ -36,6 +37,7 @@ def subdag(parent_dag_name, child_dag_name, args):
  dag_subdag = DAG(
  dag_id='%s.%s' % (parent_dag_name, child_dag_name),
  default_args=args,
+ start_date=days_ago(2),
  schedule_interval="@daily",
  )
 

diff --git a/airflow/example_dags/tutorial.py b/airflow/example_dags/tutorial.py
@@ -39,7 +39,6 @@
 default_args = {
  'owner': 'airflow',
  'depends_on_past': False,
- 'start_date': days_ago(2),
  'email': ['[email protected]'],
  'email_on_failure': False,
  'email_on_retry': False,
@@ -67,6 +66,7 @@
  default_args=default_args,
  description='A simple tutorial DAG',
  schedule_interval=timedelta(days=1),
+ start_date=days_ago(2),
  tags=['example'],
 )
 # [END instantiate_dag]

diff --git a/airflow/providers/amazon/aws/example_dags/example_ecs_fargate.py b/airflow/providers/amazon/aws/example_dags/example_ecs_fargate.py
@@ -31,7 +31,6 @@
 DEFAULT_ARGS = {
  "owner": "airflow",
  "depends_on_past": False,
- "start_date": datetime.datetime(2020, 1, 1),
  "email": ["[email protected]"],
  "email_on_failure": False,
  "email_on_retry": False,
@@ -42,6 +41,7 @@
  default_args=DEFAULT_ARGS,
  default_view="graph",
  schedule_interval=None,
+ start_date=datetime.datetime(2020, 1, 1),
  tags=["example"],
 )
 # generate dag documentation

diff --git a/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_automatic_steps.py b/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_automatic_steps.py
@@ -28,7 +28,6 @@
 DEFAULT_ARGS = {
  'owner': 'airflow',
  'depends_on_past': False,
- 'start_date': days_ago(2),
  'email': ['[email protected]'],
  'email_on_failure': False,
  'email_on_retry': False
@@ -76,6 +75,7 @@
  dag_id='emr_job_flow_automatic_steps_dag',
  default_args=DEFAULT_ARGS,
  dagrun_timeout=timedelta(hours=2),
+ start_date=days_ago(2),
  schedule_interval='0 3 * * *',
  tags=['example'],
 ) as dag:

diff --git a/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_manual_steps.py b/airflow/providers/amazon/aws/example_dags/example_emr_job_flow_manual_steps.py
@@ -33,7 +33,6 @@
 DEFAULT_ARGS = {
  'owner': 'airflow',
  'depends_on_past': False,
- 'start_date': days_ago(2),
  'email': ['[email protected]'],
  'email_on_failure': False,
  'email_on_retry': False
@@ -78,6 +77,7 @@
  dag_id='emr_job_flow_manual_steps_dag',
  default_args=DEFAULT_ARGS,
  dagrun_timeout=timedelta(hours=2),
+ start_date=days_ago(2),
  schedule_interval='0 3 * * *',
  tags=['example'],
 ) as dag:

diff --git a/airflow/providers/apache/cassandra/example_dags/example_cassandra_dag.py b/airflow/providers/apache/cassandra/example_dags/example_cassandra_dag.py
@@ -27,13 +27,13 @@
 
 args = {
  'owner': 'Airflow',
- 'start_date': days_ago(2)
 }
 
 with DAG(
  dag_id='example_cassandra_operator',
  default_args=args,
  schedule_interval=None,
+ start_date=days_ago(2),
  tags=['example']
 ) as dag:
  # [START howto_operator_cassandra_table_sensor]

diff --git a/airflow/providers/apache/hive/example_dags/example_twitter_dag.py b/airflow/providers/apache/hive/example_dags/example_twitter_dag.py
@@ -73,7 +73,6 @@ def transfertodb():
 default_args = {
  'owner': 'Ekhtiar',
  'depends_on_past': False,
- 'start_date': days_ago(5),
  'email': ['[email protected]'],
  'email_on_failure': False,
  'email_on_retry': False,
@@ -89,6 +88,7 @@ def transfertodb():
  dag_id='example_twitter_dag',
  default_args=default_args,
  schedule_interval="@daily",
+ start_date=days_ago(5),
  tags=['example'],
 ) as dag:
 

diff --git a/airflow/providers/apache/kylin/example_dags/example_kylin_dag.py b/airflow/providers/apache/kylin/example_dags/example_kylin_dag.py
@@ -27,13 +27,13 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': days_ago(1),
 }
 
 dag = DAG(
  dag_id='example_kylin_operator',
  default_args=args,
  schedule_interval=None,
+ start_date=days_ago(1),
  tags=['example']
 )
 

diff --git a/airflow/providers/apache/livy/example_dags/example_livy.py b/airflow/providers/apache/livy/example_dags/example_livy.py
@@ -28,14 +28,14 @@
 args = {
  'owner': 'airflow',
  'email': ['[email protected]'],
- 'depends_on_past': False,
- 'start_date': days_ago(5)
+ 'depends_on_past': False
 }
 
 with DAG(
  dag_id='example_livy_operator',
  default_args=args,
- schedule_interval='@daily'
+ schedule_interval='@daily',
+ start_date=days_ago(5),
 ) as dag:
 
  livy_java_task = LivyOperator(

diff --git a/airflow/providers/apache/pig/example_dags/example_pig.py b/airflow/providers/apache/pig/example_dags/example_pig.py
@@ -24,13 +24,13 @@
 
 args = {
  'owner': 'airflow',
- 'start_date': days_ago(2),
 }
 
 dag = DAG(
  dag_id='example_pig_operator',
  default_args=args,
  schedule_interval=None,
+ start_date=days_ago(2),
  tags=['example']
 )
 

diff --git a/airflow/providers/apache/spark/example_dags/example_spark_dag.py b/airflow/providers/apache/spark/example_dags/example_spark_dag.py
@@ -28,13 +28,13 @@
 
 args = {
  'owner': 'Airflow',
- 'start_date': days_ago(2)
 }
 
 with DAG(
  dag_id='example_spark_operator',
  default_args=args,
  schedule_interval=None,
+ start_date=days_ago(2),
  tags=['example']
 ) as dag:
  # [START howto_operator_spark_submit]

diff --git a/airflow/providers/cncf/kubernetes/example_dags/example_kubernetes.py b/airflow/providers/cncf/kubernetes/example_dags/example_kubernetes.py
@@ -121,13 +121,13 @@
 
 default_args = {
  'owner': 'airflow',
- 'start_date': days_ago(2)
 }
 
 with DAG(
  dag_id='example_kubernetes_operator',
  default_args=default_args,
  schedule_interval=None,
+ start_date=days_ago(2),
  tags=['example'],
 ) as dag:
  k = KubernetesPodOperator(