Add try clause to DataFusionHook.wait_for_pipeline_state (#10031)

Sometimes it may happen that the pipeline is not visible instantly in DataFusion so retrieving it will result in 404 closes: #10030
apache · Aug 3, 2020 · 27020f8 · 27020f8
1 parent 85cc2a6
commit 27020f8
Showing 1 changed file with 9 additions and 7 deletions.
diff --git a/airflow/providers/google/cloud/hooks/datafusion.py b/airflow/providers/google/cloud/hooks/datafusion.py
@@ -112,13 +112,15 @@ def wait_for_pipeline_state(
  start_time = monotonic()
  current_state = None
  while monotonic() - start_time < timeout:
- current_state = self._get_workflow_state(
- pipeline_name=pipeline_name,
- pipeline_id=pipeline_id,
- instance_url=instance_url,
- namespace=namespace,
- )
-
+ try:
+ current_state = self._get_workflow_state(
+ pipeline_name=pipeline_name,
+ pipeline_id=pipeline_id,
+ instance_url=instance_url,
+ namespace=namespace,
+ )
+ except AirflowException:
+ pass # Because the pipeline may not be visible in system yet
  if current_state in success_states:
  return
  if current_state in failure_states: