Update Error details for Generic Error Code (#32847)

Adding details of Error as discussed per Issue - Improved error logging for failed Dataflow jobs #32107 Co-authored-by: Hussein Awala <[email protected]>
apache · Aug 14, 2023 · b4102ce · b4102ce
1 parent 2950fd7
commit b4102ce
Showing 1 changed file with 5 additions and 1 deletion.
diff --git a/airflow/providers/google/cloud/hooks/dataflow.py b/airflow/providers/google/cloud/hooks/dataflow.py
@@ -1014,8 +1014,12 @@ def start_sql_job(
  self.log.info("Output: %s", proc.stdout.decode())
  self.log.warning("Stderr: %s", proc.stderr.decode())
  self.log.info("Exit code %d", proc.returncode)
+ stderr_last_20_lines = "\n".join(proc.stderr.decode().strip().splitlines()[-20:])
  if proc.returncode != 0:
- raise AirflowException(f"Process exit with non-zero exit code. Exit code: {proc.returncode}")
+ raise AirflowException(
+ f"Process exit with non-zero exit code. Exit code: {proc.returncode} Error Details : "
+ f"{stderr_last_20_lines}"
+ )
  job_id = proc.stdout.decode().strip()
 
  self.log.info("Created job ID: %s", job_id)