Fix handling of subprocess error handling in s3_file_transform and gcs (#9106)

blackw1ng · web-flow · commit 17adcea835cb · 2020-06-02T17:57:59.000+02:00
As outlined in Issue 9104 the python subprocess return code can be less than 0. The previous version only captures errors of the subprocess itself, not the negative error codes, as cause by host SIGKILL, SIGHUP, ... see https://docs.python.org/3/library/subprocess.html#subprocess. CompletedProcess.returncode We now treat all non-zero returncodes as a script failure.
diff --git a/airflow/providers/amazon/aws/operators/s3_file_transform.py b/airflow/providers/amazon/aws/operators/s3_file_transform.py
@@ -154,7 +154,7 @@ def execute(self, context):
 
                 process.wait()
 
-                if process.returncode > 0:
+                if process.returncode:
                     raise AirflowException(
                         "Transform script failed: {0}".format(process.returncode)
                     )
diff --git a/airflow/providers/google/cloud/operators/gcs.py b/airflow/providers/google/cloud/operators/gcs.py
@@ -609,7 +609,7 @@ def execute(self, context: Dict):
                     self.log.info(line.decode(self.output_encoding).rstrip())
 
             process.wait()
-            if process.returncode > 0:
+            if process.returncode:
                 raise AirflowException(
                     "Transform script failed: {0}".format(process.returncode)
                 )

Original file line number	Diff line number	Diff line change
`@@ -154,7 +154,7 @@ def execute(self, context):`
`154`	`154`
`155`	`155`	`process.wait()`
`156`	`156`
`157`		`- if process.returncode > 0:`
	`157`	`+ if process.returncode:`
`158`	`158`	`raise AirflowException(`
`159`	`159`	`"Transform script failed: {0}".format(process.returncode)`
`160`	`160`	`)`
Original file line number	Diff line number	Diff line change
`@@ -609,7 +609,7 @@ def execute(self, context: Dict):`
`609`	`609`	`self.log.info(line.decode(self.output_encoding).rstrip())`
`610`	`610`
`611`	`611`	`process.wait()`
`612`		`- if process.returncode > 0:`
	`612`	`+ if process.returncode:`
`613`	`613`	`raise AirflowException(`
`614`	`614`	`"Transform script failed: {0}".format(process.returncode)`
`615`	`615`	`)`