feat: Rename Travel Sustainability to Travel Impact Model (#351)

noman-aalian · web-flow · commit 83df285f0b47 · 2022-04-29T21:08:08.000-04:00
diff --git a/datasets/travel_impact_model/infra/flights_impact_data_pipeline.tf b/datasets/travel_impact_model/infra/flights_impact_data_pipeline.tf
@@ -15,20 +15,20 @@
  */
 
 
-resource "google_bigquery_table" "travel_sustainability_flight_emissions" {
+resource "google_bigquery_table" "travel_impact_model_flights_impact_data" {
   project     = var.project_id
-  dataset_id  = "travel_sustainability"
-  table_id    = "flight_emissions"
-  description = "Flight emissions data"
+  dataset_id  = "travel_impact_model"
+  table_id    = "flights_impact_data"
+  description = "Flights impact data"
   depends_on = [
-    google_bigquery_dataset.travel_sustainability
+    google_bigquery_dataset.travel_impact_model
   ]
 }
 
-output "bigquery_table-travel_sustainability_flight_emissions-table_id" {
-  value = google_bigquery_table.travel_sustainability_flight_emissions.table_id
+output "bigquery_table-travel_impact_model_flights_impact_data-table_id" {
+  value = google_bigquery_table.travel_impact_model_flights_impact_data.table_id
 }
 
-output "bigquery_table-travel_sustainability_flight_emissions-id" {
-  value = google_bigquery_table.travel_sustainability_flight_emissions.id
+output "bigquery_table-travel_impact_model_flights_impact_data-id" {
+  value = google_bigquery_table.travel_impact_model_flights_impact_data.id
 }
diff --git a/datasets/travel_impact_model/infra/metadata_pipeline.tf b/datasets/travel_impact_model/infra/metadata_pipeline.tf
@@ -15,20 +15,20 @@
  */
 
 
-resource "google_bigquery_table" "travel_sustainability_metadata" {
+resource "google_bigquery_table" "travel_impact_model_metadata" {
   project     = var.project_id
-  dataset_id  = "travel_sustainability"
+  dataset_id  = "travel_impact_model"
   table_id    = "metadata"
   description = "Metadata about the dataset"
   depends_on = [
-    google_bigquery_dataset.travel_sustainability
+    google_bigquery_dataset.travel_impact_model
   ]
 }
 
-output "bigquery_table-travel_sustainability_metadata-table_id" {
-  value = google_bigquery_table.travel_sustainability_metadata.table_id
+output "bigquery_table-travel_impact_model_metadata-table_id" {
+  value = google_bigquery_table.travel_impact_model_metadata.table_id
 }
 
-output "bigquery_table-travel_sustainability_metadata-id" {
-  value = google_bigquery_table.travel_sustainability_metadata.id
+output "bigquery_table-travel_impact_model_metadata-id" {
+  value = google_bigquery_table.travel_impact_model_metadata.id
 }
diff --git a/datasets/travel_impact_model/infra/provider.tf b/datasets/travel_impact_model/infra/provider.tf
diff --git a/datasets/travel_impact_model/infra/travel_impact_model_dataset.tf b/datasets/travel_impact_model/infra/travel_impact_model_dataset.tf
@@ -0,0 +1,70 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+resource "google_bigquery_dataset" "travel_impact_model" {
+  dataset_id  = "travel_impact_model"
+  project     = var.project_id
+  description = "Travel Impact Model Data"
+}
+
+data "google_iam_policy" "bq_ds__travel_impact_model" {
+  dynamic "binding" {
+    for_each = var.iam_policies["bigquery_datasets"]["travel_impact_model"]
+    content {
+      role    = binding.value["role"]
+      members = binding.value["members"]
+    }
+  }
+}
+
+resource "google_bigquery_dataset_iam_policy" "travel_impact_model" {
+  dataset_id  = google_bigquery_dataset.travel_impact_model.dataset_id
+  policy_data = data.google_iam_policy.bq_ds__travel_impact_model.policy_data
+}
+output "bigquery_dataset-travel_impact_model-dataset_id" {
+  value = google_bigquery_dataset.travel_impact_model.dataset_id
+}
+
+resource "google_storage_bucket" "travel-impact-model" {
+  name                        = "${var.bucket_name_prefix}-travel-impact-model"
+  force_destroy               = true
+  location                    = "US"
+  uniform_bucket_level_access = true
+  lifecycle {
+    ignore_changes = [
+      logging,
+    ]
+  }
+}
+
+data "google_iam_policy" "storage_bucket__travel-impact-model" {
+  dynamic "binding" {
+    for_each = var.iam_policies["storage_buckets"]["travel-impact-model"]
+    content {
+      role    = binding.value["role"]
+      members = binding.value["members"]
+    }
+  }
+}
+
+resource "google_storage_bucket_iam_policy" "travel-impact-model" {
+  bucket      = google_storage_bucket.travel-impact-model.name
+  policy_data = data.google_iam_policy.storage_bucket__travel-impact-model.policy_data
+}
+output "storage_bucket-travel-impact-model-name" {
+  value = google_storage_bucket.travel-impact-model.name
+}
diff --git a/datasets/travel_impact_model/infra/variables.tf b/datasets/travel_impact_model/infra/variables.tf
@@ -20,4 +20,7 @@ variable "bucket_name_prefix" {}
 variable "impersonating_acct" {}
 variable "region" {}
 variable "env" {}
+variable "iam_policies" {
+  default = {}
+}
 
diff --git a/datasets/travel_impact_model/pipelines/dataset.yaml b/datasets/travel_impact_model/pipelines/dataset.yaml
@@ -17,7 +17,7 @@ dataset:
   # to users of your data on the Google Cloud website.
 
   # Must be exactly the same name as the folder name your dataset.yaml is in.
-  name: travel_sustainability
+  name: travel_impact_model
 
   # A friendly, human-readable name of the dataset
   friendly_name: ~
@@ -52,8 +52,8 @@ resources:
     #   friendly_name (A user-friendly name of the dataset)
     #   description   (A user-friendly description of the dataset)
     #   location      (The geographic location where the dataset should reside)
-    dataset_id: travel_sustainability
-    description: "Travel Sustainability"
+    dataset_id: travel_impact_model
+    description: "Travel Impact Model Data"
 
   - type: storage_bucket
     # Google Cloud Storage Bucket that your pipelines need. Say, you need an
@@ -78,6 +78,6 @@ resources:
     # Optional Properties:
     #   location
     #   uniform_bucket_level_access (we suggest False for fine-grained access)
-    name: travel-sustainability
+    name: travel-impact-model
     location: US
     uniform_bucket_level_access: true
diff --git a/datasets/travel_impact_model/pipelines/flights_impact_data/flights_impact_data_dag.py b/datasets/travel_impact_model/pipelines/flights_impact_data/flights_impact_data_dag.py
@@ -24,7 +24,7 @@
 
 
 with DAG(
-    dag_id="travel_sustainability.flight_emissions",
+    dag_id="travel_impact_model.flights_impact_data",
     default_args=default_args,
     max_active_runs=1,
     schedule_interval="0 15 * * *",
@@ -33,12 +33,12 @@
 ) as dag:
 
     # Task to load CSV data to a BigQuery table
-    flight_emissions_gcs_to_bq = gcs_to_bigquery.GCSToBigQueryOperator(
-        task_id="flight_emissions_gcs_to_bq",
-        bucket="{{ var.json.travel_sustainability.source_bucket }}",
-        source_objects=["flight_emissions.csv"],
+    flights_impact_data_gcs_to_bq = gcs_to_bigquery.GCSToBigQueryOperator(
+        task_id="flights_impact_data_gcs_to_bq",
+        bucket="{{ var.json.travel_impact_model.source_bucket }}",
+        source_objects=["flights_impact_data.csv"],
         source_format="CSV",
-        destination_project_dataset_table="travel_sustainability.flight_emissions",
+        destination_project_dataset_table="travel_impact_model.flights_impact_data",
         skip_leading_rows=1,
         write_disposition="WRITE_TRUNCATE",
         schema_fields=[
@@ -123,4 +123,4 @@
         ],
     )
 
-    flight_emissions_gcs_to_bq
+    flights_impact_data_gcs_to_bq
diff --git a/datasets/travel_impact_model/pipelines/flights_impact_data/pipeline.yaml b/datasets/travel_impact_model/pipelines/flights_impact_data/pipeline.yaml
@@ -25,15 +25,15 @@ resources:
     # A Google BigQuery table to store your data. Requires a `bigquery_dataset`
     # to be specified in the config (i.e. `dataset.yaml) for the dataset that
     # this pipeline belongs in.
-    table_id: flight_emissions
-    description: "Flight emissions data"
+    table_id: flights_impact_data
+    description: "Flights impact data"
 
 dag:
   # [Required] Specify the Airflow version of the operators used by the DAG.
   airflow_version: 2
 
   initialize:
-    dag_id: flight_emissions
+    dag_id: flights_impact_data
     default_args:
       owner: "Google"
       depends_on_past: False
@@ -54,15 +54,15 @@ dag:
       # Arguments supported by this operator:
       # http://airflow.apache.org/docs/apache-airflow/stable/howto/operator/gcp/gcs.html#googlecloudstoragetobigqueryoperator
       args:
-        task_id: "flight_emissions_gcs_to_bq"
+        task_id: "flights_impact_data_gcs_to_bq"
 
         # The GCS bucket where the CSV file is located in.
-        bucket: "{{ var.json.travel_sustainability.source_bucket }}"
+        bucket: "{{ var.json.travel_impact_model.source_bucket }}"
 
         # Use the CSV file containing data from today
-        source_objects: ["flight_emissions.csv"]
+        source_objects: ["flights_impact_data.csv"]
         source_format: "CSV"
-        destination_project_dataset_table: "travel_sustainability.flight_emissions"
+        destination_project_dataset_table: "travel_impact_model.flights_impact_data"
 
         # Use this if your CSV file contains a header row
         skip_leading_rows: 1
@@ -126,4 +126,4 @@ dag:
             description: "Estimated CO2 in grams for one passenger in first cabin excluding non-CO2 effects"
 
   graph_paths:
-    - "flight_emissions_gcs_to_bq"
+    - "flights_impact_data_gcs_to_bq"
diff --git a/datasets/travel_impact_model/pipelines/metadata/metadata_dag.py b/datasets/travel_impact_model/pipelines/metadata/metadata_dag.py
@@ -24,7 +24,7 @@
 
 
 with DAG(
-    dag_id="travel_sustainability.metadata",
+    dag_id="travel_impact_model.metadata",
     default_args=default_args,
     max_active_runs=1,
     schedule_interval="0 15 * * *",
@@ -35,10 +35,10 @@
     # Task to load CSV data to a BigQuery table
     metadata_gcs_to_bq = gcs_to_bigquery.GCSToBigQueryOperator(
         task_id="metadata_gcs_to_bq",
-        bucket="{{ var.json.travel_sustainability.source_bucket }}",
+        bucket="{{ var.json.travel_impact_model.source_bucket }}",
         source_objects=["metadata.csv"],
         source_format="CSV",
-        destination_project_dataset_table="travel_sustainability.metadata",
+        destination_project_dataset_table="travel_impact_model.metadata",
         skip_leading_rows=1,
         write_disposition="WRITE_TRUNCATE",
         schema_fields=[
diff --git a/datasets/travel_impact_model/pipelines/metadata/pipeline.yaml b/datasets/travel_impact_model/pipelines/metadata/pipeline.yaml
@@ -57,12 +57,12 @@ dag:
         task_id: "metadata_gcs_to_bq"
 
         # The GCS bucket where the CSV file is located in.
-        bucket: "{{ var.json.travel_sustainability.source_bucket }}"
+        bucket: "{{ var.json.travel_impact_model.source_bucket }}"
 
         # Use the CSV file containing data from today
         source_objects: ["metadata.csv"]
         source_format: "CSV"
-        destination_project_dataset_table: "travel_sustainability.metadata"
+        destination_project_dataset_table: "travel_impact_model.metadata"
 
         # Use this if your CSV file contains a header row
         skip_leading_rows: 1
diff --git a/datasets/travel_sustainability/infra/travel_sustainability_dataset.tf b/datasets/travel_sustainability/infra/travel_sustainability_dataset.tf

Original file line number	Diff line number	Diff line change
`@@ -15,20 +15,20 @@`
`15`	`15`	`*/`
`16`	`16`
`17`	`17`
`18`		`-resource "google_bigquery_table" "travel_sustainability_metadata" {`
	`18`	`+resource "google_bigquery_table" "travel_impact_model_metadata" {`
`19`	`19`	`project = var.project_id`
`20`		`- dataset_id = "travel_sustainability"`
	`20`	`+ dataset_id = "travel_impact_model"`
`21`	`21`	`table_id = "metadata"`
`22`	`22`	`description = "Metadata about the dataset"`
`23`	`23`	`depends_on = [`
`24`		`- google_bigquery_dataset.travel_sustainability`
	`24`	`+ google_bigquery_dataset.travel_impact_model`
`25`	`25`	`]`
`26`	`26`	`}`
`27`	`27`
`28`		`-output "bigquery_table-travel_sustainability_metadata-table_id" {`
`29`		`- value = google_bigquery_table.travel_sustainability_metadata.table_id`
	`28`	`+output "bigquery_table-travel_impact_model_metadata-table_id" {`
	`29`	`+ value = google_bigquery_table.travel_impact_model_metadata.table_id`
`30`	`30`	`}`
`31`	`31`
`32`		`-output "bigquery_table-travel_sustainability_metadata-id" {`
`33`		`- value = google_bigquery_table.travel_sustainability_metadata.id`
	`32`	`+output "bigquery_table-travel_impact_model_metadata-id" {`
	`33`	`+ value = google_bigquery_table.travel_impact_model_metadata.id`
`34`	`34`	`}`