feat: Rename Travel Sustainability to Travel Impact Model (#351)

GoogleCloudPlatform · Apr 30, 2022 · 83df285 · 83df285
1 parent 03b4f89
commit 83df285
Show file tree

Hide file tree

Showing 11 changed files with 113 additions and 82 deletions.
diff --git a/...bility/infra/flight_emissions_pipeline.tf → ...del/infra/flights_impact_data_pipeline.tf b/...bility/infra/flight_emissions_pipeline.tf → ...del/infra/flights_impact_data_pipeline.tf
@@ -15,20 +15,20 @@
  */
 
 
-resource "google_bigquery_table" "travel_sustainability_flight_emissions" {
+resource "google_bigquery_table" "travel_impact_model_flights_impact_data" {
  project = var.project_id
- dataset_id = "travel_sustainability"
- table_id = "flight_emissions"
- description = "Flight emissions data"
+ dataset_id = "travel_impact_model"
+ table_id = "flights_impact_data"
+ description = "Flights impact data"
  depends_on = [
- google_bigquery_dataset.travel_sustainability
+ google_bigquery_dataset.travel_impact_model
  ]
 }
 
-output "bigquery_table-travel_sustainability_flight_emissions-table_id" {
- value = google_bigquery_table.travel_sustainability_flight_emissions.table_id
+output "bigquery_table-travel_impact_model_flights_impact_data-table_id" {
+ value = google_bigquery_table.travel_impact_model_flights_impact_data.table_id
 }
 
-output "bigquery_table-travel_sustainability_flight_emissions-id" {
- value = google_bigquery_table.travel_sustainability_flight_emissions.id
+output "bigquery_table-travel_impact_model_flights_impact_data-id" {
+ value = google_bigquery_table.travel_impact_model_flights_impact_data.id
 }
diff --git a/...sustainability/infra/metadata_pipeline.tf → ...l_impact_model/infra/metadata_pipeline.tf b/...sustainability/infra/metadata_pipeline.tf → ...l_impact_model/infra/metadata_pipeline.tf
@@ -15,20 +15,20 @@
  */
 
 
-resource "google_bigquery_table" "travel_sustainability_metadata" {
+resource "google_bigquery_table" "travel_impact_model_metadata" {
  project = var.project_id
- dataset_id = "travel_sustainability"
+ dataset_id = "travel_impact_model"
  table_id = "metadata"
  description = "Metadata about the dataset"
  depends_on = [
- google_bigquery_dataset.travel_sustainability
+ google_bigquery_dataset.travel_impact_model
  ]
 }
 
-output "bigquery_table-travel_sustainability_metadata-table_id" {
- value = google_bigquery_table.travel_sustainability_metadata.table_id
+output "bigquery_table-travel_impact_model_metadata-table_id" {
+ value = google_bigquery_table.travel_impact_model_metadata.table_id
 }
 
-output "bigquery_table-travel_sustainability_metadata-id" {
- value = google_bigquery_table.travel_sustainability_metadata.id
+output "bigquery_table-travel_impact_model_metadata-id" {
+ value = google_bigquery_table.travel_impact_model_metadata.id
 }
diff --git a/...s/travel_sustainability/infra/provider.tf → ...ets/travel_impact_model/infra/provider.tf b/...s/travel_sustainability/infra/provider.tf → ...ets/travel_impact_model/infra/provider.tf
diff --git a/datasets/travel_impact_model/infra/travel_impact_model_dataset.tf b/datasets/travel_impact_model/infra/travel_impact_model_dataset.tf
@@ -0,0 +1,70 @@
+/**
+ * Copyright 2021 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+resource "google_bigquery_dataset" "travel_impact_model" {
+ dataset_id = "travel_impact_model"
+ project = var.project_id
+ description = "Travel Impact Model Data"
+}
+
+data "google_iam_policy" "bq_ds__travel_impact_model" {
+ dynamic "binding" {
+ for_each = var.iam_policies["bigquery_datasets"]["travel_impact_model"]
+ content {
+ role = binding.value["role"]
+ members = binding.value["members"]
+ }
+ }
+}
+
+resource "google_bigquery_dataset_iam_policy" "travel_impact_model" {
+ dataset_id = google_bigquery_dataset.travel_impact_model.dataset_id
+ policy_data = data.google_iam_policy.bq_ds__travel_impact_model.policy_data
+}
+output "bigquery_dataset-travel_impact_model-dataset_id" {
+ value = google_bigquery_dataset.travel_impact_model.dataset_id
+}
+
+resource "google_storage_bucket" "travel-impact-model" {
+ name = "${var.bucket_name_prefix}-travel-impact-model"
+ force_destroy = true
+ location = "US"
+ uniform_bucket_level_access = true
+ lifecycle {
+ ignore_changes = [
+ logging,
+ ]
+ }
+}
+
+data "google_iam_policy" "storage_bucket__travel-impact-model" {
+ dynamic "binding" {
+ for_each = var.iam_policies["storage_buckets"]["travel-impact-model"]
+ content {
+ role = binding.value["role"]
+ members = binding.value["members"]
+ }
+ }
+}
+
+resource "google_storage_bucket_iam_policy" "travel-impact-model" {
+ bucket = google_storage_bucket.travel-impact-model.name
+ policy_data = data.google_iam_policy.storage_bucket__travel-impact-model.policy_data
+}
+output "storage_bucket-travel-impact-model-name" {
+ value = google_storage_bucket.travel-impact-model.name
+}
diff --git a/.../travel_sustainability/infra/variables.tf → ...ts/travel_impact_model/infra/variables.tf b/.../travel_sustainability/infra/variables.tf → ...ts/travel_impact_model/infra/variables.tf
@@ -20,4 +20,7 @@ variable "bucket_name_prefix" {}
 variable "impersonating_acct" {}
 variable "region" {}
 variable "env" {}
+variable "iam_policies" {
+ default = {}
+}
 
diff --git a/...vel_sustainability/pipelines/dataset.yaml → ...ravel_impact_model/pipelines/dataset.yaml b/...vel_sustainability/pipelines/dataset.yaml → ...ravel_impact_model/pipelines/dataset.yaml
@@ -17,7 +17,7 @@ dataset:
  # to users of your data on the Google Cloud website.
 
  # Must be exactly the same name as the folder name your dataset.yaml is in.
- name: travel_sustainability
+ name: travel_impact_model
 
  # A friendly, human-readable name of the dataset
  friendly_name: ~
@@ -52,8 +52,8 @@ resources:
  # friendly_name (A user-friendly name of the dataset)
  # description (A user-friendly description of the dataset)
  # location (The geographic location where the dataset should reside)
- dataset_id: travel_sustainability
- description: "Travel Sustainability"
+ dataset_id: travel_impact_model
+ description: "Travel Impact Model Data"
 
  - type: storage_bucket
  # Google Cloud Storage Bucket that your pipelines need. Say, you need an
@@ -78,6 +78,6 @@ resources:
  # Optional Properties:
  # location
  # uniform_bucket_level_access (we suggest False for fine-grained access)
- name: travel-sustainability
+ name: travel-impact-model
  location: US
  uniform_bucket_level_access: true
diff --git a/.../flight_emissions/flight_emissions_dag.py → ...ts_impact_data/flights_impact_data_dag.py b/.../flight_emissions/flight_emissions_dag.py → ...ts_impact_data/flights_impact_data_dag.py
@@ -24,7 +24,7 @@
 
 
 with DAG(
- dag_id="travel_sustainability.flight_emissions",
+ dag_id="travel_impact_model.flights_impact_data",
  default_args=default_args,
  max_active_runs=1,
  schedule_interval="0 15 * * *",
@@ -33,12 +33,12 @@
 ) as dag:
 
  # Task to load CSV data to a BigQuery table
- flight_emissions_gcs_to_bq = gcs_to_bigquery.GCSToBigQueryOperator(
- task_id="flight_emissions_gcs_to_bq",
- bucket="{{ var.json.travel_sustainability.source_bucket }}",
- source_objects=["flight_emissions.csv"],
+ flights_impact_data_gcs_to_bq = gcs_to_bigquery.GCSToBigQueryOperator(
+ task_id="flights_impact_data_gcs_to_bq",
+ bucket="{{ var.json.travel_impact_model.source_bucket }}",
+ source_objects=["flights_impact_data.csv"],
  source_format="CSV",
- destination_project_dataset_table="travel_sustainability.flight_emissions",
+ destination_project_dataset_table="travel_impact_model.flights_impact_data",
  skip_leading_rows=1,
  write_disposition="WRITE_TRUNCATE",
  schema_fields=[
@@ -123,4 +123,4 @@
  ],
  )
 
- flight_emissions_gcs_to_bq
+ flights_impact_data_gcs_to_bq
diff --git a/.../pipelines/flight_emissions/pipeline.yaml → ...pelines/flights_impact_data/pipeline.yaml b/.../pipelines/flight_emissions/pipeline.yaml → ...pelines/flights_impact_data/pipeline.yaml
@@ -25,15 +25,15 @@ resources:
  # A Google BigQuery table to store your data. Requires a `bigquery_dataset`
  # to be specified in the config (i.e. `dataset.yaml) for the dataset that
  # this pipeline belongs in.
- table_id: flight_emissions
- description: "Flight emissions data"
+ table_id: flights_impact_data
+ description: "Flights impact data"
 
 dag:
  # [Required] Specify the Airflow version of the operators used by the DAG.
  airflow_version: 2
 
  initialize:
- dag_id: flight_emissions
+ dag_id: flights_impact_data
  default_args:
  owner: "Google"
  depends_on_past: False
@@ -54,15 +54,15 @@ dag:
  # Arguments supported by this operator:
  # http://airflow.apache.org/docs/apache-airflow/stable/howto/operator/gcp/gcs.html#googlecloudstoragetobigqueryoperator
  args:
- task_id: "flight_emissions_gcs_to_bq"
+ task_id: "flights_impact_data_gcs_to_bq"
 
  # The GCS bucket where the CSV file is located in.
- bucket: "{{ var.json.travel_sustainability.source_bucket }}"
+ bucket: "{{ var.json.travel_impact_model.source_bucket }}"
 
  # Use the CSV file containing data from today
- source_objects: ["flight_emissions.csv"]
+ source_objects: ["flights_impact_data.csv"]
  source_format: "CSV"
- destination_project_dataset_table: "travel_sustainability.flight_emissions"
+ destination_project_dataset_table: "travel_impact_model.flights_impact_data"
 
  # Use this if your CSV file contains a header row
  skip_leading_rows: 1
@@ -126,4 +126,4 @@ dag:
  description: "Estimated CO2 in grams for one passenger in first cabin excluding non-CO2 effects"
 
  graph_paths:
- - "flight_emissions_gcs_to_bq"
+ - "flights_impact_data_gcs_to_bq"
diff --git a/...bility/pipelines/metadata/metadata_dag.py → ..._model/pipelines/metadata/metadata_dag.py b/...bility/pipelines/metadata/metadata_dag.py → ..._model/pipelines/metadata/metadata_dag.py
@@ -24,7 +24,7 @@
 
 
 with DAG(
- dag_id="travel_sustainability.metadata",
+ dag_id="travel_impact_model.metadata",
  default_args=default_args,
  max_active_runs=1,
  schedule_interval="0 15 * * *",
@@ -35,10 +35,10 @@
  # Task to load CSV data to a BigQuery table
  metadata_gcs_to_bq = gcs_to_bigquery.GCSToBigQueryOperator(
  task_id="metadata_gcs_to_bq",
- bucket="{{ var.json.travel_sustainability.source_bucket }}",
+ bucket="{{ var.json.travel_impact_model.source_bucket }}",
  source_objects=["metadata.csv"],
  source_format="CSV",
- destination_project_dataset_table="travel_sustainability.metadata",
+ destination_project_dataset_table="travel_impact_model.metadata",
  skip_leading_rows=1,
  write_disposition="WRITE_TRUNCATE",
  schema_fields=[

diff --git a/...nability/pipelines/metadata/pipeline.yaml → ...ct_model/pipelines/metadata/pipeline.yaml b/...nability/pipelines/metadata/pipeline.yaml → ...ct_model/pipelines/metadata/pipeline.yaml
@@ -57,12 +57,12 @@ dag:
  task_id: "metadata_gcs_to_bq"
 
  # The GCS bucket where the CSV file is located in.
- bucket: "{{ var.json.travel_sustainability.source_bucket }}"
+ bucket: "{{ var.json.travel_impact_model.source_bucket }}"
 
  # Use the CSV file containing data from today
  source_objects: ["metadata.csv"]
  source_format: "CSV"
- destination_project_dataset_table: "travel_sustainability.metadata"
+ destination_project_dataset_table: "travel_impact_model.metadata"
 
  # Use this if your CSV file contains a header row
  skip_leading_rows: 1

diff --git a/datasets/travel_sustainability/infra/travel_sustainability_dataset.tf b/datasets/travel_sustainability/infra/travel_sustainability_dataset.tf