Feat: Onboard Cloud Storage Geo Index (#367)

* fix: Added partitioning and clustering. Not production ready. * fix: Works for landsat_index * fix: Added sentinel-2 pipeline as well as clustering. * fix: Clean up. * fix: attempt to resolve pod not started. * fix: various fixes to pipeline.yaml which caused 'pod not started' issues. Added bigquery to requirements.txt * fix: Resolve resource issues in pipeline.yaml. * fix: Moved schema JSON files as per code review request. * fix: Exposed variable values into pipeline.yaml as per code review. * Update pipeline.yaml Co-authored-by: Adler Santos <[email protected]>
GoogleCloudPlatform · Jun 2, 2022 · 63cdb2a · 63cdb2a
1 parent ba473c6
commit 63cdb2a
Show file tree

Hide file tree

Showing 10 changed files with 1,073 additions and 498 deletions.
diff --git a/...pipelines/_images/run_csv_transform_kub/cloud_storage_geo_index_landsat_index_schema.json b/...pipelines/_images/run_csv_transform_kub/cloud_storage_geo_index_landsat_index_schema.json
@@ -0,0 +1,104 @@
+[
+ {
+ "name": "scene_id",
+ "type": "STRING",
+ "mode": "REQUIRED"
+ },
+ {
+ "name": "product_id",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "spacecraft_id",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "sensor_id",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "date_acquired",
+ "type": "DATE",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "sensing_time",
+ "type": "TIMESTAMP",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "collection_number",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "collection_category",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "data_type",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "wrs_path",
+ "type": "INTEGER",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "wrs_row",
+ "type": "INTEGER",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "cloud_cover",
+ "type": "FLOAT",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "north_lat",
+ "type": "FLOAT",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "south_lat",
+ "type": "FLOAT",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "west_lon",
+ "type": "FLOAt",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "east_lon",
+ "type": "FLOAT",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "total_size",
+ "type": "INTEGER",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "base_url",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "source_url",
+ "type": "STRING",
+ "description": "Source ",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "etl_timestamp",
+ "type": "TIMESTAMP",
+ "description": "Load time for this data row",
+ "mode": "NULLABLE"
+ }
+]
diff --git a/...ex/pipelines/_images/run_csv_transform_kub/cloud_storage_geo_index_sentinel_2_schema.json b/...ex/pipelines/_images/run_csv_transform_kub/cloud_storage_geo_index_sentinel_2_schema.json
@@ -0,0 +1,84 @@
+[
+ {
+ "name": "granule_id",
+ "type": "STRING",
+ "mode": "REQUIRED"
+ },
+ {
+ "name": "product_id",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "datatake_identifier",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "mgrs_tile",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "sensing_time",
+ "type": "TIMESTAMP",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "geometric_quality_flag",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "generation_time",
+ "type": "TIMESTAMP",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "north_lat",
+ "type": "FLOAT",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "south_lat",
+ "type": "FLOAT",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "west_lon",
+ "type": "FLOAT",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "east_lon",
+ "type": "FLOAT",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "base_url",
+ "type": "STRING",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "total_size",
+ "type": "INTEGER",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "cloud_cover",
+ "type": "FLOAT",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "source_url",
+ "type": "STRING",
+ "description": "Source ",
+ "mode": "NULLABLE"
+ },
+ {
+ "name": "etl_timestamp",
+ "type": "TIMESTAMP",
+ "description": "Load time for this data row",
+ "mode": "NULLABLE"
+ }
+]