Skip to content

Commit 3348e3e

Browse files
gcf-owl-bot[bot]feywind
gcf-owl-bot[bot]
and
feywind
authored
feat: Add SchemaViolationReason to IngestionFailureEvent (#2045)
* feat: Add SchemaViolationReason to IngestionFailureEvent docs: Standardize spelling of "acknowledgment" in Pub/Sub protos PiperOrigin-RevId: 761506540 Source-Link: googleapis/googleapis@c1af7f2 Source-Link: googleapis/googleapis-gen@260979c Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMjYwOTc5Y2M2NDg3MjJlY2UyODgxOTY2NWIyZWNkMWFiMGZiNjFhNCJ9 * build: owl-bot-staging can't be linted due to its duplicate .eslintrc.json * build: also update the owl-bot-staging copy of .eslintignore * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: feywind <[email protected]>
1 parent 62e73b8 commit 3348e3e

14 files changed

+2492
-425
lines changed

‎.eslintignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,4 @@ protos/
66
system-test/
77
samples/generated/
88
samples/typescript/
9+
owl-bot-staging/

‎.github/scripts/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
"js-yaml": "^4.1.0"
1515
},
1616
"devDependencies": {
17-
"@octokit/rest": "^21.0.0",
17+
"@octokit/rest": "^19.0.0",
1818
"mocha": "^10.0.0",
1919
"sinon": "^18.0.0"
2020
}

‎protos/google/pubsub/v1/pubsub.proto

Lines changed: 144 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2024 Google LLC
1+
// Copyright 2025 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -26,9 +26,8 @@ import "google/protobuf/field_mask.proto";
2626
import "google/protobuf/timestamp.proto";
2727
import "google/pubsub/v1/schema.proto";
2828

29-
option cc_enable_arenas = true;
3029
option csharp_namespace = "Google.Cloud.PubSub.V1";
31-
option go_package = "cloud.google.com/go/pubsub/apiv1/pubsubpb;pubsubpb";
30+
option go_package = "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb;pubsubpb";
3231
option java_multiple_files = true;
3332
option java_outer_classname = "PubsubProto";
3433
option java_package = "com.google.pubsub.v1";
@@ -550,6 +549,10 @@ message IngestionFailureEvent {
550549
// occurs, one or more Avro objects won't be ingested.
551550
message AvroFailureReason {}
552551

552+
// Set when a Pub/Sub message fails to get published due to a schema
553+
// validation violation.
554+
message SchemaViolationReason {}
555+
553556
// Failure when ingesting from a Cloud Storage source.
554557
message CloudStorageFailure {
555558
// Optional. Name of the Cloud Storage bucket used for ingestion.
@@ -573,6 +576,10 @@ message IngestionFailureEvent {
573576
// being published.
574577
ApiViolationReason api_violation_reason = 6
575578
[(google.api.field_behavior) = OPTIONAL];
579+
580+
// Optional. The Pub/Sub message failed schema validation.
581+
SchemaViolationReason schema_violation_reason = 7
582+
[(google.api.field_behavior) = OPTIONAL];
576583
}
577584
}
578585

@@ -597,6 +604,10 @@ message IngestionFailureEvent {
597604
// being published.
598605
ApiViolationReason api_violation_reason = 5
599606
[(google.api.field_behavior) = OPTIONAL];
607+
608+
// Optional. The Pub/Sub message failed schema validation.
609+
SchemaViolationReason schema_violation_reason = 6
610+
[(google.api.field_behavior) = OPTIONAL];
600611
}
601612
}
602613

@@ -621,6 +632,10 @@ message IngestionFailureEvent {
621632
// being published.
622633
ApiViolationReason api_violation_reason = 5
623634
[(google.api.field_behavior) = OPTIONAL];
635+
636+
// Optional. The Pub/Sub message failed schema validation.
637+
SchemaViolationReason schema_violation_reason = 6
638+
[(google.api.field_behavior) = OPTIONAL];
624639
}
625640
}
626641

@@ -645,6 +660,29 @@ message IngestionFailureEvent {
645660
// being published.
646661
ApiViolationReason api_violation_reason = 5
647662
[(google.api.field_behavior) = OPTIONAL];
663+
664+
// Optional. The Pub/Sub message failed schema validation.
665+
SchemaViolationReason schema_violation_reason = 6
666+
[(google.api.field_behavior) = OPTIONAL];
667+
}
668+
}
669+
670+
// Failure when ingesting from an AWS Kinesis source.
671+
message AwsKinesisFailureReason {
672+
// Optional. The stream ARN of the Kinesis stream being ingested from.
673+
string stream_arn = 1 [(google.api.field_behavior) = OPTIONAL];
674+
675+
// Optional. The partition key of the message that failed to be ingested.
676+
string partition_key = 2 [(google.api.field_behavior) = OPTIONAL];
677+
678+
// Optional. The sequence number of the message that failed to be ingested.
679+
string sequence_number = 3 [(google.api.field_behavior) = OPTIONAL];
680+
681+
// Reason why ingestion failed for the specified message.
682+
oneof reason {
683+
// Optional. The Pub/Sub message failed schema validation.
684+
SchemaViolationReason schema_violation_reason = 4
685+
[(google.api.field_behavior) = OPTIONAL];
648686
}
649687
}
650688

@@ -671,7 +709,69 @@ message IngestionFailureEvent {
671709
// Optional. Failure when ingesting from Confluent Cloud.
672710
ConfluentCloudFailureReason confluent_cloud_failure = 6
673711
[(google.api.field_behavior) = OPTIONAL];
712+
713+
// Optional. Failure when ingesting from AWS Kinesis.
714+
AwsKinesisFailureReason aws_kinesis_failure = 7
715+
[(google.api.field_behavior) = OPTIONAL];
716+
}
717+
}
718+
719+
// User-defined JavaScript function that can transform or filter a Pub/Sub
720+
// message.
721+
message JavaScriptUDF {
722+
// Required. Name of the JavasScript function that should applied to Pub/Sub
723+
// messages.
724+
string function_name = 1 [(google.api.field_behavior) = REQUIRED];
725+
726+
// Required. JavaScript code that contains a function `function_name` with the
727+
// below signature:
728+
//
729+
// ```
730+
// /**
731+
// * Transforms a Pub/Sub message.
732+
//
733+
// * @return {(Object<string, (string | Object<string, string>)>|null)} - To
734+
// * filter a message, return `null`. To transform a message return a map
735+
// * with the following keys:
736+
// * - (required) 'data' : {string}
737+
// * - (optional) 'attributes' : {Object<string, string>}
738+
// * Returning empty `attributes` will remove all attributes from the
739+
// * message.
740+
// *
741+
// * @param {(Object<string, (string | Object<string, string>)>} Pub/Sub
742+
// * message. Keys:
743+
// * - (required) 'data' : {string}
744+
// * - (required) 'attributes' : {Object<string, string>}
745+
// *
746+
// * @param {Object<string, any>} metadata - Pub/Sub message metadata.
747+
// * Keys:
748+
// * - (optional) 'message_id' : {string}
749+
// * - (optional) 'publish_time': {string} YYYY-MM-DDTHH:MM:SSZ format
750+
// * - (optional) 'ordering_key': {string}
751+
// */
752+
//
753+
// function <function_name>(message, metadata) {
754+
// }
755+
// ```
756+
string code = 2 [(google.api.field_behavior) = REQUIRED];
757+
}
758+
759+
// All supported message transforms types.
760+
message MessageTransform {
761+
// The type of transform to apply to messages.
762+
oneof transform {
763+
// Optional. JavaScript User Defined Function. If multiple JavaScriptUDF's
764+
// are specified on a resource, each must have a unique `function_name`.
765+
JavaScriptUDF javascript_udf = 2 [(google.api.field_behavior) = OPTIONAL];
674766
}
767+
768+
// Optional. This field is deprecated, use the `disabled` field to disable
769+
// transforms.
770+
bool enabled = 3 [deprecated = true, (google.api.field_behavior) = OPTIONAL];
771+
772+
// Optional. If true, the transform is disabled and will not be applied to
773+
// messages. Defaults to `false`.
774+
bool disabled = 4 [(google.api.field_behavior) = OPTIONAL];
675775
}
676776

677777
// A topic resource.
@@ -680,6 +780,8 @@ message Topic {
680780
type: "pubsub.googleapis.com/Topic"
681781
pattern: "projects/{project}/topics/{topic}"
682782
pattern: "_deleted-topic_"
783+
plural: "topics"
784+
singular: "topic"
683785
};
684786

685787
// The state of the topic.
@@ -745,6 +847,11 @@ message Topic {
745847
// Optional. Settings for ingestion from a data source into this topic.
746848
IngestionDataSourceSettings ingestion_data_source_settings = 10
747849
[(google.api.field_behavior) = OPTIONAL];
850+
851+
// Optional. Transforms to be applied to messages published to the topic.
852+
// Transforms are applied in the order specified.
853+
repeated MessageTransform message_transforms = 13
854+
[(google.api.field_behavior) = OPTIONAL];
748855
}
749856

750857
// A message that is published by publishers and consumed by subscribers. The
@@ -1064,7 +1171,7 @@ service Subscriber {
10641171
}
10651172

10661173
// Establishes a stream with the server, which sends messages down to the
1067-
// client. The client streams acknowledgements and ack deadline modifications
1174+
// client. The client streams acknowledgments and ack deadline modifications
10681175
// back to the server. The server will close the stream and return the status
10691176
// on any error. The server may close the stream with status `UNAVAILABLE` to
10701177
// reassign server-side resources, in which case, the client should
@@ -1188,6 +1295,8 @@ message Subscription {
11881295
option (google.api.resource) = {
11891296
type: "pubsub.googleapis.com/Subscription"
11901297
pattern: "projects/{project}/subscriptions/{subscription}"
1298+
plural: "subscriptions"
1299+
singular: "subscription"
11911300
};
11921301

11931302
// Possible states for a subscription.
@@ -1204,8 +1313,8 @@ message Subscription {
12041313
RESOURCE_ERROR = 2;
12051314
}
12061315

1207-
// Information about an associated Analytics Hub subscription
1208-
// (https://cloud.google.com/bigquery/docs/analytics-hub-manage-subscriptions).
1316+
// Information about an associated [Analytics Hub
1317+
// subscription](https://cloud.google.com/bigquery/docs/analytics-hub-manage-subscriptions).
12091318
message AnalyticsHubSubscriptionInfo {
12101319
// Optional. The name of the associated Analytics Hub listing resource.
12111320
// Pattern:
@@ -1328,8 +1437,8 @@ message Subscription {
13281437
//
13291438
// If not set, the default retry policy is applied. This generally implies
13301439
// that messages will be retried as soon as possible for healthy subscribers.
1331-
// RetryPolicy will be triggered on NACKs or acknowledgement deadline
1332-
// exceeded events for a given message.
1440+
// RetryPolicy will be triggered on NACKs or acknowledgment deadline exceeded
1441+
// events for a given message.
13331442
RetryPolicy retry_policy = 14 [(google.api.field_behavior) = OPTIONAL];
13341443

13351444
// Optional. Indicates whether the subscription is detached from its topic.
@@ -1344,7 +1453,7 @@ message Subscription {
13441453
// subscription:
13451454
//
13461455
// * The message sent to a subscriber is guaranteed not to be resent
1347-
// before the message's acknowledgement deadline expires.
1456+
// before the message's acknowledgment deadline expires.
13481457
// * An acknowledged message will not be resent to a subscriber.
13491458
//
13501459
// Note that subscribers may still receive multiple copies of a message
@@ -1371,14 +1480,19 @@ message Subscription {
13711480
// Only set if the subscritpion is created by Analytics Hub.
13721481
AnalyticsHubSubscriptionInfo analytics_hub_subscription_info = 23
13731482
[(google.api.field_behavior) = OUTPUT_ONLY];
1483+
1484+
// Optional. Transforms to be applied to messages before they are delivered to
1485+
// subscribers. Transforms are applied in the order specified.
1486+
repeated MessageTransform message_transforms = 25
1487+
[(google.api.field_behavior) = OPTIONAL];
13741488
}
13751489

13761490
// A policy that specifies how Pub/Sub retries message delivery.
13771491
//
13781492
// Retry delay will be exponential based on provided minimum and maximum
13791493
// backoffs. https://en.wikipedia.org/wiki/Exponential_backoff.
13801494
//
1381-
// RetryPolicy will be triggered on NACKs or acknowledgement deadline exceeded
1495+
// RetryPolicy will be triggered on NACKs or acknowledgment deadline exceeded
13821496
// events for a given message.
13831497
//
13841498
// Retry Policy is implemented on a best effort basis. At times, the delay
@@ -1418,7 +1532,7 @@ message DeadLetterPolicy {
14181532
// value must be between 5 and 100.
14191533
//
14201534
// The number of delivery attempts is defined as 1 + (the sum of number of
1421-
// NACKs and number of times the acknowledgement deadline has been exceeded
1535+
// NACKs and number of times the acknowledgment deadline has been exceeded
14221536
// for the message).
14231537
//
14241538
// A NACK is any call to ModifyAckDeadline with a 0 deadline. Note that
@@ -1680,7 +1794,7 @@ message CloudStorageConfig {
16801794

16811795
// Optional. The maximum duration that can elapse before a new Cloud Storage
16821796
// file is created. Min 1 minute, max 10 minutes, default 5 minutes. May not
1683-
// exceed the subscription's acknowledgement deadline.
1797+
// exceed the subscription's acknowledgment deadline.
16841798
google.protobuf.Duration max_duration = 6
16851799
[(google.api.field_behavior) = OPTIONAL];
16861800

@@ -1902,7 +2016,7 @@ message AcknowledgeRequest {
19022016
}
19032017

19042018
// Request for the `StreamingPull` streaming RPC method. This request is used to
1905-
// establish the initial stream as well as to stream acknowledgements and ack
2019+
// establish the initial stream as well as to stream acknowledgments and ack
19062020
// deadline modifications from the client to the server.
19072021
message StreamingPullRequest {
19082022
// Required. The subscription for which to initialize the new stream. This
@@ -1916,12 +2030,11 @@ message StreamingPullRequest {
19162030
}
19172031
];
19182032

1919-
// Optional. List of acknowledgement IDs for acknowledging previously received
2033+
// Optional. List of acknowledgment IDs for acknowledging previously received
19202034
// messages (received on this stream or a different stream). If an ack ID has
19212035
// expired, the corresponding message may be redelivered later. Acknowledging
1922-
// a message more than once will not result in an error. If the
1923-
// acknowledgement ID is malformed, the stream will be aborted with status
1924-
// `INVALID_ARGUMENT`.
2036+
// a message more than once will not result in an error. If the acknowledgment
2037+
// ID is malformed, the stream will be aborted with status `INVALID_ARGUMENT`.
19252038
repeated string ack_ids = 2 [(google.api.field_behavior) = OPTIONAL];
19262039

19272040
// Optional. The list of new ack deadlines for the IDs listed in
@@ -1938,7 +2051,7 @@ message StreamingPullRequest {
19382051
repeated int32 modify_deadline_seconds = 3
19392052
[(google.api.field_behavior) = OPTIONAL];
19402053

1941-
// Optional. List of acknowledgement IDs whose deadline will be modified based
2054+
// Optional. List of acknowledgment IDs whose deadline will be modified based
19422055
// on the corresponding element in `modify_deadline_seconds`. This field can
19432056
// be used to indicate that more time is needed to process a message by the
19442057
// subscriber, or to make the message available for redelivery if the
@@ -1987,39 +2100,39 @@ message StreamingPullRequest {
19872100
// Response for the `StreamingPull` method. This response is used to stream
19882101
// messages from the server to the client.
19892102
message StreamingPullResponse {
1990-
// Acknowledgement IDs sent in one or more previous requests to acknowledge a
2103+
// Acknowledgment IDs sent in one or more previous requests to acknowledge a
19912104
// previously received message.
19922105
message AcknowledgeConfirmation {
1993-
// Optional. Successfully processed acknowledgement IDs.
2106+
// Optional. Successfully processed acknowledgment IDs.
19942107
repeated string ack_ids = 1 [(google.api.field_behavior) = OPTIONAL];
19952108

1996-
// Optional. List of acknowledgement IDs that were malformed or whose
1997-
// acknowledgement deadline has expired.
2109+
// Optional. List of acknowledgment IDs that were malformed or whose
2110+
// acknowledgment deadline has expired.
19982111
repeated string invalid_ack_ids = 2
19992112
[(google.api.field_behavior) = OPTIONAL];
20002113

2001-
// Optional. List of acknowledgement IDs that were out of order.
2114+
// Optional. List of acknowledgment IDs that were out of order.
20022115
repeated string unordered_ack_ids = 3
20032116
[(google.api.field_behavior) = OPTIONAL];
20042117

2005-
// Optional. List of acknowledgement IDs that failed processing with
2118+
// Optional. List of acknowledgment IDs that failed processing with
20062119
// temporary issues.
20072120
repeated string temporary_failed_ack_ids = 4
20082121
[(google.api.field_behavior) = OPTIONAL];
20092122
}
20102123

2011-
// Acknowledgement IDs sent in one or more previous requests to modify the
2124+
// Acknowledgment IDs sent in one or more previous requests to modify the
20122125
// deadline for a specific message.
20132126
message ModifyAckDeadlineConfirmation {
2014-
// Optional. Successfully processed acknowledgement IDs.
2127+
// Optional. Successfully processed acknowledgment IDs.
20152128
repeated string ack_ids = 1 [(google.api.field_behavior) = OPTIONAL];
20162129

2017-
// Optional. List of acknowledgement IDs that were malformed or whose
2018-
// acknowledgement deadline has expired.
2130+
// Optional. List of acknowledgment IDs that were malformed or whose
2131+
// acknowledgment deadline has expired.
20192132
repeated string invalid_ack_ids = 2
20202133
[(google.api.field_behavior) = OPTIONAL];
20212134

2022-
// Optional. List of acknowledgement IDs that failed processing with
2135+
// Optional. List of acknowledgment IDs that failed processing with
20232136
// temporary issues.
20242137
repeated string temporary_failed_ack_ids = 3
20252138
[(google.api.field_behavior) = OPTIONAL];
@@ -2109,6 +2222,8 @@ message Snapshot {
21092222
option (google.api.resource) = {
21102223
type: "pubsub.googleapis.com/Snapshot"
21112224
pattern: "projects/{project}/snapshots/{snapshot}"
2225+
plural: "snapshots"
2226+
singular: "snapshot"
21122227
};
21132228

21142229
// Optional. The name of the snapshot.

‎protos/google/pubsub/v1/schema.proto

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright 2024 Google LLC
1+
// Copyright 2025 Google LLC
22
//
33
// Licensed under the Apache License, Version 2.0 (the "License");
44
// you may not use this file except in compliance with the License.
@@ -23,9 +23,8 @@ import "google/api/resource.proto";
2323
import "google/protobuf/empty.proto";
2424
import "google/protobuf/timestamp.proto";
2525

26-
option cc_enable_arenas = true;
2726
option csharp_namespace = "Google.Cloud.PubSub.V1";
28-
option go_package = "cloud.google.com/go/pubsub/apiv1/pubsubpb;pubsubpb";
27+
option go_package = "cloud.google.com/go/pubsub/v2/apiv1/pubsubpb;pubsubpb";
2928
option java_multiple_files = true;
3029
option java_outer_classname = "SchemaProto";
3130
option java_package = "com.google.pubsub.v1";

0 commit comments

Comments
 (0)