diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index a5cf27b97..e2d32b255 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -8,7 +8,7 @@ execution_time_limit: global_job_config: env_vars: - name: LIBRDKAFKA_VERSION - value: v2.5.0 + value: v2.5.3 prologue: commands: - checkout diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cbf09a6d..c9857786d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,34 @@ # Confluent's Python client for Apache Kafka +## v2.5.3 + +v2.5.3 is a maintenance release with the following fixes and enhancements: + +### Fixes + +* Fix an assert being triggered during push telemetry call when no metrics matched on the client side. +* Minor enhancement to JSONDeserializer to retrieve schema from Schema Registry if not provided + +confluent-kafka-python is based on librdkafka v2.5.3, see the +[librdkafka release notes](https://github.com/edenhill/librdkafka/releases/tag/v2.5.3) +for a complete list of changes, enhancements, fixes and upgrade considerations. + + + ## v2.5.0 +> [!WARNING] +This version has introduced a regression in which an assert is triggered during **PushTelemetry** call. This happens when no metric is matched on the client side among those requested by broker subscription. +> +> You won't face any problem if: +> * Broker doesn't support [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability). +> * [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) feature is disabled on the broker side. +> * [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) feature is disabled on the client side. This is enabled by default. Set configuration `enable.metrics.push` to `false`. +> * If [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) is enabled on the broker side and there is no subscription configured there. +> * If [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) is enabled on the broker side with subscriptions that match the [KIP-714](https://cwiki.apache.org/confluence/display/KAFKA/KIP-714%3A+Client+metrics+and+observability) metrics defined on the client. +> +> Having said this, we strongly recommend using `v2.5.3` and above to not face this regression at all. + v2.5.0 is a feature release with the following features, fixes and enhancements: - [KIP-107](https://cwiki.apache.org/confluence/display/KAFKA/KIP-107%3A+Add+deleteRecordsBefore%28%29+API+in+AdminClient) Added delete_records API. (#1710) diff --git a/docs/conf.py b/docs/conf.py index b82526418..995176d5f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -35,7 +35,7 @@ # built documents. # # The short X.Y version. -version = '2.5.0' +version = '2.5.3' # The full version, including alpha/beta/rc tags. release = version ###################################################################### diff --git a/examples/docker/Dockerfile.alpine b/examples/docker/Dockerfile.alpine index 7eab46f56..c50b87b6e 100644 --- a/examples/docker/Dockerfile.alpine +++ b/examples/docker/Dockerfile.alpine @@ -30,7 +30,7 @@ FROM alpine:3.12 COPY . /usr/src/confluent-kafka-python -ENV LIBRDKAFKA_VERSION v2.5.0 +ENV LIBRDKAFKA_VERSION v2.5.3 ENV KAFKACAT_VERSION master diff --git a/setup.py b/setup.py index cbc073509..fe12df2d6 100755 --- a/setup.py +++ b/setup.py @@ -75,7 +75,7 @@ def get_install_requirements(path): setup(name='confluent-kafka', # Make sure to bump CFL_VERSION* in confluent_kafka/src/confluent_kafka.h # and version in docs/conf.py. - version='2.5.0', + version='2.5.3', description='Confluent\'s Python client for Apache Kafka', author='Confluent Inc', author_email='support@confluent.io', diff --git a/src/confluent_kafka/schema_registry/json_schema.py b/src/confluent_kafka/schema_registry/json_schema.py index 656937c24..88e8d41aa 100644 --- a/src/confluent_kafka/schema_registry/json_schema.py +++ b/src/confluent_kafka/schema_registry/json_schema.py @@ -296,16 +296,18 @@ class JSONDeserializer(Deserializer): framing. Args: - schema_str (str, Schema): + schema_str (str, Schema, optional): `JSON schema definition `_ Accepts schema as either a string or a :py:class:`Schema` instance. Note that string definitions cannot reference other schemas. For referencing other schemas, - use a :py:class:`Schema` instance. + use a :py:class:`Schema` instance. If not provided, schemas will be + retrieved from schema_registry_client based on the schema ID in the + wire header of each message. from_dict (callable, optional): Callable(dict, SerializationContext) -> object. Converts a dict to a Python object instance. - schema_registry_client (SchemaRegistryClient, optional): Schema Registry client instance. Needed if ``schema_str`` is a schema referencing other schemas. + schema_registry_client (SchemaRegistryClient, optional): Schema Registry client instance. Needed if ``schema_str`` is a schema referencing other schemas or is not provided. """ # noqa: E501 __slots__ = ['_parsed_schema', '_from_dict', '_registry', '_are_references_provided', '_schema'] @@ -320,10 +322,16 @@ def __init__(self, schema_str, from_dict=None, schema_registry_client=None): if self._are_references_provided and schema_registry_client is None: raise ValueError( """schema_registry_client must be provided if "schema_str" is a Schema instance with references""") + elif schema_str is None: + if schema_registry_client is None: + raise ValueError( + """schema_registry_client must be provided if "schema_str" is not provided""" + ) + schema = schema_str else: raise TypeError('You must pass either str or Schema') - self._parsed_schema = json.loads(schema.schema_str) + self._parsed_schema = json.loads(schema.schema_str) if schema else None self._schema = schema self._registry = schema_registry_client @@ -378,7 +386,13 @@ def __call__(self, data, ctx): self._parsed_schema, store=named_schemas)) else: - validate(instance=obj_dict, schema=self._parsed_schema) + if self._parsed_schema is None: + schema = self._registry.get_schema(schema_id) + # TODO: cache the parsed schemas too? + parsed_schema = json.loads(schema.schema_str) + else: + parsed_schema = self._parsed_schema + validate(instance=obj_dict, schema=parsed_schema) except ValidationError as ve: raise SerializationError(ve.message) diff --git a/src/confluent_kafka/src/AdminTypes.c b/src/confluent_kafka/src/AdminTypes.c index 705ce36d9..43ca665ba 100644 --- a/src/confluent_kafka/src/AdminTypes.c +++ b/src/confluent_kafka/src/AdminTypes.c @@ -455,7 +455,7 @@ PyTypeObject NewPartitionsType = { " Instantiate a NewPartitions object.\n" "\n" " :param string topic: Topic name\n" - " :param int new_total_cnt: Increase the topic's partition count to this value.\n" + " :param int new_total_count: Increase the topic's partition count to this value.\n" " :param list replica_assignment: List of lists with the replication assignment for each new partition.\n" " :rtype: NewPartitions\n" "\n" diff --git a/src/confluent_kafka/src/confluent_kafka.h b/src/confluent_kafka/src/confluent_kafka.h index 8f03b95ca..d6ad31635 100644 --- a/src/confluent_kafka/src/confluent_kafka.h +++ b/src/confluent_kafka/src/confluent_kafka.h @@ -42,8 +42,8 @@ * 0xMMmmRRPP * MM=major, mm=minor, RR=revision, PP=patchlevel (not used) */ -#define CFL_VERSION 0x02050000 -#define CFL_VERSION_STR "2.5.0" +#define CFL_VERSION 0x02050300 +#define CFL_VERSION_STR "2.5.3" /** * Minimum required librdkafka version. This is checked both during @@ -51,19 +51,19 @@ * Make sure to keep the MIN_RD_KAFKA_VERSION, MIN_VER_ERRSTR and #error * defines and strings in sync. */ -#define MIN_RD_KAFKA_VERSION 0x020500ff +#define MIN_RD_KAFKA_VERSION 0x020503ff #ifdef __APPLE__ -#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v2.5.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`" +#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v2.5.3 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`" #else -#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v2.5.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html" +#define MIN_VER_ERRSTR "confluent-kafka-python requires librdkafka v2.5.3 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html" #endif #if RD_KAFKA_VERSION < MIN_RD_KAFKA_VERSION #ifdef __APPLE__ -#error "confluent-kafka-python requires librdkafka v2.5.0 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`" +#error "confluent-kafka-python requires librdkafka v2.5.3 or later. Install the latest version of librdkafka from Homebrew by running `brew install librdkafka` or `brew upgrade librdkafka`" #else -#error "confluent-kafka-python requires librdkafka v2.5.0 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html" +#error "confluent-kafka-python requires librdkafka v2.5.3 or later. Install the latest version of librdkafka from the Confluent repositories, see http://docs.confluent.io/current/installation.html" #endif #endif diff --git a/tools/smoketest.sh b/tools/smoketest.sh index acfb4ac9a..af2afd6fe 100755 --- a/tools/smoketest.sh +++ b/tools/smoketest.sh @@ -30,7 +30,7 @@ fi pyvers_tested= # Run tests with python3 -for py in 3.8 ; do +for py in 3.9 ; do echo "$0: # Smoketest with Python$py" if ! python$py -V ; then diff --git a/tools/test-manylinux.sh b/tools/test-manylinux.sh index 26ef95262..9837048ab 100755 --- a/tools/test-manylinux.sh +++ b/tools/test-manylinux.sh @@ -31,12 +31,16 @@ fi echo "$0 running from $(pwd)" function setup_ubuntu { + export DEBIAN_FRONTEND=noninteractive # Ubuntu container setup apt-get update - apt-get install -y python3.8 curl + apt-get install -y -q software-properties-common + add-apt-repository ppa:deadsnakes/ppa # python3-distutils is required on Ubuntu 18.04 and later but does # not exist on 14.04. - apt-get install -y python3.8-distutils || true + apt-get install -y -q python3.9 + apt-get install -y -q python3.9-distutils + apt-get install -y -q curl } @@ -61,7 +65,7 @@ function run_single_in_docker { # in a plethora of possibly outdated Python requirements that # might interfere with the newer packages from PyPi, such as six. # Instead install it directly from PyPa. - curl https://bootstrap.pypa.io/get-pip.py | python3.8 + curl https://bootstrap.pypa.io/get-pip.py | python3.9 /io/tools/smoketest.sh "$wheelhouse" } @@ -78,7 +82,7 @@ function run_all_with_docker { [[ ! -z $DOCKER_IMAGES ]] || \ # LTS and stable release of popular Linux distros. - DOCKER_IMAGES="ubuntu:18.04 ubuntu:20.04" + DOCKER_IMAGES="ubuntu:20.04 ubuntu:22.04" _wheels="$wheelhouse/*manylinux*.whl"