diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index e64b5e30b..a71c7535c 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -6,14 +6,25 @@ on: packageVersion: default: "2.10.0" jobs: + # + # PostgresML Python package. + # + postgresml-python: + uses: ./.github/workflows/ubuntu-postgresml-python-package.yaml + with: + packageVersion: ${{ inputs.packageVersion }} + secrets: inherit + # # PostgresML extension. # postgresml-pgml: + needs: postgresml-python strategy: fail-fast: false # Let the other job finish matrix: os: ["buildjet-4vcpu-ubuntu-2204", "buildjet-8vcpu-ubuntu-2204-arm"] + ubuntu_version: ["20.04", "22.04", "24.04"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -149,10 +160,7 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }} run: | - # Always build using latest scripts - git checkout master - - bash packages/postgresql-pgml/release.sh ${{ inputs.packageVersion }} + bash packages/postgresql-pgml/release.sh ${{ inputs.packageVersion }} ${{ matrix.ubuntu_version }} # # PostgresML meta package which installs @@ -164,6 +172,7 @@ jobs: fail-fast: false # Let the other job finish matrix: os: ["ubuntu-22.04"] + ubuntu_version: ["20.04", "22.04", "24.04"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -173,16 +182,18 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }} run: | - bash packages/postgresml/release.sh ${{ inputs.packageVersion }} + bash packages/postgresml/release.sh ${{ inputs.packageVersion }} ${{ matrix.ubuntu_version }} # # PostgresML dashboard. # postgresml-dashboard: + needs: postgresml strategy: fail-fast: false # Let the other job finish matrix: os: ["ubuntu-22.04", "buildjet-4vcpu-ubuntu-2204-arm"] + ubuntu_version: ["20.04", "22.04", "24.04"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -196,7 +207,7 @@ jobs: AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }} run: | cargo install cargo-pgml-components - bash packages/postgresml-dashboard/release.sh ${{ inputs.packageVersion }} + bash packages/postgresml-dashboard/release.sh ${{ inputs.packageVersion }} ${{ matrix.ubuntu_version }} # # PostgresML Docker image. diff --git a/.github/workflows/ubuntu-postgresml-python-package.yaml b/.github/workflows/ubuntu-postgresml-python-package.yaml index fc5eba6fc..617707e9a 100644 --- a/.github/workflows/ubuntu-postgresml-python-package.yaml +++ b/.github/workflows/ubuntu-postgresml-python-package.yaml @@ -4,14 +4,21 @@ on: workflow_dispatch: inputs: packageVersion: - default: "2.8.4" + default: "2.10.0" + workflow_call: + inputs: + packageVersion: + type: string + required: true + default: "2.10.0" jobs: postgresml-python: strategy: fail-fast: false # Let the other job finish matrix: - os: ["buildjet-4vcpu-ubuntu-2204", "buildjet-4vcpu-ubuntu-2204-arm", "ubuntu-24.04"] + os: ["buildjet-4vcpu-ubuntu-2204", "buildjet-4vcpu-ubuntu-2204-arm"] + ubuntu_version: ["20.04", "22.04", "24.04"] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -20,5 +27,22 @@ jobs: AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }} + UBUNTU_VERSION: ${{ matrix.ubuntu_version }} run: | - bash packages/postgresml-python/release.sh ${{ inputs.packageVersion }} + sudo apt update + sudo apt install -y python3-dev python3-pip python3-virtualenv software-properties-common python3-wheel-whl python3-pip-whl python3-setuptools-whl + + # Add deadsnakes PPA for all Python versions + sudo add-apt-repository -y ppa:deadsnakes/ppa + sudo apt update + + # Install Python 3.11 for all Ubuntu versions for better dependency compatibility + sudo apt install -y python3.11 python3.11-dev python3.11-venv + + # Ensure pip is updated + python3 -m pip install --upgrade pip setuptools wheel + + # Install PyTorch globally before running the build script + sudo python3 -m pip install torch + + bash packages/postgresml-python/release.sh ${{ inputs.packageVersion }} ${{ matrix.ubuntu_version }} diff --git a/docker/Dockerfile b/docker/Dockerfile index efd034649..242be9986 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM nvidia/cuda:12.1.1-devel-ubuntu22.04 +FROM nvidia/cuda:12.6.3-devel-ubuntu24.04 ENV PATH="/usr/local/cuda/bin:${PATH}" RUN apt update && \ apt install -y \ @@ -8,15 +8,25 @@ RUN apt update && \ gnupg \ coreutils \ sudo \ - openssl + openssl \ + python3-pip \ + software-properties-common + +# Add deadsnakes PPA for Python 3.11 +RUN add-apt-repository -y ppa:deadsnakes/ppa && \ + apt update && \ + apt install -y python3.11 python3.11-dev python3.11-venv python3.11-distutils + RUN echo "deb [trusted=yes] https://apt.postgresml.org $(lsb_release -cs) main" > /etc/apt/sources.list.d/postgresml.list RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null ENV TZ=UTC ENV DEBIAN_FRONTEND=noninteractive -RUN apt update -y && apt install git postgresml-15 postgresml-dashboard -y -RUN git clone --branch v0.5.0 https://github.com/pgvector/pgvector && \ +RUN apt update -y && \ + apt install -y git postgresml-python && \ + apt install -y postgresml-17 postgresml-dashboard +RUN git clone --branch v0.8.0 https://github.com/pgvector/pgvector && \ cd pgvector && \ echo "trusted = true" >> vector.control && \ make && \ @@ -25,7 +35,7 @@ echo "trusted = true" >> vector.control && \ COPY entrypoint.sh /app/entrypoint.sh COPY dashboard.sh /app/dashboard.sh -COPY --chown=postgres:postgres local_dev.conf /etc/postgresql/15/main/conf.d/01-local_dev.conf -COPY --chown=postgres:postgres pg_hba.conf /etc/postgresql/15/main/pg_hba.conf +COPY --chown=postgres:postgres local_dev.conf /etc/postgresql/17/main/conf.d/01-local_dev.conf +COPY --chown=postgres:postgres pg_hba.conf /etc/postgresql/17/main/pg_hba.conf ENTRYPOINT ["bash", "/app/entrypoint.sh"] diff --git a/packages/postgresml-dashboard/build.sh b/packages/postgresml-dashboard/build.sh index b35603b2c..7c28999ef 100644 --- a/packages/postgresml-dashboard/build.sh +++ b/packages/postgresml-dashboard/build.sh @@ -1,11 +1,24 @@ #!/bin/bash set -e +# Parse arguments +PACKAGE_VERSION=${1:-"2.10.0"} +UBUNTU_VERSION=${2:-"22.04"} + +if [[ -z "$PACKAGE_VERSION" ]]; then + echo "postgresml dashboard build script" + echo "Usage: $0 [ubuntu version]" + echo "Example: $0 2.10.0 22.04" + exit 1 +fi + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) dir="/tmp/postgresml-dashboard" deb_dir="$dir/deb-build" source_dir="$dir/source" -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -export PACKAGE_VERSION=${1:-"2.10.0"} + +export PACKAGE_VERSION +export UBUNTU_VERSION export GITHUB_STARS=$(curl -s "https://api.github.com/repos/postgresml/postgresml" | grep stargazers_count | cut -d : -f 2 | tr -d " " | tr -d ",") if [[ $(arch) == "x86_64" ]]; then export ARCH=amd64 @@ -27,7 +40,7 @@ rm "$deb_dir/release.sh" cp -R static "$deb_dir/usr/share/pgml-dashboard/dashboard-static" && \ cp -R ../pgml-cms "$deb_dir/usr/share/pgml-cms" ) -(cat ${SCRIPT_DIR}/DEBIAN/control | envsubst) > "$deb_dir/DEBIAN/control" +(cat ${SCRIPT_DIR}/DEBIAN/control | envsubst '${PACKAGE_VERSION} ${UBUNTU_VERSION} ${ARCH}') > "$deb_dir/DEBIAN/control" (cat ${SCRIPT_DIR}/etc/systemd/system/pgml-dashboard.service | envsubst) > "$deb_dir/etc/systemd/system/pgml-dashboard.service" chmod 755 ${deb_dir}/DEBIAN/post* @@ -36,6 +49,6 @@ chmod 755 ${deb_dir}/DEBIAN/pre* dpkg-deb \ --root-owner-group \ --build "$deb_dir" \ - postgresml-dashboard-${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb + "postgresml-dashboard-${PACKAGE_VERSION}-ubuntu${UBUNTU_VERSION}-${ARCH}.deb" rm -rf "$dir" diff --git a/packages/postgresml-dashboard/release.sh b/packages/postgresml-dashboard/release.sh index c06342129..8eab271b1 100644 --- a/packages/postgresml-dashboard/release.sh +++ b/packages/postgresml-dashboard/release.sh @@ -3,18 +3,34 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) package_version="$1" +target_ubuntu_version="$2" if [[ -z "$package_version" ]]; then - echo "Usage: $0 " + echo "postgresml dashboard package build and release script" + echo "Usage: $0 [ubuntu version, e.g. 22.04]" exit 1 fi +# Active LTS Ubuntu versions and their codenames +declare -A ubuntu_versions=( + ["20.04"]="focal" + ["22.04"]="jammy" + ["24.04"]="noble" +) + +# Detect current architecture if [[ $(arch) == "x86_64" ]]; then export ARCH=amd64 -else +elif [[ $(arch) == "aarch64" ]]; then export ARCH=arm64 +else + echo "Unsupported architecture: $(arch)" + exit 1 fi +echo "Building for architecture: ${ARCH}" + +# Install deb-s3 if not present if ! which deb-s3; then curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem sudo gem install deb-s3-0.11.4.gem @@ -22,18 +38,48 @@ if ! which deb-s3; then fi function package_name() { - echo "postgresml-dashboard-${package_version}-ubuntu22.04-${ARCH}.deb" + local ubuntu_version=$1 + local arch=$2 + echo "postgresml-dashboard-${package_version}-ubuntu${ubuntu_version}-${arch}.deb" } -bash ${SCRIPT_DIR}/build.sh "$package_version" +build_package() { + local ubuntu_version=$1 + local codename=$2 + + echo "Building packages for Ubuntu ${ubuntu_version} (${codename})" -if [[ ! -f $(package_name) ]]; then - echo "File $(package_name) doesn't exist" - exit 1 -fi + # Build the dashboard package + bash ${SCRIPT_DIR}/build.sh "$package_version" "$ubuntu_version" + + if [[ ! -f $(package_name ${ubuntu_version} ${ARCH}) ]]; then + echo "File $(package_name ${ubuntu_version} ${ARCH}) doesn't exist" + exit 1 + fi + + # Upload to S3 + deb-s3 upload \ + --visibility=public \ + --bucket apt.postgresml.org \ + $(package_name ${ubuntu_version} ${ARCH}) \ + --codename ${codename} -deb-s3 upload \ - --lock \ - --bucket apt.postgresml.org \ - $(package_name) \ - --codename $(lsb_release -cs) + # Clean up the package file + rm $(package_name ${ubuntu_version} ${ARCH}) +} + +# If a specific Ubuntu version is provided, only build for that version +if [[ ! -z "$target_ubuntu_version" ]]; then + if [[ -z "${ubuntu_versions[$target_ubuntu_version]}" ]]; then + echo "Error: Ubuntu version $target_ubuntu_version is not supported." + echo "Supported versions: ${!ubuntu_versions[@]}" + exit 1 + fi + + build_package "$target_ubuntu_version" "${ubuntu_versions[$target_ubuntu_version]}" +else + # If no version specified, loop through all supported Ubuntu versions + for ubuntu_version in "${!ubuntu_versions[@]}"; do + build_package "$ubuntu_version" "${ubuntu_versions[$ubuntu_version]}" + done +fi \ No newline at end of file diff --git a/packages/postgresml-python/build.sh b/packages/postgresml-python/build.sh index 602a8ca1e..492b86c01 100644 --- a/packages/postgresml-python/build.sh +++ b/packages/postgresml-python/build.sh @@ -1,21 +1,26 @@ #!/bin/bash -# -# -# set -e + SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) deb_dir="/tmp/postgresml-python/deb-build" -major=${1:-"14"} +# Parse arguments with defaults export PACKAGE_VERSION=${1:-"2.10.0"} -export PYTHON_VERSION=${2:-"3.10"} +export UBUNTU_VERSION=${2:-"22.04"} +export PYTHON_VERSION=${3:-"3.11"} +# Handle architecture if [[ $(arch) == "x86_64" ]]; then export ARCH=amd64 else export ARCH=arm64 fi +# We use Python 3.11 for all Ubuntu versions for better dependency compatibility +if [[ -z "$3" ]]; then + PYTHON_VERSION="3.11" +fi + rm -rf "$deb_dir" mkdir -p "$deb_dir" @@ -23,20 +28,25 @@ cp -R ${SCRIPT_DIR}/* "$deb_dir" rm "$deb_dir/build.sh" rm "$deb_dir/release.sh" -(cat ${SCRIPT_DIR}/DEBIAN/control | envsubst) > "$deb_dir/DEBIAN/control" -(cat ${SCRIPT_DIR}/DEBIAN/postinst | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postinst" -(cat ${SCRIPT_DIR}/DEBIAN/prerm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/prerm" -(cat ${SCRIPT_DIR}/DEBIAN/postrm | envsubst '${PGVERSION}') > "$deb_dir/DEBIAN/postrm" +(cat ${SCRIPT_DIR}/DEBIAN/control | envsubst '${PACKAGE_VERSION} ${UBUNTU_VERSION} ${ARCH} ${PYTHON_VERSION}') > "$deb_dir/DEBIAN/control" +(cat ${SCRIPT_DIR}/DEBIAN/postinst | envsubst '${PGVERSION} ${PYTHON_VERSION}') > "$deb_dir/DEBIAN/postinst" +(cat ${SCRIPT_DIR}/DEBIAN/prerm | envsubst '${PGVERSION} ${PYTHON_VERSION}') > "$deb_dir/DEBIAN/prerm" +(cat ${SCRIPT_DIR}/DEBIAN/postrm | envsubst '${PGVERSION} ${PYTHON_VERSION}') > "$deb_dir/DEBIAN/postrm" if [[ "$ARCH" == "amd64" ]]; then - cp ${SCRIPT_DIR}/../../pgml-extension/requirements.linux.txt "$deb_dir/etc/postgresml-python/requirements.txt" + # Use AMD64-specific requirements (x86_64) + cp ${SCRIPT_DIR}/../../pgml-extension/requirements.amd64.txt "$deb_dir/etc/postgresml-python/requirements.txt" else - cp ${SCRIPT_DIR}/../../pgml-extension/requirements.macos.txt "$deb_dir/etc/postgresml-python/requirements.txt" + # Use ARM64-specific requirements (aarch64) + cp ${SCRIPT_DIR}/../../pgml-extension/requirements.arm64.txt "$deb_dir/etc/postgresml-python/requirements.txt" fi -virtualenv --python="python$PYTHON_VERSION" "$deb_dir/var/lib/postgresml-python/pgml-venv" +virtualenv --python="python${PYTHON_VERSION}" "$deb_dir/var/lib/postgresml-python/pgml-venv" source "$deb_dir/var/lib/postgresml-python/pgml-venv/bin/activate" +# Install PyTorch first to help with dependency resolution +python -m pip install torch + python -m pip install -r "${deb_dir}/etc/postgresml-python/requirements.txt" deactivate @@ -48,6 +58,6 @@ dpkg-deb \ --root-owner-group \ -z1 \ --build "$deb_dir" \ - postgresml-python-${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb + "postgresml-python-${PACKAGE_VERSION}-ubuntu${UBUNTU_VERSION}-${ARCH}.deb" rm -rf "$deb_dir" diff --git a/packages/postgresml-python/release.sh b/packages/postgresml-python/release.sh index e3b7371cf..4199be41f 100644 --- a/packages/postgresml-python/release.sh +++ b/packages/postgresml-python/release.sh @@ -3,42 +3,86 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) package_version="$1" +target_ubuntu_version="$2" +if [[ -z "$package_version" ]]; then + echo "postgresml-python package build and release script" + echo "Usage: $0 [ubuntu version, e.g. 22.04]" + exit 1 +fi + +# Active LTS Ubuntu versions and their codenames +declare -A ubuntu_versions=( + ["20.04"]="focal" + ["22.04"]="jammy" + ["24.04"]="noble" +) + +# Detect current architecture if [[ $(arch) == "x86_64" ]]; then - arch=amd64 + export ARCH=amd64 +elif [[ $(arch) == "aarch64" ]]; then + export ARCH=arm64 else - arch=arm64 + echo "Unsupported architecture: $(arch)" + exit 1 fi -if [[ -z "$package_version" ]]; then - echo "postgresml-python package build and release script" - echo "usage: $0 " - exit 1 -fi +echo "Building for architecture: ${ARCH}" +# Install deb-s3 if not present if ! which deb-s3; then - curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem - sudo gem install deb-s3-0.11.4.gem - deb-s3 + curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem + sudo gem install deb-s3-0.11.4.gem + deb-s3 fi +# Install Python dependencies sudo apt install python3-pip python3 python3-virtualenv -y function package_name() { - echo "postgresml-python-$package_version-ubuntu22.04-${arch}.deb" + local ubuntu_version=$1 + local arch=$2 + echo "postgresml-python-${package_version}-ubuntu${ubuntu_version}-${arch}.deb" } -bash ${SCRIPT_DIR}/build.sh ${package_version} +build_package() { + local ubuntu_version=$1 + local codename=$2 + + echo "Building packages for Ubuntu ${ubuntu_version} (${codename})" -if [[ ! -f $(package_name ${pg}) ]]; then - echo "File $(package_name ${pg}) doesn't exist" - exit 1 -fi + # Build the Python package + bash ${SCRIPT_DIR}/build.sh "$package_version" "$ubuntu_version" + + if [[ ! -f $(package_name ${ubuntu_version} ${ARCH}) ]]; then + echo "File $(package_name ${ubuntu_version} ${ARCH}) doesn't exist" + exit 1 + fi -deb-s3 upload \ - --lock \ - --bucket apt.postgresml.org \ - $(package_name ${pg}) \ - --codename $(lsb_release -cs) + # Upload to S3 + deb-s3 upload \ + --visibility=public \ + --bucket apt.postgresml.org \ + $(package_name ${ubuntu_version} ${ARCH}) \ + --codename ${codename} -rm $(package_name ${pg}) + # Clean up the package file + rm $(package_name ${ubuntu_version} ${ARCH}) +} + +# If a specific Ubuntu version is provided, only build for that version +if [[ ! -z "$target_ubuntu_version" ]]; then + if [[ -z "${ubuntu_versions[$target_ubuntu_version]}" ]]; then + echo "Error: Ubuntu version $target_ubuntu_version is not supported." + echo "Supported versions: ${!ubuntu_versions[@]}" + exit 1 + fi + + build_package "$target_ubuntu_version" "${ubuntu_versions[$target_ubuntu_version]}" +else + # If no version specified, loop through all supported Ubuntu versions + for ubuntu_version in "${!ubuntu_versions[@]}"; do + build_package "$ubuntu_version" "${ubuntu_versions[$ubuntu_version]}" + done +fi \ No newline at end of file diff --git a/packages/postgresml/build.sh b/packages/postgresml/build.sh index b6daed532..4e0f224ba 100644 --- a/packages/postgresml/build.sh +++ b/packages/postgresml/build.sh @@ -4,7 +4,7 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) export PACKAGE_VERSION=${1:-"2.10.0"} -export PGVERSION=${2:-"14"} +export PGVERSION=${2:-"17"} export UBUNTU_VERSION=${3:-"24.04"} deb_dir="/tmp/postgresml/deb-build" diff --git a/packages/postgresml/release.sh b/packages/postgresml/release.sh index 054d18d9b..af3814612 100644 --- a/packages/postgresml/release.sh +++ b/packages/postgresml/release.sh @@ -3,13 +3,22 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) package_version="$1" +target_ubuntu_version="$2" if [[ -z "$package_version" ]]; then echo "postgresml package build and release script" - echo "usage: $0 " + echo "usage: $0 [ubuntu version, e.g. 22.04]" exit 1 fi +# Active LTS Ubuntu versions and their codenames +declare -A ubuntu_codenames=( + ["20.04"]="focal" + ["22.04"]="jammy" + ["24.04"]="noble" +) + +# Install deb-s3 if not present if ! which deb-s3; then curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem sudo gem install deb-s3-0.11.4.gem @@ -22,18 +31,10 @@ function package_name() { echo "postgresml-${pg_version}-${package_version}-ubuntu${ubuntu_version}-all.deb" } -# Active LTS Ubuntu versions -ubuntu_versions=("20.04" "22.04" "24.04") - -# Map Ubuntu versions to codenames -declare -A ubuntu_codenames=( - ["20.04"]="focal" - ["22.04"]="jammy" - ["24.04"]="noble" -) - -for ubuntu_version in "${ubuntu_versions[@]}"; do - codename=${ubuntu_codenames[$ubuntu_version]} +build_package() { + local ubuntu_version=$1 + local codename=$2 + echo "Building packages for Ubuntu ${ubuntu_version} (${codename})" for pg in {11..17}; do @@ -46,11 +47,27 @@ for ubuntu_version in "${ubuntu_versions[@]}"; do fi deb-s3 upload \ - --lock \ + --visibility=public \ --bucket apt.postgresml.org \ $(package_name ${pg} ${ubuntu_version}) \ --codename ${codename} rm $(package_name ${pg} ${ubuntu_version}) done -done +} + +# If a specific Ubuntu version is provided, only build for that version +if [[ ! -z "$target_ubuntu_version" ]]; then + if [[ -z "${ubuntu_codenames[$target_ubuntu_version]}" ]]; then + echo "Error: Ubuntu version $target_ubuntu_version is not supported." + echo "Supported versions: ${!ubuntu_codenames[@]}" + exit 1 + fi + + build_package "$target_ubuntu_version" "${ubuntu_codenames[$target_ubuntu_version]}" +else + # If no version specified, loop through all supported Ubuntu versions + for ubuntu_version in "${!ubuntu_codenames[@]}"; do + build_package "$ubuntu_version" "${ubuntu_codenames[$ubuntu_version]}" + done +fi \ No newline at end of file diff --git a/packages/postgresql-pgml/release.sh b/packages/postgresql-pgml/release.sh index 6153a7dba..9caa5947f 100644 --- a/packages/postgresql-pgml/release.sh +++ b/packages/postgresql-pgml/release.sh @@ -4,17 +4,33 @@ set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) if [[ -z "${1}" ]]; then - echo "Usage: $0 " + echo "Usage: $0 [ubuntu version, e.g. 22.04]" exit 1 fi export PACKAGE_VERSION=${1} +export TARGET_UBUNTU_VERSION=${2} + +# Active LTS Ubuntu versions and their codenames +declare -A ubuntu_versions=( + ["20.04"]="focal" + ["22.04"]="jammy" + ["24.04"]="noble" +) + +# Detect current architecture if [[ $(arch) == "x86_64" ]]; then export ARCH=amd64 -else +elif [[ $(arch) == "aarch64" ]]; then export ARCH=arm64 +else + echo "Unsupported architecture: $(arch)" + exit 1 fi +echo "Building for architecture: ${ARCH}" + +# Install deb-s3 if not present if ! which deb-s3; then curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem sudo gem install deb-s3-0.11.4.gem @@ -24,25 +40,61 @@ fi extension_dir="${SCRIPT_DIR}/../../pgml-extension" function package_name() { - echo "postgresql-pgml-${1}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb" + local pg_version=$1 + local ubuntu_version=$2 + local arch=$3 + echo "postgresql-pgml-${pg_version}_${PACKAGE_VERSION}-ubuntu${ubuntu_version}-${arch}.deb" } -for pg in {12..16}; do - release_dir="$extension_dir/target/release/pgml-pg${pg}" +build_packages() { + local ubuntu_version=$1 + local codename=$2 + + echo "Building packages for Ubuntu ${ubuntu_version} (${codename})" - mkdir -p "$release_dir/DEBIAN" + # Loop through PostgreSQL versions + for pg in {11..17}; do + echo "Building PostgreSQL ${pg} package..." - export PGVERSION=${pg} - (cat ${SCRIPT_DIR}/DEBIAN/control | envsubst '${PGVERSION} ${PACKAGE_VERSION} ${ARCH}') > "$release_dir/DEBIAN/control" + release_dir="$extension_dir/target/release/pgml-pg${pg}" + mkdir -p "$release_dir/DEBIAN" - dpkg-deb \ - --root-owner-group \ - -z1 \ - --build "$release_dir" \ - $(package_name ${pg}) + export PGVERSION=${pg} + # Update control file with Ubuntu version + (cat ${SCRIPT_DIR}/DEBIAN/control | + envsubst '${PGVERSION} ${PACKAGE_VERSION} ${ARCH}') > "$release_dir/DEBIAN/control" - deb-s3 upload \ - --bucket apt.postgresml.org \ - $(package_name ${pg}) \ - --codename $(lsb_release -cs) -done + # Build the package + dpkg-deb \ + --root-owner-group \ + -z1 \ + --build "$release_dir" \ + $(package_name ${pg} ${ubuntu_version} ${ARCH}) + + # Upload to S3 + deb-s3 upload \ + --visibility=public \ + --bucket apt.postgresml.org \ + $(package_name ${pg} ${ubuntu_version} ${ARCH}) \ + --codename ${codename} + + # Clean up the package file + rm $(package_name ${pg} ${ubuntu_version} ${ARCH}) + done +} + +# If a specific Ubuntu version is provided, only build for that version +if [[ ! -z "$TARGET_UBUNTU_VERSION" ]]; then + if [[ -z "${ubuntu_versions[$TARGET_UBUNTU_VERSION]}" ]]; then + echo "Error: Ubuntu version $TARGET_UBUNTU_VERSION is not supported." + echo "Supported versions: ${!ubuntu_versions[@]}" + exit 1 + fi + + build_packages "$TARGET_UBUNTU_VERSION" "${ubuntu_versions[$TARGET_UBUNTU_VERSION]}" +else + # If no version specified, loop through all supported Ubuntu versions + for ubuntu_version in "${!ubuntu_versions[@]}"; do + build_packages "$ubuntu_version" "${ubuntu_versions[$ubuntu_version]}" + done +fi \ No newline at end of file diff --git a/pgml-cms/docs/open-source/pgml/developers/installation.md b/pgml-cms/docs/open-source/pgml/developers/installation.md index 958ff0e81..5f4a0ecc5 100644 --- a/pgml-cms/docs/open-source/pgml/developers/installation.md +++ b/pgml-cms/docs/open-source/pgml/developers/installation.md @@ -40,6 +40,12 @@ cargo install cargo-pgrx --version 0.12.9 && \ cargo pgrx init ``` +**NOTE: You may need to set the `PGK_CONFIG_PATH` env variable:** + +```bash +export PKG_CONFIG_PATH="/opt/homebrew/opt/icu4c/lib/pkgconfig" +``` + This step will take a few minutes. Perfect opportunity to get a coffee while you wait. ### Compile and install @@ -65,14 +71,36 @@ virtualenv pgml-venv && \ source pgml-venv/bin/activate && \ pip install -r requirements.txt ``` + +PostgresML has architecture-specific requirements files: +- `requirements.amd64.txt` - For x86_64/AMD64 architectures +- `requirements.arm64.txt` - For ARM64/aarch64 architectures + +When building from source, use the appropriate file for your architecture: + +```bash +# For AMD64/x86_64 systems +pip install -r requirements.amd64.txt + +# For ARM64/aarch64 systems +pip install -r requirements.arm64.txt +``` + +These files contain frozen dependencies that have been tested with PostgresML. We recommend using Python 3.11 for optimal compatibility with all dependencies. {% endtab %} {% tab title="Globally" %} Installing Python packages globally can cause issues with your system. If you wish to proceed nonetheless, you can do so: ```bash -pip3 install -r requirements.txt +# For AMD64/x86_64 systems +pip3 install -r requirements.amd64.txt + +# For ARM64/aarch64 systems +pip3 install -r requirements.arm64.txt ``` + +We recommend using Python 3.11 for optimal compatibility with all dependencies. {% endtab %} {% endtabs %} diff --git a/pgml-extension/Brewfile b/pgml-extension/Brewfile index 27acc3a89..6a342a34c 100644 --- a/pgml-extension/Brewfile +++ b/pgml-extension/Brewfile @@ -7,3 +7,4 @@ brew "cmake" brew "pkg-config" brew "openssl" brew "virtualenv" +brew "icu4c" diff --git a/pgml-extension/examples/transformers.sql b/pgml-extension/examples/transformers.sql index d6d96c721..a61a95288 100644 --- a/pgml-extension/examples/transformers.sql +++ b/pgml-extension/examples/transformers.sql @@ -97,7 +97,10 @@ SELECT pgml.transform( ) AS result; SELECT pgml.transform( - 'summarization', + task => '{ + "task": "summarization", + "model": "facebook/bart-large-cnn" + }'::JSONB, inputs => ARRAY[ 'Dominic Cobb is the foremost practitioner of the artistic science of extraction, inserting oneself into a subject''s dreams to obtain hidden information without the subject knowing, a concept taught to him by his professor father-in-law, Dr. Stephen Miles. Dom''s associates are Miles'' former students, who Dom requires as he has given up being the dream architect for reasons he won''t disclose. Dom''s primary associate, Arthur, believes it has something to do with Dom''s deceased wife, Mal, who often figures prominently and violently in those dreams, or Dom''s want to "go home" (get back to his own reality, which includes two young children). Dom''s work is generally in corporate espionage. As the subjects don''t want the information to get into the wrong hands, the clients have zero tolerance for failure. Dom is also a wanted man, as many of his past subjects have learned what Dom has done to them. One of those subjects, Mr. Saito, offers Dom a job he can''t refuse: to take the concept one step further into inception, namely planting thoughts into the subject''s dreams without them knowing. Inception can fundamentally alter that person as a being. Saito''s target is Robert Michael Fischer, the heir to an energy business empire, which has the potential to rule the world if continued on the current trajectory. Beyond the complex logistics of the dream architecture of the case and some unknowns concerning Fischer, the biggest obstacles in success for the team become worrying about one aspect of inception which Cobb fails to disclose to the other team members prior to the job, and Cobb''s newest associate Ariadne''s belief that Cobb''s own subconscious, especially as it relates to Mal, may be taking over what happens in the dreams.' ] diff --git a/pgml-extension/requirements.amd64.txt b/pgml-extension/requirements.amd64.txt new file mode 100644 index 000000000..4a52d3150 --- /dev/null +++ b/pgml-extension/requirements.amd64.txt @@ -0,0 +1,198 @@ +accelerate==1.2.1 +aiohappyeyeballs==2.4.4 +aiohttp==3.11.11 +aiohttp-cors==0.7.0 +aiosignal==1.3.2 +airportsdata==20241001 +annotated-types==0.7.0 +anyio==4.8.0 +astor==0.8.1 +attrs==24.3.0 +auto_gptq==0.7.1 +bitsandbytes==0.45.0 +blake3==1.0.2 +cachetools==5.5.0 +catboost==1.2.7 +certifi==2024.12.14 +charset-normalizer==3.4.1 +click==8.1.8 +cloudpickle==3.1.1 +colorama==0.4.6 +coloredlogs==15.0.1 +colorful==0.5.6 +compressed-tensors==0.8.1 +contourpy==1.3.1 +ctransformers==0.2.27 +cycler==0.12.1 +datasets==3.2.0 +deepspeed==0.16.2 +depyf==0.18.0 +dill==0.3.8 +diskcache==5.6.3 +distlib==0.3.9 +distro==1.9.0 +einops==0.8.0 +evaluate==0.4.3 +fastapi==0.115.6 +filelock==3.16.1 +fonttools==4.55.3 +frozenlist==1.5.0 +fsspec==2024.9.0 +gekko==1.2.1 +gguf==0.10.0 +google-api-core==2.24.0 +google-auth==2.37.0 +googleapis-common-protos==1.66.0 +graphviz==0.20.3 +greenlet==3.1.1 +grpcio==1.69.0 +h11==0.14.0 +hjson==3.1.0 +httpcore==1.0.7 +httptools==0.6.4 +httpx==0.28.1 +huggingface-hub==0.27.1 +humanfriendly==10.0 +idna==3.10 +importlib_metadata==8.5.0 +iniconfig==2.0.0 +interegular==0.3.3 +Jinja2==3.1.5 +jiter==0.8.2 +joblib==1.4.2 +jsonpatch==1.33 +jsonpointer==3.0.0 +jsonschema==4.23.0 +jsonschema-specifications==2024.10.1 +kiwisolver==1.4.8 +langchain==0.3.14 +langchain-core==0.3.29 +langchain-text-splitters==0.3.5 +langsmith==0.2.10 +lark==1.2.2 +lightgbm==4.5.0 +linkify-it-py==2.0.3 +lm-format-enforcer==0.10.9 +lxml==5.3.0 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.0 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +memray==1.15.0 +mistral_common==1.5.1 +mpmath==1.3.0 +msgpack==1.1.0 +msgspec==0.19.0 +multidict==6.1.0 +multiprocess==0.70.16 +nest-asyncio==1.6.0 +networkx==3.4.2 +ninja==1.11.1.3 +numpy==1.26.4 +nvidia-cublas-cu12==12.4.5.8 +nvidia-cuda-cupti-cu12==12.4.127 +nvidia-cuda-nvrtc-cu12==12.4.127 +nvidia-cuda-runtime-cu12==12.4.127 +nvidia-cudnn-cu12==9.1.0.70 +nvidia-cufft-cu12==11.2.1.3 +nvidia-curand-cu12==10.3.5.147 +nvidia-cusolver-cu12==11.6.1.9 +nvidia-cusparse-cu12==12.3.1.170 +nvidia-ml-py==12.560.30 +nvidia-nccl-cu12==2.21.5 +nvidia-nvjitlink-cu12==12.4.127 +nvidia-nvtx-cu12==12.4.127 +openai==1.59.7 +opencensus==0.11.4 +opencensus-context==0.1.3 +opencv-python-headless==4.10.0.84 +optimum==1.23.3 +orjson==3.10.14 +outlines==0.1.11 +outlines_core==0.1.26 +packaging==24.2 +pandas==2.2.3 +partial-json-parser==0.2.1.1.post5 +peft==0.14.0 +pillow==10.4.0 +platformdirs==4.3.6 +plotly==5.24.1 +pluggy==1.5.0 +portalocker==3.1.1 +prometheus-fastapi-instrumentator==7.0.2 +prometheus_client==0.21.1 +propcache==0.2.1 +proto-plus==1.25.0 +protobuf==5.29.3 +psutil==6.1.1 +py-cpuinfo==9.0.0 +py-spy==0.4.0 +pyarrow==18.1.0 +pyasn1==0.6.1 +pyasn1_modules==0.4.1 +pybind11==2.13.6 +pycountry==24.6.1 +pydantic==2.10.5 +pydantic_core==2.27.2 +Pygments==2.19.1 +pyparsing==3.2.1 +pytest==8.3.4 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +pytz==2024.2 +PyYAML==6.0.2 +pyzmq==26.2.0 +ray==2.40.0 +referencing==0.35.1 +regex==2024.11.6 +requests==2.32.3 +requests-toolbelt==1.0.0 +rich==13.9.4 +rouge==1.0.1 +rpds-py==0.22.3 +rsa==4.9 +sacrebleu==2.5.1 +sacremoses==0.1.1 +safetensors==0.5.2 +scikit-learn==1.6.1 +scipy==1.15.1 +sentence-transformers==3.3.1 +sentencepiece==0.2.0 +six==1.17.0 +smart-open==7.1.0 +sniffio==1.3.1 +SQLAlchemy==2.0.37 +starlette==0.41.3 +sympy==1.13.1 +tabulate==0.9.0 +tenacity==9.0.0 +textual==1.0.0 +threadpoolctl==3.5.0 +tiktoken==0.7.0 +tokenizers==0.21.0 +torch==2.5.1 +torchaudio==2.5.1 +torchvision==0.20.1 +tqdm==4.67.1 +transformers==4.48.0 +transformers-stream-generator==0.0.5 +triton==3.1.0 +trl==0.13.0 +typing_extensions==4.12.2 +tzdata==2024.2 +uc-micro-py==1.0.3 +urllib3==2.3.0 +uvicorn==0.34.0 +uvloop==0.21.0 +virtualenv==20.28.1 +vllm==0.6.6.post1 +watchfiles==1.0.4 +websockets==14.1 +wrapt==1.17.2 +xformers==0.0.28.post3 +xgboost==2.1.3 +xgrammar==0.1.9 +xxhash==3.5.0 +yarl==1.18.3 +zipp==3.21.0 diff --git a/pgml-extension/requirements.arm64.txt b/pgml-extension/requirements.arm64.txt new file mode 100644 index 000000000..7b0d5678b --- /dev/null +++ b/pgml-extension/requirements.arm64.txt @@ -0,0 +1,110 @@ +accelerate==0.30.1 +aiohttp==3.9.5 +aiosignal==1.3.1 +annotated-types==0.6.0 +attrs==23.2.0 +bitsandbytes==0.42.0 +catboost==1.2.5 +certifi==2024.2.2 +charset-normalizer==3.3.2 +click==8.1.7 +colorama==0.4.6 +coloredlogs==15.0.1 +contourpy==1.2.1 +ctransformers==0.2.27 +cycler==0.12.1 +dataclasses-json==0.6.6 +datasets==2.16.1 +deepspeed==0.14.2 +dill==0.3.7 +docstring_parser==0.16 +einops==0.8.0 +evaluate==0.4.2 +filelock==3.14.0 +fonttools==4.51.0 +frozenlist==1.4.1 +fsspec==2023.10.0 +graphviz==0.20.3 +hjson==3.1.0 +huggingface-hub==0.23.0 +humanfriendly==10.0 +idna==3.7 +Jinja2==3.1.4 +joblib==1.4.2 +jsonpatch==1.33 +jsonpointer==2.4 +kiwisolver==1.4.5 +langchain==0.1.20 +langchain-community==0.0.38 +langchain-core==0.1.52 +langchain-text-splitters==0.0.1 +langsmith==0.1.57 +lightgbm==4.3.0 +lxml==5.2.2 +markdown-it-py==3.0.0 +MarkupSafe==2.1.5 +marshmallow==3.21.2 +matplotlib==3.8.4 +mdurl==0.1.2 +mpmath==1.3.0 +multidict==6.0.5 +multiprocess==0.70.15 +mypy-extensions==1.0.0 +networkx==3.3 +ninja==1.11.1.1 +numpy==1.26.4 +optimum==1.19.2 +orjson==3.10.3 +packaging==23.2 +pandas==2.2.2 +peft==0.10.0 +pillow==10.3.0 +plotly==5.22.0 +portalocker==2.8.2 +protobuf==5.26.1 +psutil==5.9.8 +py-cpuinfo==9.0.0 +pyarrow==11.0.0 +pyarrow-hotfix==0.6 +pydantic==2.7.1 +pydantic_core==2.18.2 +Pygments==2.18.0 +pynvml==11.5.0 +pyparsing==3.1.2 +python-dateutil==2.9.0.post0 +pytz==2024.1 +PyYAML==6.0.1 +regex==2024.5.10 +requests==2.31.0 +rich==13.7.1 +rouge==1.0.1 +sacrebleu==2.4.2 +sacremoses==0.1.1 +safetensors==0.4.3 +scikit-learn==1.4.2 +scipy==1.13.0 +sentence-transformers==2.7.0 +sentencepiece==0.2.0 +shtab==1.7.1 +six==1.16.0 +SQLAlchemy==2.0.30 +sympy==1.12 +tabulate==0.9.0 +tenacity==8.3.0 +threadpoolctl==3.5.0 +tokenizers==0.19.1 +torch==2.3.0 +torchaudio==2.3.0 +torchvision==0.18.0 +tqdm==4.66.4 +transformers==4.40.2 +transformers-stream-generator==0.0.5 +trl==0.8.6 +typing-inspect==0.9.0 +typing_extensions==4.11.0 +tyro==0.8.4 +tzdata==2024.1 +urllib3==2.2.1 +xgboost==2.0.3 +xxhash==3.4.1 +yarl==1.9.4