From ac1c6465480e178d66de8cfdc393a453ffb05687 Mon Sep 17 00:00:00 2001
From: Lev Kokotov
Date: Mon, 13 Feb 2023 13:16:34 -0800
Subject: [PATCH 01/22] Docker local dev (#536)
---
.github/workflows/package-extension.yml | 43 +++++++++
pgml-extension/Dockerfile | 110 +++++++++---------------
pgml-extension/README.md | 8 +-
3 files changed, 86 insertions(+), 75 deletions(-)
create mode 100644 .github/workflows/package-extension.yml
diff --git a/.github/workflows/package-extension.yml b/.github/workflows/package-extension.yml
new file mode 100644
index 000000000..0d622c4e3
--- /dev/null
+++ b/.github/workflows/package-extension.yml
@@ -0,0 +1,43 @@
+name: Build PostgresML Rust package
+
+on:
+ workflow_dispatch:
+
+jobs:
+ build:
+ strategy:
+ matrix:
+ os: [ubuntu-latest]
+ runs-on: ${{ matrix.os }}
+ steps:
+ - uses: actions/checkout@v3
+ - name: Install dependencies
+ run:
+ sudo apt update
+ sudo apt install -y ruby curl
+ curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
+ sudo gem install deb-s3-0.11.4.gem
+ - name: Build package
+ run:
+ docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/11/bin/pg_config
+ docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/12/bin/pg_config
+ docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/13/bin/pg_config
+ docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/14/bin/pg_config
+ docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/15/bin/pg_config
+ # - name: Publish package
+ # env:
+ # AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
+ # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ # AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
+ # working-directory: postgresml/pgml-extension
+ # run: |
+ # if [[ $(uname -a) == *"aarch64"* ]]; then
+ # ARCH="arm64"
+ # else
+ # ARCH="amd64"
+ # fi
+
+ # deb-s3 upload \
+ # --bucket apt.postgresml.org \
+ # postgresql-pgml-${{ inputs.postgresVersion }}_${{ inputs.packageVersion }}-ubuntu${{ inputs.ubuntuVersion }}-${ARCH}.deb \
+ # --codename ${{ inputs.ubuntuName }}
diff --git a/pgml-extension/Dockerfile b/pgml-extension/Dockerfile
index 1b3f2d50d..6824d47a2 100644
--- a/pgml-extension/Dockerfile
+++ b/pgml-extension/Dockerfile
@@ -1,73 +1,47 @@
-#
-# Extension builder.
-#
-ARG UBUNTU_VERSION=22.04
-FROM ubuntu:${UBUNTU_VERSION}
-
-ARG PGVERSION=14
+FROM ubuntu:jammy
+MAINTAINER team@postgresml.com
+RUN apt-get update
ARG DEBIAN_FRONTEND=noninteractive
-ARG PACKAGE_VERSION=2.0.0
-ARG PACKAGE_PYTHON=true
-ARG PACKAGE_CUDA=false
ENV TZ=Etc/UTC
-
-# Apt-fast
-RUN apt-get update && apt-get install software-properties-common -y
-
+RUN apt-get install -y software-properties-common
RUN add-apt-repository ppa:apt-fast/stable --yes
-RUN apt-get update && apt-get -y install apt-fast
-
-RUN apt-fast install apt-fast curl unzip gpg wget -y
-
-# PostgreSQL
-RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg > /dev/null
-RUN echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list
-
-# CMake
-RUN curl -L https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor | tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
-RUN echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $(lsb_release -cs) main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null
-
-RUN apt-get update && apt-fast install postgresql-${PGVERSION} libopenblas-dev cmake postgresql-server-dev-${PGVERSION} pkg-config libssl-dev build-essential libclang-dev clang libpython3-dev python3-dev -y
-
-# CUDA
-COPY docker/install_cuda.sh install_cuda.sh
-RUN bash install_cuda.sh
-
-USER postgres
-
-
-# Install Rust
+RUN apt update && apt-get install -y apt-fast
+RUN apt-get update && apt-fast install -y \
+ libopenblas-dev \
+ libssl-dev \
+ bison \
+ flex \
+ pkg-config \
+ cmake \
+ libreadline-dev \
+ libz-dev \
+ curl \
+ lsb-release \
+ tzdata \
+ sudo \
+ cmake \
+ libpq-dev \
+ libclang-dev \
+ wget \
+ postgresql-plpython3-14 \
+ postgresql-14 \
+ postgresql-server-dev-14
+RUN add-apt-repository ppa:deadsnakes/ppa --yes
+RUN apt update && apt-fast install -y \
+ python3.10 \
+ python3-pip \
+ libpython3.10-dev \
+ python3.10-dev
+RUN pip3 install xgboost scikit-learn torch lightgbm transformers datasets
+RUN useradd postgresml -m -s /bin/bash -G sudo
+RUN echo 'postgresml ALL=(ALL) NOPASSWD: ALL' >> /etc/sudoers
+USER postgresml
RUN curl https://sh.rustup.rs -sSf | sh -s -- -y
-ENV PATH="/var/lib/postgresql/.cargo/bin:${PATH}"
-
-# Install tcdi/pgx
-RUN cargo install cargo-pgx --version 0.7.1
-RUN cargo pgx init --pg${PGVERSION} /usr/bin/pg_config
-
-
-COPY --chown=postgres:postgres . /app
+RUN $HOME/.cargo/bin/cargo install cargo-pgx --version "0.7.1"
+RUN $HOME/.cargo/bin/cargo pgx init
+RUN curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
+RUN sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+RUN sudo apt update
+RUN sudo apt-get install -y postgresql-15 postgresql-13 postgresql-12 postgresql-11
+RUN sudo apt install -y postgresql-server-dev-15 postgresql-server-dev-15 postgresql-server-dev-12 postgresql-server-dev-11
WORKDIR /app
-
-
-# cargo pgx package command ignores --no-default-features sadly, so we
-# have to have a separate Cargo.toml without Python. No big deal.
-# Make sure to diff it and update it if you change the main Cargo.toml.
-ENV PACKAGE_PYTHON=${PACKAGE_PYTHON}
-RUN bash -c 'if [[ "${PACKAGE_PYTHON}" == "false" ]]; then cp docker/Cargo.toml.no-python Cargo.toml; fi'
-
-ENV PACKAGE_CUDA=${PACKAGE_CUDA}
-RUN bash -c 'if [[ "${PACKAGE_CUDA}" == "true" ]]; then cp docker/Cargo.toml.cuda Cargo.toml; fi'
-
-# Build and upload package to S3
-ENV CUDACXX=/usr/local/cuda/bin/nvcc
-RUN cargo pgx package
-
-# Deb file goes here, mount it on your local system
-VOLUME /output
-USER root
-
-ENV PACKAGE_VERSION=$PACKAGE_VERSION
-ENV OUTPUT_DIR=/output
-
-# Run
-ENTRYPOINT ["bash", "docker/build_ubuntu_deb.sh", "/output"]
diff --git a/pgml-extension/README.md b/pgml-extension/README.md
index dd1efef5a..ee6fda651 100644
--- a/pgml-extension/README.md
+++ b/pgml-extension/README.md
@@ -1,7 +1 @@
-# PostgresML Extension
-
-PostgresML is a PostgreSQL extension providing end-to-end machine learning inside your database. The extension is primarily written in Rust using [pgx](https://github.com/tcdi/pgx) and provides a SQL interface to various machine learning algorithm implementations such as [XGBoost](https://github.com/dmlc/xgboost), [LightGBM](https://github.com/microsoft/LightGBM), and [other classical methods](https://github.com/rust-ml/linfa).
-
-See [our blog](https://postgresml.org/blog/postgresml-is-moving-to-rust-for-our-2.0-release/) for a performance comparison and further motivations.
-
-Please see the [quick start instructions](https://postgresml.org/user_guides/setup/quick_start_with_docker/) for general information on installing or deploying PostgresML. A [developer guide](https://postgresml.org/developer_guide/overview/) is also available for those who would like to contribute.
+Please see the [quick start instructions](https://postgresml.org/user_guides/setup/quick_start_with_docker/) for general information on installing or deploying PostgresML. A [developer guide](https://postgresml.org/developer_guide/overview/) is also available for those who would like to contribute.
\ No newline at end of file
From cb1a6e24b0b6fe66984740d18a09b5e49055c64e Mon Sep 17 00:00:00 2001
From: Lev Kokotov
Date: Mon, 13 Feb 2023 16:40:21 -0800
Subject: [PATCH 02/22] Streamline deb build (#543)
---
.github/workflows/package-extension.yml | 58 +++++++++++++++----------
pgml-extension/Cargo.lock | 2 +-
pgml-extension/control | 8 ++--
3 files changed, 40 insertions(+), 28 deletions(-)
diff --git a/.github/workflows/package-extension.yml b/.github/workflows/package-extension.yml
index 0d622c4e3..fdbf8a46d 100644
--- a/.github/workflows/package-extension.yml
+++ b/.github/workflows/package-extension.yml
@@ -2,6 +2,9 @@ name: Build PostgresML Rust package
on:
workflow_dispatch:
+ inputs:
+ packageVersion:
+ default: "2.2.0"
jobs:
build:
@@ -9,35 +12,44 @@ jobs:
matrix:
os: [ubuntu-latest]
runs-on: ${{ matrix.os }}
+ defaults:
+ run:
+ working-directory: pgml-extension
steps:
- uses: actions/checkout@v3
- name: Install dependencies
- run:
+ run: |
sudo apt update
sudo apt install -y ruby curl
curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
sudo gem install deb-s3-0.11.4.gem
+ git submodule update --init --recursive
+ chmod 777 . -R
+ dpkg-deb --version
- name: Build package
- run:
- docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/11/bin/pg_config
- docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/12/bin/pg_config
- docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/13/bin/pg_config
- docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/14/bin/pg_config
- docker run -v $(pwd):/app levkk/postgresml:build cargo pgx package --pg-control /usr/lib/postgresql/15/bin/pg_config
- # - name: Publish package
- # env:
- # AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
- # AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- # AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
- # working-directory: postgresml/pgml-extension
- # run: |
- # if [[ $(uname -a) == *"aarch64"* ]]; then
- # ARCH="arm64"
- # else
- # ARCH="amd64"
- # fi
+ run: |
+ docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/11/bin/pg_config'
+ docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/12/bin/pg_config'
+ docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/13/bin/pg_config'
+ docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/14/bin/pg_config'
+ docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/15/bin/pg_config'
+ - name: Build debs
+ env:
+ AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
+ AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+ AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
+ run: |
+ for pg in {11..15}; do
+ export PACKAGE_VERSION=${{ inputs.packageVersion }}
+ export PGVERSION=${pg}
+ export ARCH=amd64
+
+ mkdir -p target/release/pgml-pg${pg}/DEBIAN
+ (cat control | envsubst) > target/release/pgml-pg${pg}/DEBIAN/control
+ dpkg-deb --build target/release/pgml-pg${pg} postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu1-amd64.deb
- # deb-s3 upload \
- # --bucket apt.postgresml.org \
- # postgresql-pgml-${{ inputs.postgresVersion }}_${{ inputs.packageVersion }}-ubuntu${{ inputs.ubuntuVersion }}-${ARCH}.deb \
- # --codename ${{ inputs.ubuntuName }}
+ deb-s3 upload \
+ --bucket apt.postgresml.org \
+ postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu1-amd64.deb \
+ --codename jammy
+ done
diff --git a/pgml-extension/Cargo.lock b/pgml-extension/Cargo.lock
index 3e56f8f95..de0ca2fe0 100644
--- a/pgml-extension/Cargo.lock
+++ b/pgml-extension/Cargo.lock
@@ -1680,7 +1680,7 @@ dependencies = [
[[package]]
name = "pgml"
-version = "2.1.2"
+version = "2.2.0"
dependencies = [
"anyhow",
"blas",
diff --git a/pgml-extension/control b/pgml-extension/control
index 047ad67fb..640540353 100644
--- a/pgml-extension/control
+++ b/pgml-extension/control
@@ -1,9 +1,9 @@
-Package: postgresql-pgml-PGVERSION
-Version: PACKAGE_VERSION
+Package: postgresql-pgml-${PGVERSION}
+Version: ${PACKAGE_VERSION}
Section: database
Priority: optional
-Architecture: ARCH
-Depends: postgresql-PGVERSION, libopenblas-dev, postgresql-server-dev-PGVERSION, python3, python3-dev, libpython3-dev
+Architecture: ${ARCH}
+Depends: postgresql-${PGVERSION}, libopenblas-dev, postgresql-server-dev-${PGVERSION}, python3, python3-dev, libpython3-dev
Maintainer: PostgresML
Homepage: https://postgresml.org
Description: PostgresML - machine learning with PostgreSQL
From efce953ce7e10a0ea16639c2c03d11cee6227cb8 Mon Sep 17 00:00:00 2001
From: Lev Kokotov
Date: Tue, 14 Feb 2023 12:00:17 -0800
Subject: [PATCH 03/22] Fix .deb release (#544)
---
.github/workflows/package-extension.yml | 3 +++
1 file changed, 3 insertions(+)
diff --git a/.github/workflows/package-extension.yml b/.github/workflows/package-extension.yml
index fdbf8a46d..1796cf3fc 100644
--- a/.github/workflows/package-extension.yml
+++ b/.github/workflows/package-extension.yml
@@ -44,6 +44,9 @@ jobs:
export PGVERSION=${pg}
export ARCH=amd64
+ # Fix permissions
+ sudo chown -R ${USER}:${USER} target/release
+
mkdir -p target/release/pgml-pg${pg}/DEBIAN
(cat control | envsubst) > target/release/pgml-pg${pg}/DEBIAN/control
dpkg-deb --build target/release/pgml-pg${pg} postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu1-amd64.deb
From 82869e585575644bcc75f4d4bfc36457774fb9f8 Mon Sep 17 00:00:00 2001
From: Lev Kokotov
Date: Wed, 15 Feb 2023 11:10:45 -0800
Subject: [PATCH 04/22] Fix deb again (#545)
---
.github/workflows/package-extension.yml | 119 ++++++++++++++++++++----
1 file changed, 102 insertions(+), 17 deletions(-)
diff --git a/.github/workflows/package-extension.yml b/.github/workflows/package-extension.yml
index 1796cf3fc..d262953e3 100644
--- a/.github/workflows/package-extension.yml
+++ b/.github/workflows/package-extension.yml
@@ -10,29 +10,117 @@ jobs:
build:
strategy:
matrix:
- os: [ubuntu-latest]
+ os: ["ubuntu-22.04"]
runs-on: ${{ matrix.os }}
defaults:
run:
working-directory: pgml-extension
steps:
- uses: actions/checkout@v3
+ - uses: actions-rs/toolchain@v1
+ with:
+ toolchain: stable
+ - name: Validate cargo is working
+ uses: postgresml/gh-actions-cargo@master
+ with:
+ working-directory: pgml-extension
+ command: version
- name: Install dependencies
+ env:
+ DEBIAN_FRONTEND: noninteractive
+ TZ: Etc/UTC
run: |
+ git submodule update --init --recursive
+
+ # PostgreSQL apt
+ curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/apt.postgresql.org.gpg >/dev/null
+ sudo sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
+
+ sudo apt-get install -y software-properties-common
+ sudo add-apt-repository ppa:apt-fast/stable --yes
+ sudo add-apt-repository ppa:deadsnakes/ppa --yes
sudo apt update
- sudo apt install -y ruby curl
+
+ sudo apt-get install -y apt-fast
+ sudo apt-get update && sudo apt-fast install -y \
+ libopenblas-dev \
+ libssl-dev \
+ bison \
+ flex \
+ pkg-config \
+ cmake \
+ libreadline-dev \
+ libz-dev \
+ curl \
+ lsb-release \
+ tzdata \
+ sudo \
+ cmake \
+ libpq-dev \
+ libclang-dev \
+ wget \
+ postgresql-15 \
+ postgresql-14 \
+ postgresql-13 \
+ postgresql-12 \
+ postgresql-11 \
+ postgresql-server-dev-15 \
+ postgresql-server-dev-14 \
+ postgresql-server-dev-13 \
+ postgresql-server-dev-12 \
+ postgresql-server-dev-11 \
+ lsb-release \
+ python3.10 \
+ python3-pip \
+ libpython3.10-dev \
+ python3.10-dev \
+ ruby
+
curl -sLO https://github.com/deb-s3/deb-s3/releases/download/0.11.4/deb-s3-0.11.4.gem
sudo gem install deb-s3-0.11.4.gem
- git submodule update --init --recursive
- chmod 777 . -R
dpkg-deb --version
- - name: Build package
- run: |
- docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/11/bin/pg_config'
- docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/12/bin/pg_config'
- docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/13/bin/pg_config'
- docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/14/bin/pg_config'
- docker run -v $(pwd):/app ghcr.io/postgresml/postgresml-builder:latest bash -c '$HOME/.cargo/bin/cargo pgx package --pg-config /usr/lib/postgresql/15/bin/pg_config'
+ - name: Install pgx
+ uses: postgresml/gh-actions-cargo@master
+ with:
+ working-directory: pgml-extension
+ command: install
+ args: cargo-pgx --version "0.7.1"
+ - name: pgx init
+ uses: postgresml/gh-actions-cargo@master
+ with:
+ working-directory: pgml-extension
+ command: pgx
+ args: init --pg11=/usr/lib/postgresql/11/bin/pg_config --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config
+ - name: Build Postgres 11
+ uses: postgresml/gh-actions-cargo@master
+ with:
+ working-directory: pgml-extension
+ command: pgx
+ args: package --pg-config /usr/lib/postgresql/11/bin/pg_config
+ - name: Build Postgres 12
+ uses: postgresml/gh-actions-cargo@master
+ with:
+ working-directory: pgml-extension
+ command: pgx
+ args: package --pg-config /usr/lib/postgresql/12/bin/pg_config
+ - name: Build Postgres 13
+ uses: postgresml/gh-actions-cargo@master
+ with:
+ working-directory: pgml-extension
+ command: pgx
+ args: package --pg-config /usr/lib/postgresql/13/bin/pg_config
+ - name: Build Postgres 14
+ uses: postgresml/gh-actions-cargo@master
+ with:
+ working-directory: pgml-extension
+ command: pgx
+ args: package --pg-config /usr/lib/postgresql/14/bin/pg_config
+ - name: Build Postgres 15
+ uses: postgresml/gh-actions-cargo@master
+ with:
+ working-directory: pgml-extension
+ command: pgx
+ args: package --pg-config /usr/lib/postgresql/15/bin/pg_config
- name: Build debs
env:
AWS_ACCESS_KEY_ID: ${{ vars.AWS_ACCESS_KEY_ID }}
@@ -44,15 +132,12 @@ jobs:
export PGVERSION=${pg}
export ARCH=amd64
- # Fix permissions
- sudo chown -R ${USER}:${USER} target/release
-
mkdir -p target/release/pgml-pg${pg}/DEBIAN
(cat control | envsubst) > target/release/pgml-pg${pg}/DEBIAN/control
- dpkg-deb --build target/release/pgml-pg${pg} postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu1-amd64.deb
+ dpkg-deb --root-owner-group --build target/release/pgml-pg${pg} postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-amd64.deb
deb-s3 upload \
--bucket apt.postgresml.org \
- postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu1-amd64.deb \
- --codename jammy
+ postgresql-pgml-${pg}_${PACKAGE_VERSION}-ubuntu22.04-amd64.deb \
+ --codename $(lsb_release -cs)
done
From ed959885bf9aecb4ec7a57f1746ef71ec9a1241f Mon Sep 17 00:00:00 2001
From: Lev Kokotov
Date: Wed, 15 Feb 2023 11:15:22 -0800
Subject: [PATCH 05/22] Cleanup GH actions (#548)
---
.github/workflows/ci.yml | 2 +-
.github/workflows/deploy-beta.yml | 23 ----------------------
.github/workflows/package-extension.yml | 2 +-
.github/workflows/publish-docs.yml | 4 ++--
.github/workflows/python-publish.yml | 26 -------------------------
5 files changed, 4 insertions(+), 53 deletions(-)
delete mode 100644 .github/workflows/deploy-beta.yml
delete mode 100644 .github/workflows/python-publish.yml
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 801f016ee..b07bc57f0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,4 +1,4 @@
-name: Tests
+name: tests
on:
push:
branches:
diff --git a/.github/workflows/deploy-beta.yml b/.github/workflows/deploy-beta.yml
deleted file mode 100644
index 1508b2f9b..000000000
--- a/.github/workflows/deploy-beta.yml
+++ /dev/null
@@ -1,23 +0,0 @@
-name: Deploy to beta
-
-on:
- push:
- branches:
- - master
- workflow_dispatch:
- inputs:
- name:
- description: "Just a trigger"
- required: false
- default: "Hello"
-
-jobs:
- deploy:
- runs-on: ubuntu-latest
- steps:
- - name: Install ssh key
- run: mkdir -p ~/.ssh && echo "${{ secrets.BETA_SSH_KEY }}" > ~/.ssh/id_rsa && chmod 600 ~/.ssh/id_rsa
-
- - name: Start deploy
- run: ssh -o "StrictHostKeyChecking=no" root@${{ secrets.BETA_IP }} '(cd /root; bash infrastructure/ubuntu/ubuntu_20_04_lts.sh > /var/log/deploy.log 2>&1) >&- 2>&- <&- &'
-
diff --git a/.github/workflows/package-extension.yml b/.github/workflows/package-extension.yml
index d262953e3..d9e31243d 100644
--- a/.github/workflows/package-extension.yml
+++ b/.github/workflows/package-extension.yml
@@ -1,4 +1,4 @@
-name: Build PostgresML Rust package
+name: package extension (deb)
on:
workflow_dispatch:
diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml
index 1c3625db3..5df861d6f 100644
--- a/.github/workflows/publish-docs.yml
+++ b/.github/workflows/publish-docs.yml
@@ -1,4 +1,4 @@
-name: Publish Documentation
+name: publish docs
on:
push:
@@ -18,4 +18,4 @@ jobs:
python-version: 3.x
- run: pip install -r requirements.txt
- run: mkdocs gh-deploy --force
-
\ No newline at end of file
+
diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml
deleted file mode 100644
index ead0c1aeb..000000000
--- a/.github/workflows/python-publish.yml
+++ /dev/null
@@ -1,26 +0,0 @@
-name: Upload Python Package
-
-on:
- release:
- types: [published]
-
-jobs:
- deploy:
- runs-on: ubuntu-latest
- defaults:
- run:
- working-directory: pgml-extension
- steps:
- - uses: actions/checkout@v3
- - name: Set up Python
- uses: actions/setup-python@v3
- with:
- python-version: '3.x'
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install build twine wheel
- - name: Build package
- run: python -m build
- - name: Publish package
- run: twine upload -u __token__ -p ${{ secrets.PYPI_API_TOKEN }} dist/*
From 85e2c885a51689c445c52e721bbf0220cbc10a79 Mon Sep 17 00:00:00 2001
From: Montana Low
Date: Fri, 17 Feb 2023 10:19:09 -0800
Subject: [PATCH 06/22] Fix typo
#550
---
pgml-extension/src/bindings/sklearn.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pgml-extension/src/bindings/sklearn.py b/pgml-extension/src/bindings/sklearn.py
index 5d95062aa..0d4b2930f 100644
--- a/pgml-extension/src/bindings/sklearn.py
+++ b/pgml-extension/src/bindings/sklearn.py
@@ -215,7 +215,7 @@ def calculate_metric(metric_name):
elif metric_name == "mcc":
func = matthews_corrcoef
elif metric_name == "mse":
- func = mean_squarred_error
+ func = mean_squared_error
elif metric_name == "mae":
func = mean_absolute_error
elif metric_name == "confusion_matrix":
From 4d28952b0cb4ba07eab5ea3154338657cf173c9a Mon Sep 17 00:00:00 2001
From: santiatpml <124207123+santiatpml@users.noreply.github.com>
Date: Tue, 21 Feb 2023 19:42:38 -0800
Subject: [PATCH 07/22] Dashboard snapshots fix (#552)
---
pgml-dashboard/Cargo.lock | 2 +-
pgml-dashboard/src/models.rs | 5 ++---
2 files changed, 3 insertions(+), 4 deletions(-)
diff --git a/pgml-dashboard/Cargo.lock b/pgml-dashboard/Cargo.lock
index 4d9318ea3..0039ab137 100644
--- a/pgml-dashboard/Cargo.lock
+++ b/pgml-dashboard/Cargo.lock
@@ -1529,7 +1529,7 @@ dependencies = [
[[package]]
name = "pgml-dashboard"
-version = "0.1.1"
+version = "2.2.0"
dependencies = [
"anyhow",
"bigdecimal",
diff --git a/pgml-dashboard/src/models.rs b/pgml-dashboard/src/models.rs
index 45c78f9b4..196acb871 100644
--- a/pgml-dashboard/src/models.rs
+++ b/pgml-dashboard/src/models.rs
@@ -549,8 +549,7 @@ impl Snapshot {
analysis,
created_at,
updated_at
- FROM pgml.snapshots
- "
+ FROM pgml.snapshots JOIN pg_class ON oid::regclass::text = relation_name"
)
.fetch_all(pool)
.await?)
@@ -568,7 +567,7 @@ impl Snapshot {
analysis,
created_at,
updated_at
- FROM pgml.snapshots
+ FROM pgml.snapshots JOIN pg_class ON oid::regclass::text = relation_name
WHERE id = $1",
id,
)
From 7f2e5c272c7d838cc9bb190d55464486a3b3629f Mon Sep 17 00:00:00 2001
From: santiatpml <124207123+santiatpml@users.noreply.github.com>
Date: Thu, 23 Feb 2023 15:01:35 -0800
Subject: [PATCH 08/22] Santi dashboard snapshots fix2 (#554)
---
pgml-dashboard/.gitignore | 1 +
pgml-dashboard/src/guards.rs | 5 +-
pgml-dashboard/src/lib.rs | 22 ++---
pgml-dashboard/src/main.rs | 4 +-
pgml-dashboard/src/models.rs | 124 ++++++++++++++----------
pgml-dashboard/src/templates.rs | 2 -
pgml-dashboard/templates/model.html | 4 +-
pgml-dashboard/templates/snapshot.html | 9 +-
pgml-dashboard/templates/snapshots.html | 2 +-
9 files changed, 101 insertions(+), 72 deletions(-)
diff --git a/pgml-dashboard/.gitignore b/pgml-dashboard/.gitignore
index 485dee64b..dc271fe75 100644
--- a/pgml-dashboard/.gitignore
+++ b/pgml-dashboard/.gitignore
@@ -1 +1,2 @@
.idea
+test_table.sql
diff --git a/pgml-dashboard/src/guards.rs b/pgml-dashboard/src/guards.rs
index 6c672405a..d7fb22ac1 100644
--- a/pgml-dashboard/src/guards.rs
+++ b/pgml-dashboard/src/guards.rs
@@ -49,7 +49,10 @@ impl<'r> FromRequest<'r> for Cluster {
None => return Outcome::Failure((Status::BadRequest, ())),
};
- Outcome::Success(Cluster { pool, context: shared_state.get_context(cluster_id) })
+ Outcome::Success(Cluster {
+ pool,
+ context: shared_state.get_context(cluster_id),
+ })
}
}
diff --git a/pgml-dashboard/src/lib.rs b/pgml-dashboard/src/lib.rs
index dd58ce92c..27ed0fb5b 100644
--- a/pgml-dashboard/src/lib.rs
+++ b/pgml-dashboard/src/lib.rs
@@ -31,7 +31,7 @@ use sqlx::Executor;
#[derive(Debug, Default, Clone)]
pub struct Context {
pub user: models::User,
- pub cluster: models::Cluster,
+ pub cluster: models::Cluster,
}
/// Globally shared state, saved in memory.
@@ -51,10 +51,13 @@ impl Clusters {
.max_connections(5)
.idle_timeout(std::time::Duration::from_millis(15_000))
.min_connections(0)
- .after_connect(|conn, _meta| Box::pin(async move {
- conn.execute("SET application_name = 'pgml_dashboard';").await?;
- Ok(())
- }))
+ .after_connect(|conn, _meta| {
+ Box::pin(async move {
+ conn.execute("SET application_name = 'pgml_dashboard';")
+ .await?;
+ Ok(())
+ })
+ })
.connect_lazy(database_url)?;
pools.insert(cluster_id, pool.clone());
@@ -417,18 +420,11 @@ pub async fn models_get(cluster: Cluster, id: i64) -> Result Result {
let snapshots = models::Snapshot::all(cluster.pool()).await?;
- let mut table_sizes = HashMap::new();
-
- for snapshot in &snapshots {
- let table_size = snapshot.table_size(cluster.pool()).await?;
- table_sizes.insert(snapshot.id, table_size);
- }
Ok(ResponseOk(
templates::Snapshots {
topic: "snapshots".to_string(),
snapshots,
- table_sizes,
context: cluster.context.clone(),
}
.render_once()
@@ -440,6 +436,7 @@ pub async fn snapshots_index(cluster: Cluster) -> Result Result {
let snapshot = models::Snapshot::get_by_id(cluster.pool(), id).await?;
let samples = snapshot.samples(cluster.pool(), 500).await?;
+
let models = snapshot.models(cluster.pool()).await?;
let mut projects = HashMap::new();
@@ -450,7 +447,6 @@ pub async fn snapshots_get(cluster: Cluster, id: i64) -> Result,
pub created_at: PrimitiveDateTime,
pub updated_at: PrimitiveDateTime,
+ pub exists: bool,
+ pub table_size: String,
}
impl Snapshot {
@@ -548,8 +553,22 @@ impl Snapshot {
columns,
analysis,
created_at,
- updated_at
- FROM pgml.snapshots JOIN pg_class ON oid::regclass::text = relation_name"
+ updated_at,
+ CASE
+ WHEN EXISTS (
+ SELECT 1
+ FROM pg_class c
+ WHERE c.oid::regclass::text = relation_name
+ ) THEN pg_size_pretty(pg_total_relation_size(relation_name::regclass))
+ ELSE '0 Bytes'
+ END AS \"table_size!\",
+ EXISTS (
+ SELECT 1
+ FROM pg_class c
+ WHERE c.oid::regclass::text = relation_name
+ ) AS \"exists!\"
+ FROM pgml.snapshots
+ "
)
.fetch_all(pool)
.await?)
@@ -566,25 +585,27 @@ impl Snapshot {
columns,
analysis,
created_at,
- updated_at
- FROM pgml.snapshots JOIN pg_class ON oid::regclass::text = relation_name
- WHERE id = $1",
+ updated_at,
+ CASE
+ WHEN EXISTS (
+ SELECT 1
+ FROM pg_class c
+ WHERE c.oid::regclass::text = relation_name
+ ) THEN pg_size_pretty(pg_total_relation_size(relation_name::regclass))
+ ELSE '0 Bytes'
+ END AS \"table_size!\",
+ EXISTS (
+ SELECT 1
+ FROM pg_class c
+ WHERE c.oid::regclass::text = relation_name
+ ) AS \"exists!\"
+ FROM pgml.snapshots WHERE id = $1",
id,
)
.fetch_one(pool)
.await?)
}
- pub async fn table_size(&self, pool: &PgPool) -> anyhow::Result {
- let row =
- sqlx::query("SELECT pg_size_pretty(pg_total_relation_size($1))::TEXT AS table_size")
- .bind(&self.relation_name)
- .fetch_one(pool)
- .await?;
-
- Ok(row.try_get("table_size")?)
- }
-
pub fn rows(&self) -> Option {
match self.analysis.as_ref() {
Some(analysis) => match analysis.get("samples") {
@@ -600,26 +621,28 @@ impl Snapshot {
pool: &PgPool,
rows: i64,
) -> anyhow::Result>> {
- let rows = sqlx::query(&format!(
- "SELECT row_to_json(row) r
- FROM (SELECT * FROM {} LIMIT $1) row",
- self.relation_name
- ))
- .bind(rows)
- .fetch_all(pool)
- .await?;
-
let mut samples = HashMap::new();
- rows.iter().for_each(|row| {
- let r: serde_json::Value = row.try_get("r").unwrap();
- let obj = r.as_object().unwrap();
+ if self.exists {
+ let rows = sqlx::query(&format!(
+ "SELECT row_to_json(row) r
+ FROM (SELECT * FROM {} LIMIT $1) row",
+ self.relation_name
+ ))
+ .bind(rows)
+ .fetch_all(pool)
+ .await?;
- for (key, value) in obj.iter() {
- let rf = samples.entry(key.clone()).or_insert(Vec::new());
- rf.push(value.as_f64().unwrap_or(0.) as f32);
- }
- });
+ rows.iter().for_each(|row| {
+ let r: serde_json::Value = row.try_get("r").unwrap();
+ let obj = r.as_object().unwrap();
+
+ for (key, value) in obj.iter() {
+ let rf = samples.entry(key.clone()).or_insert(Vec::new());
+ rf.push(value.as_f64().unwrap_or(0.) as f32);
+ }
+ });
+ }
Ok(samples)
}
@@ -664,7 +687,7 @@ impl Snapshot {
}
pub fn labels<'a>(&'a self) -> Option>> {
- self.columns().map(|columns|
+ self.columns().map(|columns| {
columns
.into_iter()
.filter(|column| {
@@ -672,7 +695,7 @@ impl Snapshot {
.contains(&column["name"].as_str().unwrap().to_string())
})
.collect()
- )
+ })
}
pub async fn models(&self, pool: &PgPool) -> anyhow::Result> {
@@ -680,32 +703,33 @@ impl Snapshot {
}
pub fn target_stddev(&self, name: &str) -> f32 {
- match self.analysis
+ match self
+ .analysis
.as_ref()
.unwrap()
.as_object()
.unwrap()
- .get(&format!("{}_stddev", name)) {
+ .get(&format!("{}_stddev", name))
+ {
// 2.1
Some(value) => value.as_f64().unwrap() as f32,
// 2.2+
None => {
let columns = self.columns().unwrap();
- let column = columns.iter().find(|column|
- &column["name"].as_str().unwrap() == &name
- );
+ let column = columns
+ .iter()
+ .find(|column| &column["name"].as_str().unwrap() == &name);
match column {
- Some(column) => {
- column.get("statistics")
- .unwrap()
- .as_object()
- .unwrap()
- .get("std_dev")
- .unwrap()
- .as_f64()
- .unwrap() as f32
- },
- None => 0.
+ Some(column) => column
+ .get("statistics")
+ .unwrap()
+ .as_object()
+ .unwrap()
+ .get("std_dev")
+ .unwrap()
+ .as_f64()
+ .unwrap() as f32,
+ None => 0.,
}
}
}
diff --git a/pgml-dashboard/src/templates.rs b/pgml-dashboard/src/templates.rs
index b1a612a26..ec8f4235c 100644
--- a/pgml-dashboard/src/templates.rs
+++ b/pgml-dashboard/src/templates.rs
@@ -231,7 +231,6 @@ pub struct Model {
pub struct Snapshots {
pub topic: String,
pub snapshots: Vec,
- pub table_sizes: HashMap,
pub context: Context,
}
@@ -242,7 +241,6 @@ pub struct Snapshot {
pub snapshot: models::Snapshot,
pub models: Vec,
pub projects: HashMap,
- pub table_size: String,
pub samples: HashMap>,
pub context: Context,
}
diff --git a/pgml-dashboard/templates/model.html b/pgml-dashboard/templates/model.html
index 0011b43d1..ba41a4fee 100644
--- a/pgml-dashboard/templates/model.html
+++ b/pgml-dashboard/templates/model.html
@@ -12,8 +12,8 @@ model_training<%= model.algor
- Project
- <%= project.name %>
- - Snapshot
- - <%= snapshot.relation_name %>
+ - Snapshot
+ - <%= snapshot.relation_name %>
- Created
diff --git a/pgml-dashboard/templates/snapshot.html b/pgml-dashboard/templates/snapshot.html
index 1a9ba32d4..fdd890bdd 100644
--- a/pgml-dashboard/templates/snapshot.html
+++ b/pgml-dashboard/templates/snapshot.html
@@ -13,7 +13,7 @@ storage<%= snapshot.relation_
Snapshot ID
<%= snapshot.id %>
Table Size
- <%= table_size %>
+ <%= snapshot.table_size %>
Rows
<%= snapshot.rows().unwrap() %>
Features
@@ -61,6 +61,7 @@ model_trainingModels
<% } %>
+<% if snapshot.exists { %>
label_importantLabels
<% for label in snapshot.labels().unwrap().iter() {
@@ -131,5 +132,9 @@ <%= name %> <%= feature["pg_type"].as_str().unwrap() | upper %>
}
renderCharts();
-
+<%} else { %>
+
+
Data no longer exists to plot statistics
+
+<% }%>
<% include!("footer.html"); %>
diff --git a/pgml-dashboard/templates/snapshots.html b/pgml-dashboard/templates/snapshots.html
index 24778b701..6d9bf1832 100644
--- a/pgml-dashboard/templates/snapshots.html
+++ b/pgml-dashboard/templates/snapshots.html
@@ -7,7 +7,7 @@ storageSnapshots
<%= snapshot.relation_name %>
- <%= table_sizes[&snapshot.id] %>
+ <%= snapshot.table_size %>
From 6eaa347f3b8c94f0fc23db3a486617798078757b Mon Sep 17 00:00:00 2001
From: santiatpml <124207123+santiatpml@users.noreply.github.com>
Date: Fri, 24 Feb 2023 14:33:39 -0800
Subject: [PATCH 09/22] Dashboard backend tests using rocket (#556)
---
pgml-dashboard/.gitignore | 2 +-
pgml-dashboard/src/main.rs | 68 ++++++++++++++++++++++++++++++++++++
pgml-dashboard/src/models.rs | 3 --
3 files changed, 69 insertions(+), 4 deletions(-)
diff --git a/pgml-dashboard/.gitignore b/pgml-dashboard/.gitignore
index dc271fe75..706fd07fa 100644
--- a/pgml-dashboard/.gitignore
+++ b/pgml-dashboard/.gitignore
@@ -1,2 +1,2 @@
.idea
-test_table.sql
+.vscode
diff --git a/pgml-dashboard/src/main.rs b/pgml-dashboard/src/main.rs
index 26b12726f..ea93ec056 100644
--- a/pgml-dashboard/src/main.rs
+++ b/pgml-dashboard/src/main.rs
@@ -29,3 +29,71 @@ async fn main() {
.await
.expect("failed to shut down Rocket");
}
+
+#[cfg(test)]
+mod test {
+ use rocket::local::asynchronous::Client;
+ use pgml_dashboard::Clusters;
+ use pgml_dashboard::{index, migrate, paths};
+ use rocket::fs::FileServer;
+ use rocket::{Build, Rocket};
+
+
+ async fn rocket() -> Rocket {
+ let clusters = Clusters::new();
+ clusters
+ .add(-1, &pgml_dashboard::guards::default_database_url())
+ .unwrap();
+
+ migrate(&clusters.get(-1).unwrap()).await.unwrap();
+
+ rocket::build()
+ .manage(clusters)
+ .mount("/", rocket::routes![index,])
+ .mount("/dashboard/static", FileServer::from("static"))
+ .mount("/dashboard", paths())
+ }
+
+ #[rocket::async_test]
+ async fn test_notebooks_index() {
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get("/dashboard/notebooks/").dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+
+ #[rocket::async_test]
+ async fn test_projects_index() {
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get("/dashboard/projects/").dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+
+ #[rocket::async_test]
+ async fn test_models_index() {
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get("/dashboard/models/").dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+
+ #[rocket::async_test]
+ async fn test_deployments_index() {
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get("/dashboard/deployments/").dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+
+ #[rocket::async_test]
+ async fn test_uploader() {
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get("/dashboard/uploader/").dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+
+ #[rocket::async_test]
+ async fn test_snapshots_index() {
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get("/dashboard/snapshots/").dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+}
+
diff --git a/pgml-dashboard/src/models.rs b/pgml-dashboard/src/models.rs
index 01a2c333d..9469a8861 100644
--- a/pgml-dashboard/src/models.rs
+++ b/pgml-dashboard/src/models.rs
@@ -1,11 +1,9 @@
// Markdown
use comrak::{markdown_to_html, ComrakExtensionOptions, ComrakOptions};
-use rocket::futures::TryFutureExt;
// Templates
use sailfish::TemplateOnce;
-use sqlx::postgres::PgRow;
// Database
use sqlx::postgres::types::PgInterval;
use sqlx::types::time::PrimitiveDateTime;
@@ -19,7 +17,6 @@ use tokio::io::{AsyncBufReadExt, AsyncSeekExt};
use crate::templates;
use std::collections::HashMap;
-use std::str::FromStr;
#[derive(FromRow, Debug, Clone)]
pub struct Project {
From 1e39fef285005ef6a2207ce9bd22975a412556b1 Mon Sep 17 00:00:00 2001
From: santiatpml <124207123+santiatpml@users.noreply.github.com>
Date: Mon, 27 Feb 2023 11:19:03 -0800
Subject: [PATCH 10/22] Dashboard backend tests (#557)
---
pgml-dashboard/Cargo.lock | 482 +++++++++++++++++++++++++++++++++++--
pgml-dashboard/Cargo.toml | 1 +
pgml-dashboard/src/main.rs | 98 +++++++-
3 files changed, 559 insertions(+), 22 deletions(-)
diff --git a/pgml-dashboard/Cargo.lock b/pgml-dashboard/Cargo.lock
index 0039ab137..1a1a3af61 100644
--- a/pgml-dashboard/Cargo.lock
+++ b/pgml-dashboard/Cargo.lock
@@ -49,7 +49,7 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
- "getrandom",
+ "getrandom 0.2.8",
"once_cell",
"version_check",
]
@@ -354,6 +354,12 @@ dependencies = [
"xdg",
]
+[[package]]
+name = "convert_case"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
+
[[package]]
name = "cookie"
version = "0.16.2"
@@ -365,7 +371,7 @@ dependencies = [
"hkdf",
"hmac",
"percent-encoding",
- "rand",
+ "rand 0.8.5",
"sha2",
"subtle",
"time 0.3.17",
@@ -437,10 +443,37 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
dependencies = [
"generic-array",
- "rand_core",
+ "rand_core 0.6.4",
"typenum",
]
+[[package]]
+name = "cssparser"
+version = "0.27.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
+dependencies = [
+ "cssparser-macros",
+ "dtoa-short",
+ "itoa 0.4.8",
+ "matches",
+ "phf 0.8.0",
+ "proc-macro2",
+ "quote",
+ "smallvec",
+ "syn",
+]
+
+[[package]]
+name = "cssparser-macros"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
+dependencies = [
+ "quote",
+ "syn",
+]
+
[[package]]
name = "csv-async"
version = "1.2.5"
@@ -451,7 +484,7 @@ dependencies = [
"cfg-if",
"csv-core",
"futures",
- "itoa",
+ "itoa 1.0.5",
"ryu",
"serde",
]
@@ -518,6 +551,19 @@ dependencies = [
"syn",
]
+[[package]]
+name = "derive_more"
+version = "0.99.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
+dependencies = [
+ "convert_case",
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "syn",
+]
+
[[package]]
name = "devise"
version = "0.3.1"
@@ -588,6 +634,27 @@ version = "0.15.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03d8c417d7a8cb362e0c37e5d815f5eb7c37f79ff93707329d5a194e42e54ca0"
+[[package]]
+name = "dtoa"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
+
+[[package]]
+name = "dtoa-short"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6"
+dependencies = [
+ "dtoa",
+]
+
+[[package]]
+name = "ego-tree"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591"
+
[[package]]
name = "either"
version = "1.8.1"
@@ -709,6 +776,16 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "futf"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
+dependencies = [
+ "mac",
+ "new_debug_unreachable",
+]
+
[[package]]
name = "futures"
version = "0.3.26"
@@ -809,6 +886,15 @@ dependencies = [
"slab",
]
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
[[package]]
name = "generator"
version = "0.7.2"
@@ -832,6 +918,26 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "getopts"
+version = "0.2.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5"
+dependencies = [
+ "unicode-width",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.1.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
+]
+
[[package]]
name = "getrandom"
version = "0.2.8"
@@ -962,6 +1068,20 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "html5ever"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7"
+dependencies = [
+ "log",
+ "mac",
+ "markup5ever",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "http"
version = "0.2.8"
@@ -970,7 +1090,7 @@ checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399"
dependencies = [
"bytes",
"fnv",
- "itoa",
+ "itoa 1.0.5",
]
[[package]]
@@ -1011,7 +1131,7 @@ dependencies = [
"http-body",
"httparse",
"httpdate",
- "itoa",
+ "itoa 1.0.5",
"pin-project-lite",
"socket2",
"tokio",
@@ -1120,6 +1240,12 @@ dependencies = [
"either",
]
+[[package]]
+name = "itoa"
+version = "0.4.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
+
[[package]]
name = "itoa"
version = "1.0.5"
@@ -1217,6 +1343,26 @@ dependencies = [
"tracing-subscriber",
]
+[[package]]
+name = "mac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
+
+[[package]]
+name = "markup5ever"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016"
+dependencies = [
+ "log",
+ "phf 0.10.1",
+ "phf_codegen 0.10.0",
+ "string_cache",
+ "string_cache_codegen",
+ "tendril",
+]
+
[[package]]
name = "matchers"
version = "0.1.0"
@@ -1226,6 +1372,12 @@ dependencies = [
"regex-automata",
]
+[[package]]
+name = "matches"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
+
[[package]]
name = "md-5"
version = "0.10.5"
@@ -1294,6 +1446,18 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "new_debug_unreachable"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
+
+[[package]]
+name = "nodrop"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+
[[package]]
name = "nom"
version = "7.1.3"
@@ -1537,14 +1701,107 @@ dependencies = [
"comrak",
"csv-async",
"parking_lot 0.12.1",
- "rand",
+ "rand 0.8.5",
"rocket",
"sailfish",
+ "scraper",
"serde_json",
"sqlx",
"tokio",
]
+[[package]]
+name = "phf"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
+dependencies = [
+ "phf_macros",
+ "phf_shared 0.8.0",
+ "proc-macro-hack",
+]
+
+[[package]]
+name = "phf"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"
+dependencies = [
+ "phf_shared 0.10.0",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"
+dependencies = [
+ "phf_generator 0.8.0",
+ "phf_shared 0.8.0",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"
+dependencies = [
+ "phf_generator 0.10.0",
+ "phf_shared 0.10.0",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
+dependencies = [
+ "phf_shared 0.8.0",
+ "rand 0.7.3",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
+dependencies = [
+ "phf_shared 0.10.0",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"
+dependencies = [
+ "phf_generator 0.8.0",
+ "phf_shared 0.8.0",
+ "proc-macro-hack",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+dependencies = [
+ "siphasher",
+]
+
[[package]]
name = "pin-project-lite"
version = "0.2.9"
@@ -1595,6 +1852,12 @@ version = "0.2.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+[[package]]
+name = "precomputed-hash"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+
[[package]]
name = "proc-macro-error"
version = "1.0.4"
@@ -1619,6 +1882,12 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "proc-macro-hack"
+version = "0.5.20+deprecated"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068"
+
[[package]]
name = "proc-macro2"
version = "1.0.51"
@@ -1659,6 +1928,20 @@ dependencies = [
"proc-macro2",
]
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+ "rand_pcg",
+]
+
[[package]]
name = "rand"
version = "0.8.5"
@@ -1666,8 +1949,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
]
[[package]]
@@ -1677,7 +1970,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
- "rand_core",
+ "rand_core 0.6.4",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
]
[[package]]
@@ -1686,7 +1988,25 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
dependencies = [
- "getrandom",
+ "getrandom 0.2.8",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "rand_pcg"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
+dependencies = [
+ "rand_core 0.5.1",
]
[[package]]
@@ -1704,7 +2024,7 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
dependencies = [
- "getrandom",
+ "getrandom 0.2.8",
"redox_syscall",
"thiserror",
]
@@ -1801,7 +2121,7 @@ dependencies = [
"num_cpus",
"parking_lot 0.12.1",
"pin-project-lite",
- "rand",
+ "rand 0.8.5",
"ref-cast",
"rocket_codegen",
"rocket_http",
@@ -1860,6 +2180,15 @@ dependencies = [
"uncased",
]
+[[package]]
+name = "rustc_version"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366"
+dependencies = [
+ "semver",
+]
+
[[package]]
name = "rustix"
version = "0.36.8"
@@ -1972,6 +2301,22 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
+[[package]]
+name = "scraper"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc7cb4dae083699a22a65aa9d2699c27f525e35dffaec38b10801e958ed4cf27"
+dependencies = [
+ "cssparser",
+ "ego-tree",
+ "getopts",
+ "html5ever",
+ "matches",
+ "selectors",
+ "smallvec",
+ "tendril",
+]
+
[[package]]
name = "scratch"
version = "1.0.3"
@@ -1988,6 +2333,32 @@ dependencies = [
"untrusted",
]
+[[package]]
+name = "selectors"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
+dependencies = [
+ "bitflags",
+ "cssparser",
+ "derive_more",
+ "fxhash",
+ "log",
+ "matches",
+ "phf 0.8.0",
+ "phf_codegen 0.8.0",
+ "precomputed-hash",
+ "servo_arc",
+ "smallvec",
+ "thin-slice",
+]
+
+[[package]]
+name = "semver"
+version = "1.0.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58bc9567378fc7690d6b2addae4e60ac2eeea07becb2c64b9f218b53865cba2a"
+
[[package]]
name = "serde"
version = "1.0.152"
@@ -2014,11 +2385,21 @@ version = "1.0.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a"
dependencies = [
- "itoa",
+ "itoa 1.0.5",
"ryu",
"serde",
]
+[[package]]
+name = "servo_arc"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"
+dependencies = [
+ "nodrop",
+ "stable_deref_trait",
+]
+
[[package]]
name = "sha1"
version = "0.10.5"
@@ -2065,6 +2446,12 @@ dependencies = [
"libc",
]
+[[package]]
+name = "siphasher"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de"
+
[[package]]
name = "slab"
version = "0.4.7"
@@ -2151,7 +2538,7 @@ dependencies = [
"hkdf",
"hmac",
"indexmap",
- "itoa",
+ "itoa 1.0.5",
"libc",
"log",
"md-5",
@@ -2160,7 +2547,7 @@ dependencies = [
"once_cell",
"paste",
"percent-encoding",
- "rand",
+ "rand 0.8.5",
"rustls",
"rustls-pemfile",
"serde",
@@ -2222,6 +2609,12 @@ dependencies = [
"memchr",
]
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
[[package]]
name = "state"
version = "0.5.3"
@@ -2231,6 +2624,32 @@ dependencies = [
"loom",
]
+[[package]]
+name = "string_cache"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08"
+dependencies = [
+ "new_debug_unreachable",
+ "once_cell",
+ "parking_lot 0.12.1",
+ "phf_shared 0.10.0",
+ "precomputed-hash",
+ "serde",
+]
+
+[[package]]
+name = "string_cache_codegen"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
+dependencies = [
+ "phf_generator 0.10.0",
+ "phf_shared 0.10.0",
+ "proc-macro2",
+ "quote",
+]
+
[[package]]
name = "stringprep"
version = "0.1.2"
@@ -2302,6 +2721,17 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "tendril"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
+dependencies = [
+ "futf",
+ "mac",
+ "utf-8",
+]
+
[[package]]
name = "termcolor"
version = "1.2.0"
@@ -2321,6 +2751,12 @@ dependencies = [
"windows-sys 0.42.0",
]
+[[package]]
+name = "thin-slice"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
+
[[package]]
name = "thiserror"
version = "1.0.38"
@@ -2367,7 +2803,7 @@ version = "0.3.17"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376"
dependencies = [
- "itoa",
+ "itoa 1.0.5",
"serde",
"time-core",
"time-macros",
@@ -2662,6 +3098,12 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
[[package]]
name = "uuid"
version = "1.3.0"
@@ -2701,6 +3143,12 @@ dependencies = [
"try-lock",
]
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
[[package]]
name = "wasi"
version = "0.10.0+wasi-snapshot-preview1"
diff --git a/pgml-dashboard/Cargo.toml b/pgml-dashboard/Cargo.toml
index 47523e0b6..08bb713b8 100644
--- a/pgml-dashboard/Cargo.toml
+++ b/pgml-dashboard/Cargo.toml
@@ -24,3 +24,4 @@ bigdecimal = "0.3"
chrono = "0.4"
serde_json = "1"
csv-async = "1"
+scraper = "0.14.0"
diff --git a/pgml-dashboard/src/main.rs b/pgml-dashboard/src/main.rs
index ea93ec056..05bdc6e7c 100644
--- a/pgml-dashboard/src/main.rs
+++ b/pgml-dashboard/src/main.rs
@@ -32,21 +32,22 @@ async fn main() {
#[cfg(test)]
mod test {
- use rocket::local::asynchronous::Client;
use pgml_dashboard::Clusters;
use pgml_dashboard::{index, migrate, paths};
use rocket::fs::FileServer;
+ use rocket::local::asynchronous::Client;
use rocket::{Build, Rocket};
-
+ use scraper::{Html, Selector};
+ use std::vec::Vec;
async fn rocket() -> Rocket {
let clusters = Clusters::new();
clusters
.add(-1, &pgml_dashboard::guards::default_database_url())
.unwrap();
-
+
migrate(&clusters.get(-1).unwrap()).await.unwrap();
-
+
rocket::build()
.manage(clusters)
.mount("/", rocket::routes![index,])
@@ -54,6 +55,19 @@ mod test {
.mount("/dashboard", paths())
}
+ fn get_href_links(body: &str, pattern: &str) -> Vec {
+ let document = Html::parse_document(body);
+ let selector = Selector::parse("a").unwrap();
+ let mut output = Vec::::new();
+ for element in document.select(&selector) {
+ let href = element.value().attr("href").unwrap();
+ if href.contains(pattern) && href != pattern {
+ output.push(String::from(href));
+ }
+ }
+ output
+ }
+
#[rocket::async_test]
async fn test_notebooks_index() {
let client = Client::tracked(rocket().await).await.unwrap();
@@ -95,5 +109,79 @@ mod test {
let response = client.get("/dashboard/snapshots/").dispatch().await;
assert_eq!(response.status().code, 200);
}
-}
+ #[rocket::async_test]
+ async fn test_snapshot_entries() {
+ let snapshots_endpoint = "/dashboard/snapshots/";
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get(snapshots_endpoint).dispatch().await;
+
+ let body = response.into_string().await.unwrap();
+ let snapshot_links = get_href_links(body.as_str(), snapshots_endpoint);
+
+ for link in snapshot_links {
+ let response = client.get(link.as_str()).dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+ }
+
+ #[rocket::async_test]
+ async fn test_notebook_entries() {
+ let notebooks_endpoint = "/dashboard/notebooks/";
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get(notebooks_endpoint).dispatch().await;
+
+ let body = response.into_string().await.unwrap();
+ let notebook_links = get_href_links(body.as_str(), notebooks_endpoint);
+
+ for link in notebook_links {
+ let response = client.get(link.as_str()).dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+ }
+
+ #[rocket::async_test]
+ async fn test_project_entries() {
+ let projects_endpoint = "/dashboard/projects/";
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get(projects_endpoint).dispatch().await;
+
+ let body = response.into_string().await.unwrap();
+ let project_links = get_href_links(body.as_str(), projects_endpoint);
+
+ for link in project_links {
+ let response = client.get(link.as_str()).dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+ }
+
+ #[rocket::async_test]
+ async fn test_model_entries() {
+ let models_endpoint = "/dashboard/models/";
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get(models_endpoint).dispatch().await;
+
+ let body = response.into_string().await.unwrap();
+ let model_links = get_href_links(body.as_str(), models_endpoint);
+
+ for link in model_links {
+ let response = client.get(link.as_str()).dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+ }
+
+ #[rocket::async_test]
+ async fn test_deployment_entries() {
+ let deployments_endpoint = "/dashboard/deployments/";
+ let client = Client::tracked(rocket().await).await.unwrap();
+ let response = client.get(deployments_endpoint).dispatch().await;
+
+ let body = response.into_string().await.unwrap();
+ let deployment_links = get_href_links(body.as_str(), deployments_endpoint);
+
+ for link in deployment_links {
+ let response = client.get(link.as_str()).dispatch().await;
+ assert_eq!(response.status().code, 200);
+ }
+ }
+}
From 26a7619fc5da6a695a50082ce491dad93f590a18 Mon Sep 17 00:00:00 2001
From: Montana Low
Date: Wed, 1 Mar 2023 09:27:52 -0800
Subject: [PATCH 11/22] Building the app depends on the extension being
installed (#559)
---
docker-compose.yml | 15 +++++++--------
pgml-dashboard/Dockerfile | 2 +-
2 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/docker-compose.yml b/docker-compose.yml
index 2852f8788..3d49c620b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -2,10 +2,10 @@ version: "3"
services:
postgres:
healthcheck:
- test: psql -c 'SELECT 1' -U postgres -h 127.0.0.1
- interval: 2s
- retries: 3
- timeout: 2s
+ test: [ "CMD-SHELL", "pg_isready" ]
+ interval: 1s
+ timeout: 5s
+ retries: 100
build:
context: ./pgml-extension/
dockerfile: Dockerfile.local
@@ -16,7 +16,8 @@ services:
- infinity
dashboard:
depends_on:
- - postgres
+ postgres:
+ condition: service_healthy
build:
context: ./pgml-dashboard/
dockerfile: Dockerfile
@@ -25,9 +26,7 @@ services:
environment:
ROCKET_ADDRESS: 0.0.0.0
DATABASE_URL: postgres://postgres:postgres@postgres:5432/pgml_development
- command:
- - cargo
- - run
+ command: bash -c "sqlx migrate run && cargo run"
docs:
build:
context: ./pgml-docs/
diff --git a/pgml-dashboard/Dockerfile b/pgml-dashboard/Dockerfile
index a800d772d..a6bef3507 100644
--- a/pgml-dashboard/Dockerfile
+++ b/pgml-dashboard/Dockerfile
@@ -1,4 +1,4 @@
FROM rust:1
COPY . /app
WORKDIR /app
-RUN cargo build
+RUN cargo install sqlx-cli
From b1a3aea8064234c6000931de26397ab9b5e40998 Mon Sep 17 00:00:00 2001
From: Dan <39170265+chillenberger@users.noreply.github.com>
Date: Wed, 1 Mar 2023 14:08:43 -0700
Subject: [PATCH 12/22] Add Dan to docs team page (#560)
---
pgml-docs/docs/about/team.md | 5 +++++
pgml-docs/docs/images/team/daniel.jpg | Bin 0 -> 50259 bytes
2 files changed, 5 insertions(+)
create mode 100644 pgml-docs/docs/images/team/daniel.jpg
diff --git a/pgml-docs/docs/about/team.md b/pgml-docs/docs/about/team.md
index 999c2c823..5a0027160 100644
--- a/pgml-docs/docs/about/team.md
+++ b/pgml-docs/docs/about/team.md
@@ -27,4 +27,9 @@
Santi Adavani
santiatpml
+
+
+ Daniel Illenberger
+ chillenberger
+
diff --git a/pgml-docs/docs/images/team/daniel.jpg b/pgml-docs/docs/images/team/daniel.jpg
new file mode 100644
index 0000000000000000000000000000000000000000..ab31d934fe5ae4058b8d634bb6d67272e6e0034f
GIT binary patch
literal 50259
zcmb4qWl&sEv+dvzT!Rw^cNtuRI|LalxWnM?1cJM}1a}?W65QP_!2=T@!5uE&z3;wy
ze_#LYs&jUo)2n-Tuhp-suiF3|MOg(|0Nk6g!m$DXuNwd<02(q1Dhe_hDhlemcWCIC
zMA(=Z7?@-Pgg8VWD1g8Z6jW5Sj6AHg^jr*7RBWPbT)g~3LP9hw;?m**Qapk}{Qr3f
z+`D)0FflMmv9U?{>8R-V|G(|EAApMrcLjHa00#uX8q;P=$`2HU!01h4j5efNq9e{xV2Y|;xzySc@qjUAl2&n&v-IOr9Ze(n$$%fGBLmqiJj7n;
zY3897B$t))PK8C%$&}&3o=caHCB@OhGHeRE6_vs6*j_Wfc*)>ItUrR8?7;>O|MuAw
zQH8wiPwm6A{v1!L<`I+$d3`(unM@5J>Ln83;5t~q4kAC;^Yh@;(mdM6r!Y)XOlPX&
z*nr=mTY+uNx?SB?cJ8S$v!b{sBgz45S;K8Ad>}OEi4eUqa3a&6O3#h?X)bHD6bTTS
z7NhP5lv#6fW$VuNCG#vR+2oqBqx^|k9uz~LA_A03G$t#8LI-#R>g-zK#JJ#$5|L?R
z8Yqw5vPV9P-a_>_vyc1=Z;ThmzNFf|8@TEKY!#cnFdQqs7J~c
zb(8uxOnL
ztoVcMqdSE(jx0Ia%JObCJ2ch~RqyJ`#1JQDl;qUi`I;tD0?E8II8(l{
zuK;=>qrGk&B^OUk=PT^Yt{Wa&
zgaR>@)}{M4ccfBAzM>s`&Q0E|%<&Tz=`OfI=JvZsIy(v9hUpBVk0(oOiM*kS|thMY@yn^4d#?#b@KS@_=%Is&Gu}65
z&yrZV(v#-3I#OTTlZ3s@g@na~ne(z$sG}xq5olHjbY@2&H=5PI!P(6w*8>NLXSh%<
zY1DERbv#hAY#HK$crHRd6wCTLT8$B`PFb~<1?CZ3%D}=euOR%tg`N~0*zw$#^bT!W
zMHXG10BFY|ziVW%WLpKwfS=nm;HucQi0nLS-_L7LCuU#P{+!o0H=ZN;4sufYbXoI9
zFDvE@lNjMALV@UX)piH~$-^GeSH+3r=(1kovnHE}2Toa1$0(NZH;!~p8w=uZ>wG}A
z@_1&FDv{;0gv?n<3dhon#iqmRGLa%aPIn9c8SygJxGW7}M4qww5t@A+k8cGx?yWzg=E(NtdcPi6f6z
zvO_?ppJG>0GO7)f{0zPkJjw_>V_D>B)*$@JqW&ukm6u6fza>LM7z0^2Hj@yxhpEzS
zPAD3bxR`N8;D37AeZ*Vkfkd9w_;Wh=U^!h~2Pai0(V-Q(@ntTK*~au{L?I!m2@^W8
zsPvILwxbYFFzdZz9Btx8gd(0j09^~IUZU|8AeGkRWcIJG
z&m#IfBaYy=vRr<`86V%+JOc-yCv?6!`A2A6_O}NgLSIXu4wr>n$d&-Jn;c=P)LWSR
z;eM(IG+jflCesX(X^&@U2-G8iA$np+whF-pQH1o%9~Cupf4&ec*_JBcZ4|e!6fG0i
zDbRjWj0V~WggA%LhYvCT(7=2LL`JCjjV{H%xad?^l`76}Nr9uY_Dk*e;*9_&^jjKT
z5;Cj@sr%p2uR&9spD|VoXZ*`w+)keX#tE}ZuDCYCJ@xl^f0l+TzQ
z#W?}3SL8tU_^b{?no7DaTE(-U)J;5)>1=49Y6LALtMk#A75@+l<~n`e7v=p8<8*i9
z)@yb(IJ9lx`>XzYihsOjD%T1%2W(X{W_WPniW!?DajJ@Gf3aetFrmV}jSHgH9F6|G
zh@laeN$2`+<-378O{YH3j!}Nx7s=HsTCVn@Vuk8r)&+KmlH8ETh$UjmWOFrQXxu29
zHW-18Q1d)!RTq_^I