From 6cfe79f125e44ccda2116c93d17305e4e6178724 Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Tue, 24 Oct 2023 10:10:22 -0700 Subject: [PATCH 01/11] Document packages and release process (#1113) --- packages/README.md | 105 +++++++++++++++++++++++++++++++++++ packages/dependency-tree.png | Bin 0 -> 28742 bytes 2 files changed, 105 insertions(+) create mode 100644 packages/README.md create mode 100644 packages/dependency-tree.png diff --git a/packages/README.md b/packages/README.md new file mode 100644 index 000000000..1f4c7419b --- /dev/null +++ b/packages/README.md @@ -0,0 +1,105 @@ +# Packages + +A collection of installable packages and libraries used for distributing and working with PostgresML. + +## Table of contents + +1. `cargo-pgml-components`: cargo CLI for building our web apps (e.g. `pgml-dashboard`) +2. `pgml-components`: library implementing common Rust components used in our web apps +3. `postgresml`: meta package used to install all PostgresML components on an Ubuntu system +4. `postgresml-dashboard`: web app for managing PostgresML, Ubuntu package +5. `postgresml-python`: Python dependencies shipped as a pre-built virtual environment, Ubuntu package +6. `postgresql-pgml`: PostgreSQL extension, Ubuntu package + +## Packages + +### `cargo-pgml-components` + +A cargo (Rust build tool chain) CLI for building web apps written in Rust, Rocket, Sailfish and Turbo/Stimulus. It automatically creates the necessary folder structure for a project, and bundles JavaScript and Sass files into JS and CSS bundles. See [README.md](cargo-pgml-components/README.md) for more details. + +### `pgml-components` + +A Rust library implementing common components for web apps written in Rust, Rocket and Sailfish. Used in our web apps together with the cargo CLI. + +### `postgresml` + +A Debian (Ubuntu 22.04) package which installs everything needed for PostgresML to work on an Ubuntu system. It depends on `postgresql-pgml` and `postgresml-python`, ensuring that correct versions of both are installed, as needed. + +### `postgresml-dashboard` + +A Debian (Ubuntu 22.04) package which compiles and distributes `pgml-dashboard`. It follows the same release cadence as the extension package, documented below. The dashboard is distributed separately because it's a web app and often won't run on the same system as PostgresML. + +### `postgresml-python` + +A Debian (Ubuntu 22.04) package which builds and distributes a Python virtual environment with all the required Python packages used by PostgresML. This includes HuggingFace, PyTorch, Scikit-learn, XGBoost, and many more. This package is quite large and distributed separately since we update our PostgreSQL extension more frequently than our Python dependencies. + +### `postgresql-pgml` + +A Debian (Ubuntu 22.04) package which builds and distributes the PostgreSQL extension. The extension, better known as PostgresML, is the foundation of our product suite and performs all machine learning operations. It's distributed separately from Python dependencies because it includes many algorithms that don't require Python (i.e. our `rust` runtime). Additionally, some systems manage Python dependencies outside of a virtual environment, and we don't want to mandate its use. + +## Dependency tree + +![dependency tree](./dependency-tree.png) + +## Release process + +When releasing a new version of PostgresML to the community, we follow the following release process. At the moment, there are a few manual steps documented below. There are opportunities for automation which we haven't had a moment to explore yet, but PRs and ideas here are welcome. + +### 1. Update version numbers + +#### Cargo.toml + +The version of PostgresML is set in many places, and all of them need to be updated. The first place is `Cargo.toml` (and `Cargo.lock`). Update it in `pgml-extension` and `pgml-dashboard`, making sure both of them match. This is helpful to our users because a version of the dashboard is guaranteed to work with the same version of the extension. The version in `pgml-extension/Cargo.toml` is automatically propagated to the PostgreSQL extension. Make sure to `cargo build` both packages to update the `Cargo.lock` lockfile as well. + +#### Documentation + +Additionally, we mention the version of the extension in our documentation. It would be very helpful to update it there as well, so our users are always instructed to install the latest and greatest version. Our documentation is located in `pgml-docs`. If you search it for the current version number, you should find all the places where we mention it. + +#### Github Actions + +We use Github actions to build our packages. The version of the Debian (Ubuntu) package is independently set from the extension version, but must match nonetheless to avoid confusion. The package version is passed into the build scripts using a Github Actions input variable. Currently, our build process is triggered manually, so that version should be either updated in the YAMLs, so the default is correct, or set manually by the release manager. + +Currently, we have two Github Actions: + +- `ubuntu-packages-and-docker-image`: builds `postgresml`, `postgresml-pgml`, `postgresml-dashboard`, and the Docker image +- `ubuntu-postgresml-python-package`: builds the `postgresml-python` package + +The version of the `postgresml-python` should match the version of `postgresml` only when Python dependencies change. + +### 2. Write a migration + +PostgreSQL extensions are installed into stateful databases. Therefore, a migration from the previous version to the new one is always required. A migration changes the system in a way to work with the new version of the extension. This is especially needed when we change or add an API (SQL function). Without altering/adding the SQL function definitions, PostgreSQL won't be able to use the functions we changed in the extension code. If no API changes were made, PostgreSQL extensions still require a migration file, which can be empty. This is commonly done for bug fixes for existing functionality or internal changes, e.g. swapping runtimes or adding additional options already supported by the existing kwarg-based API. + +#### How to write a migration + +A migration is a SQL (`.sql`) text file that has to be placed into the `pgml-extension/sql` folder. The name of the file is following a strict convention that needs to be followed for the migration to work: + +``` +----.sql +``` + +where: + +- `` is the name of the extension, in our case, `pgml` +- `` is the currently released version of the extension, including minor and patch version numbers +- `` is the new version of the extension being released with the PR, including minor and patch version numbers + +For example, if the current version of the extension is `2.7.10` and we would like to release the new version `2.7.11`, the migration file should be: `pgml-extension/sql/pgml--2.7.10--2.7.11.sql`. + +When the extension is packaged, the migration will be included automatically by `pgrx`, our Rust PostgreSQL extension toolkit. + +### 3. Commit and tag + +Push a commit to `master` with the changes above. Once pushed, make a new Github release. The release should be named after the version, e.g. `v2.7.11` so we and our users can more easily find the changelog. + +#### Name of the tag + +Make sure that the tag is named correctly. Tags are immutable, so if we push the wrong name, it'll stay in git forever. Tags should be named with the version number preceded by the letter "v", e.g. `v2.7.11`. + +### 4. Run Github Actions + +In this order, run the Github actions: + +1. If Python dependencies were updated, run `ubuntu-postgresml-python-package` and wait for it to complete +2. Run `ubuntu-packages-and-docker-image`. When that's finished, the new version of the extension will be released to everyone in our community. + diff --git a/packages/dependency-tree.png b/packages/dependency-tree.png new file mode 100644 index 0000000000000000000000000000000000000000..afdbf53eb4ddd7ffebafff45787a79cc69cb4b9f GIT binary patch literal 28742 zcmeIb2UJu`w;-&DV#b6hqM!spa!yK=&}4~{B{a~4Mxe>rOjK!N1A@?mCQ8m=0wkvf zB&*~sInz@|;a>0e-T%FB{+T!Ly&2bXvHH}hv#V;?uAO)3tE?z};wZ(@ef##EfXUoe z*|+Zi&%S;86AvE(Eg#4}KiRjB>^w^HF3Q^36k!G5N5>`c^A{Z_ySa@$ijM0x9Ve&Z z{rjvY2tzYFLu-3hBpd~rz3kN$3C#M<*FCCW{J3IKp&Bw~Y z&8_{jzo8i%xeHLi#uZ^@Wk|;<#ly-DKr!6q;Gp9Y2fx&f9FQo7pTFf1NC#)o=FHD) zz{~x!&CU^k2;db^28|3b4I9b_Qd4Ik@x!i~UY&C&9B8=fs zE0-i4yBGir{o|6@#SM5s8d~r6RR+st0TY6TN9^LrRm|MhT1r;NLdnup&feb1Sy@u! zAEE8Rw04%ZGrVuEU}FNe0w7GBe@=lzKwx)TCN8@z96UTfhcUC;8)$c2s6iFsx;rC| zU1XR!AWY!)yC47j5@lm!g+ko_YooCZ5(zin-LYMWhIV!~PJiuYYGbv#yxl(c!OH&v z9coZDHnjTZR!xKn%KT?vexBXwLA_<*2s87&k-^sfoTatlUgzBgdvik*8>gS&|7iMk zTYe&P51n>4HUQo)yY}nry-wE75^yUB+k1H29S0E8|NOUqV8iYYvj1NJ=V+lQgW{4< zR)DG9zb)osB%|zX!@~6wF^+~-4!g*`4-T!F9o*g;ECV7vdz8x_-tBB0kS0HlD|FVJ z%n>NK>U~2%&s?{i0I7jCa}@ZD>-Nv12z?FC1PZ=Kc|QT(H~e!wKbH7!BlSu63zYq4fXt2b_?{`-RysR z`Vc$(2b%cXjo|wyY{N|e(eEPP#tvm}V`hUiw37U@Rs7HW06awe&pz@tHefS;?1BXx zg>u{hTW_LC^pm{)SR_ zo9t~I?0ygk==Hbj=Kcu_WoTyx#KI3~f7|Z|q5j*b{##11gIgJ*5RQMH=-+MozXE%2 z1Ti~1Lx>Fb;M})CAW`n)NZUl*s{~T_#r?CCzMtdTbcTeft6S4nKaHCy1_#19y4Y-jD5XQSY|1T%x zPig#LL_6Zy(}w>0^CN+u{OIrZ!^F_u+{nhz&g7q%)_=Jz;r_oyLhP}HU$*3*&<{tN z00rcSePwKAXm5`&28{g|6XDP8`JYoN|6OGf_yYb#T?|ss|Im3j*?*DLdxz&&vth68 zKSZhgCqhUD8*S$0{*KMna4_5VMH{Lj`e zf06_+NdL5S={R|>e$_m)HyYGNf{LAaT)?b>8fF1A?jFH_Q zl)aXJ{21DzUjzm)jG-}qIS}?JI~z;5gpHMr9WaHEHb`K$hy%Bp)ju>DS|Q9JKa(-g zlHj1LIAmrcfP+MA_k}gW#00Y1|JF$VyJPnsYp3(=9h^O~+OxlZId6MSzn-_f1F%Q% z{}E>T55oNaaPI68?VoSM+)7mRr~xH9vo&>SMIs-z%oBGzhGr~-esVtGApBCmg#)ooxFH5 zGF{PvQ!AJ6oa$cXp*?-(%+7kCjXUEZdAsI5q1A@fm3iE!a(h$ZN_(@}xrS%~xJ75W zi>~uk?|npvsrUZa-cWvmrB|ep&Oa@>Z$I&=pMTySeu2M#?T=2NyQuE57x<43YbiH> z`55YT@^>!?m;X&L@#UFlspAY*YMCNuG>ymZ%Uho_9jPjbvcYXPC3A9f;X_G*$y-(r z51qhD${y>kFfLU~Q5Li4NT)N%*p^+H8#G8C>PXj8ZMr!m-(Bbx>%P6*RkaZ?!}k*Z z`FxW59{B)5@?&htHB-XrBxf?;EfQUs?pAG3+WubPx;$C?#{Wrere3K@owTardiQL9 zX}!zXtLy&f*@7)PQejgX86)MZgXra{F7heo4DI}2IkpP5&J2U=R9l-5jXbJXqGT+r zj;RX&LWfo7##N1y+%$**>kz%N`k4y#)%@<=l#nGXr#bf=(|Btb1jRjN;vMXD=G)WVqc#8p%qiWYc( zK}Ol>qAl`px%%@gBmJubq5ez0?aS=q< zuk(0pEWJ#Q_3Y_0wy&9SiZGEdgFiWb<=V0H%rEj>5N(+rLfxfKS<|1hC8YEO>Kew& zmREFwT{TkiX*$ick`9yjms44l1GKkGrl*Ioqi9_0r{b|!mIHjqKK|Oey5LJTWvvb0 z&5O1;Kp~%QfdKLvMQ~5d0wMJ;~=mG2ut)lhmuOFT!S{Z58e){xc%)osBpI{lDX;(`Z{;4fd4rWm% zz=dG=3u*BS;9#lICczZ_t<#-4$WU1_rrNZua*Hz*<%C4rq3Zb3$i`P(y?syZ*Wuw& zkrXkK#Y@3*iDu7V<+#+FqNB$f8W+gw{pBrC)zaaq2(6M~c>~O0>;4jT!Mr(Y6{+EM za!(nHJb4x!KCXKS-bRPQ3V&hIU;IUFgE31>D8%PCuOk_2cZ7A2l{1dD4Gr9@MdNMk z3KqgUYsSTkO+s`cYz9}nzS$!1DK8@@TD2}K+*5poYjv}35*VYj_iBYt8{fXE zjkL4N>88C`7s6<|-VG1erci5gn3u_74%T&nNisIVK$AU17&}2gzlsj*`I% zKHH{+dF#hU>(jMzP4FpqM(KA72R(b4&rbE^r={zaYAh@)U^TP!N2R>__bbGz+J6gc zLJ!7Th2diOzTeDUVrCO*wp!g92{*u)wMGOI9X*4)-0LJW)05xGAsoGMm5Yb&2}y+G?KCEJ?8h-@f)^;Q4DwA#9@;jd98 zA2W<=f<(g7YFpaia*b{I8Z0`#NWKgm!Ku#XK`%o2`c2Y7W^3#^ac$&8#_!5R1?(9lXaK6M@c&wqtF_37^6@;tIM z)2q3Ww6dJmdey@E@d8;xTs*FXS??E21 zJ!XMLpp)WrdCI=2iaODiPS&M0uo{U(;@0EuB_(~ia!Ds^VWd8a#P~|1fQ6&P;)JBi zkZK-+X>aP@-aJ04{iTQ*gTn+usL~7-=-5?gZHk~2>`~n*wWjDzvSbuhy}wY0+@rQ+@<(Qj^AxopkX(6Naq zJ<;b^b4ATv1u~?OS*7~xajiVR_H$6|`pkrxt#~8ELn|nUP!=o~d6g-WJR_#d&XxHVb+8C!kMo8Y*Tyc{(ZNgln z;0(sJJPLmDCn>Bk$B_MMUeclJ^P4~Pql2CQ}9_ZgF_#xC*)KM7-KN_{kYm6OV9lpd-WHz zpJRE^cC)HaNwrW>FLGi2+~U0!_HuyQa|%5V{#cr~=xHJheq2Ieb-LR)ECFt6`lQYm zORx47%kc}Swd^@ka7e{i{FH?CW>-~Frzp?k5_I@9Y zUl>cYKBR)ElxkSx)JwzEGIh1ioIQKhUY?qbO$jm$=LX7iZ03XoF!uOt3YE*iym@#x7IFvxiZu1zvJfVSrNjdqllIN@-+Uf%)uc4;Vbo_j~)>V zpvGUMYnRJ*dgLmegj4$EnFv1zZ0(T-;7NI}6Ct7ZmXrvx;`s8jA|qQbdLJM;`RUay zH`?phfqYrKXD3h1arI7JI2#)zkuW?Km?FS(Wwo_2?uGM0H5PIXkP5~JR@V3IKtANp zNI%St#NSm>Rc$040-Cf|g;B8y9IhpB{_!EU7ttZiF{P+5;quLhqQ)2@%>wJe^IJX) zr-Af)W)Ju&KPh#GGytMV7sthY%Wa8UF2b}ql0~Nwd5}r*AchxlV|#NoeE3Ff&i$|A z-9@&>(FKnzAg1gHW-wi(as%-8Q>y#RZA{nRd{yDiPM3FeI&(xpKH+r3yC`ntH5%z4 z?)pO_(F?ClOiYMX*%Kk=ef1PLoMNXCMBwB5Io?ezsF@yhpz_cm7Tfk6^%bqG*v5KQ z?&O%)UehXbww2MA_%y4PE;QfbR+PM+QwOkE z|Cz(MQKRqdAjaI(`lmbSoYkmSu4&_gMjDqNw#OsMxH})DIKTOw&e;LF=A>oTlSHk& zA+Lj(#Tyim(JPPPhB;36`5c2Jm#EJ^*oox*&_X$V6k`Nyy>m%El`Xk&&a@-0T3&9v z$m#s7`@~GWmO-5#8PmQW>LfMpDvI79#oK38 zAGfqV@zsJd1NuOg@vXc9C?QKG=)3bR!^^&ZJNELN`egI@ z>$q%$@}%FQ7_>+60J5a6pA=Y))!}i{DOSA_8B^0FX1xM6z}yH!?Ah`b5Kk$0T?r0j z(ec}H3%#Lk(um>7x%ZW*yV9!?pQ)$gx;n`M86xTFKU7$Vjh(5Zl}T2Nfvn#6TrSJ5 z%uq>xa#&$z|L|y(O<0CbL7l3S2;Jt&40^!BRn9-;JbgC5>rx8Ds%je>Ll2#xkOAf> zors7*7@Gm>@?;lR?v0HDUR8BuIY#S81)^Q<_T(XHa5$Vwvf}0L3XjqTz@r@Jx~&I# zQ`MPpceCSZX=x)|<-~SAuQU|oIL+xt^4Yu+cvTMzY@&6T=#vaaqZ^B}-Byv8lBwa> zAgTjf$4`Fd#v?#ufak-|^mg@)g)&pT8StiPf#6rKke+$u^n3sxskE9e2L~73F0$YA z#%U@l2H}D0FR3|#wOu%dp8Q0)OrEFP>U1dJ`?69}PvkwlawzMC=rwgcALCQi<$|sV z+~`QvMB`MGF|_gQxQnZ8{OIE`4xL(~iyQjei=_nDt~`&=Zw%>Sks@9dFgtEBS$Ua| zxk`fmQq4|-7FQ$b_DdQm+0ud;c@=yfLKPMjP7m1%I-d6asA+sFtu|w$s=Uz~yAIjf z##)}v_GG2-W)XrevgqrfF;j~?j~GG6V8GX9DtESrr}7QzbDM85=@eRzOhHR`z6wse zWCVg1$R5~Vjqg=>eOPUDF(JGUu-%S^C+;ijW}Nn;4Jo{OXbi~}=eG@*n`>Vl=9o2K zDPB*CqGJ?*RjhqGb`iBh-Cb;%EFZ!Cj`keifO7)m(yQ2B(Lv*ZM=o+$!pwSQx}T=E z0Eq@33E)-KF+uj|Ui4TtUzFHx-@2drFTh9{wgb{Lstm-kFd~Hbr+P zJJPNz32$Q~TT6ykgybJAw$@P9K08e_{C4NSLi0Q!1UN1+kMkxa=wgez%dx{CLh`Y7+x`4^V<&tdBgleLQdkU@DH1%B)*XOfD3LL5>(3iHq zW{FrZsi&UH#g0~N&T_%3B?yZzw2nGX(1ioBMRM}Wi^-1EDC|gkvv8>0NL}r=SBXIo z`PN)U;Q;5H}nRd-1h^LE?C@Q)uqQlT*lQM|P-2b$3rxy&N8&?_@N^${G#@+#Px zsh-5NGM)*R?rh_yTN4!`u-v4^0H01VGi%P1@J~}V707{PuYn5dQmfBz-bGv%M9Cm4 zw;j=sZp{y$<^0~~GfEX>*yc4`Hl&Et$j8XLTgK};bf_!R@=%!R-3agD>!9TlT!e@9 zWZX5p*_F<6SGb1h22m8B4X-aB8_)?L_BK5&z)AG_bju^t@y8MhDhVPdUAoCrlq()A z&GzfKu1qrm0^%XTC+e4M$%ZRs=v1;6z!csg;iR{=BvcZ(7__?{l8|CP#3wdT z%V@XRaNqthS;Ee?!eyYySev?n^J^M>P&1f9&o#L=vdM2^TW&)TxU-pci&|(OHMD&u zT$wz*6y0gRh*lEjAT%G_Pi#a3`28iPAV1L*;-xD`6f6p?28?FKzQ7IQJT_PCIzlyJ zH9?nSqHrAb%|{(sHs?eLg<9fC_03C9W9d(pn8=^aI>JWIEMnP)-|EL?bwa+^MD=AULs59c2?iEKNvcZPw@^0B{@j;VEf7aQ`<>hlQYMboqM@Ys+g15_?v!t=byX&gbMr!1V$oD(@^MnUC@jbFkOx zD00R_4x6uUb==}{bU&zhfh=!ZOPy%ZGB@XcFaIOZa{g|!Ez#TWo zxtFOOshE!=+pB|oB2Ch|9Pj&Ru%+0ECb+9t;6OyyDrtv}vVp$GvU)Cu_rUQ=j~Yju zetAiB>*#Ux3-fO(5qS6f6()gFLbu)5$K;sE_^mScMSVTL0*j6t=;M>KRwWG?$e0rP z;?_q~3-xTz)3ByH57v!laf~u;4Ttu7Y(@@0akH5AB2l_Yv~7SY7oZ?Y2F{1CeL!n` zcJKn8hMxJ({>P=U6Qu5X6zUNTE4)%YyPfO3!?rO>dDn)(b&GW6~QTn_EP!A*=$5KP8V+_k9Y;{IU3!#{d)M6+Ls7RIB z$s>n}>EFg(xd5j=2&swVR#$0xkdL42G-o&~U-AML##Z1h7HMjBuTD!%8*>?2AcC5~P9PbX$kJm1_X$OhkBRW3mbLWuCDM&MK zIxns=q+vcz=<_4VpVtJs@ZjfmJ7S`<_moKkIA1(~v~X0xsbGpiaS6hh#?6$#N`Td! z>Q`Yjry}+}biEhibM7|KxD#?8Lnxy7Pe)NivC{TlF2Mg*bScLQd(h@9rOL;0^lyD& z(j@|>bcoM^L$W<(+_2c#E1A_*hL+=mZ|vr30wDlLua7eYic^;WWVbHw6eobF9&E|N z)77;^eiPNxTpm;ToXh9x!>2O96#ySikIXTu^5akQSWA^7kXJ;s#jl;C#`!iE-LWK& zK66jm=>}!n8$*eUwpE8A-*nl^h9y!xsOL5_`v#`9?ONhvc~`UG>CeS9-xe6CIrnc= ze%e~&CK3bMDKQ;P;IK0@M9klrLb9k6qyixaZ7r|_=(w2LGmff6sT_k0SqI}RG;k?I z^4rD(cXCJxv+#w0ecJ}?K_L%Bv0yzfk1n_npE`BOVO825)2#05 z4Vge$x|T`Ee_>|j3KvxL`s%jljp-ezjE!dVn;dbm;qa*P+;fDu+4uXWc zFekmwC-yLVN~wzq7K@c>zH)2i`_~Vq)lWWFH}aDuYhYTU`w3Q1aQE>s^15n;Oa6-- z17{Bc)G$!nA13w#2VM0z?Eo-LlgR;oSC1p`Z(d@PlKckzIY3YaXmgP!rgqN*OHXe$ zV-S5MjuRG+LhZw9g|-6));eWsgZ{2<1eepM6cIk7*G_~aAntd8td!P({s7-=Gj|0HiN2Wh;$hrs6l{_r<)epAlx z)A5^i{AQ28Wyf#%_*;4Stswo@9)D}dzs-l=Cem;F@wbiq+kx<}IY~GaEN-Z!Q0H5A zglXqn!ba*s={PuAsn+=s2z~umCT3>!1Lbb;!2u6D?lJRdd|$uCllii_mWqd{>6SYh znc9q?$ij}iGJPsbL!qLNbh2veAxhQ|5%XJ>te zu_hCJiMdM?{WTx0XZwnCtTzkzaURpDwJuabo76!KE>w@M!Pjnx637Q%CKLSU*48yP zQ|NeH6O%v?2T3`9Z4#ysA}ojh0LGADW}MO^oA3!zFfLRJm7QF+F8e)PiO z+jO|rix^PQ&lUonrG3V-5yMA-(*!b;H2|yS@s+M+D|R zT-4MiO9!sqH?J~7qzGfOv?rcATGHVWLiyk(xJ zzLb&9MZR^R8u;9na}PLWAK0R@&uxg1Cy2)!J(zx-Cy;_S__Sugi{-)0a7_a@vh^27 zbnTaU+H&A>k=%$MPHK}3CJ%f5IXN40(%9j$=T8~#)Z9G+)+Q?~Sd&Qb58{$$EcRZ; zCaESXa(s4H7-;NGvieLi^3qB#Ni1h_3gSw zB93v}0D)hV=OzZTf=Y7m!Z=QkYSbcW*8L;LDfyKToxET^8!G$5cQ4sO(X@H^`?oI% z(W{U%_W~F26i2j5kGx&-x??y>op|caD?F6&52dZddhA4k8o=Ch>ccdw$D-|51USdH z#5ld)xkpCvc8#P;WcapUt~meP z5c54XzEt3F{V~KiAs!6j&j|e4@(cqSdHZ56!2+KTo@jETN>GgQ2k|3SUEG>G)wY3q zb>mY|#ISY>?r)6+@Uop<1$4N_%5z`8HaZDXr1(z}`>V$ha-Yeyo(7SF+aGJ5PV~uG zxNn`mKd^bhp+{F2WG{S^ts2_|2`S88r@50F5Kzd7aM$ zA^v@XnC#V4b=V~L^@X-HGd3vaZoEbepB+2T`T}Gf%0Y!Qx%NGI2pTOdt(<%B zh_v!uLqW9gl*bvnEdBD^W@eq4dQ4Dd9a68f7R14sbqcLQlV5qtf*j53<(o@k=NVb# zjck|3@2n$-oPwjGm`+e|zo-knAqUbonGGs*Cp)stOx3w-wX*FVr{Ap*KWTO)j5niK z#=;^Mk5iQ{uo+~vA8QKjcXNzd1+i$^8ye{hM~)mx8c=ybm47QYW4gaobF?waEJ#{K z)i3v!AC;l<^J|XBPgccUIeF)SaRoZy1hZLKdV1ScF(F%pk=Cv30A*H?qu14g-U7AT_g?<615 z3J7RHf~r`cxUEFNGE&&1#C>Od%ax1B*fn6)e=(~ zwheHf*Obo@gJt(Q4SLYmI*)r@;goyFT0X3jOFQ%S-2s(yZ3lFo`$l)NN~W$rF+pjy ze*OWWe7RF+)T1$qFC1jvMkE>t4h2UU*qHP%@^t~1x7KOJLK@k-kPy)S{AV^H^ z^W2=FKjQYS_PWWJ;scE$f*-JiK9`ypl?)4>$hFSf#gnON@+w@QmT;oKv`{Tezn|GD zs`6e%q?TW&e^Z7z;?S~rj8C?}$uSSBJ^>4NNcu@f%j(E&RJPU43-l6wRkS$j$mBNu zl`5DbMy!>JKtg(j+gHp`MYX_6tC&zZ%h#lIYdNGiHqrQLP+OjsDpbt}qgWCSyOSUl z8Hs7s;f=yp(^4FSDS$d&lj(e8_%a5-(o;!-HNHDQGV*G$pC*ukFF2CR471?C*gdKc z)u_Uyx6+v?M-K8XK{gQg)per{a??Z7XpGq_52>ch_cNE~1_$wmE79GiNy*yz7WC`a zaM{}(&(hrwEq~h5C2&71nK~WsnGLB0i51(%?SyW=(Zh*n4_m7khsBajYp)D??bzf# zk`IqXa3=UZWWCtFsZU>~`(`U0rn=~4S)olM7tR_A%Ia$4CH-lcgyd$+SM_tPrkA@6 z!kIFQ-qZ46zKq-D21fDLU9zb#JVUvi%gWe}l?o3o-(KmB=MpR&uuCgu8|yZU<;lJN zYkp4~FUwq%O~p$~X#*yZtjQ&m7GvKeKqqVQc{?3-y<$d*Bc=CzkoEfU$?WB^apWTo$$#!{L{5VcYzO9=h&CtzaEt$+$vp_>99~& z__naKITv-gMoO1M9O$(79AWJ-^-DVxm%W4{Aa=5gu9t(4@;VkPd*_Vv|ekXoy~T}DRh6#;WV42tgWy2 z*Cu;#ll7r1{(ZI2H)ZOuZG+FTOio5r1L-bYIMvc1EynoL1gR-W0yzsJNL9tB1ys2W zdKAJ;P^(=1ujSE)$_H;38h2WUEbm}Iur`V%hMZN`eAbsyvmzQb*Qp0eYfWeKTix_V z6Pvkem9hrPha0$>xR3bKUl(kTFHJ>|XjkfctZ(@@Hx@MYyc=I>>3Ij@!>|(6?1J&1 zc|C0)uL^y4f|I+PPs+?2BrDPbZpSUco3H3<^LzlEFoGN?>t^1|?RoXoCTt1yd|=a9 zASE_4)V29;!jnLg1rl_l)kt%>)lHiT($$@Jy8LTX280{Z_VG>x%@;LrWkNC6)5=w? zUyuNz+Qlz6nuO+_>B|Im`tPu5w^1fziC!yM{E_$*iXYCK>q_f;*mQ~3EZX)-H%ior zueZ=C)*VU$gJEpg1PkM7uHfS`eT$J&tsuugm!-IAVa;dSbL$P_;W=^4Q|77OWP5z*o5*80|6|xzJ;o6&bTHiW^}R4ELA_NTly9 zITpwI6hCZ$-khu0R0M_YX0KjjS6@u543~@)v3BFJ6+n%ruk9*f(a*UX#;ku0WY%%% zpr@)1^qWht(N7PQ_Z~0PBy3DxLt|ns(}c-o*@#mf4mc&{;s(<ZmCiT#^kU*` zI{dSH3{r=_Rrxn2&@2u5FBOQrKr?s=4$vBlLy_=UDe9RhiA16MC$;dMbwkB{WgF z!PK@_rA?UdB>h>;__*0}Wi#fM9BcVf8x_gz?XPiBchs`(=;d(cVs%!Lrj9CeM@q-z z{pE`VRdEl~PZm*FUbb?0gWs${U0MV-ZR8y0f;6I|CBS{t7*+yO;3CPmh(`+`2r&;wuV6V|qQ^qM#J`h)0rMpDm4Y-K*>( zmzroN`f#~g)fvqm?G;LzO!6_j9Sz)5b~ACWn36}9r+S-pPqq2eyWWwWimNxtf1W$9 zTm1FIllpN>sVGlXO%r^kdhc{}>XDd%tWxVuUYDK6V+!I4!kBv#lF?_;6y=(lY$+B7 zS>$g~iJ$7(Dy?|0-KbO89Gc219atK2&6chq(bHNO{m3^0Qg2x`A|?oaRUe~_sDbiI z|62aR#5k0<#`Ljl`bbN0x01zd#T2qT+x}^!gS?__STC})@%TMJ=O^7wuU9IG@YFuo zSqFn~!wPrUorhbcEi_Ic~4e!V08{@qF=7|OYjBx?I*GPwEgM7fi^#a==DOK_q zN!Qu+u?BD4TvyD&SkH}#c6y#h5) zDjry7A7^TZU{Qg=$ie8#Z;ImFxVGPxd?;HOKn zyCSjT=yh=I1TPm;$;HP_Md|wa+EPK3h(otM?x%;Pk?-4CXfbH8ePD8c+I2=BsD1YQQqDoz;)LXNqQ}aDD0W z1l{z}LM5SIYUwVIWe4}6jVm!#t;=JsiJE2wPmW&%xr3I{lBEeZlrcc|f1V_D_IQ)O zQ`A)(*(rC8-{zO&D|W%;wY0)hPpPFQ|u-H^Tb1_o4^Fpqo%yu1YlKj4CC0s-!PIvaP%;U&eR{G7CjX z+pkXa#lOIBrGiwlyW;8*`0+(Q6g%EWlcYb~z(cE(jJdfD{!;ME6sdQ!@Mc3gpAGPe zt?!SkKhOQT#bE-NK^>X~){cj^>Ip(uaf}gqE~^G}m^Al|i{*BytE1zyUZ4iNk)CYh zqF!%?oa9oo>zto{EB=uTUvC7DWtdA*deijQOlxH%9Ze>-_hP&2^gPBj0ij_m+89q+GxS41@=e-(t2JN|k7pfIg(4?89rg4xS4k%iUWPEf zBYXC^UR@e2aAnME?E{tkV5V zye4oV$YW$lfYx9_)M*3(7ezC{$y1*71Y12~L*~E~f9IXbg5}dcFMr%sJSa><0N4m`xn-+H` z$C5FNFhvx%j9hRHW#zlBX=i6|0!nLuCeh?QPL1Od!nltwaK#w5roCjVSc~so`u5yw zEn!`Md@+i9f6FEcwH~2=hh%knPM~ql!2}3u7l(mc?Rut#;*HE!z;HE8)8KUnHfvB3wj%^ za)rlrz*!#z&P_wAOmdWB-*!4XE>h4T_2Mx+%T=# zL{j^XOT!MMVsnVr#XIPO{dCFt{jR3xCAy?YFR;IzK34!=JfF*|leIO0nW9&S`t;`Y z2NTHLdC}P&G(2nqDQUWdsHVdc6tnLESBN@6CEQ0#<~pJ_n04uZ`}aB0hHah=eFa^V zloa2gwQ=q?Ehg7Tnwofx&954edUT+cNUrkt16Ml0_1Rtdvh+rU31c|PVA{bna-RxrnOnF5v>tUIic-}8E0unS$NjK5So zSF*^O>*lL}q_saD1$&{d&_?Bu+~cY#WRmf`sbu`Q()K%EKI=@Yp1fN5$5WvUGoS*a zi^OU|dkEb-zqQyL&M$CmQkNE&9g83Dv1>GQ$S6kk;`p|v?xpE4O}Ql2`Xi6V!Jajp znf#JM&}*i;{2leZ%SS8xviUcAjY+LwTZRwQ-`lf|rR`^@;!^oWk%I;O4ft_|ZC9m$ zolh;{NZvTh%H`C!ouU>(rqF?jPa4t`#(!)oXLc&Q! zwpzBl?vM`R;j};lpjR{xGZH9Vb=|X7}Hs|_h6beH-pZZR* zMpn7B63C=|S3inVFTnyG)k-cW%r2q6zZF}Qkg+MzDY3$p0e_OIpGKi?hoYnx!vyyd zbgPkIc6>o<2UC?R*Fdcf!{qkS5_tOTC+zCE5Eji&THeQ{JbLc)RU+N4t1({XG!9dE z9?jnTgx1?~Q*otQ6X%NbU$?@AH;fg!N+;cxj&9Y?I4`wI!z_&EX9OBKtyYm(#gC~Ow0I(UKG7lmz5<^24B=<+re%77i|4`qFq5s+o;LszHPN5v6&f| z=o6Ie%9DI4Hklfeu?|VH;rNMip;k|h6ltC4p(?Z}Y*VDaRsQS|J)@+dd&+EGvCui8 zzXB9gFLd-;o7u{ElW7Sp@|(_CuZ2~yi=j!Bz$cS3A8CI0kua_`Ew3adot_Iw&h*F= zKm0Jw4HrZ_%Zh!qv+hovXikuR&Z}u};5tE<+wRL+BVe1R9TU?^rJvj*ZVO$6lv?wU zRM@<(Y<)DUG0>|-xKLvABtVj8_>qYZCzo#&4p6-&J&TVbb}0W2)y1usH66XRu`1no zJZPsl#c^wd&Fd=8x4JNqxQ1aO&NDSWiWgzgB31u=rV=6e+Q6I=2chw1Vt)K@p#pt5@Ag<igz}q62S#^3*QH?@bc+RA=xGKC4gx$Q7$GHBY{Fn%7vkn zMbvLDZM6>%eQKpK?N%?V$h*AJmS|~c&NSvx_R1G8L_tJ1LqNG$0mDKu}FbVCmDSaV~35 zu~O)33f`z1Mmb!>^muQCd%tEd*C zeT6T9D+{`A_UrN;ZHb9TG4?=nliI(Lej)~_m8L+qysVGH1qt8M;^StJw^)j#d3@YaA9EySd96VqFaG-B*fMqNgzCVZp|4So0X=Uz ztYc<_l6*suGH)DbwC(K+2bAnnGn|eUe(rcATbPZvPRy?;DfXu42Bi*MN)(+Zh!i_c z_DdCFNl%}c)Dau22Fk#g%FWOZE>udb@tLjXLDfuwGwL~TO6u7JX}Ira10gH>?@hr+ zTZ&)eLzbeCeAombP;i~oJvmTe5te>wq>WH~n#5vW6%4o_Pty*%WSHSC00KSQJ(~Af2CZ4b*XxjYeF4 zw8Lh)L>Ym7V*fVjM>MW+vA!cmU`{Xa__CcC81*s3=d=UcvA1+&gXFyf1IVZXMyLSsWfybfEHDX`Af(qv~{BA6q4nGK8u2}B28Wx#r3SEk! zn`xmoRqaBhKAS=XvIHiFDcOUdpb5j>#1s%KVr*HHQG;T_;I^}nACc;*u6oO?%JZR8 zwlBty1XP&j<;ftfq6Z>rg&b#}4ftFZIWty1wBPTIE0`Y9!n`u@d_NW}6DN?*u2tJl z0Z3$ZnlC_1p*GvN#$;=4J~%S;VzWMen;2ZoWm1fsb_jm)g8J-r#gOPeO7|U2??vIV zWTG(-XJfhg2dCo*$EsVaR6yBJz)^=A=RlZw+{blk{3=uhs+DV2+gEJgO6n_0@Z3%` zsPu%2dbP?-z%3q> zB(-q#>V9f%)(49Noa|L@OrYq>b9{o(_xv;uvo>`S?1b1K*9tNHFCacFPeY3 z&&*p?9+aCJ)iuBS^=@Kn6~NqN4zB;@BI1uwsc+I2B_+6z1cLS( Date: Tue, 24 Oct 2023 10:13:09 -0700 Subject: [PATCH 02/11] Release 12 through 16, remove 11 --- .github/workflows/ubuntu-packages-and-docker-image.yml | 6 ------ packages/postgresml/release.sh | 2 +- packages/postgresql-pgml/release.sh | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index 339f356cf..8bc459cc2 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -105,12 +105,6 @@ jobs: working-directory: pgml-extension command: pgrx args: init --pg11=/usr/lib/postgresql/11/bin/pg_config --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config - - name: Build Postgres 11 - uses: postgresml/gh-actions-cargo@master - with: - working-directory: pgml-extension - command: pgrx - args: package --pg-config /usr/lib/postgresql/11/bin/pg_config - name: Build Postgres 12 uses: postgresml/gh-actions-cargo@master with: diff --git a/packages/postgresml/release.sh b/packages/postgresml/release.sh index 9e1775cec..07a684523 100644 --- a/packages/postgresml/release.sh +++ b/packages/postgresml/release.sh @@ -20,7 +20,7 @@ function package_name() { echo "postgresml-$1-$package_version-ubuntu22.04-all.deb" } -for pg in {11..15}; do +for pg in {12..16}; do bash ${SCRIPT_DIR}/build.sh ${package_version} ${pg} if [[ ! -f $(package_name ${pg}) ]]; then diff --git a/packages/postgresql-pgml/release.sh b/packages/postgresql-pgml/release.sh index 7f945a79c..139fb7694 100644 --- a/packages/postgresql-pgml/release.sh +++ b/packages/postgresql-pgml/release.sh @@ -27,7 +27,7 @@ function package_name() { echo "postgresql-pgml-${1}_${PACKAGE_VERSION}-ubuntu22.04-${ARCH}.deb" } -for pg in {11..15}; do +for pg in {12..16}; do release_dir="$extension_dir/target/release/pgml-pg${pg}" mkdir -p "$release_dir/DEBIAN" From b0f91e3d4d0aada9b26fa976a468a8ad705b13a0 Mon Sep 17 00:00:00 2001 From: Lev Date: Tue, 24 Oct 2023 10:14:35 -0700 Subject: [PATCH 03/11] Setup pg16 --- .github/workflows/ubuntu-packages-and-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index 8bc459cc2..069f10b33 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -104,7 +104,7 @@ jobs: with: working-directory: pgml-extension command: pgrx - args: init --pg11=/usr/lib/postgresql/11/bin/pg_config --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config + args: --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config --pg16=/usr/lib/postgresql/16/bin/pg_config - name: Build Postgres 12 uses: postgresml/gh-actions-cargo@master with: From 044ec86142940057dfd937a39d96d92f59f3019c Mon Sep 17 00:00:00 2001 From: Dan <39170265+chillenberger@users.noreply.github.com> Date: Tue, 24 Oct 2023 11:25:30 -0600 Subject: [PATCH 04/11] Dan updates for post mvp dbs (#1117) --- .../inputs/range_group/range_group.scss | 11 --------- .../inputs/range_group/template.html | 2 ++ .../inputs/select/select_controller.js | 12 ++++++++-- .../components/inputs/select/template.html | 4 ++-- .../src/components/inputs/switch/mod.rs | 4 ++-- .../src/components/inputs/switch/switch.scss | 3 ++- .../text/editable_header/editable_header.scss | 6 +++++ .../editable_header_controller.js | 2 ++ .../inputs/text/editable_header/template.html | 2 +- .../src/components/lists/item/mod.rs | 23 +++++++++++++++++++ .../src/components/lists/item/template.html | 4 ++++ pgml-dashboard/src/components/lists/mod.rs | 6 +++++ pgml-dashboard/src/components/mod.rs | 3 +++ .../static/css/scss/abstracts/variables.scss | 13 ++++++++++- .../static/css/scss/components/_badges.scss | 3 +-- .../static/css/scss/components/_forms.scss | 17 ++++++++++++++ .../templates/content/playground.html | 4 ++-- .../templates/layout/web_app_base.html | 2 +- 18 files changed, 96 insertions(+), 25 deletions(-) create mode 100644 pgml-dashboard/src/components/lists/item/mod.rs create mode 100644 pgml-dashboard/src/components/lists/item/template.html create mode 100644 pgml-dashboard/src/components/lists/mod.rs diff --git a/pgml-dashboard/src/components/inputs/range_group/range_group.scss b/pgml-dashboard/src/components/inputs/range_group/range_group.scss index da68a1172..cc2186951 100644 --- a/pgml-dashboard/src/components/inputs/range_group/range_group.scss +++ b/pgml-dashboard/src/components/inputs/range_group/range_group.scss @@ -4,19 +4,8 @@ div[data-controller="inputs-range-group"] { } .hourly-rate { - display: flex; - flex-direction: row; background-color: #{$gray-900}; - border-radius: $border-radius; - padding: 8px 4px; - color: #{$gray-400}; - text-align: center; - font-size: 18px; - font-style: normal; - font-weight: 700; - line-height: 24px; - letter-spacing: 0.18px; } .cost { diff --git a/pgml-dashboard/src/components/inputs/range_group/template.html b/pgml-dashboard/src/components/inputs/range_group/template.html index f70e5cc8c..14f38b536 100644 --- a/pgml-dashboard/src/components/inputs/range_group/template.html +++ b/pgml-dashboard/src/components/inputs/range_group/template.html @@ -42,12 +42,14 @@
<%- title %>
+
<% for info in item { %> <% } %>
+
<% for info in item { %> diff --git a/pgml-dashboard/src/components/inputs/select/select_controller.js b/pgml-dashboard/src/components/inputs/select/select_controller.js index e7c712dba..d5321f1b0 100644 --- a/pgml-dashboard/src/components/inputs/select/select_controller.js +++ b/pgml-dashboard/src/components/inputs/select/select_controller.js @@ -4,8 +4,16 @@ export default class extends Controller { static targets = ["input", "value"] choose(e) { - this.inputTarget.value = e.target.innerHTML - this.valueTarget.innerHTML = e.target.innerHTML + this.setValue(e.target.innerHTML) + } + + resetSelect() { + this.setValue(this.element.dataset.initial) + } + + setValue(value) { + this.inputTarget.value = value + this.valueTarget.innerHTML = value this.inputTarget.dispatchEvent(new Event('change')) } } diff --git a/pgml-dashboard/src/components/inputs/select/template.html b/pgml-dashboard/src/components/inputs/select/template.html index b158717bc..4bc33ecd4 100644 --- a/pgml-dashboard/src/components/inputs/select/template.html +++ b/pgml-dashboard/src/components/inputs/select/template.html @@ -2,7 +2,7 @@ use crate::components::dropdown::Dropdown; use crate::components::stimulus::stimulus_target::StimulusTarget; %> -
+
<% let mut dropdown = Dropdown::new() .items(options) @@ -29,5 +29,5 @@ <%+ dropdown %> - data-action="<%- action %>" /> + data-action="<%- action %> reset->inputs-select#resetSelect" />
diff --git a/pgml-dashboard/src/components/inputs/switch/mod.rs b/pgml-dashboard/src/components/inputs/switch/mod.rs index 389fde568..7db04ae71 100644 --- a/pgml-dashboard/src/components/inputs/switch/mod.rs +++ b/pgml-dashboard/src/components/inputs/switch/mod.rs @@ -60,8 +60,8 @@ impl Switch { self } - pub fn start_toggled(mut self) -> Switch { - self.initial_state = State::Right; + pub fn default_position(mut self, state: State) -> Switch { + self.initial_state = state; self } diff --git a/pgml-dashboard/src/components/inputs/switch/switch.scss b/pgml-dashboard/src/components/inputs/switch/switch.scss index ac8ffe4d9..af6e97af8 100644 --- a/pgml-dashboard/src/components/inputs/switch/switch.scss +++ b/pgml-dashboard/src/components/inputs/switch/switch.scss @@ -7,11 +7,12 @@ div[data-controller="inputs-switch"] { } .label { - padding: 8px 20px; + padding: 8px 40px; border-radius: 5rem; text-align: center; display: flex; justify-content: center; + align-items: center; @extend .gap-2; } diff --git a/pgml-dashboard/src/components/inputs/text/editable_header/editable_header.scss b/pgml-dashboard/src/components/inputs/text/editable_header/editable_header.scss index 49b36cad9..709658e68 100644 --- a/pgml-dashboard/src/components/inputs/text/editable_header/editable_header.scss +++ b/pgml-dashboard/src/components/inputs/text/editable_header/editable_header.scss @@ -33,4 +33,10 @@ div[data-controller="inputs-text-editable-header"] { padding: 0px; margin-bottom: -2px; // compensate for border space } + + #title { + &.error { + border-bottom: 1px solid #{$error} + } + } } diff --git a/pgml-dashboard/src/components/inputs/text/editable_header/editable_header_controller.js b/pgml-dashboard/src/components/inputs/text/editable_header/editable_header_controller.js index e00b8cefb..b5195a087 100644 --- a/pgml-dashboard/src/components/inputs/text/editable_header/editable_header_controller.js +++ b/pgml-dashboard/src/components/inputs/text/editable_header/editable_header_controller.js @@ -31,9 +31,11 @@ export default class extends Controller { error(e) { this.errorTarget.innerHTML = e.detail this.errorTarget.style.display = "block" + this.headerTarget.classList.add("error") } clear() { this.errorTarget.style.display = "none" + this.headerTarget.classList.remove("error") } } diff --git a/pgml-dashboard/src/components/inputs/text/editable_header/template.html b/pgml-dashboard/src/components/inputs/text/editable_header/template.html index eb320ed5a..31c879a7b 100644 --- a/pgml-dashboard/src/components/inputs/text/editable_header/template.html +++ b/pgml-dashboard/src/components/inputs/text/editable_header/template.html @@ -5,7 +5,7 @@
<<%= header_type.to_string() %> class="align-items-center <%= header_type.to_string() %> d-flex gap-3"> - + <%= value %> diff --git a/pgml-dashboard/src/components/lists/item/mod.rs b/pgml-dashboard/src/components/lists/item/mod.rs new file mode 100644 index 000000000..8a0ff1645 --- /dev/null +++ b/pgml-dashboard/src/components/lists/item/mod.rs @@ -0,0 +1,23 @@ +use pgml_components::component; +use sailfish::TemplateOnce; + +#[derive(TemplateOnce, Default)] +#[template(path = "lists/item/template.html")] +pub struct Item { + value: String, +} + +impl Item { + pub fn new() -> Item { + Item { + value: String::from("Your list item"), + } + } + + pub fn value(mut self, value: &str) -> Item { + self.value = value.into(); + self + } +} + +component!(Item); diff --git a/pgml-dashboard/src/components/lists/item/template.html b/pgml-dashboard/src/components/lists/item/template.html new file mode 100644 index 000000000..7df3b5ec6 --- /dev/null +++ b/pgml-dashboard/src/components/lists/item/template.html @@ -0,0 +1,4 @@ +
+ Checkmark + <%- value %> +
diff --git a/pgml-dashboard/src/components/lists/mod.rs b/pgml-dashboard/src/components/lists/mod.rs new file mode 100644 index 000000000..ac438d6cd --- /dev/null +++ b/pgml-dashboard/src/components/lists/mod.rs @@ -0,0 +1,6 @@ +// This file is automatically generated. +// You shouldn't modify it manually. + +// src/components/lists/item +pub mod item; +pub use item::Item; diff --git a/pgml-dashboard/src/components/mod.rs b/pgml-dashboard/src/components/mod.rs index bb92bae97..4db70e0da 100644 --- a/pgml-dashboard/src/components/mod.rs +++ b/pgml-dashboard/src/components/mod.rs @@ -32,6 +32,9 @@ pub mod inputs; pub mod left_nav_menu; pub use left_nav_menu::LeftNavMenu; +// src/components/lists +pub mod lists; + // src/components/modal pub mod modal; pub use modal::Modal; diff --git a/pgml-dashboard/static/css/scss/abstracts/variables.scss b/pgml-dashboard/static/css/scss/abstracts/variables.scss index 943b80b54..41e35b75b 100644 --- a/pgml-dashboard/static/css/scss/abstracts/variables.scss +++ b/pgml-dashboard/static/css/scss/abstracts/variables.scss @@ -97,13 +97,24 @@ $slate-shade-800: #2B274C; $slate-shade-900: #1D1A33; $slate-shade-1000: #0E0D19; +$magenta-shade-100: #E6008A; +$magenta-shade-200: #cf007c; +$magenta-shade-300: #b8006e; +$magenta-shade-400: #a10060; +$magenta-shade-500: #8a0052; +$magenta-shade-600: #730045; +$magenta-shade-700: #5c0037; +$magenta-shade-800: #450029; +$magenta-shade-900: #2e001b; +$magenta-shade-1000: #17000d; + // Colors $primary: #0D0D0E; $secondary: $gray-100; $danger: $peach-shade-100; $error: $peach-shade-100; $purple: $slate-tint-100; -$pink: #e7477c; +$pink: $magenta-shade-100; $hp-white: #{$gray-200}; // Background Colors diff --git a/pgml-dashboard/static/css/scss/components/_badges.scss b/pgml-dashboard/static/css/scss/components/_badges.scss index 851ca68aa..ebb3ac4a5 100644 --- a/pgml-dashboard/static/css/scss/components/_badges.scss +++ b/pgml-dashboard/static/css/scss/components/_badges.scss @@ -5,8 +5,7 @@ align-items: center; } -.version-badge { - @extend .badge; +.eyebrow-badge { text-transform: uppercase; color: #{$pink}; } diff --git a/pgml-dashboard/static/css/scss/components/_forms.scss b/pgml-dashboard/static/css/scss/components/_forms.scss index cc11d237c..14231ce0c 100644 --- a/pgml-dashboard/static/css/scss/components/_forms.scss +++ b/pgml-dashboard/static/css/scss/components/_forms.scss @@ -256,3 +256,20 @@ background: transparent; caret-color: #{$input-color}; } + +.hourly-rate { + display: flex; + flex-direction: row; + gap: .5rem; + background-color: #{$gray-900}; + border-radius: $border-radius; + padding: 16px 20px; + + color: #{$gray-400}; + text-align: center; + font-size: 18px; + font-style: normal; + font-weight: 700; + line-height: 24px; + letter-spacing: 0.18px; +} diff --git a/pgml-dashboard/templates/content/playground.html b/pgml-dashboard/templates/content/playground.html index f90591c6a..84248a6eb 100644 --- a/pgml-dashboard/templates/content/playground.html +++ b/pgml-dashboard/templates/content/playground.html @@ -7,7 +7,7 @@ use crate::components::stimulus::stimulus_action::StimulusAction; // use crate::components::stimulus::stimulus_action::StimulusEvents; use crate::components::inputs::select::Select; -use crate::components::inputs::switch::Switch; +use crate::components::inputs::switch::{Switch, State}; %>
@@ -223,7 +223,7 @@

Inputs

.name("switch")) .left("CPU", "memory") .right("GPU", "mode_fan") - .start_toggled() %> + .default_position(State::Right) %>
diff --git a/pgml-dashboard/templates/layout/web_app_base.html b/pgml-dashboard/templates/layout/web_app_base.html index a84ce5e8e..e3ababb5e 100644 --- a/pgml-dashboard/templates/layout/web_app_base.html +++ b/pgml-dashboard/templates/layout/web_app_base.html @@ -30,7 +30,7 @@ <%+ WebAppLeftNav::new( upper_left_nav, lower_left_nav, dropdown_nav ) %>
-
+
<%- Breadcrumbs::render( breadcrumbs ) %>
From dcbc7d47447c905af249f98fa3fb551c8a37338e Mon Sep 17 00:00:00 2001 From: santiatpml <124207123+santiatpml@users.noreply.github.com> Date: Tue, 24 Oct 2023 12:09:31 -0700 Subject: [PATCH 05/11] pgml chat with history + additional functionality (#1047) --- pgml-apps/pgml-chat/.env.template | 10 +- pgml-apps/pgml-chat/.gitignore | 5 +- pgml-apps/pgml-chat/README.md | 38 +-- pgml-apps/pgml-chat/pgml_chat/.gitignore | 2 + pgml-apps/pgml-chat/pgml_chat/main.py | 309 +++++++++++++++--- .../pgml-chat/pgml_chat/pgml_playground.py | 116 +++++++ pgml-apps/pgml-chat/poetry.lock | 292 +++++++++++------ pgml-apps/pgml-chat/pyproject.toml | 11 +- 8 files changed, 617 insertions(+), 166 deletions(-) create mode 100644 pgml-apps/pgml-chat/pgml_chat/.gitignore create mode 100644 pgml-apps/pgml-chat/pgml_chat/pgml_playground.py diff --git a/pgml-apps/pgml-chat/.env.template b/pgml-apps/pgml-chat/.env.template index 2d582a412..f08a1b630 100644 --- a/pgml-apps/pgml-chat/.env.template +++ b/pgml-apps/pgml-chat/.env.template @@ -1,14 +1,6 @@ OPENAI_API_KEY= DATABASE_URL= -MODEL=hkunlp/instructor-xl -MODEL_PARAMS={"instruction": "Represent the Wikipedia document for retrieval: "} -QUERY_PARAMS={"instruction": "Represent the Wikipedia question for retrieving supporting documents: "} -SYSTEM_PROMPT="You are an assistant to answer questions about an open source software named PostgresML. Your name is PgBot. You are based out of San Francisco, California." -BASE_PROMPT="Given relevant parts of a document and a question, create a final answer.\ - Include a SQL query in the answer wherever possible. \ - Use the following portion of a long document to see if any of the text is relevant to answer the question.\ - \nReturn any relevant text verbatim.\n{context}\nQuestion: {question}\n \ - If the context is empty then ask for clarification and suggest user to send an email to team@postgresml.org or join PostgresML [Discord](https://discord.gg/DmyJP3qJ7U)." + SLACK_BOT_TOKEN= SLACK_APP_TOKEN= DISCORD_BOT_TOKEN= \ No newline at end of file diff --git a/pgml-apps/pgml-chat/.gitignore b/pgml-apps/pgml-chat/.gitignore index 6769e21d9..6b45645ee 100644 --- a/pgml-apps/pgml-chat/.gitignore +++ b/pgml-apps/pgml-chat/.gitignore @@ -157,4 +157,7 @@ cython_debug/ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ \ No newline at end of file +#.idea/ + +pgml_chat/pgml_playground.py +pgml_chat/llama2.py \ No newline at end of file diff --git a/pgml-apps/pgml-chat/README.md b/pgml-apps/pgml-chat/README.md index 64c925f8e..c1d570035 100644 --- a/pgml-apps/pgml-chat/README.md +++ b/pgml-apps/pgml-chat/README.md @@ -34,23 +34,16 @@ wget https://raw.githubusercontent.com/postgresml/postgresml/master/pgml-apps/pg ```bash OPENAI_API_KEY= DATABASE_URL= -MODEL=hkunlp/instructor-xl -MODEL_PARAMS={"instruction": "Represent the Wikipedia document for retrieval: "} -QUERY_PARAMS={"instruction": "Represent the Wikipedia question for retrieving supporting documents: "} -SYSTEM_PROMPT="You are an assistant to answer questions about an open source software named PostgresML. Your name is PgBot. You are based out of San Francisco, California." -BASE_PROMPT="Given relevant parts of a document and a question, create a final answer.\ - Include a SQL query in the answer wherever possible. \ - Use the following portion of a long document to see if any of the text is relevant to answer the question.\ - \nReturn any relevant text verbatim.\n{context}\nQuestion: {question}\n \ - If the context is empty then ask for clarification and suggest user to send an email to team@postgresml.org or join PostgresML [Discord](https://discord.gg/DmyJP3qJ7U)." ``` # Usage You can get help on the command line interface by running: ```bash -(pgml-bot-builder-py3.9) pgml-chat % pgml-chat --help -usage: pgml-chat [-h] --collection_name COLLECTION_NAME [--root_dir ROOT_DIR] [--stage {ingest,chat}] [--chat_interface {cli,slack}] +(pgml-bot-builder-py3.9) pgml-chat % pgml-chat % pgml-chat --help +usage: pgml-chat [-h] --collection_name COLLECTION_NAME [--root_dir ROOT_DIR] [--stage {ingest,chat}] [--chat_interface {cli,slack,discord}] + [--chat_history CHAT_HISTORY] [--bot_name BOT_NAME] [--bot_language BOT_LANGUAGE] [--bot_topic BOT_TOPIC] + [--bot_topic_primary_language BOT_TOPIC_PRIMARY_LANGUAGE] [--bot_persona BOT_PERSONA] PostgresML Chatbot Builder @@ -61,8 +54,19 @@ optional arguments: --root_dir ROOT_DIR Input folder to scan for markdown files. Required for ingest stage. Not required for chat stage (default: None) --stage {ingest,chat} Stage to run (default: chat) - --chat_interface {cli, slack, discord} + --chat_interface {cli,slack,discord} Chat interface to use (default: cli) + --chat_history CHAT_HISTORY + Number of messages from history used for generating response (default: 1) + --bot_name BOT_NAME Name of the bot (default: PgBot) + --bot_language BOT_LANGUAGE + Language of the bot (default: English) + --bot_topic BOT_TOPIC + Topic of the bot (default: PostgresML) + --bot_topic_primary_language BOT_TOPIC_PRIMARY_LANGUAGE + Primary programming language of the topic (default: ) + --bot_persona BOT_PERSONA + Persona of the bot (default: Engineer) ``` ## Ingest In this step, we ingest documents, chunk documents, generate embeddings and index these embeddings for fast query. @@ -161,14 +165,8 @@ pip install . -# Options -You can control the behavior of the chatbot by setting the following environment variables: -- `SYSTEM_PROMPT`: This is the prompt that is used to initialize the chatbot. You can customize this prompt to change the behavior of the chatbot. For example, you can change the name of the chatbot or the location of the chatbot. -- `BASE_PROMPT`: This is the prompt that is used to generate responses to user queries. You can customize this prompt to change the behavior of the chatbot. -- `MODEL`: This is the open source embedding model used to generate embeddings for the documents. You can change this to use a different model. - # Roadmap -- ~~`hyerbot --chat_interface {cli, slack, discord}` that supports Slack, and Discord.~~ +- ~~Use a collection for chat history that can be retrieved and used to generate responses.~~ - Support for file formats like rst, html, pdf, docx, etc. - Support for open source models in addition to OpenAI for chat completion. -- Support for multi-turn converstaions using converstaion buffer. Use a collection for chat history that can be retrieved and used to generate responses. +- Support for multi-turn converstaions using converstaion buffer. diff --git a/pgml-apps/pgml-chat/pgml_chat/.gitignore b/pgml-apps/pgml-chat/pgml_chat/.gitignore new file mode 100644 index 000000000..655ada3a8 --- /dev/null +++ b/pgml-apps/pgml-chat/pgml_chat/.gitignore @@ -0,0 +1,2 @@ +pgml_playground.py +llama2.py \ No newline at end of file diff --git a/pgml-apps/pgml-chat/pgml_chat/main.py b/pgml-apps/pgml-chat/pgml_chat/main.py index 4b731b8bc..3dce96836 100644 --- a/pgml-apps/pgml-chat/pgml_chat/main.py +++ b/pgml-apps/pgml-chat/pgml_chat/main.py @@ -1,5 +1,5 @@ import asyncio -from pgml import Collection, Model, Splitter, Pipeline, migrate, init_logger +from pgml import Collection, Model, Splitter, Pipeline, migrate, init_logger, Builtins import logging from rich.logging import RichHandler from rich.progress import track @@ -11,6 +11,8 @@ from time import time import openai import signal +from uuid import uuid4 +import pendulum import ast from slack_bolt.async_app import AsyncApp @@ -61,6 +63,54 @@ def handler(signum, frame): help="Chat interface to use", ) +parser.add_argument( + "--chat_history", + dest="chat_history", + type=int, + default=0, + help="Number of messages from history used for generating response", +) + +parser.add_argument( + "--bot_name", + dest="bot_name", + type=str, + default="PgBot", + help="Name of the bot", +) + +parser.add_argument( + "--bot_language", + dest="bot_language", + type=str, + default="English", + help="Language of the bot", +) + +parser.add_argument( + "--bot_topic", + dest="bot_topic", + type=str, + default="PostgresML", + help="Topic of the bot", +) +parser.add_argument( + "--bot_topic_primary_language", + dest="bot_topic_primary_language", + type=str, + default="SQL", + help="Primary programming language of the topic", +) + +parser.add_argument( + "--bot_persona", + dest="bot_persona", + type=str, + default="Engineer", + help="Persona of the bot", +) + + args = parser.parse_args() FORMAT = "%(message)s" @@ -77,9 +127,19 @@ def handler(signum, frame): # The code is using the `argparse` module to parse command line arguments. +chat_history_collection_name = args.collection_name + "_chat_history" collection = Collection(args.collection_name) +chat_collection = Collection(chat_history_collection_name) stage = args.stage chat_interface = args.chat_interface +chat_history = args.chat_history + +# Get all bot related environment variables +bot_name = args.bot_name +bot_language = args.bot_language +bot_persona = args.bot_persona +bot_topic = args.bot_topic +bot_topic_primary_language = args.bot_topic_primary_language # The above code is retrieving environment variables and assigning their values to various variables. database_url = os.environ.get("DATABASE_URL") @@ -87,16 +147,73 @@ def handler(signum, frame): splitter_params = os.environ.get( "SPLITTER_PARAMS", {"chunk_size": 1500, "chunk_overlap": 40} ) + splitter = Splitter(splitter_name, splitter_params) -model_name = os.environ.get("MODEL", "intfloat/e5-small") -model_params = ast.literal_eval(os.environ.get("MODEL_PARAMS", {})) +model_name = "hkunlp/instructor-xl" +model_embedding_instruction = "Represent the %s document for retrieval: " % (bot_topic) +model_params = {"instruction": model_embedding_instruction} +# model_name = "BAAI/bge-large-en-v1.5" +# model_params = {} model = Model(model_name, "pgml", model_params) pipeline = Pipeline(args.collection_name + "_pipeline", model, splitter) -query_params = ast.literal_eval(os.environ.get("QUERY_PARAMS", {})) -system_prompt = os.environ.get("SYSTEM_PROMPT") -base_prompt = os.environ.get("BASE_PROMPT") +chat_history_pipeline = Pipeline( + chat_history_collection_name + "_pipeline", model, splitter +) + +query_params_instruction = ( + "Represent the %s question for retrieving supporting documents: " % (bot_topic) +) +query_params = {"instruction": query_params_instruction} +# query_params = {} + +default_system_prompt_template = """ +You are an assistant to answer questions about {topic}. +Your name is {name}. You speak like {persona} in {language}. Use the given list of documents to answer user's question. +Use the conversation history if it is applicable to answer the question. +Use the following steps: + +1. Identify if the user input is really a question. +2. If the user input is not related to the {topic} then respond that it is not related to the {topic}. +3. If the user input is related to the {topic} then first identify relevant documents from the list of documents. +4. If the documents that you found relevant have information to completely and accurately answers the question then respond with the answer. +5. If the documents that you found relevant have code snippets then respond with the code snippets. +6. Most importantly, don't make up code snippets that are not present in the documents. +7. If the user input is generic like Cool, Thanks, Hello, etc. then respond with a generic answer. +""" + +default_system_prompt = default_system_prompt_template.format( + topic=bot_topic, + name=bot_name, + persona=bot_persona, + language=bot_language, + response_programming_language=bot_topic_primary_language, +) + +system_prompt = default_system_prompt + +base_prompt = """ +{conversation_history} +#### +Documents +#### +{context} +### +User: {question} +### + +Helpful Answer:""" + openai_api_key = os.environ.get("OPENAI_API_KEY") +system_prompt_document = [ + { + "text": system_prompt, + "id": str(uuid4())[:8], + "interface": chat_interface, + "role": "system", + "timestamp": pendulum.now().timestamp(), + } +] async def upsert_documents(folder: str) -> int: log.info("Scanning " + folder + " for markdown files") @@ -117,13 +234,112 @@ async def upsert_documents(folder: str) -> int: return len(md_files) +async def generate_chat_response( + user_input, + system_prompt, + openai_api_key, + temperature=0.7, + max_tokens=256, + top_p=0.9, + user_name="", +): + messages = [] + messages.append({"role": "system", "content": system_prompt}) + + chat_history_messages = await chat_collection.get_documents( { + "limit" : chat_history*2, + "order_by": {"timestamp": "desc"}, + "filter": { + "metadata": { + "$and" : [ + { + "$or": + [ + {"role": {"$eq": "assistant"}}, + {"role": {"$eq": "user"}} + ] + }, + { + "interface" : { + "$eq" : chat_interface + } + }, + { + "user_name" : { + "$eq" : user_name + } + } + ] + } + } + } + ) + + # Reverse the order so that user messages are first + + chat_history_messages.reverse() + + conversation_history = "" + for entry in chat_history_messages: + document = entry["document"] + if document["role"] == "user": + conversation_history += "User: " + document["text"] + "\n" + if document["role"] == "assistant": + conversation_history += "Assistant: " + document["text"] + "\n" + + log.info(conversation_history) + + history_documents = [] + user_message_id = str(uuid4())[:8] + _document = { + "text": user_input, + "id": user_message_id, + "interface": chat_interface, + "role": "user", + "timestamp": pendulum.now().timestamp(), + "user_name": user_name, + } + history_documents.append(_document) + + if user_input: + query = await get_prompt(user_input,conversation_history) + + messages.append({"role": "user", "content": query}) + + log.info(messages) + + response = await generate_response( + messages, + openai_api_key, + max_tokens=max_tokens, + temperature=temperature, + top_p=top_p, + ) + + _document = { + "text": response, + "id": str(uuid4())[:8], + "parent_message_id" : user_message_id, + "interface": chat_interface, + "role": "assistant", + "timestamp": pendulum.now().timestamp(), + "user_name": user_name, + } + history_documents.append(_document) + + await chat_collection.upsert_documents(history_documents) + + return response + + async def generate_response( messages, openai_api_key, temperature=0.7, max_tokens=256, top_p=0.9 ): openai.api_key = openai_api_key log.debug("Generating response from OpenAI API: " + str(messages)) response = openai.ChatCompletion.create( - model="gpt-3.5-turbo", + # model="gpt-3.5-turbo-16k", + model="gpt-4", messages=messages, temperature=temperature, max_tokens=max_tokens, @@ -137,43 +353,58 @@ async def generate_response( async def ingest_documents(folder: str): # Add the pipeline to the collection, does nothing if we have already added it await collection.add_pipeline(pipeline) + await chat_collection.add_pipeline(chat_history_pipeline) # This will upsert, chunk, and embed the contents in the folder total_docs = await upsert_documents(folder) log.info("Total documents: " + str(total_docs)) -async def get_prompt(user_input: str = ""): +async def get_prompt(user_input: str = "", conversation_history: str = "") -> str: + query_input = "In the context of " + bot_topic + ", " + user_input vector_results = ( await collection.query() - .vector_recall(user_input, pipeline, query_params) - .limit(2) + .vector_recall(query_input, pipeline, query_params) + .limit(5) .fetch_all() ) log.info(vector_results) - context = "" - for result in vector_results: - context += result[1] + "\n" - - query = base_prompt.format(context=context, question=user_input) + context = "" + for id, result in enumerate(vector_results): + if result[0] > 0.6: + context += "#### \n Document %d: " % (id) + result[1] + "\n" + + if conversation_history: + conversation_history = "#### \n Conversation History: \n" + conversation_history + + query = base_prompt.format( + conversation_history=conversation_history, + context=context, + question=user_input, + topic=bot_topic, + persona=bot_persona, + language=bot_language, + response_programming_language=bot_topic_primary_language, + ) return query async def chat_cli(): - user_input = "Who are you?" + user_name = os.environ.get("USER") while True: try: - messages = [{"role": "system", "content": system_prompt}] - if user_input: - query = await get_prompt(user_input) - messages.append({"role": "user", "content": query}) - response = await generate_response( - messages, openai_api_key, max_tokens=512, temperature=0.0 - ) - log.info("PgBot: " + response) - user_input = input("User (Ctrl-C to exit): ") + response = await generate_chat_response( + user_input, + system_prompt, + openai_api_key, + max_tokens=512, + temperature=0.3, + top_p=0.9, + user_name=user_name, + ) + print("PgBot: " + response) except KeyboardInterrupt: print("Exiting...") break @@ -191,15 +422,17 @@ async def chat_slack(): @app.message(f"<@{bot_user_id}>") async def message_hello(message, say): print("Message received... ") - messages = [{"role": "system", "content": system_prompt}] user_input = message["text"] - - query = await get_prompt(user_input) - messages.append({"role": "user", "content": query}) - response = await generate_response( - messages, openai_api_key, max_tokens=512, temperature=1.0 - ) user = message["user"] + response = await generate_chat_response( + user_input, + system_prompt, + openai_api_key, + max_tokens=512, + temperature=0.7, + user_name = user, + ) + await say(text=f"<@{user}> {response}") @@ -218,21 +451,18 @@ async def message_hello(message, say): @client.event async def on_ready(): + await chat_collection.upsert_documents(system_prompt_document) print(f"We have logged in as {client.user}") @client.event async def on_message(message): bot_mention = f"<@{client.user.id}>" - messages = [{"role": "system", "content": system_prompt}] if message.author != client.user and bot_mention in message.content: print("Discord response in progress ..") user_input = message.content - query = await get_prompt(user_input) - - messages.append({"role": "user", "content": query}) - response = await generate_response( - messages, openai_api_key, max_tokens=512, temperature=1.0 + response = await generate_chat_response( + user_input, system_prompt, openai_api_key, max_tokens=512, temperature=0.7,user_name=message.author.name ) await message.channel.send(response) @@ -243,6 +473,7 @@ async def run(): chunks, and logs the total number of documents and chunks. """ log.info("Starting pgml-chat.... ") + await chat_collection.upsert_documents(system_prompt_document) # await migrate() if stage == "ingest": root_dir = args.root_dir @@ -265,4 +496,6 @@ def main(): client.run(os.environ["DISCORD_BOT_TOKEN"]) else: asyncio.run(run()) + + main() diff --git a/pgml-apps/pgml-chat/pgml_chat/pgml_playground.py b/pgml-apps/pgml-chat/pgml_chat/pgml_playground.py new file mode 100644 index 000000000..866310ed4 --- /dev/null +++ b/pgml-apps/pgml-chat/pgml_chat/pgml_playground.py @@ -0,0 +1,116 @@ +from pgml import Collection, Builtins, Pipeline +import asyncio +from rich import print + + # chat_history_user_messages = await chat_collection.query().vector_recall(user_input, chat_history_pipeline, query_params)( + # { + # "limit" : chat_history, + # "filter": { + # "metadata": { + # "$and": [ + # {{"role": {"$eq": "user"}}, + # {{"interface": {"$eq": chat_interface}}, + # ] + # } + # }, + # } + # ).fetch_all() + + # chat_history_assistant_messages = await chat_collection.query().vector_recall(user_input, chat_history_pipeline, query_params)( + # { + # "limit" : chat_history, + # "filter": { + # "metadata": { + # "$and": [ + # {"role": {"$eq": "assistant"}}, + # {"interface": {"$eq": chat_interface}}, + # ] + # } + # }, + # } + # ).fetch_all() + + +async def main(): + collection = Collection("pgml_chat_all_docs_4_chat_history") + builtins = Builtins() + query = """SELECT metadata->>'role' as role, text as content from %s.documents + WHERE metadata @> '{\"interface\" : \"cli\"}'::JSONB + AND metadata @> '{\"role\" : \"user\"}'::JSONB + OR metadata @> '{\"role\" : \"assistant\"}'::JSONB + ORDER BY metadata->>'timestamp' DESC LIMIT %d""" % ( + "pgml_chat_readme_1_chat_history", + 4, + ) + results = await builtins.query(query).fetch_all() + results.reverse() + # print(results) + documents = await collection.get_documents( + { "limit": 3, + "order_by": {"timestamp": "desc"}, + "filter": { + "metadata": { + "$and": [ + # {"role": {"$eq": "assistant"}}, + {"interface": {"$eq": "cli"}}, + ] + } + } + } + ) + print(documents) + pipeline = Pipeline("pgml_chat_all_docs_4_chat_history_pipeline") + chat_history_user_messages = ( + await collection.query() + .vector_recall( + "how do I use xgboost", + pipeline, + { + "instruction": "Represent the question for retrieving supporting documents: " + }, + ) + .limit(2) + .filter( + { + "metadata": { + "$and": [ + {"role": {"$eq": "user"}}, + {"interface": {"$eq": "discord"}}, + ] + } + } + ) + .fetch_all() + ) + + # print(chat_history_user_messages) + + results = ( + await collection.query() + .vector_recall( + "PostgresML on your Ubuntu machine", + pipeline, + { + "instruction": "Represent the question for retrieving supporting documents: " + }, + ) + .limit(10) + .filter( + { + "metadata": { + "$and": [ + {"role": {"$eq": "assistant"}}, + {"interface": {"$eq": "cli"}}, + ] + } + } + ) + .fetch_all() + ) + # print(results) + + # llama2-7b-chat + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pgml-apps/pgml-chat/poetry.lock b/pgml-apps/pgml-chat/poetry.lock index 682eacb13..dc95a2fea 100644 --- a/pgml-apps/pgml-chat/poetry.lock +++ b/pgml-apps/pgml-chat/poetry.lock @@ -126,14 +126,14 @@ frozenlist = ">=1.1.0" [[package]] name = "async-timeout" -version = "4.0.2" +version = "4.0.3" description = "Timeout context manager for asyncio programs" category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, - {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, + {file = "async-timeout-4.0.3.tar.gz", hash = "sha256:4640d96be84d82d02ed59ea2b7105a0f7b33abe8703703cd0ab0bf87c427522f"}, + {file = "async_timeout-4.0.3-py3-none-any.whl", hash = "sha256:7405140ff1230c310e51dc27b3145b9092d659ce68ff733fb0cefe3ee42be028"}, ] [[package]] @@ -155,36 +155,41 @@ docs = ["furo", "myst-parser", "sphinx", "sphinx-notfound-page", "sphinxcontrib- tests = ["attrs[tests-no-zope]", "zope-interface"] tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "pytest-xdist[psutil]"] +[package.source] +type = "legacy" +url = "https://test.pypi.org/simple" +reference = "testpypi" + [[package]] name = "black" -version = "23.7.0" +version = "23.9.1" description = "The uncompromising code formatter." category = "main" optional = false python-versions = ">=3.8" files = [ - {file = "black-23.7.0-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:5c4bc552ab52f6c1c506ccae05681fab58c3f72d59ae6e6639e8885e94fe2587"}, - {file = "black-23.7.0-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:552513d5cd5694590d7ef6f46e1767a4df9af168d449ff767b13b084c020e63f"}, - {file = "black-23.7.0-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:86cee259349b4448adb4ef9b204bb4467aae74a386bce85d56ba4f5dc0da27be"}, - {file = "black-23.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:501387a9edcb75d7ae8a4412bb8749900386eaef258f1aefab18adddea1936bc"}, - {file = "black-23.7.0-cp310-cp310-win_amd64.whl", hash = "sha256:fb074d8b213749fa1d077d630db0d5f8cc3b2ae63587ad4116e8a436e9bbe995"}, - {file = "black-23.7.0-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:b5b0ee6d96b345a8b420100b7d71ebfdd19fab5e8301aff48ec270042cd40ac2"}, - {file = "black-23.7.0-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:893695a76b140881531062d48476ebe4a48f5d1e9388177e175d76234ca247cd"}, - {file = "black-23.7.0-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:c333286dc3ddca6fdff74670b911cccedacb4ef0a60b34e491b8a67c833b343a"}, - {file = "black-23.7.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:831d8f54c3a8c8cf55f64d0422ee875eecac26f5f649fb6c1df65316b67c8926"}, - {file = "black-23.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:7f3bf2dec7d541b4619b8ce526bda74a6b0bffc480a163fed32eb8b3c9aed8ad"}, - {file = "black-23.7.0-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:f9062af71c59c004cd519e2fb8f5d25d39e46d3af011b41ab43b9c74e27e236f"}, - {file = "black-23.7.0-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:01ede61aac8c154b55f35301fac3e730baf0c9cf8120f65a9cd61a81cfb4a0c3"}, - {file = "black-23.7.0-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:327a8c2550ddc573b51e2c352adb88143464bb9d92c10416feb86b0f5aee5ff6"}, - {file = "black-23.7.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d1c6022b86f83b632d06f2b02774134def5d4d4f1dac8bef16d90cda18ba28a"}, - {file = "black-23.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:27eb7a0c71604d5de083757fbdb245b1a4fae60e9596514c6ec497eb63f95320"}, - {file = "black-23.7.0-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:8417dbd2f57b5701492cd46edcecc4f9208dc75529bcf76c514864e48da867d9"}, - {file = "black-23.7.0-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:47e56d83aad53ca140da0af87678fb38e44fd6bc0af71eebab2d1f59b1acf1d3"}, - {file = "black-23.7.0-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:25cc308838fe71f7065df53aedd20327969d05671bac95b38fdf37ebe70ac087"}, - {file = "black-23.7.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:642496b675095d423f9b8448243336f8ec71c9d4d57ec17bf795b67f08132a91"}, - {file = "black-23.7.0-cp39-cp39-win_amd64.whl", hash = "sha256:ad0014efc7acf0bd745792bd0d8857413652979200ab924fbf239062adc12491"}, - {file = "black-23.7.0-py3-none-any.whl", hash = "sha256:9fd59d418c60c0348505f2ddf9609c1e1de8e7493eab96198fc89d9f865e7a96"}, - {file = "black-23.7.0.tar.gz", hash = "sha256:022a582720b0d9480ed82576c920a8c1dde97cc38ff11d8d8859b3bd6ca9eedb"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_arm64.whl", hash = "sha256:d6bc09188020c9ac2555a498949401ab35bb6bf76d4e0f8ee251694664df6301"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_universal2.whl", hash = "sha256:13ef033794029b85dfea8032c9d3b92b42b526f1ff4bf13b2182ce4e917f5100"}, + {file = "black-23.9.1-cp310-cp310-macosx_10_16_x86_64.whl", hash = "sha256:75a2dc41b183d4872d3a500d2b9c9016e67ed95738a3624f4751a0cb4818fe71"}, + {file = "black-23.9.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13a2e4a93bb8ca74a749b6974925c27219bb3df4d42fc45e948a5d9feb5122b7"}, + {file = "black-23.9.1-cp310-cp310-win_amd64.whl", hash = "sha256:adc3e4442eef57f99b5590b245a328aad19c99552e0bdc7f0b04db6656debd80"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_arm64.whl", hash = "sha256:8431445bf62d2a914b541da7ab3e2b4f3bc052d2ccbf157ebad18ea126efb91f"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_universal2.whl", hash = "sha256:8fc1ddcf83f996247505db6b715294eba56ea9372e107fd54963c7553f2b6dfe"}, + {file = "black-23.9.1-cp311-cp311-macosx_10_16_x86_64.whl", hash = "sha256:7d30ec46de88091e4316b17ae58bbbfc12b2de05e069030f6b747dfc649ad186"}, + {file = "black-23.9.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:031e8c69f3d3b09e1aa471a926a1eeb0b9071f80b17689a655f7885ac9325a6f"}, + {file = "black-23.9.1-cp311-cp311-win_amd64.whl", hash = "sha256:538efb451cd50f43aba394e9ec7ad55a37598faae3348d723b59ea8e91616300"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_arm64.whl", hash = "sha256:638619a559280de0c2aa4d76f504891c9860bb8fa214267358f0a20f27c12948"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_universal2.whl", hash = "sha256:a732b82747235e0542c03bf352c126052c0fbc458d8a239a94701175b17d4855"}, + {file = "black-23.9.1-cp38-cp38-macosx_10_16_x86_64.whl", hash = "sha256:cf3a4d00e4cdb6734b64bf23cd4341421e8953615cba6b3670453737a72ec204"}, + {file = "black-23.9.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cf99f3de8b3273a8317681d8194ea222f10e0133a24a7548c73ce44ea1679377"}, + {file = "black-23.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:14f04c990259576acd093871e7e9b14918eb28f1866f91968ff5524293f9c573"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_arm64.whl", hash = "sha256:c619f063c2d68f19b2d7270f4cf3192cb81c9ec5bc5ba02df91471d0b88c4c5c"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_universal2.whl", hash = "sha256:6a3b50e4b93f43b34a9d3ef00d9b6728b4a722c997c99ab09102fd5efdb88325"}, + {file = "black-23.9.1-cp39-cp39-macosx_10_16_x86_64.whl", hash = "sha256:c46767e8df1b7beefb0899c4a95fb43058fa8500b6db144f4ff3ca38eb2f6393"}, + {file = "black-23.9.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50254ebfa56aa46a9fdd5d651f9637485068a1adf42270148cd101cdf56e0ad9"}, + {file = "black-23.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:403397c033adbc45c2bd41747da1f7fc7eaa44efbee256b53842470d4ac5a70f"}, + {file = "black-23.9.1-py3-none-any.whl", hash = "sha256:6ccd59584cc834b6d127628713e4b6b968e5f79572da66284532525a042549f9"}, + {file = "black-23.9.1.tar.gz", hash = "sha256:24b6b3ff5c6d9ea08a8888f6977eae858e1f340d7260cf56d70a49823236b62d"}, ] [package.dependencies] @@ -194,7 +199,7 @@ packaging = ">=22.0" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} +typing-extensions = {version = ">=4.0.1", markers = "python_version < \"3.11\""} [package.extras] colorama = ["colorama (>=0.4.3)"] @@ -301,19 +306,24 @@ files = [ [[package]] name = "click" -version = "8.1.6" +version = "8.1.7" description = "Composable command line interface toolkit" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "click-8.1.6-py3-none-any.whl", hash = "sha256:fa244bb30b3b5ee2cae3da8f55c9e5e0c0e86093306301fb418eb9dc40fbded5"}, - {file = "click-8.1.6.tar.gz", hash = "sha256:48ee849951919527a045bfe3bf7baa8a959c423134e1a5b98c05c20ba75a1cbd"}, + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} +[package.source] +type = "legacy" +url = "https://test.pypi.org/simple" +reference = "testpypi" + [[package]] name = "colorama" version = "0.4.6" @@ -326,16 +336,21 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[package.source] +type = "legacy" +url = "https://test.pypi.org/simple" +reference = "testpypi" + [[package]] name = "discord-py" -version = "2.3.1" +version = "2.3.2" description = "A Python wrapper for the Discord API" category = "main" optional = false python-versions = ">=3.8.0" files = [ - {file = "discord.py-2.3.1-py3-none-any.whl", hash = "sha256:149652f24da299706270bf8c03c2fcf80cf1caf3a480744c61d5b001688b380d"}, - {file = "discord.py-2.3.1.tar.gz", hash = "sha256:8eb4fe66b5d503da6de3a8425e23012711dc2fbcd7a782107a92beac15ee3459"}, + {file = "discord.py-2.3.2-py3-none-any.whl", hash = "sha256:9da4679fc3cb10c64b388284700dc998663e0e57328283bbfcfc2525ec5960a6"}, + {file = "discord.py-2.3.2.tar.gz", hash = "sha256:4560f70f2eddba7e83370ecebd237ac09fbb4980dc66507482b0c0e5b8f76b9c"}, ] [package.dependencies] @@ -565,14 +580,14 @@ files = [ [[package]] name = "openai" -version = "0.27.8" +version = "0.27.10" description = "Python client library for the OpenAI API" category = "main" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-0.27.8-py3-none-any.whl", hash = "sha256:e0a7c2f7da26bdbe5354b03c6d4b82a2f34bd4458c7a17ae1a7092c3e397e03c"}, - {file = "openai-0.27.8.tar.gz", hash = "sha256:2483095c7db1eee274cebac79e315a986c4e55207bb4fa7b82d185b3a2ed9536"}, + {file = "openai-0.27.10-py3-none-any.whl", hash = "sha256:beabd1757e3286fa166dde3b70ebb5ad8081af046876b47c14c41e203ed22a14"}, + {file = "openai-0.27.10.tar.gz", hash = "sha256:60e09edf7100080283688748c6803b7b3b52d5a55d21890f3815292a0552d83b"}, ] [package.dependencies] @@ -600,84 +615,145 @@ files = [ [[package]] name = "pathspec" -version = "0.11.1" +version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] +[[package]] +name = "pendulum" +version = "2.1.2" +description = "Python datetimes made easy" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +files = [ + {file = "pendulum-2.1.2-cp27-cp27m-macosx_10_15_x86_64.whl", hash = "sha256:b6c352f4bd32dff1ea7066bd31ad0f71f8d8100b9ff709fb343f3b86cee43efe"}, + {file = "pendulum-2.1.2-cp27-cp27m-win_amd64.whl", hash = "sha256:318f72f62e8e23cd6660dbafe1e346950281a9aed144b5c596b2ddabc1d19739"}, + {file = "pendulum-2.1.2-cp35-cp35m-macosx_10_15_x86_64.whl", hash = "sha256:0731f0c661a3cb779d398803655494893c9f581f6488048b3fb629c2342b5394"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:3481fad1dc3f6f6738bd575a951d3c15d4b4ce7c82dce37cf8ac1483fde6e8b0"}, + {file = "pendulum-2.1.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9702069c694306297ed362ce7e3c1ef8404ac8ede39f9b28b7c1a7ad8c3959e3"}, + {file = "pendulum-2.1.2-cp35-cp35m-win_amd64.whl", hash = "sha256:fb53ffa0085002ddd43b6ca61a7b34f2d4d7c3ed66f931fe599e1a531b42af9b"}, + {file = "pendulum-2.1.2-cp36-cp36m-macosx_10_15_x86_64.whl", hash = "sha256:c501749fdd3d6f9e726086bf0cd4437281ed47e7bca132ddb522f86a1645d360"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:c807a578a532eeb226150d5006f156632df2cc8c5693d778324b43ff8c515dd0"}, + {file = "pendulum-2.1.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:2d1619a721df661e506eff8db8614016f0720ac171fe80dda1333ee44e684087"}, + {file = "pendulum-2.1.2-cp36-cp36m-win_amd64.whl", hash = "sha256:f888f2d2909a414680a29ae74d0592758f2b9fcdee3549887779cd4055e975db"}, + {file = "pendulum-2.1.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:e95d329384717c7bf627bf27e204bc3b15c8238fa8d9d9781d93712776c14002"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:4c9c689747f39d0d02a9f94fcee737b34a5773803a64a5fdb046ee9cac7442c5"}, + {file = "pendulum-2.1.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:1245cd0075a3c6d889f581f6325dd8404aca5884dea7223a5566c38aab94642b"}, + {file = "pendulum-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:db0a40d8bcd27b4fb46676e8eb3c732c67a5a5e6bfab8927028224fbced0b40b"}, + {file = "pendulum-2.1.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:f5e236e7730cab1644e1b87aca3d2ff3e375a608542e90fe25685dae46310116"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_i686.whl", hash = "sha256:de42ea3e2943171a9e95141f2eecf972480636e8e484ccffaf1e833929e9e052"}, + {file = "pendulum-2.1.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7c5ec650cb4bec4c63a89a0242cc8c3cebcec92fcfe937c417ba18277d8560be"}, + {file = "pendulum-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:33fb61601083f3eb1d15edeb45274f73c63b3c44a8524703dc143f4212bf3269"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_i686.whl", hash = "sha256:29c40a6f2942376185728c9a0347d7c0f07905638c83007e1d262781f1e6953a"}, + {file = "pendulum-2.1.2-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:94b1fc947bfe38579b28e1cccb36f7e28a15e841f30384b5ad6c5e31055c85d7"}, + {file = "pendulum-2.1.2.tar.gz", hash = "sha256:b06a0ca1bfe41c990bbf0c029f0b6501a7f2ec4e38bfec730712015e8860f207"}, +] + +[package.dependencies] +python-dateutil = ">=2.6,<3.0" +pytzdata = ">=2020.1" + [[package]] name = "pgml" -version = "0.8.0" +version = "0.9.4" description = "Python SDK is designed to facilitate the development of scalable vector search applications on PostgreSQL databases." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "pgml-0.8.0-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:9308a53e30121df0c428a15cff93f8a6c5d6ba936f31c4f4b8c066fbdca9c8cc"}, - {file = "pgml-0.8.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4e92a014be000c99de6f97ff9f4f63f40af4712a7a480c914283e63b804024c2"}, - {file = "pgml-0.8.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:4d84992b5c5834334f390e4b517e2fb1af6e47f37f244f01867d4f32918b5e47"}, - {file = "pgml-0.8.0-cp310-cp310-manylinux_2_34_aarch64.whl", hash = "sha256:e042422d836b4afd584b63746a53d68a84b370a5b716c1f557fee68ea904a2f5"}, - {file = "pgml-0.8.0-cp310-cp310-manylinux_2_34_x86_64.whl", hash = "sha256:05fcb19667f48093cd5bfbbef34d76b87efa36f5c9f8aa8f52b21958134d9507"}, - {file = "pgml-0.8.0-cp310-none-win_amd64.whl", hash = "sha256:42d5c4bbd0bca75c346b9f4a70301e692eb213e7ac0e394f8f44ee90a08f1f8b"}, - {file = "pgml-0.8.0-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:12855bacf6df2ac8d0039453755bcc778c3781e857010713ed811a9726617080"}, - {file = "pgml-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:46830c9971ee9f2d01ca181d196ca8a2e30d2d9c3d5a106595456534cee7f313"}, - {file = "pgml-0.8.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:34f9ec58369fe6ed05b2ddce7cd212055bb679dd1badb42fa876148bba3e455f"}, - {file = "pgml-0.8.0-cp311-cp311-manylinux_2_34_aarch64.whl", hash = "sha256:da072fe1b0eb3999a01fcd1b1b7e180cbd14eb6a1d65fa32f0f5977bed8ed1a7"}, - {file = "pgml-0.8.0-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:fbcceacc564b80852f8a33098169546fa741ff5ee8e1cd3207b2a3cdbe23345e"}, - {file = "pgml-0.8.0-cp311-none-win_amd64.whl", hash = "sha256:dd6b7fe356bc440179d2b3cdb58ee517140978f671cbdb27459b9309d074b01d"}, - {file = "pgml-0.8.0-cp37-cp37m-manylinux_2_28_x86_64.whl", hash = "sha256:854c913c0549f5fdde34783f2035256b07873ca8d93e637dd56939e9ac4dfc70"}, - {file = "pgml-0.8.0-cp37-cp37m-manylinux_2_34_aarch64.whl", hash = "sha256:16e64df9b259361bd63f0f9aa52100ee85a4bf678c7d03fcc1d0df082469336f"}, - {file = "pgml-0.8.0-cp37-cp37m-manylinux_2_34_x86_64.whl", hash = "sha256:c42f2a92d5c05c390b2b6c34aadf6faa0cfb4243d5244c44bd699f75a28757b1"}, - {file = "pgml-0.8.0-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:a5bb83ff9bece5021c7d0a078138c87f3e59aaf51208166266b82c439a54bd51"}, - {file = "pgml-0.8.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:5e1e22f64fc536c20d026e9bf4a58797535de6d4cde18858ba14f6c28ca6dc9b"}, - {file = "pgml-0.8.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:08050b4b35c90034fb49d96ea74edda130a494f2cfabd956bd6c0d68d02f5d35"}, - {file = "pgml-0.8.0-cp38-cp38-manylinux_2_34_aarch64.whl", hash = "sha256:d71a17e0458747c87534004acdfa586fb978b76e4688611deac4ee677e651f64"}, - {file = "pgml-0.8.0-cp38-cp38-manylinux_2_34_x86_64.whl", hash = "sha256:2b059ee7b9173698c0bad8a6f14d35ee90cd6b28c2fb80a7a30396935c0bdab0"}, - {file = "pgml-0.8.0-cp38-none-win_amd64.whl", hash = "sha256:ca3c6e8c570a3ec78ccae14efb8a19aeb73f41f569f162b76750be5d40b40016"}, - {file = "pgml-0.8.0-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:b9ad06ad7b4284539844effdae31d444402afe53f887974b1a88138af6715422"}, - {file = "pgml-0.8.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:592f6364e69194db819fde66072ffdeec349ebca00af9efad6fbc4e23b18fb26"}, - {file = "pgml-0.8.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:cba5a4b5e7fd32d35635ac83f8472f669f5ea49ca0059f8d50671ac9c76dca63"}, - {file = "pgml-0.8.0-cp39-cp39-manylinux_2_34_aarch64.whl", hash = "sha256:ae8c63d577c060cfeb46f7adc2e6b60c2b2f7478205e455bde1c233df3ed581c"}, - {file = "pgml-0.8.0-cp39-cp39-manylinux_2_34_x86_64.whl", hash = "sha256:c9832807832f325338a0783e27ee58ebf65b960d3b629e816ffff3de30308519"}, - {file = "pgml-0.8.0-cp39-none-win_amd64.whl", hash = "sha256:acb82bf88ce2f7945cae3ae95ad4e37e24576e478ba50754c61230dc52c91630"}, + {file = "pgml-0.9.4-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:d517ad306bf1522145bfc779bd2d727426c631168ac72b69640a2c097a83dcad"}, + {file = "pgml-0.9.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3f2154e4ced996ba15f65061035f9dc4c2c1f54b9671b7df2219bfdfe6a83e47"}, + {file = "pgml-0.9.4-cp310-cp310-manylinux_2_31_x86_64.whl", hash = "sha256:6c4732346c7e7ca3de03fbb3493fa8a00032220158c0909729ac85d56d41ed2b"}, + {file = "pgml-0.9.4-cp310-cp310-manylinux_2_34_aarch64.whl", hash = "sha256:b494f4d8dab4dd89b44d618a682ccf02c658ce035a4f16d03021d7e62b0acc8e"}, + {file = "pgml-0.9.4-cp310-cp310-manylinux_2_34_x86_64.whl", hash = "sha256:d5c4ff491979c61b8958de3fdc5c57a0e1cd81a5966ec897a4f9500aded40dfa"}, + {file = "pgml-0.9.4-cp310-none-win_amd64.whl", hash = "sha256:e44557115b4235662161654d1163ff78cbf8b03bee85505ee077f14f60232cd9"}, + {file = "pgml-0.9.4-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:340d661a05364a7306e62dd9f7b577a2e3f4606e52a73c5a66cb51c8d0531298"}, + {file = "pgml-0.9.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:6b658abbd698b4f9bb10510fdd1ae155c09bbe11dbff7e53747ad0367df23013"}, + {file = "pgml-0.9.4-cp311-cp311-manylinux_2_31_x86_64.whl", hash = "sha256:d2bc9a6b2e7d0df7ca1b023f3cb4a0e733c0573704e0311ef1e6c65813fe4a08"}, + {file = "pgml-0.9.4-cp311-cp311-manylinux_2_34_aarch64.whl", hash = "sha256:eb88f16ae5e114cefa96f5c9e256f46d920625ee56acc3725de2f3d2237c4c9a"}, + {file = "pgml-0.9.4-cp311-cp311-manylinux_2_34_x86_64.whl", hash = "sha256:e3f8cf74b910888454eea73ca6aa40f713b9b492e1e4186a7e3e8c6870a380bc"}, + {file = "pgml-0.9.4-cp311-none-win_amd64.whl", hash = "sha256:12e5e0da59252bab278b9f6f84299699db136d20f7f22526d2919be92e051b48"}, + {file = "pgml-0.9.4-cp37-cp37m-manylinux_2_31_x86_64.whl", hash = "sha256:ea09c16eaf1d159e5a19e148fee962e7fcfab663d0df704ea5aa563ebaff82c4"}, + {file = "pgml-0.9.4-cp37-cp37m-manylinux_2_34_aarch64.whl", hash = "sha256:37adb2533a2995e000d0814070bfd8f100938644dd2d90be7a2914eb4dc55925"}, + {file = "pgml-0.9.4-cp37-cp37m-manylinux_2_34_x86_64.whl", hash = "sha256:d1a236fe0f2b9a33109c572ebd0261b8b8f1194de6892525cd2811084fb93672"}, + {file = "pgml-0.9.4-cp38-cp38-macosx_10_7_x86_64.whl", hash = "sha256:6e2f78ce5350ba20f780e0753ba068794277ec7ef3ea13f8592babab1f8d9cc1"}, + {file = "pgml-0.9.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:e9db6baffdd9edda409a44f209a4c8ba282f885c257912586f440401f2df290c"}, + {file = "pgml-0.9.4-cp38-cp38-manylinux_2_31_x86_64.whl", hash = "sha256:16b13e104a009490677715e77696bef090cfb12b3f4ff2b5ff6d38e6da0fabb1"}, + {file = "pgml-0.9.4-cp38-cp38-manylinux_2_34_aarch64.whl", hash = "sha256:0a6fc0223790d4ddf2a5db09fe016fe06f88a6769caa23d1251bafcc12ca0b17"}, + {file = "pgml-0.9.4-cp38-cp38-manylinux_2_34_x86_64.whl", hash = "sha256:a509020d10dcb8efada9e37a53f4e4fab80e80036dd4ce76dd1f1ea791a6834e"}, + {file = "pgml-0.9.4-cp38-none-win_amd64.whl", hash = "sha256:1c1b8fa68f735f15615cdad92f7fed5ed7159892f892d316425ab24e020c971f"}, + {file = "pgml-0.9.4-cp39-cp39-macosx_10_7_x86_64.whl", hash = "sha256:56061c1ae34721ca2863bf30a6b173e98736cfc698d618932efa154c562582b1"}, + {file = "pgml-0.9.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:79d4ab494a8d0006d16f33ff5c6ebb6039f0ca655b54af22b908cbaf306cabbf"}, + {file = "pgml-0.9.4-cp39-cp39-manylinux_2_31_x86_64.whl", hash = "sha256:eb001d7800e3fb189359d2d81e7fabbb12322a0c9c4277d2ad491a85a3b2594b"}, + {file = "pgml-0.9.4-cp39-cp39-manylinux_2_34_aarch64.whl", hash = "sha256:d2ff28950f98e3dc3b377c855fb39cf10db11e67191b1ea07897c4196f7b8cd4"}, + {file = "pgml-0.9.4-cp39-cp39-manylinux_2_34_x86_64.whl", hash = "sha256:c292dba6b3a3ee4f93a4ed44633959cfae70ffa33ec8775ec35e44583edf20bc"}, + {file = "pgml-0.9.4-cp39-none-win_amd64.whl", hash = "sha256:668286b5d7f0896cde03bacb97396295dfdcaf7dd71e9b7a664cf4d6f86ab072"}, + {file = "pgml-0.9.4.tar.gz", hash = "sha256:17af7d22bfcc2f1b73ce449276cfa29edc22a859dac8e8079b6913b8825439b0"}, ] +[package.source] +type = "legacy" +url = "https://test.pypi.org/simple" +reference = "testpypi" + [[package]] name = "platformdirs" -version = "3.9.1" +version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.9.1-py3-none-any.whl", hash = "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f"}, - {file = "platformdirs-3.9.1.tar.gz", hash = "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421"}, + {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, + {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, ] [package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] [[package]] name = "pygments" -version = "2.15.1" +version = "2.16.1" description = "Pygments is a syntax highlighting package written in Python." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "Pygments-2.15.1-py3-none-any.whl", hash = "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"}, - {file = "Pygments-2.15.1.tar.gz", hash = "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c"}, + {file = "Pygments-2.16.1-py3-none-any.whl", hash = "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692"}, + {file = "Pygments-2.16.1.tar.gz", hash = "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"}, ] [package.extras] plugins = ["importlib-metadata"] +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:104a4ff9f1ece23d8a31582156ea3ae928afe7121fac9fed3e967a1e2d6cf6ed"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:1efd93a2e222eb7360b5396108fdfa04e9753637d24143b8026dfb48ffbc755b"}, +] + +[package.dependencies] +six = ">=1.5" + +[package.source] +type = "legacy" +url = "https://test.pypi.org/simple" +reference = "testpypi" + [[package]] name = "python-dotenv" version = "1.0.0" @@ -693,6 +769,18 @@ files = [ [package.extras] cli = ["click (>=5.0)"] +[[package]] +name = "pytzdata" +version = "2020.1" +description = "The Olson timezone database for Python." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pytzdata-2020.1-py2.py3-none-any.whl", hash = "sha256:e1e14750bcf95016381e4d472bad004eef710f2d6417240904070b3d6654485f"}, + {file = "pytzdata-2020.1.tar.gz", hash = "sha256:3efa13b335a00a8de1d345ae41ec78dd11c9f8807f522d39850f2dd828681540"}, +] + [[package]] name = "requests" version = "2.31.0" @@ -717,14 +805,14 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "rich" -version = "13.4.2" +version = "13.5.3" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" category = "main" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.4.2-py3-none-any.whl", hash = "sha256:8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec"}, - {file = "rich-13.4.2.tar.gz", hash = "sha256:d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898"}, + {file = "rich-13.5.3-py3-none-any.whl", hash = "sha256:9257b468badc3d347e146a4faa268ff229039d4c2d176ab0cffb4c4fbc73d5d9"}, + {file = "rich-13.5.3.tar.gz", hash = "sha256:87b43e0543149efa1253f485cd845bb7ee54df16c9617b8a893650ab84b4acb6"}, ] [package.dependencies] @@ -735,6 +823,18 @@ typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9 [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] + [[package]] name = "slack-bolt" version = "1.18.0" @@ -759,19 +859,19 @@ testing-without-asyncio = ["Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "Werk [[package]] name = "slack-sdk" -version = "3.21.3" +version = "3.22.0" description = "The Slack API Platform SDK for Python" category = "main" optional = false python-versions = ">=3.6.0" files = [ - {file = "slack_sdk-3.21.3-py2.py3-none-any.whl", hash = "sha256:de3c07b92479940b61cd68c566f49fbc9974c8f38f661d26244078f3903bb9cc"}, - {file = "slack_sdk-3.21.3.tar.gz", hash = "sha256:20829bdc1a423ec93dac903470975ebf3bc76fd3fd91a4dadc0eeffc940ecb0c"}, + {file = "slack_sdk-3.22.0-py2.py3-none-any.whl", hash = "sha256:f102a4902115dff3b97c3e8883ad4e22d54732221886fc5ef29bfc290f063b4a"}, + {file = "slack_sdk-3.22.0.tar.gz", hash = "sha256:6eacce0fa4f8cfb4d84eac0d7d7e1b1926040a2df654ae86b94179bdf2bc4d8c"}, ] [package.extras] optional = ["SQLAlchemy (>=1.4,<3)", "aiodns (>1.0)", "aiohttp (>=3.7.3,<4)", "boto3 (<=2)", "websocket-client (>=1,<2)", "websockets (>=10,<11)"] -testing = ["Flask (>=1,<2)", "Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "Werkzeug (<2)", "black (==22.8.0)", "boto3 (<=2)", "click (==8.0.4)", "databases (>=0.5)", "flake8 (>=5,<6)", "itsdangerous (==1.1.0)", "moto (>=3,<4)", "psutil (>=5,<6)", "pytest (>=6.2.5,<7)", "pytest-asyncio (<1)", "pytest-cov (>=2,<3)"] +testing = ["Flask (>=1,<2)", "Flask-Sockets (>=0.2,<1)", "Jinja2 (==3.0.3)", "Werkzeug (<2)", "black (==22.8.0)", "boto3 (<=2)", "click (==8.0.4)", "flake8 (>=5,<6)", "itsdangerous (==1.1.0)", "moto (>=3,<4)", "psutil (>=5,<6)", "pytest (>=6.2.5,<7)", "pytest-asyncio (<1)", "pytest-cov (>=2,<3)"] [[package]] name = "tomli" @@ -787,47 +887,47 @@ files = [ [[package]] name = "tqdm" -version = "4.65.0" +version = "4.66.1" description = "Fast, Extensible Progress Meter" category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, - {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, + {file = "tqdm-4.66.1-py3-none-any.whl", hash = "sha256:d302b3c5b53d47bce91fea46679d9c3c6508cf6332229aa1e7d8653723793386"}, + {file = "tqdm-4.66.1.tar.gz", hash = "sha256:d88e651f9db8d8551a62556d3cff9e3034274ca5d66e93197cf2490e2dcb69c7"}, ] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} [package.extras] -dev = ["py-make (>=0.1.0)", "twine", "wheel"] +dev = ["pytest (>=6)", "pytest-cov", "pytest-timeout", "pytest-xdist"] notebook = ["ipywidgets (>=6)"] slack = ["slack-sdk"] telegram = ["requests"] [[package]] name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" +version = "4.8.0" +description = "Backported and Experimental Type Hints for Python 3.8+" category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, + {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, + {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, ] [[package]] name = "urllib3" -version = "2.0.4" +version = "2.0.5" description = "HTTP library with thread-safe connection pooling, file post, and more." category = "main" optional = false python-versions = ">=3.7" files = [ - {file = "urllib3-2.0.4-py3-none-any.whl", hash = "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"}, - {file = "urllib3-2.0.4.tar.gz", hash = "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11"}, + {file = "urllib3-2.0.5-py3-none-any.whl", hash = "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"}, + {file = "urllib3-2.0.5.tar.gz", hash = "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594"}, ] [package.extras] @@ -927,4 +1027,4 @@ multidict = ">=4.0" [metadata] lock-version = "2.0" python-versions = ">=3.8,<4.0" -content-hash = "790dc81785c78af605fe97847b90d3ff2cbf901c28ec6e0a055f4c01858892ea" +content-hash = "1412f9786ed6bf30332e934385e4c63c4847ed6f198a26d8f1003bc00d518a0c" diff --git a/pgml-apps/pgml-chat/pyproject.toml b/pgml-apps/pgml-chat/pyproject.toml index 10f9c95e9..4dc061d57 100644 --- a/pgml-apps/pgml-chat/pyproject.toml +++ b/pgml-apps/pgml-chat/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "pgml-chat" -version = "0.1.1" +version = "0.2.0" description = "PostgresML bot builder for all your documentation" authors = ["PostgresML "] license = "MIT" @@ -11,14 +11,21 @@ packages = [{include = "pgml_chat"}] python = ">=3.8,<4.0" openai = "^0.27.8" rich = "^13.4.2" -pgml = "^0.9.0" python-dotenv = "^1.0.0" click = "^8.1.6" black = "^23.7.0" slack-bolt = "^1.18.0" discord-py = "^2.3.1" +pendulum = "^2.1.2" +pgml = "^0.9.4" +[[tool.poetry.source]] +name = "testpypi" +url = "https://test.pypi.org/simple/" +default = false +secondary = false + [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" From 8b054abd85d1b34ebad3127a05c266a6dd8d7600 Mon Sep 17 00:00:00 2001 From: Dan <39170265+chillenberger@users.noreply.github.com> Date: Tue, 24 Oct 2023 16:24:11 -0600 Subject: [PATCH 06/11] Dan update select value (#1119) --- pgml-dashboard/src/components/inputs/select/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pgml-dashboard/src/components/inputs/select/mod.rs b/pgml-dashboard/src/components/inputs/select/mod.rs index 5bfbe3dc1..9e6d33c1e 100644 --- a/pgml-dashboard/src/components/inputs/select/mod.rs +++ b/pgml-dashboard/src/components/inputs/select/mod.rs @@ -57,6 +57,11 @@ impl Select { self } + pub fn value(mut self, value: &str) -> Self { + self.value = value.to_owned(); + self + } + pub fn name(mut self, name: &str) -> Self { self.name = name.to_owned(); self From d9c5cdcadbf57a9434dde016b7e85a053be99373 Mon Sep 17 00:00:00 2001 From: Philippe SEGATORI Date: Wed, 25 Oct 2023 16:16:43 +0200 Subject: [PATCH 07/11] Fix ubuntu package and docker image deployment workflow (#1120) --- .github/workflows/ubuntu-packages-and-docker-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index 069f10b33..0fd6b575a 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -104,7 +104,7 @@ jobs: with: working-directory: pgml-extension command: pgrx - args: --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config --pg16=/usr/lib/postgresql/16/bin/pg_config + args: init --pg12=/usr/lib/postgresql/12/bin/pg_config --pg13=/usr/lib/postgresql/13/bin/pg_config --pg14=/usr/lib/postgresql/14/bin/pg_config --pg15=/usr/lib/postgresql/15/bin/pg_config --pg16=/usr/lib/postgresql/16/bin/pg_config - name: Build Postgres 12 uses: postgresml/gh-actions-cargo@master with: From d7913387590f20ca1fd091deb488c8ff18abd6e9 Mon Sep 17 00:00:00 2001 From: Ruixiang Tan <819464715@qq.com> Date: Wed, 25 Oct 2023 23:52:46 +0800 Subject: [PATCH 08/11] fix: replace obsolete links (#1121) --- pgml-extension/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pgml-extension/README.md b/pgml-extension/README.md index 8bc280337..fb0117060 100644 --- a/pgml-extension/README.md +++ b/pgml-extension/README.md @@ -1 +1 @@ -Please see the [quick start instructions](https://postgresml.org/guides/setup/quick_start_with_docker/) for general information on installing or deploying PostgresML. A [developer guide](https://postgresml.org/docs/guides/setup/developers) is also available for those who would like to contribute. +Please see the [quick start instructions](https://postgresml.org/docs/guides/developer-docs/quick-start-with-docker) for general information on installing or deploying PostgresML. A [developer guide](https://postgresml.org/docs/guides/developer-docs/contributing) is also available for those who would like to contribute. From 38ae87cc3edf5349f4092af92d85b2235e4b90aa Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Wed, 25 Oct 2023 13:15:06 -0700 Subject: [PATCH 09/11] Fix cargo-watch (#1123) --- pgml-dashboard/.ignore | 2 ++ pgml-dashboard/build.rs | 24 +++++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) create mode 100644 pgml-dashboard/.ignore diff --git a/pgml-dashboard/.ignore b/pgml-dashboard/.ignore new file mode 100644 index 000000000..4eff5c86c --- /dev/null +++ b/pgml-dashboard/.ignore @@ -0,0 +1,2 @@ +*package-lock.json +node_modules/ diff --git a/pgml-dashboard/build.rs b/pgml-dashboard/build.rs index 8017afafb..b96470c58 100644 --- a/pgml-dashboard/build.rs +++ b/pgml-dashboard/build.rs @@ -11,15 +11,21 @@ fn main() { let git_hash = String::from_utf8(output.stdout).unwrap(); println!("cargo:rustc-env=GIT_SHA={}", git_hash); - let status = Command::new("cargo") - .arg("pgml-components") - .arg("bundle") - .arg("--lock") - .status() - .expect("failed to run 'cargo pgml-bundle'"); - - if !status.success() { - panic!("failed to run 'cargo pgml-bundle'"); + for i in 0..5 { + let status = Command::new("cargo") + .arg("pgml-components") + .arg("bundle") + .arg("--lock") + .status() + .expect("failed to run 'cargo pgml-bundle'"); + + if !status.success() { + if i < 4 { + println!("cargo:warning=failed to run 'cargo pgml-bundle', retrying"); + } else { + panic!("failed to run 'cargo pgml-bundle'"); + } + } } let css_version = From 0e939143ab1e1edb445e6b5085349d86b1116045 Mon Sep 17 00:00:00 2001 From: Silas Marvin <19626586+SilasMarvin@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:11:27 -0700 Subject: [PATCH 10/11] Some updates from QA (#1124) --- .../src/components/accordian/accordian.scss | 10 +---- .../accordian/accordian_controller.js | 39 ++++++++++++++----- .../src/components/chatbot/chatbot.scss | 4 +- .../components/chatbot/chatbot_controller.js | 2 +- .../src/components/chatbot/template.html | 4 +- pgml-dashboard/src/components/star/mod.rs | 4 ++ .../icons/stars/give_it_a_spin_outline.svg | 9 +++++ .../icons/stars/give_it_a_spin_text.svg | 24 ++++++++++++ 8 files changed, 73 insertions(+), 23 deletions(-) create mode 100644 pgml-dashboard/static/images/icons/stars/give_it_a_spin_outline.svg create mode 100644 pgml-dashboard/static/images/icons/stars/give_it_a_spin_text.svg diff --git a/pgml-dashboard/src/components/accordian/accordian.scss b/pgml-dashboard/src/components/accordian/accordian.scss index e90cb1ac5..dc1a279ce 100644 --- a/pgml-dashboard/src/components/accordian/accordian.scss +++ b/pgml-dashboard/src/components/accordian/accordian.scss @@ -4,13 +4,7 @@ div[data-controller="accordian"] { } .accordian-body { - display: none; - height: 0px; - transition: all 0.5s ease-in-out; - } - - .accordian-body.selected { - display: block; - height: auto; + overflow: hidden; + transition: all 0.3s ease-in-out; } } diff --git a/pgml-dashboard/src/components/accordian/accordian_controller.js b/pgml-dashboard/src/components/accordian/accordian_controller.js index b9eace831..d91ba65f6 100644 --- a/pgml-dashboard/src/components/accordian/accordian_controller.js +++ b/pgml-dashboard/src/components/accordian/accordian_controller.js @@ -1,18 +1,37 @@ import { Controller } from "@hotwired/stimulus"; export default class extends Controller { + initialize() { + this.bodies = document.getElementsByClassName("accordian-body"); + this.headers = document.getElementsByClassName("accordian-header"); + + this.heights = new Map(); + for (let i = 0; i < this.bodies.length; i++) { + this.heights.set(this.bodies[i], this.bodies[i].offsetHeight); + if (i > 0) { + this.bodies[i].style.maxHeight = "0px"; + } else { + this.bodies[i].style.maxHeight = this.bodies[i].offsetHeight + "px"; + } + } + } + + titleClick(e) { let target = e.currentTarget.getAttribute("data-value"); - let elements = document.getElementsByClassName("accordian-body"); - for (let i = 0; i < elements.length; i++) { - elements[i].classList.remove("selected"); - } - elements = document.getElementsByClassName("accordian-header"); - for (let i = 0; i < elements.length; i++) { - elements[i].classList.remove("selected"); - } - let element = document.querySelector(`[data-accordian-target="${target}"]`); - element.classList.add("selected"); e.currentTarget.classList.add("selected"); + + let body = document.querySelector(`[data-accordian-target="${target}"]`); + body.classList.add("selected"); + body.style.maxHeight = this.heights.get(body) + "px"; + + for (let i = 0; i < this.bodies.length; i++) { + if (body != this.bodies[i]) { + this.bodies[i].classList.remove("selected"); + this.bodies[i].style.maxHeight = "0px"; + } + if (e.currentTarget != this.headers[i]) + this.headers[i].classList.remove("selected"); + } } } diff --git a/pgml-dashboard/src/components/chatbot/chatbot.scss b/pgml-dashboard/src/components/chatbot/chatbot.scss index ded625965..f5964dc8f 100644 --- a/pgml-dashboard/src/components/chatbot/chatbot.scss +++ b/pgml-dashboard/src/components/chatbot/chatbot.scss @@ -59,8 +59,8 @@ div[data-controller="chatbot"] { } .chatbot-brain-option-logo { - height: 34px; - width: 34px; + height: 30px; + width: 30px; background-position: center; background-repeat: no-repeat; background-size: contain; diff --git a/pgml-dashboard/src/components/chatbot/chatbot_controller.js b/pgml-dashboard/src/components/chatbot/chatbot_controller.js index 515dea535..ef6703b33 100644 --- a/pgml-dashboard/src/components/chatbot/chatbot_controller.js +++ b/pgml-dashboard/src/components/chatbot/chatbot_controller.js @@ -65,7 +65,7 @@ const knowledgeBaseIdToName = (knowledgeBase) => { const createKnowledgeBaseNotice = (knowledgeBase) => { return ` -
Chatting with Knowledge Base ${knowledgeBaseIdToName( +
Chatting with Knowledge Base ${knowledgeBaseIdToName( knowledgeBase, )}
`; diff --git a/pgml-dashboard/src/components/chatbot/template.html b/pgml-dashboard/src/components/chatbot/template.html index cd37f4ad5..2a0b06a99 100644 --- a/pgml-dashboard/src/components/chatbot/template.html +++ b/pgml-dashboard/src/components/chatbot/template.html @@ -1,6 +1,6 @@
-
+
Knowledge Base:
@@ -101,7 +101,7 @@
Knowledge Base:
-

Chatbot

+

Chatbot

diff --git a/pgml-dashboard/src/components/star/mod.rs b/pgml-dashboard/src/components/star/mod.rs index 670577ab9..9494cf1ab 100644 --- a/pgml-dashboard/src/components/star/mod.rs +++ b/pgml-dashboard/src/components/star/mod.rs @@ -26,6 +26,10 @@ const SVGS: Lazy> = Lazy::new(|| { "give_it_a_spin", include_str!("../../../static/images/icons/stars/give_it_a_spin.svg"), ); + map.insert( + "give_it_a_spin_outline", + include_str!("../../../static/images/icons/stars/give_it_a_spin_outline.svg"), + ); map }); diff --git a/pgml-dashboard/static/images/icons/stars/give_it_a_spin_outline.svg b/pgml-dashboard/static/images/icons/stars/give_it_a_spin_outline.svg new file mode 100644 index 000000000..e03aca3b7 --- /dev/null +++ b/pgml-dashboard/static/images/icons/stars/give_it_a_spin_outline.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/pgml-dashboard/static/images/icons/stars/give_it_a_spin_text.svg b/pgml-dashboard/static/images/icons/stars/give_it_a_spin_text.svg new file mode 100644 index 000000000..9e465d703 --- /dev/null +++ b/pgml-dashboard/static/images/icons/stars/give_it_a_spin_text.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + From ae57bacd5e702568a70eb05c0cf4ecde80681bbd Mon Sep 17 00:00:00 2001 From: Lev Kokotov Date: Thu, 26 Oct 2023 11:08:40 -0700 Subject: [PATCH 11/11] Mistral AI and GGUF support (#1125) --- .github/workflows/ubuntu-packages-and-docker-image.yml | 2 +- .github/workflows/ubuntu-postgresml-python-package.yaml | 2 +- packages/postgresml/DEBIAN/control | 2 +- pgml-extension/Cargo.lock | 2 +- pgml-extension/Cargo.toml | 2 +- pgml-extension/requirements.txt | 6 ++++-- pgml-extension/sql/pgml--2.7.12--2.7.13.sql | 0 pgml-extension/src/bindings/transformers/transformers.py | 2 +- 8 files changed, 10 insertions(+), 8 deletions(-) create mode 100644 pgml-extension/sql/pgml--2.7.12--2.7.13.sql diff --git a/.github/workflows/ubuntu-packages-and-docker-image.yml b/.github/workflows/ubuntu-packages-and-docker-image.yml index 0fd6b575a..0d9df7a7e 100644 --- a/.github/workflows/ubuntu-packages-and-docker-image.yml +++ b/.github/workflows/ubuntu-packages-and-docker-image.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "2.7.12" + default: "2.7.13" jobs: # # PostgresML extension. diff --git a/.github/workflows/ubuntu-postgresml-python-package.yaml b/.github/workflows/ubuntu-postgresml-python-package.yaml index ede7a100a..1af8ef614 100644 --- a/.github/workflows/ubuntu-postgresml-python-package.yaml +++ b/.github/workflows/ubuntu-postgresml-python-package.yaml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: packageVersion: - default: "2.7.12" + default: "2.7.13" jobs: postgresml-python: diff --git a/packages/postgresml/DEBIAN/control b/packages/postgresml/DEBIAN/control index 7e7daff92..dc36d743c 100644 --- a/packages/postgresml/DEBIAN/control +++ b/packages/postgresml/DEBIAN/control @@ -3,7 +3,7 @@ Version: ${PACKAGE_VERSION} Section: database Priority: optional Architecture: all -Depends: postgresml-python (>= 2.7.12), postgresql-pgml-${PGVERSION} (>= ${PACKAGE_VERSION}) +Depends: postgresml-python (>= 2.7.13), postgresql-pgml-${PGVERSION} (>= ${PACKAGE_VERSION}) Maintainer: PostgresML Homepage: https://postgresml.org Description: PostgresML - Generative AI and Simple ML inside PostgreSQL diff --git a/pgml-extension/Cargo.lock b/pgml-extension/Cargo.lock index a63f6c5ca..fe83c4c3c 100644 --- a/pgml-extension/Cargo.lock +++ b/pgml-extension/Cargo.lock @@ -1723,7 +1723,7 @@ dependencies = [ [[package]] name = "pgml" -version = "2.7.12" +version = "2.7.13" dependencies = [ "anyhow", "blas", diff --git a/pgml-extension/Cargo.toml b/pgml-extension/Cargo.toml index 4dd01e4b1..58d949533 100644 --- a/pgml-extension/Cargo.toml +++ b/pgml-extension/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pgml" -version = "2.7.12" +version = "2.7.13" edition = "2021" [lib] diff --git a/pgml-extension/requirements.txt b/pgml-extension/requirements.txt index 89b3c2742..6dd8b4690 100644 --- a/pgml-extension/requirements.txt +++ b/pgml-extension/requirements.txt @@ -16,13 +16,15 @@ sacremoses==0.0.53 scikit-learn==1.3.0 sentencepiece==0.1.99 sentence-transformers==2.2.2 -tokenizers==0.13.3 +tokenizers==0.14.1 torch==2.0.1 torchaudio==2.0.2 torchvision==0.15.2 tqdm==4.66.1 -transformers==4.33.1 +transformers==4.34.1 xgboost==2.0.0 langchain==0.0.287 einops==0.6.1 pynvml==11.5.0 +transformers-stream-generator==0.0.4 +optimum==1.13.2 diff --git a/pgml-extension/sql/pgml--2.7.12--2.7.13.sql b/pgml-extension/sql/pgml--2.7.12--2.7.13.sql new file mode 100644 index 000000000..e69de29bb diff --git a/pgml-extension/src/bindings/transformers/transformers.py b/pgml-extension/src/bindings/transformers/transformers.py index 98c843691..8b1d1a43d 100644 --- a/pgml-extension/src/bindings/transformers/transformers.py +++ b/pgml-extension/src/bindings/transformers/transformers.py @@ -197,7 +197,7 @@ def create_pipeline(task): lower = model_name.lower() else: lower = None - if lower and "-ggml" in lower: + if lower and ("-ggml" in lower or "-gguf" in lower): pipe = GGMLPipeline(model_name, **task) elif lower and "-gptq" in lower: pipe = GPTQPipeline(model_name, **task)