Merge branch 'main' into registry-tracing-logging

.github/workflows/extensions.yaml

@@ -122,7 +122,7 @@ jobs:
         run: |
           su postgres -c '/usr/lib/postgresql/15/bin/postgres &'
           sleep 5
-          export EXTENSIONS=$(psql postgres://postgres:postgres@localhost:5432 -tA -c "select name from pg_available_extensions where name NOT IN ('plpgsql', 'plperlu', 'plperl', 'pltcl', 'plpython3u', 'pltclu', 'pg_stat_statements')";)
+          export EXTENSIONS=$(psql postgres://postgres:postgres@localhost:5432 -tA -c "select name from pg_available_extensions where name NOT IN ('plpgsql', 'plperlu', 'plperl', 'pltcl', 'plpython3u', 'pltclu', 'pg_stat_statements', 'pgml')";)
           for EXTENSION in $EXTENSIONS; do
             psql postgres://postgres:postgres@localhost:5432 -c "create extension if not exists \"$EXTENSION\" cascade;"
           done

cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "pg-trunk"
-version = "0.12.25"
+version = "0.12.26"
 edition = "2021"
 authors = ["Steven Miller", "Ian Stanton", "Vinícius Miguel"]
 description = "A package manager for PostgreSQL extensions"
@@ -56,6 +56,8 @@ tokio-stream = "0.1.12"
 tokio-task-manager = "0.2.0"
 toml = "0.7.2"
 which = "4.4.0"
+lazy_static = "1.5.0"
+fastrand = "2.1.0"
 
 [dev-dependencies]
 assert_cmd = "2.0.8"

cli/src/commands/install.rs

@@ -1,6 +1,7 @@
 use super::SubCommand;
 use crate::control_file::ControlFile;
 use crate::manifest::{Manifest, PackagedFile};
+use crate::retry::get_retry;
 use crate::semver::compare_by_semver;
 use crate::v1::TrunkProjectView;
 use anyhow::{anyhow, bail, ensure, Context};
@@ -323,7 +324,7 @@ async fn fetch_archive_from_v1(
 async fn fetch_archive_legacy(registry: &str, name: &str, version: &str) -> anyhow::Result<Url> {
     let endpoint = format!("{}/extensions/{}/{}/download", registry, name, version);
 
-    let response = reqwest::get(endpoint).await?;
+    let response = get_retry(&endpoint).await?;
     let status = response.status();
 
     if status.is_success() {
@@ -423,7 +424,7 @@ async fn install<'name: 'async_recursion>(
     let temp_dir = tempfile::tempdir()?;
     let dest_path = temp_dir.path().join(file_name);
 
-    let response = reqwest::get(url).await?;
+    let response = get_retry(url).await?;
 
     let mut dest_file = File::create(&dest_path)?;
     // write the response body to the file

cli/src/commands/publish.rs

@@ -3,6 +3,7 @@ use crate::commands::categories::VALID_CATEGORY_SLUGS;
 use crate::commands::publish::PublishError::InvalidExtensionName;
 use crate::config::{self, ExtensionConfiguration, LoadableLibrary};
 use crate::manifest::Manifest;
+use crate::retry::get_retry;
 use crate::trunk_toml::{cli_or_trunk, cli_or_trunk_opt, SystemDependencies};
 use anyhow::anyhow;
 use async_trait::async_trait;
@@ -213,14 +214,13 @@ impl SubCommand for PublishCommand {
         // Validate categories input if provided
         if publish_settings.categories.is_some() {
             let response =
-                reqwest::get(&format!("{}/categories/all", publish_settings.registry)).await?;
+                get_retry(&format!("{}/categories/all", publish_settings.registry)).await?;
             match response.status() {
                 StatusCode::OK => {
-                    let response_body = response.text().await?;
-                    let resp: Vec<Category> = serde_json::from_str(&response_body)?;
+                    let categories: Vec<Category> = response.json().await?;
                     // Collect list of valid category slugs
-                    for r in resp {
-                        slugs.push(r.slug);
+                    for category in categories {
+                        slugs.push(category.slug);
                     }
                 }
                 _ => {

cli/src/main.rs

@@ -2,6 +2,7 @@ mod commands;
 mod config;
 mod control_file;
 mod manifest;
+mod retry;
 mod semver;
 mod sync_utils;
 mod trunk_toml;

cli/src/retry.rs

@@ -0,0 +1,39 @@
+use std::time::Duration;
+
+use anyhow::bail;
+use lazy_static::lazy_static;
+use reqwest::{IntoUrl, Response};
+
+lazy_static! {
+    static ref CLIENT: reqwest::Client = reqwest::Client::new();
+}
+
+pub async fn get_retry<U: IntoUrl>(url: U) -> anyhow::Result<Response> {
+    let url = url.into_url()?;
+
+    let max_retries = 5;
+    let max_interval = Duration::from_secs(15);
+    let mut interval = Duration::from_millis(500);
+
+    for retry in 1..=max_retries {
+        match CLIENT.get(url.clone()).send().await {
+            Ok(resp) => {
+                if resp.status().is_success() {
+                    return Ok(resp);
+                }
+                let err_msg = resp.text().await.unwrap_or_default();
+                eprintln!("Failed to GET {url}: {err_msg}. Attempt no. {retry}");
+            }
+            Err(err) => {
+                eprintln!("Network error on GET {url}: {err}. Attempt no. {retry}");
+            }
+        }
+
+        let jitter = Duration::from_millis(fastrand::u64(0..1500));
+        interval = std::cmp::min(interval * 2, max_interval); // Double the interval for exponential backoff
+
+        tokio::time::sleep(interval + jitter).await;
+    }
+
+    bail!("Failed to GET {url} after {max_retries} retries.");
+}

contrib/jsonschema/Dockerfile

@@ -2,7 +2,7 @@ ARG PG_VERSION=16
 FROM quay.io/coredb/pgrx-builder:pg${PG_VERSION}-pgrx0.11.2
 USER root
 
-ARG RELEASE=0.1.0
+ARG RELEASE=0.1.1
 
 # Extension build dependencies
 RUN apt-get update && apt-get install -y build-essential \

contrib/jsonschema/Trunk.toml

@@ -1,6 +1,6 @@
 [extension]
 name = "jsonschema"
-version = "0.1.0"
+version = "0.1.1"
 repository = "https://github.com/tembo-io/pg-jsonschema-boon"
 license = "MIT"
 description = "The jsonschema extension provides JSON Schema validation functions supporting drafts 2020-12, 2019-09, 7, 6, and 4"
@@ -15,4 +15,4 @@ apt = ["libc6"]
 postgres_version = "16"
 platform = "linux/amd64"
 dockerfile = "Dockerfile"
-install_command = "cd jsonschema-0.1.0 && make install"
+install_command = "cd jsonschema-0.1.1 && make install"

contrib/postgresml/Dockerfile

@@ -1,44 +1,43 @@
-ARG PG_VERSION=15
-FROM quay.io/coredb/pgrx-builder:pg${PG_VERSION}-pgrx0.9.7
+ARG PG_VERSION=16
+FROM quay.io/coredb/pgrx-builder:pg${PG_VERSION}-pgrx0.11.0
 USER root
-# quay.io/coredb/pgrx-builder:pg15-pgrx0.9.7
+
+ARG RELEASE=v2.7.13
+ARG PG_VERSION=16
+
 # Extension build dependencies
 RUN apt-get update && apt-get install -y \
 	build-essential \
-	libreadline-dev \
-	zlib1g-dev \
-	flex \
-	bison \
-	libxml2-dev \
-	libxslt-dev \
-	libssl-dev \
-	libxml2-utils \
-	xsltproc \
-	ccache \
-	python3-dev \
 	clang \
+	libopenblas-dev \
+	libssl-dev \
+	bison \
+	flex \
+	pkg-config \
 	cmake \
+	libreadline-dev \
+	libz-dev \
+	tzdata \
+	sudo \
+	libpq-dev \
 	libclang-dev \
-	libopenblas-dev \
+	postgresql-${PG_VERSION} \
+	postgresql-server-dev-${PG_VERSION} \
+	python3 \
+	python3-pip \
 	libpython3-dev \
-	pkg-config
-
-# Install Rust
-RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+	lld
 
-# Set default Rust version
-RUN /root/.cargo/bin/rustup default stable
+RUN rustup default 1.78.0-x86_64-unknown-linux-gnu
 
 # Clone repository
 RUN git clone https://github.com/postgresml/postgresml.git
 
-ARG RELEASE=v2.7.1
-
 # Build the extension
 RUN cd postgresml && \
 	git submodule update --init --recursive && \
 	git fetch origin ${RELEASE} && \
 	git checkout ${RELEASE} && \
 	cd pgml-extension && \
-    cargo pgrx init --pg15 /usr/bin/pg_config && \
+    cargo pgrx init --pg${PG_VERSION} /usr/bin/pg_config && \
     cargo pgrx package

registry/src/extensions.rs

@@ -41,14 +41,12 @@ pub async fn extension_owners(
     conn: Data<Pool<Postgres>>,
 ) -> Result<Vec<Value>, ExtensionRegistryError> {
     let mut extension_owners: Vec<Value> = Vec::new();
-    // Create a transaction on the database
-    let mut tx = conn.begin().await?;
 
     let owners = sqlx::query!(
         "SELECT * FROM extension_owners WHERE extension_id = $1;",
         extension_id
     )
-    .fetch_all(&mut *tx)
+    .fetch_all(conn.as_ref())
     .await?;
 
     for row in owners.into_iter() {

registry/src/readme.rs

@@ -50,6 +50,7 @@ mod markdown_parsing {
     #[derive(Debug)]
     #[cfg_attr(test, derive(PartialEq))]
     pub struct MarkdownLink<'a> {
+        #[allow(unused)]
         pub alt_text: &'a str,
         pub path_or_link: &'a str,
     }

registry/src/routes/categories.rs

@@ -8,9 +8,8 @@ pub async fn get_all_categories(
     conn: web::Data<Pool<Postgres>>,
 ) -> Result<HttpResponse, ExtensionRegistryError> {
     // Create a database transaction
-    let mut tx = conn.begin().await?;
     let mut rows = sqlx::query!("SELECT * FROM categories")
-        .fetch_all(&mut *tx)
+        .fetch_all(conn.as_ref())
         .await?;
     let mut categories: Vec<Value> = Vec::with_capacity(rows.len());
 

registry/src/routes/extensions.rs

@@ -443,26 +443,24 @@ pub async fn get_version_history(
     let name = path.into_inner();
     let mut versions: Vec<Value> = Vec::new();
 
-    // Create a database transaction
-    let mut tx = conn.begin().await?;
     // Get extension information
     let row = sqlx::query!("SELECT * FROM extensions WHERE name = $1", name)
-        .fetch_one(&mut *tx)
+        .fetch_one(conn.as_ref())
         .await?;
     let extension_id: i32 = row.id as i32;
     let description = row.description.to_owned();
     let homepage = row.homepage.to_owned();
     let documentation = row.documentation.to_owned();
     let repository = row.repository.to_owned();
     let owners = extension_owners(extension_id, conn.clone()).await?;
-    let categories = get_categories_for_extension(extension_id, conn).await?;
+    let categories = get_categories_for_extension(extension_id, conn.clone()).await?;
 
     // Get information for all versions of extension
     let rows = sqlx::query!(
         "SELECT * FROM versions WHERE extension_id = $1",
         extension_id
     )
-    .fetch_all(&mut *tx)
+    .fetch_all(conn.as_ref())
     .await?;
 
     // TODO(ianstanton) DRY