Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(cargo-pgrx): add support for downloading specific pg versions #1605

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

feat(cargo-pgrx): add support for downloading specific pg versions #1605

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
292 changes: 229 additions & 63 deletions cargo-pgrx/src/command/init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,17 +7,6 @@
//LICENSE All rights reserved.
//LICENSE
//LICENSE Use of this source code is governed by the MIT license that can be found in the LICENSE file.
use crate::command::stop::stop_postgres;
use crate::command::version::pgrx_default;
use crate::CommandExecute;
use bzip2::bufread::BzDecoder;
use eyre::{eyre, WrapErr};
use owo_colors::OwoColorize;
use pgrx_pg_config::{
get_c_locale_flags, prefix_path, ConfigToml, PgConfig, PgConfigSelector, Pgrx, PgrxHomeError,
};
use tar::Archive;

use std::collections::HashMap;
use std::fs::File;
use std::io::{Read, Write};
Expand All @@ -26,6 +15,22 @@ use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::OnceLock;

use bzip2::bufread::BzDecoder;
use eyre::{bail, eyre, WrapErr};
use owo_colors::OwoColorize;
use pgrx_pg_config::{
get_c_locale_flags, prefix_path, ConfigToml, PgConfig, PgConfigSelector, PgMinorVersion,
PgVersion, Pgrx, PgrxHomeError,
};
use tar::Archive;
use url::Url;

use crate::command::stop::stop_postgres;
use crate::command::version::pgrx_default;
use crate::CommandExecute;

use super::generate_ftp_download_url;

static PROCESS_ENV_DENYLIST: &[&str] = &[
"DEBUG",
"MAKEFLAGS",
Expand All @@ -41,42 +46,196 @@ static PROCESS_ENV_DENYLIST: &[&str] = &[
"LIBRARY_PATH", // see https://github.com/pgcentralfoundation/pgrx/issues/16
];

/// Specifies how to retrieve/init a Postgres version
pub(crate) enum PostgresRetrievalMethod {
/// The path to a postgres pg_config directory
PgConfigPath(PathBuf),
/// When the postgres version should be downloaded from wherever cargo-pgrx wants
ManagedDownload,
/// A more fully specified version (possibly a point release, rc or beta)
///
/// (ex. `download.1` or `download.beta2`)
ManagedDownloadWithMinor(String),
}

impl PostgresRetrievalMethod {
fn from_method_with_minor(m: &str, minor_version: &Option<String>) -> eyre::Result<Self> {
match (m, minor_version) {
// If the path is 'download' then we can prompt a managed download,
// of the latest version by default
("download", None) => Ok(Self::ManagedDownload),
// Versions that are specified with dots are likely point releases, betas or RCs
("download", Some(v)) => Ok(Self::ManagedDownloadWithMinor(v.clone())),
// If the method is the path to a valid file, then we can use it
(p, _) if std::path::Path::new(p).exists() => Ok(Self::PgConfigPath(PathBuf::from(p))),
// All other methods should be skipped
(m, _) => bail!(
"unrecognized input [{m}], please specify a valid PG_CONFIG path, or 'download'"
),
}
}
}

/// Default [`Pgrx`] instance that is used by implementation methods
static DEFAULT_PGRX: OnceLock<eyre::Result<Pgrx>> = OnceLock::new();
fn get_default_pgrx() -> &'static eyre::Result<Pgrx> {
DEFAULT_PGRX.get_or_init(|| pgrx_default())
}

impl PostgresRetrievalMethod {
/// Given a Postgres major version, resolve this version input (which can take many forms)
/// into a [`PgConfig`] which can be used as part of a [`Pgrx`]
///
/// # Arguments
///
/// * `pg_major_version` - Postgres major version, with the 'pg' prefix (ex. `"pg16"`)
/// * `pgrx_home` - A Path to the PGRX home directory
/// * `pg_init` - Init command configuration
fn resolve(
&self,
pg_major_version: &str,
pgrx_home: impl AsRef<Path>,
pg_init: &Init,
) -> eyre::Result<PgConfig> {
// Build a PgConfig based on the versions that were provided
let pg_config = match self {
// If a config path was specified, then we can look it up
Self::PgConfigPath(pg_config_path) => {
PgConfig::new_with_defaults(pg_config_path.clone())
}
// If 'download' was specified, we can download teh version of
Self::ManagedDownload => {
let Ok(default_pgrx) = get_default_pgrx().as_ref() else {
bail!("failed to generate default pgrx config");
};
default_pgrx.get(pg_major_version).wrap_err_with(|| {
format!("{pg_major_version} is not a known Postgres major version")
})?
}
// If a specialized version was used (ex. a point release, beta or rc), then
// use that more specified version rather than the simply the major version
Self::ManagedDownloadWithMinor(minor) => {
let version = PgVersion::try_from((
format!("{pg_major_version}.{minor}").as_ref(),
Some(Url::parse(&generate_ftp_download_url(
pg_major_version.trim_start_matches("pg"),
minor,
))?),
))?;

// Beta and RC versions cannot be downloaded this way (sources aren't avaiable via FTP, anyway)
if let PgMinorVersion::Rc(_) | PgMinorVersion::Beta(_) = version.minor {
bail!("managed download not supported for Rc and Beta versions");
}

download_postgres(&PgConfig::from(version), pgrx_home.as_ref(), pg_init)?
}
};

// Check the label after building, only necessary if we're not doing managed download
// to ensure we didn't somehow give a pg_config with the wrong version
if let Self::ManagedDownloadWithMinor(_) | Self::PgConfigPath(_) = self {
let label = pg_config.label().ok();
if label.is_some() && label.as_deref() != Some(pg_major_version) {
bail!(
"wrong `pg_config` given to `--{pg_major_version}` `{:?}` is for PostgreSQL {}",
pg_config.path(),
pg_config.major_version()?,
);
}
}

Ok(pg_config)
}
}

/// Initialize pgrx development environment for the first time
#[derive(clap::Args, Debug)]
#[clap(author)]
pub(crate) struct Init {
/// If installed locally, the path to PG12's `pgconfig` tool, or `download` to have pgrx download/compile/install it
#[clap(env = "PG12_PG_CONFIG", long)]
pg12: Option<String>,

/// Specify a minor version for PG12
#[clap(
env = "PG12_MINOR_VERSION",
long,
help = "Postgres 12 minor version (ex. '1', 'beta2', 'rc3')"
)]
pg12_minor_version: Option<String>,

/// If installed locally, the path to PG13's `pgconfig` tool, or `download` to have pgrx download/compile/install it
#[clap(env = "PG13_PG_CONFIG", long)]
pg13: Option<String>,

/// Specify a minor version for PG13
#[clap(
env = "PG13_MINOR_VERSION",
long,
help = "Postgres 13 minor version (ex. '1', 'beta2', 'rc3')"
)]
pg13_minor_version: Option<String>,

/// If installed locally, the path to PG14's `pgconfig` tool, or `download` to have pgrx download/compile/install it
#[clap(env = "PG14_PG_CONFIG", long)]
pg14: Option<String>,

/// Specify a minor version for PG14
#[clap(
env = "PG14_MINOR_VERSION",
long,
help = "Postgres 14 minor version (ex. '1', 'beta2', 'rc3')"
)]
pg14_minor_version: Option<String>,

/// If installed locally, the path to PG15's `pgconfig` tool, or `download` to have pgrx download/compile/install it
#[clap(env = "PG15_PG_CONFIG", long)]
pg15: Option<String>,

/// Specify a minor version for PG15
#[clap(
env = "PG15_MINOR_VERSION",
long,
help = "Postgres 15 minor version (ex. '1', 'beta2', 'rc3')"
)]
pg15_minor_version: Option<String>,

/// If installed locally, the path to PG16's `pgconfig` tool, or `download` to have pgrx download/compile/install it
#[clap(env = "PG16_PG_CONFIG", long)]
pg16: Option<String>,

/// Specify a minor version for PG16
#[clap(
env = "PG16_MINOR_VERSION",
long,
help = "Postgres 16 minor version (ex. '1', 'beta2', 'rc3')"
)]
pg16_minor_version: Option<String>,

#[clap(from_global, action = ArgAction::Count)]
verbose: u8,

#[clap(long, help = "Base port number")]
base_port: Option<u16>,

#[clap(long, help = "Base testing port number")]
base_testing_port: Option<u16>,

#[clap(long, help = "Additional flags to pass to the configure script")]
configure_flag: Vec<String>,

/// Compile PostgreSQL with the necessary flags to detect a good amount of
/// memory errors when run under Valgrind.
///
/// Building PostgreSQL with these flags requires that Valgrind be
/// installed, but the resulting build is usable without valgrind.
#[clap(long)]
valgrind: bool,

#[clap(long, short, help = "Allow N make jobs at once")]
jobs: Option<usize>,

#[clap(skip)]
jobserver: OnceLock<jobslot::Client>,
}
Expand All @@ -97,79 +256,86 @@ impl CommandExecute for Init {
)
.unwrap();

// Parse versions provided into a lookup of requested versions (if present) to the
// retrieval method that should be used for retreiving them
let mut versions = HashMap::new();

if let Some(ref version) = self.pg12 {
versions.insert("pg12", version.clone());
// Since arguments to options like --pg12 are *methods* of obtaining the source
// (i.e. 'download' or a path on disk to a pg_config), we use them to generate a method
// which we will later `resolve()`
if let Some(ref method) = self.pg12 {
versions.insert(
"pg12",
PostgresRetrievalMethod::from_method_with_minor(method, &self.pg12_minor_version)?,
);
}
if let Some(ref version) = self.pg13 {
versions.insert("pg13", version.clone());
if let Some(ref method) = self.pg13 {
versions.insert(
"pg13",
PostgresRetrievalMethod::from_method_with_minor(method, &self.pg13_minor_version)?,
);
}
if let Some(ref version) = self.pg14 {
versions.insert("pg14", version.clone());
if let Some(ref method) = self.pg14 {
versions.insert(
"pg14",
PostgresRetrievalMethod::from_method_with_minor(method, &self.pg14_minor_version)?,
);
}
if let Some(ref version) = self.pg15 {
versions.insert("pg15", version.clone());
if let Some(ref method) = self.pg15 {
versions.insert(
"pg15",
PostgresRetrievalMethod::from_method_with_minor(method, &self.pg15_minor_version)?,
);
}
if let Some(ref version) = self.pg16 {
versions.insert("pg16", version.clone());
if let Some(ref method) = self.pg16 {
versions.insert(
"pg16",
PostgresRetrievalMethod::from_method_with_minor(method, &self.pg16_minor_version)?,
);
}

// If versions were not specified at all, install defaults
if versions.is_empty() {
// no arguments specified, so we'll just install our defaults
init_pgrx(&pgrx_default()?, &self)
} else {
// user specified arguments, so we'll only install those versions of Postgres
let mut default_pgrx = None;
let mut pgrx = Pgrx::default();

for (pgver, pg_config_path) in versions {
let config = if pg_config_path == "download" {
if default_pgrx.is_none() {
default_pgrx = Some(pgrx_default()?);
}
default_pgrx
.as_ref()
.unwrap() // We just set this
.get(pgver)
.wrap_err_with(|| format!("{pgver} is not a known Postgres version"))?
.clone()
} else {
let config = PgConfig::new_with_defaults(pg_config_path.as_str().into());
let label = config.label().ok();
// We allow None in case it's configured via the environment or something.
if label.is_some() && label.as_deref() != Some(pgver) {
return Err(eyre!(
"wrong `pg_config` given to `--{pgver}` `{pg_config_path:?}` is for PostgreSQL {}",
config.major_version()?,
));
}
config
};
pgrx.push(config);
}
init_pgrx(
get_default_pgrx()
.as_ref()
.map_err(|e| eyre!("failed to retreive default pgrx settings: {e}"))?,
&self,
)?;
}

init_pgrx(&pgrx, &self)
// Resolve all versions required and then push them into the config
let pgrx_home = get_pgrx_home()?;
let mut pgrx = Pgrx::default();
for (pg_version, version_input) in versions.into_iter() {
pgrx.push(version_input.resolve(pg_version, &pgrx_home, &self)?)
}

init_pgrx(&pgrx, &self)
}
}

#[tracing::instrument(skip_all)]
pub(crate) fn init_pgrx(pgrx: &Pgrx, init: &Init) -> eyre::Result<()> {
let pgrx_home = match Pgrx::home() {
Ok(path) => path,
/// Determine the home that pgrx should use
pub(crate) fn get_pgrx_home() -> eyre::Result<PathBuf> {
match Pgrx::home() {
Ok(path) => Ok(path),
Err(e) => match e {
PgrxHomeError::NoHomeDirectory => return Err(e.into()),
PgrxHomeError::IoError(e) => return Err(e.into()),
PgrxHomeError::NoHomeDirectory => Err(e.into()),
PgrxHomeError::IoError(e) => Err(e.into()),
PgrxHomeError::MissingPgrxHome(path) => {
// $PGRX_HOME doesn't exist, but that's okay as `cargo pgrx init` is the right time
// to try and create it
println!("{} PGRX_HOME at `{}`", " Creating".bold().green(), path.display());
std::fs::create_dir_all(&path)?;
path
Ok(path)
}
},
};
}
}

#[tracing::instrument(skip_all)]
pub(crate) fn init_pgrx(pgrx: &Pgrx, init: &Init) -> eyre::Result<()> {
let pgrx_home = get_pgrx_home()?;

let mut output_configs = std::thread::scope(|s| -> eyre::Result<Vec<_>> {
let span = tracing::Span::current();
Expand All @@ -186,7 +352,7 @@ pub(crate) fn init_pgrx(pgrx: &Pgrx, init: &Init) -> eyre::Result<()> {
let mut pg_config = pg_config.clone();
stop_postgres(&pg_config).ok(); // no need to fail on errors trying to stop postgres while initializing
if !pg_config.is_real() {
pg_config = match download_postgres(&pg_config, &pgrx_home, init) {
pg_config = match download_postgres(&pg_config, pgrx_home.as_path(), init) {
Ok(pg_config) => pg_config,
Err(e) => return Err(eyre!(e)),
}
Expand Down
9 changes: 9 additions & 0 deletions cargo-pgrx/src/command/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,4 +39,13 @@ fn build_agent_for_url(url: &str) -> eyre::Result<Agent> {
}
}

/// Generate the FTP download for a Postgres tarball, given the major and minor versions
fn generate_ftp_download_url(major: impl ToString, minor: impl ToString) -> String {
let major = major.to_string();
let minor = minor.to_string();
format!(
"https://ftp.postgresql.org/pub/source/v{major}.{minor}/postgresql-{major}.{minor}.tar.bz2"
)
}

// TODO: Abstract over the repeated `fn perform`?
Loading
Loading