breaking(sqlite): libsqlite3-sys versioning, feature flags, extension loading unsafe

# Changes to unreleased code (fold together with related PRs)

* Rename config key `common.drivers.sqlite.load-extensions` to `drivers.sqlite.unsafe-load-extensions`
* Added `drivers.mysql`, `drivers.postgres`, and support for future external drivers

# Breaking Changes

* Changed `libsqlite3-sys` versioning policy to use version ranges
    * Diesel has been using this successfully for many years: https://github.com/diesel-rs/diesel/blob/cdf7c51c35da7ebc6fb12e71857bfba8487c20cd/diesel/Cargo.toml#L53
* Mark `SqliteConnectOptions::extension()` and `::extension_with_entrypoint()` as `unsafe`
* Added new non-default features corresponding to conditionally compiled SQLite APIs:
    * `sqlite-deserialize` enabling `SqliteConnection::serialize()` and `SqliteConnection::deserialize()`
    * `sqlite-load-extension` enabling `SqliteConnectOptions::extension()` and `::extension_with_entrypoint()`
    * `sqlite-unlock-notify` enables internal use of `sqlite3_unlock_notify()`

# TODO

- [x] `sqlite3_value` is not safe to access concurrently
- [ ] list all the issues this fixes

Author: abonander

Cargo.lock

@@ -54,8 +54,9 @@ authors.workspace = true
 repository.workspace = true
 rust-version.workspace = true
 
+# Note: written so that it may be copy-pasted to other crates
 [package.metadata.docs.rs]
-features = ["all-databases", "_unstable-all-types", "_unstable-doc", "sqlite-preupdate-hook"]
+features = ["_unstable-docs"]
 rustdoc-args = ["--cfg", "docsrs"]
 
 [features]
@@ -83,8 +84,13 @@ _unstable-all-types = [
     "bit-vec",
     "bstr"
 ]
+
 # Render documentation that wouldn't otherwise be shown (e.g. `sqlx_core::config`).
-_unstable-doc = []
+_unstable-docs = [
+    "all-databases",
+    "_unstable-all-types",
+    "sqlx-sqlite/_unstable-docs"
+]
 
 # Base runtime features without TLS
 runtime-async-std = ["_rt-async-std", "sqlx-core/_rt-async-std", "sqlx-macros?/_rt-async-std"]
@@ -110,10 +116,32 @@ _sqlite = []
 any = ["sqlx-core/any", "sqlx-mysql?/any", "sqlx-postgres?/any", "sqlx-sqlite?/any"]
 postgres = ["sqlx-postgres", "sqlx-macros?/postgres"]
 mysql = ["sqlx-mysql", "sqlx-macros?/mysql"]
-sqlite = ["_sqlite", "sqlx-sqlite/bundled", "sqlx-macros?/sqlite"]
+sqlite = ["sqlite-bundled", "sqlite-deserialize", "sqlite-load-extension", "sqlite-unlock-notify"]
+
+# SQLite base features
+sqlite-bundled = ["_sqlite", "sqlx-sqlite/bundled", "sqlx-macros?/sqlite"]
 sqlite-unbundled = ["_sqlite", "sqlx-sqlite/unbundled", "sqlx-macros?/sqlite-unbundled"]
+
+# SQLite features using conditionally compiled APIs
+# Note: these assume `sqlite-bundled` or `sqlite-unbundled` is also enabled
+#
+# Enable `SqliteConnection::deserialize()` and `::serialize()`
+# Cannot be used with `-DSQLITE_OMIT_DESERIALIZE`; requires `-DSQLITE_ENABLE_DESERIALIZE` on SQLite < 3.36.0
+sqlite-deserialize = ["sqlx-sqlite/deserialize"]
+
+# Enable `SqliteConnectOptions::extension()` and `::extension_with_entrypoint()`.
+# Also required to use `drivers.sqlite.unsafe-load-extensions` from `sqlx.toml`.
+# Cannot be used with `-DSQLITE_OMIT_LOAD_EXTENSION`
+sqlite-load-extension = ["sqlx-sqlite/load-extension", "sqlx-macros?/sqlite-load-extension"]
+
+# Enables `sqlite3_preupdate_hook`
+# Requires `-DSQLITE_ENABLE_PREUPDATE_HOOK` (set automatically with `sqlite-bundled`)
 sqlite-preupdate-hook = ["sqlx-sqlite/preupdate-hook"]
 
+# Enable internal handling of `SQLITE_LOCKED_SHAREDCACHE`
+# Requires `-DSQLITE_ENABLE_UNLOCK_NOTIFY` (set automatically with `sqlite-bundled`)
+sqlite-unlock-notify = ["sqlx-sqlite/unlock-notify"]
+
 # types
 json = ["sqlx-core/json", "sqlx-macros?/json", "sqlx-mysql?/json", "sqlx-postgres?/json", "sqlx-sqlite?/json"]
 
@@ -211,8 +239,14 @@ cast_sign_loss = 'deny'
 # See `clippy.toml`
 disallowed_methods = 'deny'
 
-[lints.rust]
-unexpected_cfgs = { level = 'warn', check-cfg = ['cfg(mariadb, values(any()))'] }
+
+[lints.rust.unexpected_cfgs]
+level = 'warn'
+check-cfg = [
+    'cfg(mariadb, values(any()))',
+    'cfg(sqlite_ipaddr)',
+    'cfg(sqlite_test_sqlcipher)',
+]
 
 #
 # Any

Cargo.toml

@@ -2,11 +2,11 @@
 # Including the full path to the extension is somewhat unusual,
 # because normally an extension will be installed in a standard
 # directory which is part of the library search path. If that were the
-# case here, the load-extensions value could just be `["ipaddr"]`
+# case here, the unsafe-load-extensions value could just be `["ipaddr"]`
 #
 # When the extension file is installed in a non-standard location, as
 # in this example, there are two options:
 # * Provide the full path the the extension, as seen below.
 # * Add the non-standard location to the library search path, which on
 #   Linux means adding it to the LD_LIBRARY_PATH environment variable.
-load-extensions = ["/tmp/sqlite3-lib/ipaddr"]
+unsafe-load-extensions = ["/tmp/sqlite3-lib/ipaddr"]

examples/sqlite/extension/sqlx.toml

@@ -177,7 +177,7 @@ async fn do_run(opt: Opt) -> anyhow::Result<()> {
         } => {
             let config = config.load_config().await?;
             connect_opts.populate_db_url(&config)?;
-            prepare::run(check, all, workspace, connect_opts, args).await?
+            prepare::run(&config, check, all, workspace, connect_opts, args).await?
         }
 
         #[cfg(feature = "completions")]
@@ -188,27 +188,9 @@ async fn do_run(opt: Opt) -> anyhow::Result<()> {
 }
 
 /// Attempt to connect to the database server, retrying up to `ops.connect_timeout`.
-async fn connect(opts: &ConnectOpts) -> anyhow::Result<AnyConnection> {
+async fn connect(config: &Config, opts: &ConnectOpts) -> anyhow::Result<AnyConnection> {
     retry_connect_errors(opts, move |url| {
-        // This only handles the default case. For good support of
-        // the new command line options, we need to work out some
-        // way to make the appropriate ConfigOpt available here. I
-        // suspect that that infrastructure would be useful for
-        // other things in the future, as well, but it also seems
-        // like an extensive and intrusive change.
-        //
-        // On the other hand, the compile-time checking macros
-        // can't be configured to use a different config file at
-        // all, so I believe this is okay for the time being.
-        let config = Some(std::path::PathBuf::from("sqlx.toml")).and_then(|p| {
-            if p.exists() {
-                Some(p)
-            } else {
-                None
-            }
-        });
-
-        async move { AnyConnection::connect_with_config(url, config.clone()).await }
+        AnyConnection::connect_with_driver_config(url, &config.drivers)
     })
     .await
 }

sqlx-cli/src/lib.rs

@@ -130,7 +130,7 @@ pub async fn info(
 ) -> anyhow::Result<()> {
     let migrator = migration_source.resolve(config).await?;
 
-    let mut conn = crate::connect(connect_opts).await?;
+    let mut conn = crate::connect(config, connect_opts).await?;
 
     // FIXME: we shouldn't actually be creating anything here
     for schema_name in &config.migrate.create_schemas {
@@ -229,7 +229,7 @@ pub async fn run(
         }
     }
 
-    let mut conn = crate::connect(connect_opts).await?;
+    let mut conn = crate::connect(config, connect_opts).await?;
 
     for schema_name in &config.migrate.create_schemas {
         conn.create_schema_if_not_exists(schema_name).await?;
@@ -331,7 +331,7 @@ pub async fn revert(
         }
     }
 
-    let mut conn = crate::connect(connect_opts).await?;
+    let mut conn = crate::connect(config, connect_opts).await?;
 
     // FIXME: we should not be creating anything here if it doesn't exist
     for schema_name in &config.migrate.create_schemas {

sqlx-cli/src/migrate.rs

@@ -5,14 +5,15 @@ use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
+use crate::metadata::{manifest_dir, Metadata};
+use crate::opt::ConnectOpts;
+use crate::Config;
 use anyhow::{bail, Context};
 use console::style;
 use sqlx::Connection;
 
-use crate::metadata::{manifest_dir, Metadata};
-use crate::opt::ConnectOpts;
-
-pub struct PrepareCtx {
+pub struct PrepareCtx<'a> {
+    pub config: &'a Config,
     pub workspace: bool,
     pub all: bool,
     pub cargo: OsString,
@@ -21,7 +22,7 @@ pub struct PrepareCtx {
     pub connect_opts: ConnectOpts,
 }
 
-impl PrepareCtx {
+impl PrepareCtx<'_> {
     /// Path to the directory where cached queries should be placed.
     fn prepare_dir(&self) -> anyhow::Result<PathBuf> {
         if self.workspace {
@@ -33,6 +34,7 @@ impl PrepareCtx {
 }
 
 pub async fn run(
+    config: &Config,
     check: bool,
     all: bool,
     workspace: bool,
@@ -50,6 +52,7 @@ hint: This command only works in the manifest directory of a Cargo package or wo
 
     let metadata: Metadata = Metadata::from_current_directory(&cargo)?;
     let ctx = PrepareCtx {
+        config,
         workspace,
         all,
         cargo,
@@ -65,9 +68,9 @@ hint: This command only works in the manifest directory of a Cargo package or wo
     }
 }
 
-async fn prepare(ctx: &PrepareCtx) -> anyhow::Result<()> {
+async fn prepare(ctx: &PrepareCtx<'_>) -> anyhow::Result<()> {
     if ctx.connect_opts.database_url.is_some() {
-        check_backend(&ctx.connect_opts).await?;
+        check_backend(ctx.config, &ctx.connect_opts).await?;
     }
 
     let prepare_dir = ctx.prepare_dir()?;
@@ -93,9 +96,9 @@ async fn prepare(ctx: &PrepareCtx) -> anyhow::Result<()> {
     Ok(())
 }
 
-async fn prepare_check(ctx: &PrepareCtx) -> anyhow::Result<()> {
+async fn prepare_check(ctx: &PrepareCtx<'_>) -> anyhow::Result<()> {
     if ctx.connect_opts.database_url.is_some() {
-        check_backend(&ctx.connect_opts).await?;
+        check_backend(ctx.config, &ctx.connect_opts).await?;
     }
 
     // Re-generate and store the queries in a separate directory from both the prepared
@@ -359,8 +362,8 @@ fn load_json_file(path: impl AsRef<Path>) -> anyhow::Result<serde_json::Value> {
     Ok(serde_json::from_slice(&file_bytes)?)
 }
 
-async fn check_backend(opts: &ConnectOpts) -> anyhow::Result<()> {
-    crate::connect(opts).await?.close().await?;
+async fn check_backend(config: &Config, opts: &ConnectOpts) -> anyhow::Result<()> {
+    crate::connect(config, opts).await?.close().await?;
     Ok(())
 }
 

sqlx-cli/src/prepare.rs

@@ -76,7 +76,6 @@ futures-util = { version = "0.3.19", default-features = false, features = ["allo
 log = { version = "0.4.18", default-features = false }
 memchr = { version = "2.4.1", default-features = false }
 percent-encoding = "2.1.0"
-regex = { version = "1.5.5", optional = true }
 serde = { version = "1.0.132", features = ["derive", "rc"], optional = true }
 serde_json = { version = "1.0.73", features = ["raw_value"], optional = true }
 toml = { version = "0.8.16", optional = true }

sqlx-core/Cargo.toml

@@ -5,11 +5,11 @@ use crate::any::{Any, AnyConnectOptions};
 use crate::connection::{ConnectOptions, Connection};
 use crate::error::Error;
 
+use crate::config;
 use crate::database::Database;
 use crate::sql_str::SqlSafeStr;
-pub use backend::AnyConnectionBackend;
-
 use crate::transaction::Transaction;
+pub use backend::AnyConnectionBackend;
 
 mod backend;
 mod executor;
@@ -37,40 +37,45 @@ impl AnyConnection {
     pub(crate) fn connect(options: &AnyConnectOptions) -> BoxFuture<'_, crate::Result<Self>> {
         Box::pin(async {
             let driver = crate::any::driver::from_url(&options.database_url)?;
-            (driver.connect)(options).await
+            (*driver.connect)(options, None).await
         })
     }
 
     /// UNSTABLE: for use with `sqlx-cli`
     ///
     /// Connect to the database, and instruct the nested driver to
     /// read options from the sqlx.toml file as appropriate.
-    #[cfg(feature = "sqlx-toml")]
     #[doc(hidden)]
-    pub fn connect_with_config(
+    pub async fn connect_with_driver_config(
         url: &str,
-        path: Option<std::path::PathBuf>,
-    ) -> BoxFuture<'static, Result<Self, Error>>
+        driver_config: &config::drivers::Config,
+    ) -> Result<Self, Error>
     where
         Self: Sized,
     {
-        let options: Result<AnyConnectOptions, Error> = url.parse();
+        let options: AnyConnectOptions = url.parse()?;
 
-        Box::pin(async move { Self::connect_with(&options?.with_config_file(path)).await })
+        let driver = crate::any::driver::from_url(&options.database_url)?;
+        (*driver.connect)(&options, Some(driver_config)).await
     }
 
-    pub(crate) fn connect_with_db<DB: Database>(
-        options: &AnyConnectOptions,
-    ) -> BoxFuture<'_, crate::Result<Self>>
+    pub(crate) fn connect_with_db<'a, DB: Database>(
+        options: &'a AnyConnectOptions,
+        driver_config: Option<&'a config::drivers::Config>,
+    ) -> BoxFuture<'a, crate::Result<Self>>
     where
         DB::Connection: AnyConnectionBackend,
         <DB::Connection as Connection>::Options:
-            for<'a> TryFrom<&'a AnyConnectOptions, Error = Error>,
+            for<'b> TryFrom<&'b AnyConnectOptions, Error = Error>,
     {
         let res = TryFrom::try_from(options);
 
-        Box::pin(async {
-            let options: <DB::Connection as Connection>::Options = res?;
+        Box::pin(async move {
+            let mut options: <DB::Connection as Connection>::Options = res?;
+
+            if let Some(config) = driver_config {
+                options = options.__unstable_apply_driver_config(config)?;
+            }
 
             Ok(AnyConnection {
                 backend: Box::new(options.connect().await?),

sqlx-core/src/any/connection/mod.rs

@@ -3,9 +3,8 @@ use crate::any::{AnyConnectOptions, AnyConnection};
 use crate::common::DebugFn;
 use crate::connection::Connection;
 use crate::database::Database;
-use crate::Error;
+use crate::{config, Error};
 use futures_core::future::BoxFuture;
-use futures_util::FutureExt;
 use std::fmt::{Debug, Formatter};
 use std::sync::OnceLock;
 use url::Url;
@@ -29,8 +28,12 @@ macro_rules! declare_driver_with_optional_migrate {
 pub struct AnyDriver {
     pub(crate) name: &'static str,
     pub(crate) url_schemes: &'static [&'static str],
-    pub(crate) connect:
-        DebugFn<fn(&AnyConnectOptions) -> BoxFuture<'_, crate::Result<AnyConnection>>>,
+    pub(crate) connect: DebugFn<
+        for<'a> fn(
+            &'a AnyConnectOptions,
+            Option<&'a config::drivers::Config>,
+        ) -> BoxFuture<'a, crate::Result<AnyConnection>>,
+    >,
     pub(crate) migrate_database: Option<AnyMigrateDatabase>,
 }
 
@@ -68,10 +71,10 @@ impl AnyDriver {
     {
         Self {
             migrate_database: Some(AnyMigrateDatabase {
-                create_database: DebugFn(|url| DB::create_database(url).boxed()),
-                database_exists: DebugFn(|url| DB::database_exists(url).boxed()),
-                drop_database: DebugFn(|url| DB::drop_database(url).boxed()),
-                force_drop_database: DebugFn(|url| DB::force_drop_database(url).boxed()),
+                create_database: DebugFn(|url| Box::pin(DB::create_database(url))),
+                database_exists: DebugFn(|url| Box::pin(DB::database_exists(url))),
+                drop_database: DebugFn(|url| Box::pin(DB::drop_database(url))),
+                force_drop_database: DebugFn(|url| Box::pin(DB::force_drop_database(url))),
             }),
             ..Self::without_migrate::<DB>()
         }
@@ -131,6 +134,7 @@ pub fn install_drivers(
         .map_err(|_| "drivers already installed".into())
 }
 
+#[cfg(feature = "migrate")]
 pub(crate) fn from_url_str(url: &str) -> crate::Result<&'static AnyDriver> {
     from_url(&url.parse().map_err(Error::config)?)
 }