improve the 'simple select' optimization to take pseudo-functions into account.

lovasoa · lovasoa · commit 39a11aae179e · 2024-04-15T20:40:34.000+02:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,9 @@
  - [Updated sqlparser](https://github.com/sqlparser-rs/sqlparser-rs/blob/main/CHANGELOG.md)
    - adds support for named windows in window functions
  - New icons with tabler icons 3.2: https://tabler.io/icons/changelog
+ - Optimize queries like `select 'xxx' as component, sqlpage.some_function(...) as parameter`
+   to avoid making an unneeded database query. 
+   This is especially important for the performance of `sqlpage.run_sql` and the `dynamic` component.
 
 ## 0.20.2 (2024-04-01)
 
diff --git a/examples/official-site/examples/chart.sql b/examples/official-site/examples/chart.sql
@@ -1,31 +1,20 @@
-select 
-    'chart'             as component,
-    'Quarterly Revenue' as title,
-    'area'              as type,
-    'indigo'            as color,
-    5                   as marker,
-    TRUE                as time;
-select 
-    '2022-01-01T00:00:00Z' as x,
-    15                     as y;
-select 
-    '2022-04-01T00:00:00Z' as x,
-    46                     as y;
-select 
-    '2022-07-01T00:00:00Z' as x,
-    23                     as y;
-select 
-    '2022-10-01T00:00:00Z' as x,
-    70                     as y;
-select 
-    '2023-01-01T00:00:00Z' as x,
-    35                     as y;
-select 
-    '2023-04-01T00:00:00Z' as x,
-    106                    as y;
-select 
-    '2023-07-01T00:00:00Z' as x,
-    53                     as y;
-
+set n=coalesce($n, 1);
 
+select 
+    'chart'                    as component,
+    'Syracuse Sequence'        as title,
+    coalesce($type, 'area')    as type,
+    coalesce($color, 'indigo')  as color,
+    5                          as marker;
+with recursive seq(x, y) as (
+    select 0, CAST($n as integer)
+    union all
+    select x+1, case
+        when y % 2 = 0 then y/2
+        else 3*y+1
+    end
+    from seq
+    where x<10
+)
+select x, y from seq;
 
diff --git a/examples/official-site/sqlpage/migrations/01_documentation.sql b/examples/official-site/sqlpage/migrations/01_documentation.sql
@@ -544,7 +544,22 @@ The `color` property sets the color of each series separately, in order.
         { "series": "PostgreSQL", "x": "2000", "y": 48},{ "series": "SQLite", "x": "2000", "y": 14},{ "series": "MySQL", "x": "2000", "y": 78},
         { "series": "PostgreSQL", "x": "2010", "y": 65},{ "series": "SQLite", "x": "2010", "y": 22},{ "series": "MySQL", "x": "2010", "y": 83},
         { "series": "PostgreSQL", "x": "2020", "y": 73},{ "series": "SQLite", "x": "2020", "y": 28},{ "series": "MySQL", "x": "2020", "y": 87}
-      ]'));
+      ]')),
+    ('chart', '
+## Multiple charts on the same line
+
+You can create information-dense dashboards by using the [card component](?component=card#component)
+to put multiple charts on the same line.
+
+For this, create one sql file per visualization you want to show,
+and set the `embed` attribute of the [card](?component=card#component) component
+to the path of the file you want to include, followed by `?_sqlpage_embed`.
+',
+        json('[
+            {"component":"card", "title":"A dashboard with multiple graphs on the same line", "columns": 2},
+            {"embed": "/examples/chart.sql?color=green&n=42&_sqlpage_embed", "footer_md": "You can find the sql file that generates the chart [here](https://github.com/lovasoa/SQLpage/tree/main/examples/official-site/examples/chart.sql)" },
+            {"embed": "/examples/chart.sql?_sqlpage_embed" },
+        ]'));
 
 INSERT INTO component(name, icon, description) VALUES
     ('table', 'table', 'A table with optional filtering and sorting.
diff --git a/src/webserver/database/execute_queries.rs b/src/webserver/database/execute_queries.rs
@@ -6,14 +6,15 @@ use std::collections::HashMap;
 use std::pin::Pin;
 
 use super::csv_import::run_csv_import;
-use super::sql::{ParsedSqlFile, ParsedStatement, StmtWithParams};
+use super::sql::{ParsedSqlFile, ParsedStatement, SimpleSelectValue, StmtWithParams};
 use crate::dynamic_component::parse_dynamic_rows;
+use crate::utils::add_value_to_map;
 use crate::webserver::database::sql_pseudofunctions::extract_req_param;
 use crate::webserver::database::sql_to_json::row_to_string;
 use crate::webserver::http::SingleOrVec;
 use crate::webserver::http_request_info::RequestInfo;
 
-use super::sql_pseudofunctions::StmtParam;
+use super::sql_pseudofunctions::{extract_req_param_as_json, StmtParam};
 use super::{highlight_sql_error, Database, DbItem};
 use sqlx::any::{AnyArguments, AnyQueryResult, AnyRow, AnyStatement, AnyTypeInfo};
 use sqlx::pool::PoolConnection;
@@ -68,7 +69,7 @@ pub fn stream_query_results<'a>(
                     )?;
                 },
                 ParsedStatement::StaticSimpleSelect(value) => {
-                    for i in parse_dynamic_rows(DbItem::Row(value.clone().into())) {
+                    for i in parse_dynamic_rows(DbItem::Row(exec_static_simple_select(value, request).await?)) {
                         yield i;
                     }
                 }
@@ -79,6 +80,22 @@ pub fn stream_query_results<'a>(
     .map(|res| res.unwrap_or_else(DbItem::Error))
 }
 
+/// Executes the sqlpage pseudo-functions contained in a static simple select
+async fn exec_static_simple_select(
+    columns: &[(String, SimpleSelectValue)],
+    req: &RequestInfo,
+) -> anyhow::Result<serde_json::Value> {
+    let mut map = serde_json::Map::with_capacity(columns.len());
+    for (name, value) in columns {
+        let value = match value {
+            SimpleSelectValue::Static(s) => s.clone(),
+            SimpleSelectValue::Dynamic(p) => extract_req_param_as_json(p, req).await?,
+        };
+        map = add_value_to_map(map, (name.clone(), value));
+    }
+    Ok(serde_json::Value::Object(map))
+}
+
 /// This function is used to create a pinned boxed stream of query results.
 /// This allows recursive calls.
 pub fn stream_query_results_boxed<'a>(
diff --git a/src/webserver/database/sql.rs b/src/webserver/database/sql.rs
@@ -1,7 +1,6 @@
 use super::csv_import::{extract_csv_copy_statement, CsvImport};
 use super::sql_pseudofunctions::{func_call_to_param, StmtParam};
 use crate::file_cache::AsyncFromStrWithState;
-use crate::utils::add_value_to_map;
 use crate::{AppState, Database};
 use anyhow::Context;
 use async_trait::async_trait;
@@ -59,7 +58,7 @@ pub(super) struct StmtWithParams {
 #[derive(Debug)]
 pub(super) enum ParsedStatement {
     StmtWithParams(StmtWithParams),
-    StaticSimpleSelect(serde_json::Map<String, serde_json::Value>),
+    StaticSimpleSelect(Vec<(String, SimpleSelectValue)>),
     SetVariable {
         variable: StmtParam,
         value: StmtWithParams,
@@ -68,6 +67,12 @@ pub(super) enum ParsedStatement {
     Error(anyhow::Error),
 }
 
+#[derive(Debug, PartialEq)]
+pub(super) enum SimpleSelectValue {
+    Static(serde_json::Value),
+    Dynamic(StmtParam),
+}
+
 fn parse_sql<'a>(
     dialect: &'a dyn Dialect,
     sql: &'a str,
@@ -93,11 +98,6 @@ fn parse_single_statement(parser: &mut Parser<'_>, db_kind: AnyKind) -> Option<P
     };
     log::debug!("Parsed statement: {stmt}");
     while parser.consume_token(&SemiColon) {}
-    if let Some(static_statement) = extract_static_simple_select(&stmt) {
-        log::debug!("Optimised a static simple select to avoid a trivial database query: {stmt} optimized to {static_statement:?}");
-        return Some(ParsedStatement::StaticSimpleSelect(static_statement));
-    }
-
     let params = ParameterExtractor::extract_parameters(&mut stmt, db_kind);
     if let Some((variable, query)) = extract_set_variable(&mut stmt) {
         return Some(ParsedStatement::SetVariable {
@@ -108,6 +108,10 @@ fn parse_single_statement(parser: &mut Parser<'_>, db_kind: AnyKind) -> Option<P
     if let Some(csv_import) = extract_csv_copy_statement(&mut stmt) {
         return Some(ParsedStatement::CsvImport(csv_import));
     }
+    if let Some(static_statement) = extract_static_simple_select(&stmt, &params) {
+        log::debug!("Optimised a static simple select to avoid a trivial database query: {stmt} optimized to {static_statement:?}");
+        return Some(ParsedStatement::StaticSimpleSelect(static_statement));
+    }
     let query = stmt.to_string();
     log::debug!("Final transformed statement: {stmt}");
     Some(ParsedStatement::StmtWithParams(StmtWithParams {
@@ -174,7 +178,8 @@ fn map_param(mut name: String) -> StmtParam {
 
 fn extract_static_simple_select(
     stmt: &Statement,
-) -> Option<serde_json::Map<String, serde_json::Value>> {
+    params: &[StmtParam],
+) -> Option<Vec<(String, SimpleSelectValue)>> {
     let set_expr = match stmt {
         Statement::Query(q)
             if q.limit.is_none()
@@ -208,22 +213,52 @@ fn extract_static_simple_select(
         }
         _ => return None,
     };
-    let mut map = serde_json::Map::with_capacity(select_items.len());
+    let mut items = Vec::with_capacity(select_items.len());
+    let mut params_iter = params.iter().cloned();
     for select_item in select_items {
         let sqlparser::ast::SelectItem::ExprWithAlias { expr, alias } = select_item else {
             return None;
         };
+        use serde_json::Value::*;
+        use SimpleSelectValue::*;
         let value = match expr {
-            Expr::Value(Value::Boolean(b)) => serde_json::Value::Bool(*b),
-            Expr::Value(Value::Number(n, _)) => serde_json::Value::Number(n.parse().ok()?),
-            Expr::Value(Value::SingleQuotedString(s)) => serde_json::Value::String(s.clone()),
-            Expr::Value(Value::Null) => serde_json::Value::Null,
-            _ => return None,
+            Expr::Value(Value::Boolean(b)) => Static(Bool(*b)),
+            Expr::Value(Value::Number(n, _)) => Static(Number(n.parse().ok()?)),
+            Expr::Value(Value::SingleQuotedString(s)) => Static(String(s.clone())),
+            Expr::Value(Value::Null) => Static(Null),
+            e if is_simple_select_placeholder(e) => {
+                if let Some(p) = params_iter.next() {
+                    Dynamic(p)
+                } else {
+                    log::error!("Parameter not extracted for placehorder: {expr:?}");
+                    return None;
+                }
+            }
+            other => {
+                log::trace!("Cancelling simple select optimization because of expr: {other:?}");
+                return None;
+            }
         };
         let key = alias.value.clone();
-        map = add_value_to_map(map, (key, value));
+        items.push((key, value));
+    }
+    if let Some(p) = params_iter.next() {
+        log::error!("static select extraction failed because of extraneous parameter: {p:?}");
+        return None;
+    }
+    Some(items)
+}
+
+fn is_simple_select_placeholder(e: &Expr) -> bool {
+    match e {
+        Expr::Value(Value::Placeholder(_)) => true,
+        Expr::Cast {
+            expr,
+            data_type: DataType::Text | DataType::Varchar(_) | DataType::Char(_),
+            format: None,
+        } if is_simple_select_placeholder(expr) => true,
+        _ => false,
     }
-    Some(map)
 }
 
 fn extract_set_variable(stmt: &mut Statement) -> Option<(StmtParam, String)> {
@@ -705,59 +740,109 @@ mod test {
 
     #[test]
     fn test_static_extract() {
+        use SimpleSelectValue::Static;
+
         assert_eq!(
-            extract_static_simple_select(&parse_postgres_stmt(
-                "select 'hello' as hello, 42 as answer, null as nothing, 'world' as hello"
-            )),
-            Some(
-                serde_json::json!({
-                    "hello": ["hello", "world"],
-                    "answer": 42,
-                    "nothing": (),
-                })
-                .as_object()
-                .unwrap()
-                .clone()
-            )
+            extract_static_simple_select(
+                &parse_postgres_stmt(
+                    "select 'hello' as hello, 42 as answer, null as nothing, 'world' as hello"
+                ),
+                &[]
+            ),
+            Some(vec![
+                ("hello".into(), Static("hello".into())),
+                ("answer".into(), Static(42.into())),
+                ("nothing".into(), Static(().into())),
+                ("hello".into(), Static("world".into())),
+            ])
+        );
+    }
+
+    #[test]
+    fn test_simple_select_with_sqlpage_pseudofunction() {
+        let sql = "select 'text' as component, $x as contents, $y as title";
+        let dialects: &[&dyn Dialect] = &[
+            &PostgreSqlDialect {},
+            &SQLiteDialect {},
+            &MySqlDialect {},
+            &MsSqlDialect {},
+        ];
+        for &dialect in dialects {
+            let parsed: Vec<ParsedStatement> = parse_sql(dialect, sql).unwrap().collect();
+            use SimpleSelectValue::{Dynamic, Static};
+            use StmtParam::GetOrPost;
+            match &parsed[..] {
+                [ParsedStatement::StaticSimpleSelect(q)] => assert_eq!(
+                    q,
+                    &[
+                        ("component".into(), Static("text".into())),
+                        ("contents".into(), Dynamic(GetOrPost("x".into()))),
+                        ("title".into(), Dynamic(GetOrPost("y".into()))),
+                    ]
+                ),
+                other => panic!("failed to extract simple select in {dialect:?}: {other:?}"),
+            }
+        }
+    }
+
+    #[test]
+    fn test_simple_select_only_extraction() {
+        use SimpleSelectValue::{Dynamic, Static};
+        use StmtParam::Cookie;
+        assert_eq!(
+            extract_static_simple_select(
+                &parse_postgres_stmt("select 'text' as component, $1 as contents"),
+                &[Cookie("cook".into())]
+            ),
+            Some(vec![
+                ("component".into(), Static("text".into())),
+                ("contents".into(), Dynamic(Cookie("cook".into()))),
+            ])
         );
     }
 
     #[test]
     fn test_static_extract_doesnt_match() {
         assert_eq!(
-            extract_static_simple_select(&parse_postgres_stmt(
-                "select 'hello' as hello, 42 as answer limit 0"
-            )),
+            extract_static_simple_select(
+                &parse_postgres_stmt("select 'hello' as hello, 42 as answer limit 0"),
+                &[]
+            ),
             None
         );
         assert_eq!(
-            extract_static_simple_select(&parse_postgres_stmt(
-                "select 'hello' as hello, 42 as answer order by 1"
-            )),
+            extract_static_simple_select(
+                &parse_postgres_stmt("select 'hello' as hello, 42 as answer order by 1"),
+                &[]
+            ),
             None
         );
         assert_eq!(
-            extract_static_simple_select(&parse_postgres_stmt(
-                "select 'hello' as hello, 42 as answer offset 1"
-            )),
+            extract_static_simple_select(
+                &parse_postgres_stmt("select 'hello' as hello, 42 as answer offset 1"),
+                &[]
+            ),
             None
         );
         assert_eq!(
-            extract_static_simple_select(&parse_postgres_stmt(
-                "select 'hello' as hello, 42 as answer where 1 = 0"
-            )),
+            extract_static_simple_select(
+                &parse_postgres_stmt("select 'hello' as hello, 42 as answer where 1 = 0"),
+                &[]
+            ),
             None
         );
         assert_eq!(
-            extract_static_simple_select(&parse_postgres_stmt(
-                "select 'hello' as hello, 42 as answer FROM t"
-            )),
+            extract_static_simple_select(
+                &parse_postgres_stmt("select 'hello' as hello, 42 as answer FROM t"),
+                &[]
+            ),
             None
         );
         assert_eq!(
-            extract_static_simple_select(&parse_postgres_stmt(
-                "select x'CAFEBABE' as hello, 42 as answer"
-            )),
+            extract_static_simple_select(
+                &parse_postgres_stmt("select x'CAFEBABE' as hello, 42 as answer"),
+                &[]
+            ),
             None
         );
     }
diff --git a/src/webserver/database/sql_pseudofunctions.rs b/src/webserver/database/sql_pseudofunctions.rs
