enable arbitrary precision in the internal representation of values

- Enabled arbitrary precision for `serde_json` to handle large numbers without rounding.
- Refactored decimal handling in `sql_to_json.rs` to utilize a new function for converting decimals to JSON format.

Fixes #1052

Author: lovasoa

CHANGELOG.md

@@ -20,6 +20,7 @@
    - See the [function documentation](https://sql-page.com/functions.sql?function=hmac) for detailed examples
  - Fixed a slight spacing issue in the list components empty value display.
  - Improved performance of setting a variable to a literal value. `SET x = 'hello'` is now executed locally by SQLPage and does not send anything to the database. This completely removes the cost of extracting static values into variables for cleaner SQL files.
+ - Enable arbitrary precision in the internal representation of numbers. This guarantees zero precision loss when the database returns very large or very small DECIMAL or NUMERIC values.
 
 ## v0.37.1
  - fixed decoding of UUID values

Cargo.lock

@@ -44,7 +44,11 @@ tokio = { version = "1.24.1", features = ["macros", "rt", "process", "sync"] }
 tokio-stream = "0.1.9"
 anyhow = "1"
 serde = "1"
-serde_json = { version = "1.0.82", features = ["preserve_order", "raw_value"] }
+serde_json = { version = "1.0.82", features = [
+    "preserve_order",
+    "raw_value",
+    "arbitrary_precision",
+] }
 lambda-web = { version = "0.2.1", features = ["actix4"], optional = true }
 sqlparser = { version = "0.59.0", default-features = false, features = [
     "std",
@@ -53,7 +57,7 @@ sqlparser = { version = "0.59.0", default-features = false, features = [
 async-stream = "0.3"
 async-trait = "0.1.61"
 async-recursion = "1.0.0"
-bigdecimal = "0.4.8"
+bigdecimal = { version = "0.4.8", features = ["serde-json"] }
 include_dir = "0.7.2"
 config = { version = "0.15.4", features = ["json"] }
 markdown = { version = "1.0.0-alpha.23", features = ["log"] }
@@ -85,7 +89,7 @@ lambda-web = ["dep:lambda-web", "odbc-static"]
 
 
 [patch.crates-io]
-odbc-sys = { git = "https://github.com/sqlpage/odbc-sys", branch="no-autotools" }
+odbc-sys = { git = "https://github.com/sqlpage/odbc-sys", branch = "no-autotools" }
 
 [build-dependencies]
 awc = { version = "3", features = ["rustls-0_23-webpki-roots"] }

Cargo.toml

@@ -1,6 +1,6 @@
 use crate::utils::add_value_to_map;
 use crate::webserver::database::blob_to_data_url;
-use bigdecimal::{BigDecimal, ToPrimitive};
+use bigdecimal::BigDecimal;
 use chrono::{DateTime, FixedOffset, NaiveDateTime};
 use serde_json::{self, Map, Value};
 use sqlx::any::{AnyColumn, AnyRow, AnyTypeInfo, AnyTypeInfoKind};
@@ -68,6 +68,13 @@ fn decode_raw<'a, T: Decode<'a, sqlx::any::Any> + Default>(
     }
 }
 
+fn decimal_to_json(decimal: &BigDecimal) -> Value {
+    // to_plain_string always returns a valid JSON string
+    Value::Number(serde_json::Number::from_string_unchecked(
+        decimal.normalized().to_plain_string(),
+    ))
+}
+
 pub fn sql_nonnull_to_json<'r>(mut get_ref: impl FnMut() -> sqlx::any::AnyValueRef<'r>) -> Value {
     use AnyTypeInfoKind::{Mssql, MySql};
     let raw_value = get_ref();
@@ -77,18 +84,7 @@ pub fn sql_nonnull_to_json<'r>(mut get_ref: impl FnMut() -> sqlx::any::AnyValueR
     let AnyTypeInfo(ref db_type) = *type_info;
     match type_name {
         "REAL" | "FLOAT" | "FLOAT4" | "FLOAT8" | "DOUBLE" => decode_raw::<f64>(raw_value).into(),
-        "NUMERIC" | "DECIMAL" => {
-            let decimal = decode_raw::<BigDecimal>(raw_value);
-            if decimal.is_integer() {
-                if let Some(int) = decimal.to_i64() {
-                    return int.into();
-                }
-            }
-            if let Some(float) = decimal.to_f64() {
-                return float.into();
-            }
-            decimal.to_string().into()
-        }
+        "NUMERIC" | "DECIMAL" => decimal_to_json(&decode_raw(raw_value)),
         "INT8" | "BIGINT" | "SERIAL8" | "BIGSERIAL" | "IDENTITY" | "INT64" | "INTEGER8"
         | "BIGINT SIGNED" => decode_raw::<i64>(raw_value).into(),
         "INT" | "INT4" | "INTEGER" | "MEDIUMINT" | "YEAR" => decode_raw::<i32>(raw_value).into(),
@@ -209,6 +205,7 @@ mod tests {
                 42::INT8 as big_int,
                 42.25::FLOAT4 as float4,
                 42.25::FLOAT8 as float8,
+                123456789123456789123456789::NUMERIC as numeric,
                 TRUE as boolean,
                 '2024-03-14'::DATE as date,
                 '13:14:15'::TIME as time,
@@ -237,6 +234,7 @@ mod tests {
                 "big_int": 42,
                 "float4": 42.25,
                 "float8": 42.25,
+                "numeric": 123456789123456789123456789_u128,
                 "boolean": true,
                 "date": "2024-03-14",
                 "time": "13:14:15",

src/webserver/database/sql_to_json.rs

@@ -0,0 +1,7 @@
+SET test_stored_number = 123456789123456789123456789;
+select 'text' as component,
+    case $test_stored_number 
+        when '123456789123456789123456789' then 'It works !'
+        else 'It failed ! Expected 123456789123456789123456789 but got ' || COALESCE($test_stored_number, 'NULL')
+    end
+AS contents;