From b7e53e885a84697cead6324c95c165944f5a3783 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20B=C3=B8ving?= Date: Sat, 22 Feb 2020 12:17:47 +0100 Subject: [PATCH] First draft of JSON/JSONB support for postgres This introduces two new wrapper types `Json` and `Jsonb`, currently exported by `sqlx::postgres::{Json, Jsonb}` and adds serde and serde_json as optional dependencies, under the feature flag `json`. Wrapping types in `Json[b]` that implement `serde::Deserialize` and `serde::Serialize` allows them to be decoded and encoded respectivly. --- Cargo.lock | 3 + Cargo.toml | 4 +- sqlx-core/Cargo.toml | 3 + sqlx-core/src/postgres/mod.rs | 3 + sqlx-core/src/postgres/protocol/type_id.rs | 5 ++ sqlx-core/src/postgres/types/json.rs | 93 ++++++++++++++++++++++ sqlx-core/src/postgres/types/mod.rs | 7 ++ tests/postgres-types-json.rs | 66 +++++++++++++++ tests/postgres.rs | 2 + 9 files changed, 185 insertions(+), 1 deletion(-) create mode 100644 sqlx-core/src/postgres/types/json.rs create mode 100644 tests/postgres-types-json.rs diff --git a/Cargo.lock b/Cargo.lock index 656f8323..d4fcf1ad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1656,6 +1656,7 @@ dependencies = [ "env_logger", "futures 0.3.4", "paste", + "serde", "sqlx-core 0.3.0-alpha.1", "sqlx-macros 0.3.0-alpha.1", "sqlx-test", @@ -1722,6 +1723,8 @@ dependencies = [ "num-bigint", "percent-encoding 2.1.0", "rand", + "serde", + "serde_json", "sha-1", "sha2", "tokio 0.2.13", diff --git a/Cargo.toml b/Cargo.toml index ecaffb86..4cd1f35a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ authors = [ ] [package.metadata.docs.rs] -features = [ "tls", "postgres", "mysql", "uuid", "chrono" ] +features = [ "tls", "postgres", "mysql", "uuid", "chrono", "json" ] rustdoc-args = ["--cfg", "docsrs"] [features] @@ -50,6 +50,7 @@ bigdecimal = ["sqlx-core/bigdecimal_bigint", "sqlx-macros/bigdecimal"] chrono = [ "sqlx-core/chrono", "sqlx-macros/chrono" ] ipnetwork = [ "sqlx-core/ipnetwork", "sqlx-macros/ipnetwork" ] uuid = [ "sqlx-core/uuid", "sqlx-macros/uuid" ] +json = [ "sqlx-core/json" ] [dependencies] sqlx-core = { version = "0.3.0-alpha.1", path = "sqlx-core", default-features = false } @@ -65,6 +66,7 @@ dotenv = "0.15.0" trybuild = "1.0.24" sqlx-test = { path = "./sqlx-test" } paste = "0.1.7" +serde = { version = "1.0", features = [ "derive" ] } [[test]] name = "postgres-macros" diff --git a/sqlx-core/Cargo.toml b/sqlx-core/Cargo.toml index 0c12b49c..39602d3c 100644 --- a/sqlx-core/Cargo.toml +++ b/sqlx-core/Cargo.toml @@ -19,6 +19,7 @@ unstable = [] # `bigdecimal` uses types from it but does not reexport (tsk tsk) bigdecimal_bigint = ["bigdecimal", "num-bigint"] postgres = [ "md-5", "sha2", "base64", "sha-1", "rand", "hmac", "futures-channel/sink", "futures-util/sink" ] +json = ["serde", "serde_json"] mysql = [ "sha-1", "sha2", "generic-array", "num-bigint", "base64", "digest", "rand" ] sqlite = [ "libsqlite3-sys" ] tls = [ "async-native-tls" ] @@ -56,6 +57,8 @@ sha2 = { version = "0.8.1", default-features = false, optional = true } tokio = { version = "0.2.13", default-features = false, features = [ "dns", "fs", "time", "tcp" ], optional = true } url = { version = "2.1.1", default-features = false } uuid = { version = "0.8.1", default-features = false, optional = true, features = [ "std" ] } +serde = { version = "1.0", features = [ "derive" ], optional = true } +serde_json = { version = "1.0", optional = true } # [dependencies.libsqlite3-sys] diff --git a/sqlx-core/src/postgres/mod.rs b/sqlx-core/src/postgres/mod.rs index 2b22388e..a06b64f1 100644 --- a/sqlx-core/src/postgres/mod.rs +++ b/sqlx-core/src/postgres/mod.rs @@ -9,6 +9,9 @@ pub use listen::{PgListener, PgNotification}; pub use row::{PgRow, PgValue}; pub use types::PgTypeInfo; +#[cfg(feature = "json")] +pub use types::{Json, Jsonb}; + mod arguments; mod connection; mod cursor; diff --git a/sqlx-core/src/postgres/protocol/type_id.rs b/sqlx-core/src/postgres/protocol/type_id.rs index 5a6b9578..4c14ed98 100644 --- a/sqlx-core/src/postgres/protocol/type_id.rs +++ b/sqlx-core/src/postgres/protocol/type_id.rs @@ -62,4 +62,9 @@ impl TypeId { pub(crate) const ARRAY_CIDR: TypeId = TypeId(651); pub(crate) const ARRAY_INET: TypeId = TypeId(1041); + + // JSON + + pub(crate) const JSON: TypeId = TypeId(114); + pub(crate) const JSONB: TypeId = TypeId(3802); } diff --git a/sqlx-core/src/postgres/types/json.rs b/sqlx-core/src/postgres/types/json.rs new file mode 100644 index 00000000..c0986dd5 --- /dev/null +++ b/sqlx-core/src/postgres/types/json.rs @@ -0,0 +1,93 @@ +use crate::decode::{Decode, DecodeError}; +use crate::encode::Encode; +use crate::io::{Buf, BufMut}; +use crate::postgres::protocol::TypeId; +use crate::postgres::types::PgTypeInfo; +use crate::postgres::Postgres; +use crate::types::HasSqlType; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +impl HasSqlType for Postgres { + fn type_info() -> PgTypeInfo { + PgTypeInfo::new(TypeId::JSON) + } +} + +impl Encode for Value { + fn encode(&self, buf: &mut Vec) { + Json(self).encode(buf) + } +} + +impl Decode for Value { + fn decode(buf: &[u8]) -> Result { + let Json(item) = Decode::decode(buf)?; + Ok(item) + } +} + +#[derive(Debug, PartialEq)] +pub struct Json(pub T); + +impl HasSqlType> for Postgres { + fn type_info() -> PgTypeInfo { + PgTypeInfo::new(TypeId::JSON) + } +} + +impl Encode for Json +where + T: Serialize, +{ + fn encode(&self, buf: &mut Vec) { + serde_json::to_writer(buf, &self.0) + .expect("failed to serialize json for encoding to database"); + } +} + +impl Decode for Json +where + T: for<'a> Deserialize<'a>, +{ + fn decode(buf: &[u8]) -> Result { + let item = serde_json::from_slice(buf)?; + Ok(Json(item)) + } +} + +#[derive(Debug, PartialEq)] +pub struct Jsonb(pub T); + +impl HasSqlType> for Postgres { + fn type_info() -> PgTypeInfo { + PgTypeInfo::new(TypeId::JSONB) + } +} + +impl Encode for Jsonb +where + T: Serialize, +{ + fn encode(&self, buf: &mut Vec) { + // TODO: I haven't been figure out what this byte is, but it is required or else we get the error: + // Error: unsupported jsonb version number 34 + buf.put_u8(1); + + serde_json::to_writer(buf, &self.0) + .expect("failed to serialize json for encoding to database"); + } +} + +impl Decode for Jsonb +where + T: for<'a> Deserialize<'a>, +{ + fn decode(mut buf: &[u8]) -> Result { + // TODO: I don't know what this byte is, similarly to Encode + let _ = buf.get_u8()?; + + let item = serde_json::from_slice(buf)?; + Ok(Jsonb(item)) + } +} diff --git a/sqlx-core/src/postgres/types/mod.rs b/sqlx-core/src/postgres/types/mod.rs index 1f85a615..8ea22307 100644 --- a/sqlx-core/src/postgres/types/mod.rs +++ b/sqlx-core/src/postgres/types/mod.rs @@ -80,9 +80,15 @@ mod chrono; #[cfg(feature = "uuid")] mod uuid; +#[cfg(feature = "json")] +pub mod json; + #[cfg(feature = "ipnetwork")] mod ipnetwork; +#[cfg(feature = "json")] +pub use json::{Json, Jsonb}; + /// Type information for a Postgres SQL type. #[derive(Debug, Clone)] pub struct PgTypeInfo { @@ -149,6 +155,7 @@ impl TypeInfo for PgTypeInfo { | (TypeId::INET, TypeId::CIDR) | (TypeId::ARRAY_CIDR, TypeId::ARRAY_INET) | (TypeId::ARRAY_INET, TypeId::ARRAY_CIDR) => true, + _ => { // TODO: 99% of postgres types are direct equality for [compatible]; when we add something that isn't (e.g, JSON/JSONB), fix this here self.id.0 == other.id.0 diff --git a/tests/postgres-types-json.rs b/tests/postgres-types-json.rs new file mode 100644 index 00000000..d84d5ddb --- /dev/null +++ b/tests/postgres-types-json.rs @@ -0,0 +1,66 @@ +use sqlx::{postgres::{PgConnection, Json, Jsonb}, Connection as _, Row}; +use serde::{Deserialize, Serialize}; + +async fn connect() -> anyhow::Result { + Ok(PgConnection::open(dotenv::var("DATABASE_URL")?).await?) +} + +macro_rules! test { + ($name:ident: $ty:ty: $($text:literal == $value:expr),+) => { + mod $name { + use super::*; + + #[cfg_attr(feature = "runtime-async-std", async_std::test)] + #[cfg_attr(feature = "runtime-tokio", tokio::test)] + async fn json () -> anyhow::Result<()> { + let mut conn = connect().await?; + + // Always use jsonb for the comparison, as json does not support equality + $( + let row = sqlx::query(&format!("SELECT {}::json::jsonb = $1::jsonb, $1 as _1", $text)) + .bind(Json($value)) + .fetch_one(&mut conn) + .await?; + + assert!(row.get::(0)); + assert!(Json($value) == row.get::, _>("_1")); + )+ + + Ok(()) + } + #[cfg_attr(feature = "runtime-async-std", async_std::test)] + #[cfg_attr(feature = "runtime-tokio", tokio::test)] + async fn jsonb () -> anyhow::Result<()> { + let mut conn = connect().await?; + + $( + let row = sqlx::query(&format!("SELECT {}::jsonb = $1::jsonb, $1 as _1", $text)) + .bind(Jsonb($value)) + .fetch_one(&mut conn) + .await?; + + assert!(row.get::(0)); + assert!(Jsonb($value) == row.get::, _>("_1")); + )+ + + Ok(()) + } + } + } +} + +test!(postgres_json_string: String: "'\"Hello, World!\"'" == "Hello, World!".to_string()); + +test!(postgres_json_emoji_simple: String: "'\"😎\"'" == "😎".to_string()); +test!(postgres_json_emoji_multi: String: "'\"🙋‍♀️\"'" == "🙋‍♀️".to_string()); + +test!(postgres_json_vec: Vec: "'[\"Hello\", \"World!\"]'" == vec!["Hello".to_string(), "World!".to_string()]); + +#[derive(Deserialize, Serialize, Debug, PartialEq)] +struct Friend { + name: String, + age: u32, +} + +test!(postgres_json_struct: Friend: "'{\"name\":\"Joe\",\"age\":33}'" == Friend { name: "Joe".to_string(), age: 33 }); + diff --git a/tests/postgres.rs b/tests/postgres.rs index 98c563ee..565ab886 100644 --- a/tests/postgres.rs +++ b/tests/postgres.rs @@ -4,6 +4,8 @@ use sqlx::{Connection, Executor, Postgres, Row}; use sqlx_test::new; use std::time::Duration; +// TODO: As soon as I tried to deserialize a json value in a function, inferance for this test stopped working. I am at a loss as to how to resolve this. +#[cfg(not(feature = "json"))] #[cfg_attr(feature = "runtime-async-std", async_std::test)] #[cfg_attr(feature = "runtime-tokio", tokio::test)] async fn it_connects() -> anyhow::Result<()> {