Compare commits

...

15 Commits

Author SHA1 Message Date
9bdb767685 version: 1.0.82 2026-03-20 18:05:43 -04:00
bdd89fe695 cleanup 2026-03-20 18:05:37 -04:00
8135d80045 cleanup 2026-03-20 18:05:18 -04:00
9255439d53 added support for root schema compiled properties for the mixer 2026-03-20 18:04:49 -04:00
9038607729 version: 1.0.81 2026-03-20 15:53:59 -04:00
9f6c27c3b8 support ad-hoc refing without entity types 2026-03-20 15:53:48 -04:00
75aac41362 version: 1.0.80 2026-03-20 06:48:19 -04:00
dbcef42401 merger fixes 2026-03-20 06:48:08 -04:00
b6c5561d2f version: 1.0.79 2026-03-20 05:58:53 -04:00
e01b778d68 jsob and test array handling improved in merger 2026-03-20 05:58:43 -04:00
6eb134c0d6 test checkpoint 2026-03-20 05:17:28 -04:00
7ccc4b7cce version: 1.0.78 2026-03-20 04:41:46 -04:00
77bfa4cd18 historical and notify respected 2026-03-20 04:41:35 -04:00
b47a5abd26 version: 1.0.77 2026-03-20 01:59:56 -04:00
fcd8310ed8 added new and old to changes and pg notify 2026-03-20 01:59:48 -04:00
12 changed files with 800 additions and 1746 deletions

View File

File diff suppressed because it is too large Load Diff

1439
out.txt

File diff suppressed because it is too large Load Diff

View File

@ -23,6 +23,7 @@ use relation::Relation;
use schema::Schema;
use serde_json::Value;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use r#type::Type;
pub struct Database {
@ -310,12 +311,84 @@ impl Database {
}
fn compile_schemas(&mut self) {
// Pass 3: compile_internals across pure structure
let schema_ids: Vec<String> = self.schemas.keys().cloned().collect();
let mut compiled_names_map: HashMap<String, Vec<String>> = HashMap::new();
let mut compiled_props_map: HashMap<String, std::collections::BTreeMap<String, Arc<Schema>>> =
HashMap::new();
for id in &schema_ids {
if let Some(schema) = self.schemas.get(id) {
let mut visited = HashSet::new();
let merged = self.merged_properties(schema, &mut visited);
let mut names: Vec<String> = merged.keys().cloned().collect();
if !names.is_empty() {
names.sort();
compiled_names_map.insert(id.clone(), names);
compiled_props_map.insert(id.clone(), merged);
}
}
}
for id in schema_ids {
if let Some(schema) = self.schemas.get_mut(&id) {
if let Some(names) = compiled_names_map.remove(&id) {
schema.obj.compiled_property_names = Some(names);
}
if let Some(props) = compiled_props_map.remove(&id) {
schema.obj.compiled_properties = Some(props);
}
schema.compile_internals();
}
}
}
pub fn merged_properties(
&self,
schema: &Schema,
visited: &mut HashSet<String>,
) -> std::collections::BTreeMap<String, Arc<Schema>> {
if let Some(props) = &schema.obj.compiled_properties {
return props.clone();
}
let mut props = std::collections::BTreeMap::new();
if let Some(id) = &schema.obj.id {
if !visited.insert(id.clone()) {
return props;
}
}
if let Some(ref_id) = &schema.obj.r#ref {
if let Some(parent_schema) = self.schemas.get(ref_id) {
props.extend(self.merged_properties(parent_schema, visited));
}
}
if let Some(all_of) = &schema.obj.all_of {
for ao in all_of {
props.extend(self.merged_properties(ao, visited));
}
}
if let Some(then_schema) = &schema.obj.then_ {
props.extend(self.merged_properties(then_schema, visited));
}
if let Some(else_schema) = &schema.obj.else_ {
props.extend(self.merged_properties(else_schema, visited));
}
if let Some(local_props) = &schema.obj.properties {
for (k, v) in local_props {
props.insert(k.clone(), v.clone());
}
}
if let Some(id) = &schema.obj.id {
visited.remove(id);
}
props
}
}

View File

@ -167,6 +167,13 @@ pub struct SchemaObject {
#[serde(skip_serializing_if = "Option::is_none")]
pub extensible: Option<bool>,
#[serde(rename = "compiledProperties")]
#[serde(skip_serializing_if = "Option::is_none")]
pub compiled_property_names: Option<Vec<String>>,
#[serde(skip)]
pub compiled_properties: Option<BTreeMap<String, Arc<Schema>>>,
#[serde(skip)]
pub compiled_format: Option<CompiledFormat>,
#[serde(skip)]

View File

@ -15,6 +15,8 @@ pub struct Type {
#[serde(default)]
pub historical: bool,
#[serde(default)]
pub notify: bool,
#[serde(default)]
pub sensitive: bool,
#[serde(default)]
pub ownable: bool,

View File

@ -3,6 +3,7 @@
pub mod cache;
use crate::database::r#type::Type;
use crate::database::Database;
use serde_json::Value;
use std::sync::Arc;
@ -175,7 +176,7 @@ impl Merger {
// Attempt to extract relative object type name
let relative_type_name = match relative.get("type").and_then(|v| v.as_str()) {
Some(t) => t,
Some(t) => t.to_string(),
None => continue,
};
@ -184,7 +185,7 @@ impl Merger {
// Call central Database O(1) graph logic
let relative_relation = self.db.get_relation(
&type_def.name,
relative_type_name,
&relative_type_name,
&relation_name,
Some(&relative_keys),
);
@ -199,11 +200,16 @@ impl Merger {
}
}
let merged_relative = match self.merge_internal(Value::Object(relative), notifications)? {
let mut merged_relative = match self.merge_internal(Value::Object(relative), notifications)? {
Value::Object(m) => m,
_ => continue,
};
merged_relative.insert(
"type".to_string(),
Value::String(relative_type_name),
);
Self::apply_entity_relation(
&mut entity_fields,
&relation.source_columns,
@ -321,8 +327,9 @@ impl Merger {
}
}
// 7. Perform change tracking
// 7. Perform change tracking dynamically suppressing noise based on type bounds!
let notify_sql = self.merge_entity_change(
type_def,
&entity_fields,
entity_fetched.as_ref(),
entity_change_kind.as_deref(),
@ -620,11 +627,7 @@ impl Merger {
for key in &sorted_keys {
columns.push(format!("\"{}\"", key));
let val = entity_pairs.get(key).unwrap();
if val.as_str() == Some("") {
values.push("NULL".to_string());
} else {
values.push(Self::quote_literal(val));
}
values.push(Self::format_sql_value(val, key, entity_type));
}
if columns.is_empty() {
@ -658,7 +661,11 @@ impl Merger {
if val.as_str() == Some("") {
set_clauses.push(format!("\"{}\" = NULL", key));
} else {
set_clauses.push(format!("\"{}\" = {}", key, Self::quote_literal(val)));
set_clauses.push(format!(
"\"{}\" = {}",
key,
Self::format_sql_value(val, key, entity_type)
));
}
}
@ -680,6 +687,7 @@ impl Merger {
fn merge_entity_change(
&self,
type_obj: &Type,
entity_fields: &serde_json::Map<String, Value>,
entity_fetched: Option<&serde_json::Map<String, Value>>,
entity_change_kind: Option<&str>,
@ -694,7 +702,8 @@ impl Merger {
let id_str = entity_fields.get("id").unwrap();
let type_name = entity_fields.get("type").unwrap();
let mut changes = serde_json::Map::new();
let mut old_vals = serde_json::Map::new();
let mut new_vals = serde_json::Map::new();
let is_update = change_kind == "update" || change_kind == "delete";
if !is_update {
@ -707,7 +716,7 @@ impl Merger {
];
for (k, v) in entity_fields {
if !system_keys.contains(k) {
changes.insert(k.clone(), v.clone());
new_vals.insert(k.clone(), v.clone());
}
}
} else {
@ -724,12 +733,13 @@ impl Merger {
if let Some(fetched) = entity_fetched {
let old_val = fetched.get(k).unwrap_or(&Value::Null);
if v != old_val {
changes.insert(k.clone(), v.clone());
new_vals.insert(k.clone(), v.clone());
old_vals.insert(k.clone(), old_val.clone());
}
}
}
}
changes.insert("type".to_string(), type_name.clone());
new_vals.insert("type".to_string(), type_name.clone());
}
let mut complete = entity_fields.clone();
@ -743,33 +753,48 @@ impl Merger {
}
}
let new_val_obj = Value::Object(new_vals);
let old_val_obj = if old_vals.is_empty() {
Value::Null
} else {
Value::Object(old_vals)
};
let mut notification = serde_json::Map::new();
notification.insert("complete".to_string(), Value::Object(complete));
if is_update {
notification.insert("changes".to_string(), Value::Object(changes.clone()));
notification.insert("new".to_string(), new_val_obj.clone());
if old_val_obj != Value::Null {
notification.insert("old".to_string(), old_val_obj.clone());
}
let change_sql = format!(
"INSERT INTO agreego.change (changes, entity_id, id, kind, modified_at, modified_by) VALUES ({}, {}, {}, {}, {}, {})",
Self::quote_literal(&Value::Object(changes)),
Self::quote_literal(id_str),
Self::quote_literal(&Value::String(uuid::Uuid::new_v4().to_string())),
Self::quote_literal(&Value::String(change_kind.to_string())),
Self::quote_literal(&Value::String(timestamp.to_string())),
Self::quote_literal(&Value::String(user_id.to_string()))
);
let mut notify_sql = None;
if type_obj.historical {
let change_sql = format!(
"INSERT INTO agreego.change (\"old\", \"new\", entity_id, id, kind, modified_at, modified_by) VALUES ({}, {}, {}, {}, {}, {}, {})",
Self::quote_literal(&old_val_obj),
Self::quote_literal(&new_val_obj),
Self::quote_literal(id_str),
Self::quote_literal(&Value::String(uuid::Uuid::new_v4().to_string())),
Self::quote_literal(&Value::String(change_kind.to_string())),
Self::quote_literal(&Value::String(timestamp.to_string())),
Self::quote_literal(&Value::String(user_id.to_string()))
);
let notify_sql = format!(
"SELECT pg_notify('entity', {})",
Self::quote_literal(&Value::String(Value::Object(notification).to_string()))
);
self
.db
.execute(&change_sql, None)
.map_err(|e| format!("Executor Error in change: {:?}", e))?;
}
self
.db
.execute(&change_sql, None)
.map_err(|e| format!("Executor Error in change: {:?}", e))?;
if type_obj.notify {
notify_sql = Some(format!(
"SELECT pg_notify('entity', {})",
Self::quote_literal(&Value::String(Value::Object(notification).to_string()))
));
}
Ok(Some(notify_sql))
Ok(notify_sql)
}
fn compare_entities(
@ -821,6 +846,34 @@ impl Merger {
}
}
fn format_sql_value(val: &Value, key: &str, entity_type: &Type) -> String {
if val.as_str() == Some("") {
return "NULL".to_string();
}
let mut is_pg_array = false;
if let Some(field_types_map) = entity_type.field_types.as_ref().and_then(|v| v.as_object()) {
if let Some(t_val) = field_types_map.get(key) {
if let Some(t_str) = t_val.as_str() {
if t_str.starts_with('_') {
is_pg_array = true;
}
}
}
}
if is_pg_array && val.is_array() {
let mut s = val.to_string();
if s.starts_with('[') && s.ends_with(']') {
s.replace_range(0..1, "{");
s.replace_range(s.len() - 1..s.len(), "}");
}
Self::quote_literal(&Value::String(s))
} else {
Self::quote_literal(val)
}
}
fn quote_literal(val: &Value) -> String {
match val {
Value::Null => "NULL".to_string(),

View File

@ -19,11 +19,7 @@ pub struct Node<'a> {
impl<'a> Compiler<'a> {
/// Compiles a JSON schema into a nested PostgreSQL query returning JSONB
pub fn compile(
&self,
schema_id: &str,
filter_keys: &[String],
) -> Result<String, String> {
pub fn compile(&self, schema_id: &str, filter_keys: &[String]) -> Result<String, String> {
let schema = self
.db
.schemas
@ -251,8 +247,7 @@ impl<'a> Compiler<'a> {
let mut bypass_node = node.clone();
bypass_node.schema = std::sync::Arc::new(bypass_schema);
let mut bypassed_args =
self.compile_select_clause(r#type, table_aliases, bypass_node)?;
let mut bypassed_args = self.compile_select_clause(r#type, table_aliases, bypass_node)?;
select_args.append(&mut bypassed_args);
} else {
let mut family_schemas = Vec::new();
@ -400,7 +395,9 @@ impl<'a> Compiler<'a> {
) -> Result<Vec<String>, String> {
let mut select_args = Vec::new();
let grouped_fields = r#type.grouped_fields.as_ref().and_then(|v| v.as_object());
let merged_props = self.get_merged_properties(node.schema.as_ref());
let merged_props = self
.db
.merged_properties(node.schema.as_ref(), &mut std::collections::HashSet::new());
let mut sorted_keys: Vec<&String> = merged_props.keys().collect();
sorted_keys.sort();
@ -417,9 +414,9 @@ impl<'a> Compiler<'a> {
_ => false,
};
let is_primitive = prop_schema.obj.r#ref.is_none()
&& !is_object_or_array
&& prop_schema.obj.family.is_none()
let is_primitive = prop_schema.obj.r#ref.is_none()
&& !is_object_or_array
&& prop_schema.obj.family.is_none()
&& prop_schema.obj.one_of.is_none();
if is_primitive {
@ -494,7 +491,13 @@ impl<'a> Compiler<'a> {
where_clauses.push(format!("NOT {}.archived", entity_alias));
self.compile_filter_conditions(r#type, type_aliases, &node, &base_alias, &mut where_clauses);
self.compile_relation_conditions(r#type, type_aliases, &node, &base_alias, &mut where_clauses)?;
self.compile_relation_conditions(
r#type,
type_aliases,
&node,
&base_alias,
&mut where_clauses,
)?;
Ok(where_clauses)
}
@ -509,7 +512,10 @@ impl<'a> Compiler<'a> {
for (t_name, fields_val) in gf {
if let Some(fields_arr) = fields_val.as_array() {
if fields_arr.iter().any(|v| v.as_str() == Some(field_name)) {
return type_aliases.get(t_name).cloned().unwrap_or_else(|| base_alias.to_string());
return type_aliases
.get(t_name)
.cloned()
.unwrap_or_else(|| base_alias.to_string());
}
}
}
@ -606,13 +612,31 @@ impl<'a> Compiler<'a> {
));
} else {
let sql_op = match op {
"$eq" => if is_ilike { "ILIKE" } else { "=" },
"$ne" => if is_ilike { "NOT ILIKE" } else { "!=" },
"$eq" => {
if is_ilike {
"ILIKE"
} else {
"="
}
}
"$ne" => {
if is_ilike {
"NOT ILIKE"
} else {
"!="
}
}
"$gt" => ">",
"$gte" => ">=",
"$lt" => "<",
"$lte" => "<=",
_ => if is_ilike { "ILIKE" } else { "=" },
_ => {
if is_ilike {
"ILIKE"
} else {
"="
}
}
};
let param_sql = if is_ilike && (op == "$eq" || op == "$ne") {
@ -643,7 +667,9 @@ impl<'a> Compiler<'a> {
let mut child_relation_alias = base_alias.to_string();
if let Some(parent_type) = node.parent_type {
let merged_props = self.get_merged_properties(node.schema.as_ref());
let merged_props = self
.db
.merged_properties(node.schema.as_ref(), &mut std::collections::HashSet::new());
let relative_keys: Vec<String> = merged_props.keys().cloned().collect();
let (relation, is_parent_source) = self
@ -695,25 +721,4 @@ impl<'a> Compiler<'a> {
}
Ok(())
}
fn get_merged_properties(
&self,
schema: &crate::database::schema::Schema,
) -> std::collections::BTreeMap<String, Arc<crate::database::schema::Schema>> {
let mut props = std::collections::BTreeMap::new();
if let Some(ref_id) = &schema.obj.r#ref {
if let Some(parent_schema) = self.db.schemas.get(ref_id) {
props.extend(self.get_merged_properties(parent_schema));
}
}
if let Some(local_props) = &schema.obj.properties {
for (k, v) in local_props {
props.insert(k.clone(), v.clone());
}
}
props
}
}

View File

@ -8536,3 +8536,9 @@ fn test_merger_0_7() {
let path = format!("{}/fixtures/merger.json", env!("CARGO_MANIFEST_DIR"));
crate::tests::runner::run_test_case(&path, 0, 7).unwrap();
}
#[test]
fn test_merger_0_8() {
let path = format!("{}/fixtures/merger.json", env!("CARGO_MANIFEST_DIR"));
crate::tests::runner::run_test_case(&path, 0, 8).unwrap();
}

View File

@ -62,7 +62,8 @@ fn test_library_api() {
"properties": {
"name": { "type": "string" }
},
"required": ["name"]
"required": ["name"],
"compiledProperties": ["name"]
}
}
})

View File

@ -13,28 +13,40 @@ impl<'a> ValidationContext<'a> {
) -> Result<bool, ValidationError> {
let current = self.instance;
if let Some(obj) = current.as_object() {
// Entity Bound Implicit Type Validation
if let Some(lookup_key) = self.schema.id.as_ref().or(self.schema.r#ref.as_ref()) {
let base_type_name = lookup_key.split('.').next_back().unwrap_or("").to_string();
if let Some(type_def) = self.db.types.get(&base_type_name)
&& let Some(type_val) = obj.get("type")
// Entity implicit type validation
// Use the specific schema id or ref as a fallback
if let Some(identifier) = self.schema.id.as_ref().or(self.schema.r#ref.as_ref()) {
// Kick in if the data object has a type field
if let Some(type_val) = obj.get("type")
&& let Some(type_str) = type_val.as_str()
{
if type_def.variations.contains(type_str) {
// Ensure it passes strict mode
result.evaluated_keys.insert("type".to_string());
// Get the string or the final segment as the base
let base = identifier.split('.').next_back().unwrap_or("").to_string();
// Check if the base is a global type name
if let Some(type_def) = self.db.types.get(&base) {
// Ensure the instance type is a variation of the global type
if type_def.variations.contains(type_str) {
// Ensure it passes strict mode
result.evaluated_keys.insert("type".to_string());
} else {
result.errors.push(ValidationError {
code: "CONST_VIOLATED".to_string(), // Aligning with original const override errors
message: format!(
"Type '{}' is not a valid descendant for this entity bound schema",
type_str
),
path: format!("{}/type", self.path),
});
}
} else {
result.errors.push(ValidationError {
code: "CONST_VIOLATED".to_string(), // Aligning with original const override errors
message: format!(
"Type '{}' is not a valid descendant for this entity bound schema",
type_str
),
path: format!("{}/type", self.path),
});
// Ad-Hoc schemas natively use strict schema discriminator strings instead of variation inheritance
if type_str == identifier {
result.evaluated_keys.insert("type".to_string());
}
}
}
}
if let Some(min) = self.schema.min_properties
&& (obj.len() as f64) < min
{
@ -44,6 +56,7 @@ impl<'a> ValidationContext<'a> {
path: self.path.to_string(),
});
}
if let Some(max) = self.schema.max_properties
&& (obj.len() as f64) > max
{
@ -53,6 +66,7 @@ impl<'a> ValidationContext<'a> {
path: self.path.to_string(),
});
}
if let Some(ref req) = self.schema.required {
for field in req {
if !obj.contains_key(field) {
@ -114,10 +128,14 @@ impl<'a> ValidationContext<'a> {
// Entity Bound Implicit Type Interception
if key == "type"
&& let Some(lookup_key) = sub_schema.id.as_ref().or(sub_schema.r#ref.as_ref())
&& let Some(schema_bound) = sub_schema.id.as_ref().or(sub_schema.r#ref.as_ref())
{
let base_type_name = lookup_key.split('.').next_back().unwrap_or("").to_string();
if let Some(type_def) = self.db.types.get(&base_type_name)
let physical_type_name = schema_bound
.split('.')
.next_back()
.unwrap_or("")
.to_string();
if let Some(type_def) = self.db.types.get(&physical_type_name)
&& let Some(instance_type) = child_instance.as_str()
&& type_def.variations.contains(instance_type)
{

View File

@ -1 +1 @@
1.0.76
1.0.82