validation progress

.agent/workflows/jspg.md

@@ -4,9 +4,11 @@ description: jspg work preparation
 
 This workflow will get you up-to-speed on the JSPG custom json-schema-based cargo pgrx postgres validation extension. Everything you read will be in the jspg directory/project.
 
-Read over this entire workflow and commit to every section of work in a task list, so that you don't stop half way through before reviewing all of the directories and files mentioned. Do not ask for confirmation after generating this task list and proceed through all sections in your list.
+Read over this entire workflow and commit to every section of work in a fresh task list (DO THIS FIRST), so that you don't stop half way through before reviewing all of the directories and files mentioned. Do not ask for confirmation after generating this task list and proceed through all sections in your list.
 
-Please analyze the files and directories and do not use cat, find, or the terminal to discover or read in any of these files. Analyze every file mentioned. If a directory is mentioned or a /*, please analyze the directory, every single file at its root, and recursively analyze every subdirectory and every single file in every subdirectory to capture not just critical files, but the entirety of what is requested. I state again, DO NOT just review a cherry picking of files in any folder or wildcard specified. Review 100% of all files discovered recursively!
+Please analyze the files and directories and do not use cat, find, or the terminal AT ALL to discover or read in any of these files! USE YOUR TOOLS ONLY. Analyze every file mentioned. If a directory is mentioned or a /*, please analyze the directory, every single file at its root, and recursively analyze every subdirectory and every single file in every subdirectory to capture not just critical files, but the entirety of what is requested. I state again, DO NOT just review a cherry picking of files in any folder or wildcard specified. Review 100% of all files discovered recursively!
+
+Do not make any code changes. Just focus on your task list and reading files!
 
 Section 1: Various Documentation
 
@@ -48,7 +50,6 @@ Now, review some punc type and enum source in the api project with api/ these fi
 - api/punc/sql/tables.sql
 - api/punc/sql/domains.sql
 - api/punc/sql/indexes.sql
-- api/punc/sql/functions/entity.sql
 - api/punc/sql/functions/puncs.sql
 - api/punc/sql/puncs/entity.sql
 - api/punc/sql/puncs/persons.sql

fixtures/invoice.json

@@ -0,0 +1,220 @@
+[
+  {
+    "description": "Invoice Attachment Reproducer",
+    "database": {
+      "puncs": [
+        {
+          "name": "get_invoice",
+          "schemas": [
+            {
+              "$id": "get_invoice.response",
+              "oneOf": [
+                { "$ref": "invoice" },
+                { "type": "null" }
+              ]
+            }
+          ]
+        }
+      ],
+      "enums": [],
+      "relations": [
+        {
+          "id": "10000000-0000-0000-0000-000000000001",
+          "type": "relation",
+          "constraint": "fk_attachment_attachable_entity",
+          "source_type": "attachment",
+          "source_columns": ["attachable_id", "attachable_type"],
+          "destination_type": "entity",
+          "destination_columns": ["id", "type"],
+          "prefix": null
+        }
+      ],
+      "types": [
+        {
+          "name": "entity",
+          "schemas": [
+            {
+              "$id": "entity",
+              "type": "object",
+              "properties": {
+                "id": { "type": "string", "format": "uuid" },
+                "type": { "type": "string" },
+                "archived": { "type": "boolean" },
+                "created_at": { "type": "string", "format": "date-time" }
+              }
+            }
+          ],
+          "hierarchy": ["entity"],
+          "variations": ["entity", "activity", "invoice", "attachment"],
+          "fields": ["id", "type", "archived", "created_at"],
+          "grouped_fields": {
+            "entity": ["id", "type", "archived", "created_at"]
+          },
+          "field_types": {
+            "id": "uuid",
+            "type": "text",
+            "archived": "boolean",
+            "created_at": "timestamptz"
+          },
+          "lookup_fields": [],
+          "historical": false,
+          "notify": false,
+          "relationship": false
+        },
+        {
+          "name": "activity",
+          "schemas": [
+            {
+              "$id": "activity",
+              "$ref": "entity",
+              "properties": {
+                "start_date": { "type": "string", "format": "date-time" }
+              }
+            }
+          ],
+          "hierarchy": ["activity", "entity"],
+          "variations": ["activity", "invoice"],
+          "fields": ["id", "type", "archived", "created_at", "start_date"],
+          "grouped_fields": {
+            "entity": ["id", "type", "archived", "created_at"],
+            "activity": ["start_date"]
+          },
+          "field_types": {
+            "id": "uuid",
+            "type": "text",
+            "archived": "boolean",
+            "created_at": "timestamptz",
+            "start_date": "timestamptz"
+          },
+          "lookup_fields": [],
+          "historical": false,
+          "notify": false,
+          "relationship": false
+        },
+        {
+          "name": "invoice",
+          "schemas": [
+            {
+              "$id": "invoice",
+              "$ref": "activity",
+              "properties": {
+                "status": { "type": "string" },
+                "attachments": {
+                  "type": "array",
+                  "items": { "$ref": "attachment" }
+                }
+              }
+            }
+          ],
+          "hierarchy": ["invoice", "activity", "entity"],
+          "variations": ["invoice"],
+          "fields": ["id", "type", "archived", "created_at", "start_date", "status"],
+          "grouped_fields": {
+            "entity": ["id", "type", "archived", "created_at"],
+            "activity": ["start_date"],
+            "invoice": ["status"]
+          },
+          "field_types": {
+            "id": "uuid",
+            "type": "text",
+            "archived": "boolean",
+            "created_at": "timestamptz",
+            "start_date": "timestamptz",
+            "status": "text"
+          },
+          "lookup_fields": [],
+          "historical": false,
+          "notify": false,
+          "relationship": false
+        },
+        {
+          "name": "attachment",
+          "schemas": [
+            {
+              "$id": "attachment",
+              "$ref": "entity",
+              "properties": {
+                "name": { "type": "string" },
+                "attachable_id": { "type": "string", "format": "uuid" },
+                "attachable_type": { "type": "string" },
+                "kind": { "type": "string" },
+                "file": { "type": "string" },
+                "data": { "type": "object" }
+              }
+            }
+          ],
+          "hierarchy": ["attachment", "entity"],
+          "variations": ["attachment"],
+          "fields": ["id", "type", "archived", "created_at", "attachable_id", "attachable_type", "kind", "file", "data", "name"],
+          "grouped_fields": {
+            "entity": ["id", "type", "archived", "created_at"],
+            "attachment": ["attachable_id", "attachable_type", "kind", "file", "data", "name"]
+          },
+          "field_types": {
+            "id": "uuid",
+            "type": "text",
+            "archived": "boolean",
+            "created_at": "timestamptz",
+            "attachable_id": "uuid",
+            "attachable_type": "text",
+            "kind": "text",
+            "file": "text",
+            "data": "jsonb",
+            "name": "text"
+          },
+          "lookup_fields": [],
+          "historical": false,
+          "notify": false,
+          "relationship": false
+        }
+      ]
+    },
+    "tests": [
+      {
+        "description": "Invoice with an empty array of attachments",
+        "schema_id": "get_invoice.response",
+        "data": {
+          "id": "11111111-1111-1111-1111-111111111111",
+          "type": "invoice",
+          "archived": false,
+          "created_at": "2023-01-01T00:00:00Z",
+          "status": "draft",
+          "attachments": []
+        },
+        "action": "validate",
+        "expect": {
+          "success": true
+        }
+      },
+      {
+        "description": "Invoice with a valid attachment with null data",
+        "schema_id": "get_invoice.response",
+        "data": {
+          "id": "11111111-1111-1111-1111-111111111111",
+          "type": "invoice",
+          "archived": false,
+          "created_at": "2023-01-01T00:00:00Z",
+          "start_date": null,
+          "status": "draft",
+          "attachments": [
+            {
+              "id": "22222222-2222-2222-2222-222222222222",
+              "type": "attachment",
+              "archived": false,
+              "created_at": "2023-01-01T00:00:00Z",
+              "name": "receipt",
+              "attachable_id": "11111111-1111-1111-1111-111111111111",
+              "attachable_type": "invoice",
+              "kind": "document",
+              "file": "path/to/doc.pdf"
+            }
+          ]
+        },
+        "action": "validate",
+        "expect": {
+          "success": true
+        }
+      }
+    ]
+  }
+]

src/queryer/compiler.rs

@@ -185,9 +185,9 @@ impl<'a> Compiler<'a> {
     select_args.append(&mut poly_args);
 
     let jsonb_obj_sql = if select_args.is_empty() {
-      "jsonb_build_object()".to_string()
+      "jsonb_strip_nulls(jsonb_build_object())".to_string()
     } else {
-      format!("jsonb_build_object({})", select_args.join(", "))
+      format!("jsonb_strip_nulls(jsonb_build_object({}))", select_args.join(", "))
     };
 
     // 3. Build WHERE clauses
@@ -325,7 +325,7 @@ impl<'a> Compiler<'a> {
       }
       build_args.push(format!("'{}', {}", k, child_sql));
     }
-    let combined = format!("jsonb_build_object({})", build_args.join(", "));
+    let combined = format!("jsonb_strip_nulls(jsonb_build_object({}))", build_args.join(", "));
     Ok((combined, "object".to_string()))
   }
 

src/tests/fixtures.rs

@@ -3623,6 +3623,18 @@ fn test_pattern_1_0() {
     crate::tests::runner::run_test_case(&path, 1, 0).unwrap();
 }
 
+#[test]
+fn test_invoice_0_0() {
+    let path = format!("{}/fixtures/invoice.json", env!("CARGO_MANIFEST_DIR"));
+    crate::tests::runner::run_test_case(&path, 0, 0).unwrap();
+}
+
+#[test]
+fn test_invoice_0_1() {
+    let path = format!("{}/fixtures/invoice.json", env!("CARGO_MANIFEST_DIR"));
+    crate::tests::runner::run_test_case(&path, 0, 1).unwrap();
+}
+
 #[test]
 fn test_max_properties_0_0() {
     let path = format!("{}/fixtures/maxProperties.json", env!("CARGO_MANIFEST_DIR"));

src/validator/rules/combinators.rs

@@ -17,6 +17,7 @@ impl<'a> ValidationContext<'a> {
 
     if let Some(ref one_of) = self.schema.one_of {
       let mut passed_candidates: Vec<(Option<String>, usize, ValidationResult)> = Vec::new();
+      let mut failed_errors: Vec<ValidationError> = Vec::new();
 
       for sub in one_of {
         let derived = self.derive_for_schema(sub, true);
@@ -28,6 +29,8 @@ impl<'a> ValidationContext<'a> {
             .and_then(|id| self.db.depths.get(id).copied())
             .unwrap_or(0);
           passed_candidates.push((child_id, depth, sub_res));
+        } else {
+          failed_errors.extend(sub_res.errors);
         }
       }
 
@@ -39,6 +42,7 @@ impl<'a> ValidationContext<'a> {
           message: "Matches none of oneOf schemas".to_string(),
           path: self.path.to_string(),
         });
+        result.errors.extend(failed_errors);
       } else {
         // Apply depth heuristic tie-breaker
         let mut best_depth: Option<usize> = None;