[core] Negate predicates like Java so NULL rows stay excluded

crates/integrations/datafusion/src/filter_pushdown.rs

@@ -1350,7 +1350,7 @@ mod tests {
                 .pushed_predicate
                 .expect("supported conjunct should still translate")
                 .to_string(),
-            "(dt = '2024-01-01' AND NOT (hr = 10))"
+            "(dt = '2024-01-01' AND hr != 10)"
         );
         assert!(analysis.requires_residual);
     }
@@ -1368,7 +1368,7 @@ mod tests {
                 .pushed_predicate
                 .expect("NOT partition predicate should translate inexactly")
                 .to_string(),
-            "NOT (dt = '2024-01-01')"
+            "dt != '2024-01-01'"
         );
         assert!(analysis.requires_residual);
     }
@@ -1530,7 +1530,7 @@ mod tests {
             build_pushed_predicate(&[filter], &fields)
                 .expect("NOT should translate as an inexact pushed predicate")
                 .to_string(),
-            "NOT (dt = '2024-01-01')"
+            "dt != '2024-01-01'"
         );
     }
 

crates/integrations/datafusion/src/physical_plan/scan.rs

@@ -452,11 +452,10 @@ fn predicate_covers_candidate(predicate: &Predicate, candidate: &Predicate) -> b
                 op: PredicateOperator::NotBetween,
                 literals,
             },
-            Predicate::Not(inner),
+            // `Predicate::negate` turns NOT(x >= lo AND x <= hi) into
+            // x < lo OR x > hi.
+            Predicate::Or(children),
         ) if literals.len() == 2 => {
-            let Predicate::And(children) = inner.as_ref() else {
-                return false;
-            };
             let matches_bound = |candidate: &Predicate,
                                  expected_op: PredicateOperator,
                                  expected_literal: &Datum| {
@@ -476,12 +475,13 @@ fn predicate_covers_candidate(predicate: &Predicate, candidate: &Predicate) -> b
                         && candidate_literals[0] == *expected_literal
                 )
             };
-            children
-                .iter()
-                .any(|child| matches_bound(child, PredicateOperator::GtEq, &literals[0]))
+            children.len() == 2
+                && children
+                    .iter()
+                    .any(|child| matches_bound(child, PredicateOperator::Lt, &literals[0]))
                 && children
                     .iter()
-                    .any(|child| matches_bound(child, PredicateOperator::LtEq, &literals[1]))
+                    .any(|child| matches_bound(child, PredicateOperator::Gt, &literals[1]))
         }
         _ => false,
     }

crates/paimon/src/arrow/residual.rs

@@ -166,9 +166,16 @@ pub(crate) fn evaluate_predicate_mask(
             else {
                 return Ok(None);
             };
-            Ok(Some(boolean_mask_from_predicate(mask.len(), |row_index| {
-                !mask.value(row_index)
-            })))
+            let negated =
+                boolean_mask_from_predicate(mask.len(), |row_index| !mask.value(row_index));
+            if let Some(guard) = inner.not_null_guard() {
+                let Some(guard) = evaluate_predicate_mask(batch, &guard, file_fields, scan_fields)?
+                else {
+                    return Ok(None);
+                };
+                return Ok(Some(combine_filter_masks(&negated, &guard, false)));
+            }
+            Ok(Some(negated))
         }
         Predicate::Leaf {
             column,
@@ -2125,6 +2132,22 @@ mod tests {
         assert_eq!(int_values(&out), vec![10, 20]);
     }
 
+    #[test]
+    fn test_not_excludes_null_row() {
+        // NOT(age > 25) on [10, null, 30] -> [10]; NOT(unknown) is unknown.
+        let f = int_field(0, "age");
+        let b = int_batch("age", vec![Some(10), None, Some(30)]);
+        let pred = Predicate::Not(Box::new(leaf(
+            0,
+            DataType::Int(IntType::new()),
+            PredicateOperator::Gt,
+            vec![Datum::Int(25)],
+        )));
+        let fp = file_predicates(vec![pred], vec![f.clone()]);
+        let out = filter_record_batch_by_predicates(b, &fp, &[f]).unwrap();
+        assert_eq!(int_values(&out), vec![10]);
+    }
+
     #[test]
     fn test_null_row_excluded() {
         // age = [10, null, 30]; age > 5 -> [10, 30] (null → false).

crates/paimon/src/predicate_stats.rs

@@ -1054,13 +1054,14 @@ mod tests {
         let dt = DataType::Int(IntType::new());
         let fields = vec![DataField::new(0, "id".to_string(), dt.clone())];
         let mapping = vec![Some(0)];
-        let predicate = Predicate::negate(Predicate::Leaf {
+        // Built directly: `Predicate::negate` would flip the leaf to `<=`.
+        let predicate = Predicate::Not(Box::new(Predicate::Leaf {
             column: "id".to_string(),
             index: 0,
             data_type: dt,
             op: PredicateOperator::Gt,
             literals: vec![Datum::Int(5)],
-        });
+        }));
 
         let with_nulls = MockStats {
             row_count: 5,