From 64a0d7d1dddd25e3ec92b4731ebfa23da1048688 Mon Sep 17 00:00:00 2001 From: Thy Tran <58045538+ThyTran1402@users.noreply.github.com> Date: Fri, 27 Mar 2026 15:02:40 -0400 Subject: [PATCH 1/3] added isnotnull condition support Signed-off-by: Thy Tran <58045538+ThyTran1402@users.noreply.github.com> --- docs/user/ppl/functions/condition.md | 20 +++++++++++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 ++ ppl/src/main/antlr/OpenSearchPPLParser.g4 | 7 +++++++ .../sql/ppl/parser/AstExpressionBuilder.java | 10 ++++++++++ .../ppl/parser/AstExpressionBuilderTest.java | 18 +++++++++++++++++ 5 files changed, 57 insertions(+) diff --git a/docs/user/ppl/functions/condition.md b/docs/user/ppl/functions/condition.md index 759fa09a1e5..c717cc85338 100644 --- a/docs/user/ppl/functions/condition.md +++ b/docs/user/ppl/functions/condition.md @@ -7,6 +7,8 @@ PPL conditional functions enable global filtering of query results based on spec Returns `TRUE` if the field is `NULL`, `FALSE` otherwise. +The `field IS NULL` predicate syntax is also supported as a synonym. + The `isnull()` function is commonly used: - In `eval` expressions to create conditional fields. - With the `if()` function to provide default values. @@ -69,6 +71,14 @@ source=accounts | where isnull(employer) | fields account_number, firstname, employer ``` + +The `IS NULL` predicate syntax can be used as an equivalent alternative: + +```ppl +source=accounts +| where employer IS NULL +| fields account_number, firstname, employer +``` The query returns the following results: @@ -87,6 +97,8 @@ fetched rows / total rows = 1/1 Returns `TRUE` if the field is NOT `NULL`, `FALSE` otherwise. +The `field IS NOT NULL` predicate syntax is also supported as a synonym. + The `isnotnull()` function is commonly used: - In `eval` expressions to create Boolean flags. - In `where` clauses to filter out null values. @@ -130,6 +142,14 @@ source=accounts | where not isnotnull(employer) | fields account_number, employer ``` + +The `IS NOT NULL` predicate syntax can be used as an equivalent alternative: + +```ppl +source=accounts +| where employer IS NOT NULL +| fields account_number, employer +``` The query returns the following results: diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 36b52cf8a1b..08f85508289 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -187,7 +187,9 @@ PATH: 'PATH'; CASE: 'CASE'; ELSE: 'ELSE'; IN: 'IN'; +IS: 'IS'; EXISTS: 'EXISTS'; +NULL: 'NULL'; // Geo IP eval function GEOIP: 'GEOIP'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 7e3862a8683..2ac1a2e4462 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -911,6 +911,11 @@ expression | left = expression comparisonOperator right = expression # compareExpr | expression NOT? IN LT_PRTHS valueList RT_PRTHS # inExpr | expression NOT? BETWEEN expression AND expression # between + | expression IS nullNotnull # isNullPredicate + ; + +nullNotnull + : NOT? NULL ; @@ -1594,6 +1599,8 @@ wildcard keywordsCanBeId : searchableKeyWord | IN + | IS + | NULL ; searchableKeyWord diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index c58eca20575..a15c7d8fa31 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -249,6 +249,16 @@ public UnresolvedExpression visitInExpr(InExprContext ctx) { return ctx.NOT() != null ? new Not(expr) : expr; } + @Override + public UnresolvedExpression visitIsNullPredicate( + OpenSearchPPLParser.IsNullPredicateContext ctx) { + return new Function( + ctx.nullNotnull().NOT() == null + ? IS_NULL.getName().getFunctionName() + : IS_NOT_NULL.getName().getFunctionName(), + Arrays.asList(visit(ctx.expression()))); + } + /** Value Expression. */ @Override public UnresolvedExpression visitBinaryArithmetic(BinaryArithmeticContext ctx) { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index af10b53defb..ce7a120ff56 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -299,6 +299,24 @@ public void testBooleanIsNotNullFunction() { filter(relation("t"), function("is not null", field("a")))); } + @Test + public void testIsNullPredicate() { + assertEqual( + "source=t | where a is null", filter(relation("t"), function("is null", field("a")))); + assertEqual( + "source=t | where a IS NULL", filter(relation("t"), function("is null", field("a")))); + } + + @Test + public void testIsNotNullPredicate() { + assertEqual( + "source=t | where a is not null", + filter(relation("t"), function("is not null", field("a")))); + assertEqual( + "source=t | where a IS NOT NULL", + filter(relation("t"), function("is not null", field("a")))); + } + /** Todo. search operator should not include functionCall, need to change antlr. */ @Ignore("search operator should not include functionCall, need to change antlr") public void testEvalExpr() { From 11227dc8371e0a45f1b0f419e2664a345cff9543 Mon Sep 17 00:00:00 2001 From: Thy Tran <58045538+ThyTran1402@users.noreply.github.com> Date: Fri, 27 Mar 2026 19:13:58 -0400 Subject: [PATCH 2/3] added tests for ingesttest Signed-off-by: Thy Tran <58045538+ThyTran1402@users.noreply.github.com> --- .../opensearch/sql/ppl/WhereCommandIT.java | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java index a386987e532..ad4c3475818 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/WhereCommandIT.java @@ -193,6 +193,27 @@ public void testIsNotNullFunction() throws IOException { verifyDataRows(result, rows("Amber JOHnny")); } + @Test + public void testIsNullPredicate() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where age IS NULL | fields firstname", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifyDataRows(result, rows("Virginia")); + } + + @Test + public void testIsNotNullPredicate() throws IOException { + JSONObject result = + executeQuery( + String.format( + "source=%s | where age IS NOT NULL and like(firstname, 'Ambe_%%') | fields" + + " firstname", + TEST_INDEX_BANK_WITH_NULL_VALUES)); + verifyDataRows(result, rows("Amber JOHnny")); + } + @Test public void testWhereWithMetadataFields() throws IOException { JSONObject result = From 98b70caddce8c7f46194c3aaa55975f578d09ab0 Mon Sep 17 00:00:00 2001 From: Thy Tran <58045538+ThyTran1402@users.noreply.github.com> Date: Fri, 3 Apr 2026 14:05:28 -0400 Subject: [PATCH 3/3] fixed IS NOT NULL doctest output in condition.md Signed-off-by: Thy Tran <58045538+ThyTran1402@users.noreply.github.com> --- docs/user/ppl/functions/condition.md | 19 ++++++++++++++++--- .../sql/ppl/parser/AstExpressionBuilder.java | 3 +-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/docs/user/ppl/functions/condition.md b/docs/user/ppl/functions/condition.md index c717cc85338..95f805ade94 100644 --- a/docs/user/ppl/functions/condition.md +++ b/docs/user/ppl/functions/condition.md @@ -142,8 +142,19 @@ source=accounts | where not isnotnull(employer) | fields account_number, employer ``` + +The query returns the following results: + +```text +fetched rows / total rows = 1/1 ++----------------+----------+ +| account_number | employer | +|----------------+----------| +| 18 | null | ++----------------+----------+ +``` -The `IS NOT NULL` predicate syntax can be used as an equivalent alternative: +The `IS NOT NULL` predicate syntax is equivalent to `isnotnull()`: ```ppl source=accounts @@ -154,11 +165,13 @@ source=accounts The query returns the following results: ```text -fetched rows / total rows = 1/1 +fetched rows / total rows = 3/3 +----------------+----------+ | account_number | employer | |----------------+----------| -| 18 | null | +| 1 | Pyrami | +| 6 | Netagy | +| 13 | Quility | +----------------+----------+ ``` diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index a15c7d8fa31..e2d7570aeaf 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -250,8 +250,7 @@ public UnresolvedExpression visitInExpr(InExprContext ctx) { } @Override - public UnresolvedExpression visitIsNullPredicate( - OpenSearchPPLParser.IsNullPredicateContext ctx) { + public UnresolvedExpression visitIsNullPredicate(OpenSearchPPLParser.IsNullPredicateContext ctx) { return new Function( ctx.nullNotnull().NOT() == null ? IS_NULL.getName().getFunctionName()