SearchDslParser.java
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.nereids.search.SearchLexer;
import org.apache.doris.nereids.search.SearchParser;
import org.apache.doris.nereids.search.SearchParserBaseVisitor;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.antlr.v4.runtime.ANTLRInputStream;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.tree.ParseTree;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
/**
* Search DSL Parser using ANTLR-generated parser.
* Parses DSL syntax and extracts field bindings for FE->BE communication.
* <p>
* Supported syntax:
* - field:term
* - field:"quoted term"
* - field:prefix*
* - field:*wildcard*
* - field:/regexp/
* - AND/OR/NOT operators
* - Parentheses for grouping
* - Range queries: field:[1 TO 10], field:{1 TO 10}
* - List queries: field:IN(value1 value2)
* - Any/All queries: field:ANY(value), field:ALL(value)
*/
public class SearchDslParser {
private static final Logger LOG = LogManager.getLogger(SearchDslParser.class);
private static final ObjectMapper JSON_MAPPER = new ObjectMapper();
/**
* Parse DSL string and return intermediate representation
*/
public static QsPlan parseDsl(String dsl) {
return parseDsl(dsl, null, null, null);
}
/**
* Parse DSL string with default field and operator support
*
* @param dsl DSL query string
* @param defaultField Default field name when DSL doesn't specify field (optional)
* @param defaultOperator Default operator ("and" or "or") for multi-term queries (optional, defaults to "or")
* @return Parsed QsPlan
*/
public static QsPlan parseDsl(String dsl, String defaultField, String defaultOperator) {
return parseDsl(dsl, defaultField, defaultOperator, null);
}
/**
* Parse DSL string with full options support
*
* @param dsl DSL query string
* @param defaultField Default field name when DSL doesn't specify field (optional)
* @param defaultOperator Default operator ("and" or "or") for multi-term queries (optional, defaults to "or")
* @param options JSON options string (optional), e.g., '{"mode":"lucene","minimum_should_match":0}'
* @return Parsed QsPlan
*/
public static QsPlan parseDsl(String dsl, String defaultField, String defaultOperator, String options) {
// Parse options
SearchOptions searchOptions = parseOptions(options);
// Use Lucene mode parser if specified
if (searchOptions.isLuceneMode()) {
return parseDslLuceneMode(dsl, defaultField, defaultOperator, searchOptions);
}
// Standard mode parsing
return parseDslStandardMode(dsl, defaultField, defaultOperator);
}
/**
* Standard mode parsing (original behavior)
*/
private static QsPlan parseDslStandardMode(String dsl, String defaultField, String defaultOperator) {
if (dsl == null || dsl.trim().isEmpty()) {
return new QsPlan(new QsNode(QsClauseType.TERM, "error", "empty_dsl"), new ArrayList<>());
}
// Expand simplified DSL if default field is provided
String expandedDsl = dsl;
if (defaultField != null && !defaultField.trim().isEmpty()) {
expandedDsl = expandSimplifiedDsl(dsl.trim(), defaultField.trim(),
normalizeDefaultOperator(defaultOperator));
}
try {
// Create ANTLR lexer and parser
SearchLexer lexer = new SearchLexer(new ANTLRInputStream(expandedDsl));
CommonTokenStream tokens = new CommonTokenStream(lexer);
SearchParser parser = new SearchParser(tokens);
// Add error listener to detect parsing errors
parser.removeErrorListeners();
parser.addErrorListener(new org.antlr.v4.runtime.BaseErrorListener() {
@Override
public void syntaxError(org.antlr.v4.runtime.Recognizer<?, ?> recognizer,
Object offendingSymbol,
int line, int charPositionInLine,
String msg, org.antlr.v4.runtime.RecognitionException e) {
throw new RuntimeException("Invalid search DSL syntax at line " + line
+ ":" + charPositionInLine + " " + msg);
}
});
// Parse the search query
ParseTree tree = parser.search();
// Check if parsing was successful
if (tree == null) {
throw new RuntimeException("Invalid search DSL syntax");
}
// Build AST using visitor pattern
QsAstBuilder visitor = new QsAstBuilder();
QsNode root = visitor.visit(tree);
// Extract field bindings
Set<String> fieldNames = visitor.getFieldNames();
List<QsFieldBinding> bindings = new ArrayList<>();
int slotIndex = 0;
for (String fieldName : fieldNames) {
bindings.add(new QsFieldBinding(fieldName, slotIndex++));
}
return new QsPlan(root, bindings);
} catch (Exception e) {
LOG.error("Failed to parse search DSL: '{}' (expanded: '{}')", dsl, expandedDsl, e);
throw new RuntimeException("Invalid search DSL syntax: " + dsl + ". Error: " + e.getMessage(), e);
}
}
/**
* Normalize default operator to lowercase "and" or "or"
*/
private static String normalizeDefaultOperator(String operator) {
if (operator == null || operator.trim().isEmpty()) {
return "or"; // Default to OR
}
String normalized = operator.trim().toLowerCase();
if ("and".equals(normalized) || "or".equals(normalized)) {
return normalized;
}
throw new IllegalArgumentException("Invalid default operator: " + operator
+ ". Must be 'and' or 'or'");
}
/**
* Expand simplified DSL to full DSL format
* <p>
* Examples:
* - "foo bar" + field="tags" + operator="and" → "tags:ALL(foo bar)"
* - "foo* bar*" + field="tags" + operator="and" → "tags:foo* AND tags:bar*"
* - "foo OR bar" + field="tags" → "tags:foo OR tags:bar"
* - "EXACT(foo bar)" + field="tags" → "tags:EXACT(foo bar)"
*
* @param dsl Simple DSL string
* @param defaultField Default field name
* @param defaultOperator "and" or "or"
* @return Expanded full DSL
*/
private static String expandSimplifiedDsl(String dsl, String defaultField, String defaultOperator) {
// 1. If DSL already contains field names (colon), return as-is
if (containsFieldReference(dsl)) {
return dsl;
}
// 2. Check if DSL starts with a function keyword (EXACT, ANY, ALL, IN)
if (startsWithFunction(dsl)) {
return defaultField + ":" + dsl;
}
// 3. Check for explicit boolean operators in DSL
if (containsExplicitOperators(dsl)) {
return addFieldPrefixToOperatorExpression(dsl, defaultField);
}
// 4. Tokenize and analyze terms
List<String> terms = tokenizeDsl(dsl);
if (terms.isEmpty()) {
return defaultField + ":" + dsl;
}
// 5. Single term - simple case
if (terms.size() == 1) {
return defaultField + ":" + terms.get(0);
}
// 6. Multiple terms - check for wildcards
boolean hasWildcard = terms.stream().anyMatch(SearchDslParser::containsWildcard);
if (hasWildcard) {
// Wildcards cannot be tokenized - must create separate field queries
String operator = "and".equals(defaultOperator) ? " AND " : " OR ";
return terms.stream()
.map(term -> defaultField + ":" + term)
.collect(java.util.stream.Collectors.joining(operator));
} else {
// Regular multi-term query - use ANY/ALL
String clauseType = "and".equals(defaultOperator) ? "ALL" : "ANY";
return defaultField + ":" + clauseType + "(" + dsl + ")";
}
}
/**
* Check if DSL contains field references (has colon not in quoted strings or escaped)
*/
private static boolean containsFieldReference(String dsl) {
boolean inQuotes = false;
boolean inRegex = false;
for (int i = 0; i < dsl.length(); i++) {
char c = dsl.charAt(i);
// Handle escape sequences - skip the escaped character
if (c == '\\' && i + 1 < dsl.length()) {
i++; // Skip next character (it's escaped)
continue;
}
if (c == '"') {
inQuotes = !inQuotes;
} else if (c == '/' && !inQuotes) {
inRegex = !inRegex;
} else if (c == ':' && !inQuotes && !inRegex) {
return true;
}
}
return false;
}
/**
* Check if DSL starts with function keywords
*/
private static boolean startsWithFunction(String dsl) {
String upper = dsl.toUpperCase();
return upper.startsWith("EXACT(")
|| upper.startsWith("ANY(")
|| upper.startsWith("ALL(")
|| upper.startsWith("IN(");
}
/**
* Check if DSL contains explicit boolean operators (AND/OR/NOT)
*/
private static boolean containsExplicitOperators(String dsl) {
// Look for standalone AND/OR/NOT keywords (not part of field names)
String upper = dsl.toUpperCase();
return upper.matches(".*\\s+(AND|OR)\\s+.*")
|| upper.matches("^NOT\\s+.*")
|| upper.matches(".*\\s+NOT\\s+.*");
}
/**
* Add field prefix to expressions with explicit operators
* Example: "foo AND bar" → "field:foo AND field:bar"
* Handles escape sequences properly (e.g., "First\ Value" stays as single term)
*/
private static String addFieldPrefixToOperatorExpression(String dsl, String defaultField) {
StringBuilder result = new StringBuilder();
StringBuilder currentTerm = new StringBuilder();
int i = 0;
while (i < dsl.length()) {
// Skip whitespace (but not escaped whitespace)
while (i < dsl.length() && Character.isWhitespace(dsl.charAt(i))) {
i++;
}
if (i >= dsl.length()) {
break;
}
// Handle escape sequences - include both backslash and next char
if (dsl.charAt(i) == '\\' && i + 1 < dsl.length()) {
currentTerm.append(dsl.charAt(i));
currentTerm.append(dsl.charAt(i + 1));
i += 2;
continue;
}
// Try to match operators
String remaining = dsl.substring(i);
String upperRemaining = remaining.toUpperCase();
if (upperRemaining.startsWith("AND ") || upperRemaining.startsWith("AND\t")
|| (upperRemaining.equals("AND") && i + 3 >= dsl.length())) {
// Found AND operator
if (currentTerm.length() > 0) {
if (result.length() > 0) {
result.append(" ");
}
result.append(defaultField).append(":").append(currentTerm.toString().trim());
currentTerm.setLength(0);
}
if (result.length() > 0) {
result.append(" ");
}
result.append(dsl.substring(i, i + 3)); // Preserve original case
i += 3;
continue;
} else if (upperRemaining.startsWith("OR ") || upperRemaining.startsWith("OR\t")
|| (upperRemaining.equals("OR") && i + 2 >= dsl.length())) {
// Found OR operator
if (currentTerm.length() > 0) {
if (result.length() > 0) {
result.append(" ");
}
result.append(defaultField).append(":").append(currentTerm.toString().trim());
currentTerm.setLength(0);
}
if (result.length() > 0) {
result.append(" ");
}
result.append(dsl.substring(i, i + 2)); // Preserve original case
i += 2;
continue;
} else if (upperRemaining.startsWith("NOT ") || upperRemaining.startsWith("NOT\t")
|| (upperRemaining.equals("NOT") && i + 3 >= dsl.length())) {
// Found NOT operator
if (currentTerm.length() > 0) {
if (result.length() > 0) {
result.append(" ");
}
result.append(defaultField).append(":").append(currentTerm.toString().trim());
currentTerm.setLength(0);
}
if (result.length() > 0) {
result.append(" ");
}
result.append(dsl.substring(i, i + 3)); // Preserve original case
i += 3;
continue;
}
// Not an operator, accumulate term
currentTerm.append(dsl.charAt(i));
i++;
}
// Add last term
if (currentTerm.length() > 0) {
if (result.length() > 0) {
result.append(" ");
}
result.append(defaultField).append(":").append(currentTerm.toString().trim());
}
return result.toString().trim();
}
/**
* Tokenize DSL into terms (split by whitespace, respecting quotes, escapes, and functions)
*/
private static List<String> tokenizeDsl(String dsl) {
List<String> terms = new ArrayList<>();
StringBuilder currentTerm = new StringBuilder();
boolean inQuotes = false;
boolean inParens = false;
int parenDepth = 0;
for (int i = 0; i < dsl.length(); i++) {
char c = dsl.charAt(i);
if (c == '"' && (i == 0 || dsl.charAt(i - 1) != '\\')) {
inQuotes = !inQuotes;
currentTerm.append(c);
} else if (c == '(' && !inQuotes) {
parenDepth++;
inParens = true;
currentTerm.append(c);
} else if (c == ')' && !inQuotes) {
parenDepth--;
if (parenDepth == 0) {
inParens = false;
}
currentTerm.append(c);
} else if (c == '\\' && i + 1 < dsl.length()) {
// Escape sequence - include both backslash and next char in term
currentTerm.append(c);
currentTerm.append(dsl.charAt(i + 1));
i++; // Skip next character
} else if (Character.isWhitespace(c) && !inQuotes && !inParens) {
// End of term (only if not escaped - handled above)
if (currentTerm.length() > 0) {
terms.add(currentTerm.toString());
currentTerm = new StringBuilder();
}
} else {
currentTerm.append(c);
}
}
// Add last term
if (currentTerm.length() > 0) {
terms.add(currentTerm.toString());
}
return terms;
}
/**
* Check if a term contains wildcard characters (* or ?)
* Escaped wildcards (\* or \?) are not counted.
*/
private static boolean containsWildcard(String term) {
// Ignore wildcards in quoted strings or regex
if (term.startsWith("\"") && term.endsWith("\"")) {
return false;
}
if (term.startsWith("/") && term.endsWith("/")) {
return false;
}
// Check for unescaped wildcards
for (int i = 0; i < term.length(); i++) {
char c = term.charAt(i);
if (c == '\\' && i + 1 < term.length()) {
// Skip escaped character
i++;
continue;
}
if (c == '*' || c == '?') {
return true;
}
}
return false;
}
/**
* Clause types supported
*/
public enum QsClauseType {
TERM, // field:value
PHRASE, // field:"phrase search"
PREFIX, // field:prefix*
WILDCARD, // field:*wild*card*
REGEXP, // field:/pattern/
RANGE, // field:[1 TO 10] or field:{1 TO 10}
LIST, // field:IN(value1 value2)
ANY, // field:ANY(value) - any match
ALL, // field:ALL(value) - all match
EXACT, // field:EXACT(value) - exact match without tokenization
AND, // clause1 AND clause2 (standard boolean algebra)
OR, // clause1 OR clause2 (standard boolean algebra)
NOT, // NOT clause (standard boolean algebra)
OCCUR_BOOLEAN // Lucene-style boolean query with MUST/SHOULD/MUST_NOT
}
/**
* Occur type for Lucene-style boolean queries
*/
public enum QsOccur {
MUST, // Term must appear (equivalent to +term)
SHOULD, // Term should appear (optional)
MUST_NOT // Term must not appear (equivalent to -term)
}
/**
* ANTLR visitor to build QsNode AST from parse tree
*/
private static class QsAstBuilder extends SearchParserBaseVisitor<QsNode> {
private final Set<String> fieldNames = new HashSet<>();
// Context stack to track current field name during parsing
private String currentFieldName = null;
public Set<String> getFieldNames() {
return fieldNames;
}
@Override
public QsNode visitSearch(SearchParser.SearchContext ctx) {
QsNode result = visit(ctx.clause());
if (result == null) {
throw new RuntimeException("Invalid search clause");
}
return result;
}
@Override
public QsNode visitOrClause(SearchParser.OrClauseContext ctx) {
if (ctx.andClause().size() == 1) {
QsNode result = visit(ctx.andClause(0));
if (result == null) {
throw new RuntimeException("Invalid OR clause operand");
}
return result;
}
List<QsNode> children = new ArrayList<>();
for (SearchParser.AndClauseContext andCtx : ctx.andClause()) {
QsNode child = visit(andCtx);
if (child == null) {
throw new RuntimeException("Invalid OR clause operand");
}
children.add(child);
}
return new QsNode(QsClauseType.OR, children);
}
@Override
public QsNode visitAndClause(SearchParser.AndClauseContext ctx) {
if (ctx.notClause().size() == 1) {
QsNode result = visit(ctx.notClause(0));
if (result == null) {
throw new RuntimeException("Invalid AND clause operand");
}
return result;
}
List<QsNode> children = new ArrayList<>();
for (SearchParser.NotClauseContext notCtx : ctx.notClause()) {
QsNode child = visit(notCtx);
if (child == null) {
throw new RuntimeException("Invalid AND clause operand");
}
children.add(child);
}
return new QsNode(QsClauseType.AND, children);
}
@Override
public QsNode visitNotClause(SearchParser.NotClauseContext ctx) {
if (ctx.NOT() != null) {
QsNode child = visit(ctx.atomClause());
if (child == null) {
throw new RuntimeException("Invalid NOT clause: missing operand");
}
List<QsNode> children = new ArrayList<>();
children.add(child);
return new QsNode(QsClauseType.NOT, children);
}
QsNode result = visit(ctx.atomClause());
if (result == null) {
throw new RuntimeException("Invalid atom clause");
}
return result;
}
@Override
public QsNode visitAtomClause(SearchParser.AtomClauseContext ctx) {
if (ctx.clause() != null) {
// Parenthesized clause
QsNode result = visit(ctx.clause());
if (result == null) {
throw new RuntimeException("Invalid parenthesized clause");
}
return result;
}
if (ctx.fieldQuery() == null) {
throw new RuntimeException("Invalid atom clause: missing field query");
}
QsNode result = visit(ctx.fieldQuery());
if (result == null) {
throw new RuntimeException("Invalid field query");
}
return result;
}
@Override
public QsNode visitFieldQuery(SearchParser.FieldQueryContext ctx) {
if (ctx.fieldPath() == null) {
throw new RuntimeException("Invalid field query: missing field path");
}
// Build complete field path from segments (support field.subcolumn syntax)
StringBuilder fullPath = new StringBuilder();
List<SearchParser.FieldSegmentContext> segments = ctx.fieldPath().fieldSegment();
for (int i = 0; i < segments.size(); i++) {
if (i > 0) {
fullPath.append('.');
}
String segment = segments.get(i).getText();
// Remove quotes if present
if (segment.startsWith("\"") && segment.endsWith("\"")) {
segment = segment.substring(1, segment.length() - 1);
}
fullPath.append(segment);
}
String fieldPath = fullPath.toString();
fieldNames.add(fieldPath);
// Set current field context before visiting search value
String previousFieldName = currentFieldName;
currentFieldName = fieldPath;
try {
if (ctx.searchValue() == null) {
throw new RuntimeException("Invalid field query: missing search value");
}
QsNode result = visit(ctx.searchValue());
if (result == null) {
throw new RuntimeException("Invalid search value");
}
return result;
} finally {
// Restore previous context
currentFieldName = previousFieldName;
}
}
@Override
public QsNode visitSearchValue(SearchParser.SearchValueContext ctx) {
String fieldName = getCurrentFieldName();
// Handle each search value type independently for better readability
if (ctx.TERM() != null) {
return createTermNode(fieldName, ctx.TERM().getText());
}
if (ctx.PREFIX() != null) {
return createPrefixNode(fieldName, ctx.PREFIX().getText());
}
if (ctx.WILDCARD() != null) {
return createWildcardNode(fieldName, ctx.WILDCARD().getText());
}
if (ctx.REGEXP() != null) {
return createRegexpNode(fieldName, ctx.REGEXP().getText());
}
if (ctx.QUOTED() != null) {
return createPhraseNode(fieldName, ctx.QUOTED().getText());
}
if (ctx.rangeValue() != null) {
return createRangeNode(fieldName, ctx.rangeValue());
}
if (ctx.listValue() != null) {
return createListNode(fieldName, ctx.listValue());
}
if (ctx.anyAllValue() != null) {
return createAnyAllNode(fieldName, ctx.anyAllValue().getText());
}
if (ctx.exactValue() != null) {
return createExactNode(fieldName, ctx.exactValue().getText());
}
// Fallback for unknown types
return createTermNode(fieldName, ctx.getText());
}
private QsNode createTermNode(String fieldName, String value) {
return new QsNode(QsClauseType.TERM, fieldName, unescapeTermValue(value));
}
private QsNode createPrefixNode(String fieldName, String value) {
return new QsNode(QsClauseType.PREFIX, fieldName, unescapeTermValue(value));
}
private QsNode createWildcardNode(String fieldName, String value) {
return new QsNode(QsClauseType.WILDCARD, fieldName, unescapeTermValue(value));
}
private QsNode createRegexpNode(String fieldName, String regexpText) {
String regexp = regexpText;
// Remove surrounding slashes
if (regexp.startsWith("/") && regexp.endsWith("/")) {
regexp = regexp.substring(1, regexp.length() - 1);
}
return new QsNode(QsClauseType.REGEXP, fieldName, regexp);
}
private QsNode createPhraseNode(String fieldName, String quotedText) {
String quoted = quotedText;
// Remove surrounding quotes
if (quoted.startsWith("\"") && quoted.endsWith("\"")) {
quoted = quoted.substring(1, quoted.length() - 1);
}
return new QsNode(QsClauseType.PHRASE, fieldName, quoted);
}
private QsNode createRangeNode(String fieldName, SearchParser.RangeValueContext ctx) {
// Reconstruct the original range text with spaces preserved
String rangeText;
if (ctx.LBRACKET() != null) {
rangeText = "[" + ctx.rangeEndpoint(0).getText() + " TO " + ctx.rangeEndpoint(1).getText() + "]";
} else {
rangeText = "{" + ctx.rangeEndpoint(0).getText() + " TO " + ctx.rangeEndpoint(1).getText() + "}";
}
return new QsNode(QsClauseType.RANGE, fieldName, rangeText);
}
private QsNode createListNode(String fieldName, SearchParser.ListValueContext ctx) {
// Reconstruct the original list text with spaces preserved
StringBuilder listText = new StringBuilder("IN(");
for (int i = 0; i < ctx.LIST_TERM().size(); i++) {
if (i > 0) {
listText.append(" ");
}
listText.append(ctx.LIST_TERM(i).getText());
}
listText.append(")");
return new QsNode(QsClauseType.LIST, fieldName, listText.toString());
}
private QsNode createAnyAllNode(String fieldName, String anyAllText) {
// Extract content between parentheses
String innerContent = extractParenthesesContent(anyAllText);
String sanitizedContent = stripOuterQuotes(innerContent);
if (anyAllText.toUpperCase().startsWith("ANY(") || anyAllText.toLowerCase().startsWith("any(")) {
return new QsNode(QsClauseType.ANY, fieldName, sanitizedContent);
}
if (anyAllText.toUpperCase().startsWith("ALL(") || anyAllText.toLowerCase().startsWith("all(")) {
return new QsNode(QsClauseType.ALL, fieldName, sanitizedContent);
}
// Fallback to ANY for unknown cases
return new QsNode(QsClauseType.ANY, fieldName, sanitizedContent);
}
private QsNode createExactNode(String fieldName, String exactText) {
// Extract content between parentheses
String innerContent = extractParenthesesContent(exactText);
return new QsNode(QsClauseType.EXACT, fieldName, innerContent);
}
private String extractParenthesesContent(String text) {
int openParen = text.indexOf('(');
int closeParen = text.lastIndexOf(')');
if (openParen >= 0 && closeParen > openParen) {
return text.substring(openParen + 1, closeParen).trim();
}
return "";
}
private String getCurrentFieldName() {
// Use the current field name from parsing context
return currentFieldName != null ? currentFieldName : "_all";
}
private String stripOuterQuotes(String text) {
if (text == null || text.length() < 2) {
return text;
}
char first = text.charAt(0);
char last = text.charAt(text.length() - 1);
if ((first == '"' && last == '"') || (first == '\'' && last == '\'')) {
return text.substring(1, text.length() - 1);
}
return text;
}
}
/**
* Intermediate Representation for search DSL parsing result
*/
public static class QsPlan {
@JsonProperty("root")
public QsNode root;
@JsonProperty("fieldBindings")
public List<QsFieldBinding> fieldBindings;
@JsonCreator
public QsPlan(@JsonProperty("root") QsNode root,
@JsonProperty("fieldBindings") List<QsFieldBinding> fieldBindings) {
this.root = root;
this.fieldBindings = fieldBindings != null ? fieldBindings : new ArrayList<>();
}
/**
* Parse QsPlan from JSON string
*/
public static QsPlan fromJson(String json) {
try {
return JSON_MAPPER.readValue(json, QsPlan.class);
} catch (JsonProcessingException e) {
LOG.warn("Failed to parse QsPlan from JSON: {}", json, e);
return new QsPlan(new QsNode(QsClauseType.TERM, "error", null), new ArrayList<>());
}
}
/**
* Serialize QsPlan to JSON string
*/
public String toJson() {
try {
return JSON_MAPPER.writeValueAsString(this);
} catch (JsonProcessingException e) {
LOG.warn("Failed to serialize QsPlan to JSON", e);
return "{}";
}
}
@Override
public int hashCode() {
return Objects.hash(root, fieldBindings);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
QsPlan qsPlan = (QsPlan) o;
return Objects.equals(root, qsPlan.root)
&& Objects.equals(fieldBindings, qsPlan.fieldBindings);
}
}
/**
* Search AST node representing a clause in the DSL
*/
public static class QsNode {
@JsonProperty("type")
public QsClauseType type;
@JsonProperty("field")
public String field;
@JsonProperty("value")
public String value;
@JsonProperty("children")
public List<QsNode> children;
@JsonProperty("occur")
public QsOccur occur;
@JsonProperty("minimumShouldMatch")
public Integer minimumShouldMatch;
/**
* Constructor for JSON deserialization
*
* @param type the clause type
* @param field the field name
* @param value the field value
* @param children the child nodes
* @param occur the occurrence type
* @param minimumShouldMatch the minimum should match value
*/
@JsonCreator
public QsNode(@JsonProperty("type") QsClauseType type,
@JsonProperty("field") String field,
@JsonProperty("value") String value,
@JsonProperty("children") List<QsNode> children,
@JsonProperty("occur") QsOccur occur,
@JsonProperty("minimumShouldMatch") Integer minimumShouldMatch) {
this.type = type;
this.field = field;
this.value = value;
this.children = children != null ? children : new ArrayList<>();
this.occur = occur;
this.minimumShouldMatch = minimumShouldMatch;
}
public QsNode(QsClauseType type, String field, String value) {
this.type = type;
this.field = field;
this.value = value;
this.children = new ArrayList<>();
}
public QsNode(QsClauseType type, List<QsNode> children) {
this.type = type;
this.children = children != null ? children : new ArrayList<>();
}
public QsNode(QsClauseType type, List<QsNode> children, Integer minimumShouldMatch) {
this.type = type;
this.children = children != null ? children : new ArrayList<>();
this.minimumShouldMatch = minimumShouldMatch;
}
public QsNode withOccur(QsOccur occur) {
this.occur = occur;
return this;
}
@Override
public int hashCode() {
return Objects.hash(type, field, value, children, occur, minimumShouldMatch);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
QsNode qsNode = (QsNode) o;
return type == qsNode.type
&& Objects.equals(field, qsNode.field)
&& Objects.equals(value, qsNode.value)
&& Objects.equals(children, qsNode.children)
&& occur == qsNode.occur
&& Objects.equals(minimumShouldMatch, qsNode.minimumShouldMatch);
}
}
/**
* Field binding information extracted from DSL
*/
public static class QsFieldBinding {
@JsonProperty("fieldName")
public String fieldName;
@JsonProperty("slotIndex")
public int slotIndex;
@JsonCreator
public QsFieldBinding(@JsonProperty("fieldName") String fieldName,
@JsonProperty("slotIndex") int slotIndex) {
this.fieldName = fieldName;
this.slotIndex = slotIndex;
}
@Override
public int hashCode() {
return Objects.hash(fieldName, slotIndex);
}
@Override
public boolean equals(Object o) {
if (this == o) {
return true;
}
if (o == null || getClass() != o.getClass()) {
return false;
}
QsFieldBinding that = (QsFieldBinding) o;
return slotIndex == that.slotIndex
&& Objects.equals(fieldName, that.fieldName);
}
}
/**
* Search options parsed from JSON
*/
public static class SearchOptions {
private String mode = "standard";
private Integer minimumShouldMatch = null;
public boolean isLuceneMode() {
return "lucene".equalsIgnoreCase(mode);
}
public String getMode() {
return mode;
}
public void setMode(String mode) {
this.mode = mode;
}
public Integer getMinimumShouldMatch() {
return minimumShouldMatch;
}
public void setMinimumShouldMatch(Integer minimumShouldMatch) {
this.minimumShouldMatch = minimumShouldMatch;
}
}
/**
* Parse options JSON string
*/
private static SearchOptions parseOptions(String optionsJson) {
SearchOptions options = new SearchOptions();
if (optionsJson == null || optionsJson.trim().isEmpty()) {
return options;
}
try {
// Parse JSON using Jackson
com.fasterxml.jackson.databind.JsonNode jsonNode = JSON_MAPPER.readTree(optionsJson);
if (jsonNode.has("mode")) {
options.setMode(jsonNode.get("mode").asText());
}
if (jsonNode.has("minimum_should_match")) {
options.setMinimumShouldMatch(jsonNode.get("minimum_should_match").asInt());
}
} catch (Exception e) {
LOG.warn("Failed to parse search options JSON: {}", optionsJson, e);
}
return options;
}
/**
* Lucene mode parsing - implements ES/Lucene-style boolean query semantics.
* <p>
* Key differences from standard mode:
* - Operators are processed left-to-right as modifiers (not traditional boolean algebra)
* - AND marks preceding and current terms as MUST (+)
* - OR marks preceding and current terms as SHOULD
* - NOT marks current term as MUST_NOT (-)
* - When minimum_should_match=0 and there are MUST clauses, SHOULD clauses are ignored
* <p>
* Examples:
* - "a AND b OR c" → +a (with minimum_should_match=0, SHOULD terms discarded)
* - "a AND b OR NOT c AND d" → +a -c +d
* - "a OR b OR c" → a b c (at least one must match)
*/
private static QsPlan parseDslLuceneMode(String dsl, String defaultField, String defaultOperator,
SearchOptions options) {
if (dsl == null || dsl.trim().isEmpty()) {
return new QsPlan(new QsNode(QsClauseType.TERM, "error", "empty_dsl"), new ArrayList<>());
}
// Expand simplified DSL if default field is provided
String expandedDsl = dsl;
if (defaultField != null && !defaultField.trim().isEmpty()) {
expandedDsl = expandSimplifiedDsl(dsl.trim(), defaultField.trim(),
normalizeDefaultOperator(defaultOperator));
}
try {
// Create ANTLR lexer and parser
SearchLexer lexer = new SearchLexer(new ANTLRInputStream(expandedDsl));
CommonTokenStream tokens = new CommonTokenStream(lexer);
SearchParser parser = new SearchParser(tokens);
// Add error listener
parser.removeErrorListeners();
parser.addErrorListener(new org.antlr.v4.runtime.BaseErrorListener() {
@Override
public void syntaxError(org.antlr.v4.runtime.Recognizer<?, ?> recognizer,
Object offendingSymbol,
int line, int charPositionInLine,
String msg, org.antlr.v4.runtime.RecognitionException e) {
throw new RuntimeException("Invalid search DSL syntax at line " + line
+ ":" + charPositionInLine + " " + msg);
}
});
// Parse using standard parser first
ParseTree tree = parser.search();
if (tree == null) {
throw new RuntimeException("Invalid search DSL syntax");
}
// Build AST using Lucene-mode visitor
QsLuceneModeAstBuilder visitor = new QsLuceneModeAstBuilder(options);
QsNode root = visitor.visit(tree);
// Extract field bindings
Set<String> fieldNames = visitor.getFieldNames();
List<QsFieldBinding> bindings = new ArrayList<>();
int slotIndex = 0;
for (String fieldName : fieldNames) {
bindings.add(new QsFieldBinding(fieldName, slotIndex++));
}
return new QsPlan(root, bindings);
} catch (Exception e) {
LOG.error("Failed to parse search DSL in Lucene mode: '{}' (expanded: '{}')", dsl, expandedDsl, e);
throw new RuntimeException("Invalid search DSL syntax: " + dsl + ". Error: " + e.getMessage(), e);
}
}
/**
* ANTLR visitor for Lucene-mode AST building.
* Transforms standard boolean expressions into Lucene-style OCCUR_BOOLEAN queries.
*/
private static class QsLuceneModeAstBuilder extends SearchParserBaseVisitor<QsNode> {
private final Set<String> fieldNames = new HashSet<>();
private final SearchOptions options;
private String currentFieldName = null;
public QsLuceneModeAstBuilder(SearchOptions options) {
this.options = options;
}
public Set<String> getFieldNames() {
return fieldNames;
}
@Override
public QsNode visitSearch(SearchParser.SearchContext ctx) {
QsNode result = visit(ctx.clause());
if (result == null) {
throw new RuntimeException("Invalid search clause");
}
return result;
}
@Override
public QsNode visitOrClause(SearchParser.OrClauseContext ctx) {
// In Lucene mode, we need to process the entire OR chain together
// to correctly assign MUST/SHOULD/MUST_NOT based on operator sequence
return processLuceneBooleanChain(ctx);
}
/**
* Process the entire boolean expression chain in Lucene mode.
* This is the core of Lucene-style boolean parsing.
*/
private QsNode processLuceneBooleanChain(SearchParser.OrClauseContext ctx) {
// Collect all terms and operators from the expression tree
List<TermWithOccur> terms = new ArrayList<>();
collectTermsWithOperators(ctx, terms, QsOccur.MUST); // default_operator = AND means MUST
if (terms.isEmpty()) {
throw new RuntimeException("No terms found in boolean expression");
}
if (terms.size() == 1) {
TermWithOccur singleTerm = terms.get(0);
if (singleTerm.isNegated) {
// Single negated term - must wrap in OCCUR_BOOLEAN for BE to handle MUST_NOT
singleTerm.node.occur = QsOccur.MUST_NOT;
List<QsNode> children = new ArrayList<>();
children.add(singleTerm.node);
return new QsNode(QsClauseType.OCCUR_BOOLEAN, children, 0);
}
// Single non-negated term - return directly without wrapper
return singleTerm.node;
}
// Apply Lucene boolean logic
applyLuceneBooleanLogic(terms);
// Determine minimum_should_match
Integer minShouldMatch = options.getMinimumShouldMatch();
if (minShouldMatch == null) {
// Default: 0 if there are MUST clauses, 1 if only SHOULD
boolean hasMust = terms.stream().anyMatch(t -> t.occur == QsOccur.MUST);
boolean hasMustNot = terms.stream().anyMatch(t -> t.occur == QsOccur.MUST_NOT);
minShouldMatch = (hasMust || hasMustNot) ? 0 : 1;
}
// Filter out SHOULD clauses if minimum_should_match=0 and there are MUST clauses
final int finalMinShouldMatch = minShouldMatch;
if (minShouldMatch == 0) {
boolean hasMust = terms.stream().anyMatch(t -> t.occur == QsOccur.MUST);
if (hasMust) {
terms = terms.stream()
.filter(t -> t.occur != QsOccur.SHOULD)
.collect(java.util.stream.Collectors.toList());
}
}
if (terms.isEmpty()) {
throw new RuntimeException("All terms filtered out in Lucene boolean logic");
}
if (terms.size() == 1) {
TermWithOccur remainingTerm = terms.get(0);
if (remainingTerm.occur == QsOccur.MUST_NOT) {
// Single MUST_NOT term - must wrap in OCCUR_BOOLEAN for BE to handle
remainingTerm.node.occur = QsOccur.MUST_NOT;
List<QsNode> children = new ArrayList<>();
children.add(remainingTerm.node);
return new QsNode(QsClauseType.OCCUR_BOOLEAN, children, 0);
}
return remainingTerm.node;
}
// Build OCCUR_BOOLEAN node
List<QsNode> children = new ArrayList<>();
for (TermWithOccur term : terms) {
term.node.occur = term.occur;
children.add(term.node);
}
return new QsNode(QsClauseType.OCCUR_BOOLEAN, children, finalMinShouldMatch);
}
/**
* Collect terms from the parse tree with their positions
*/
private void collectTermsWithOperators(ParseTree ctx, List<TermWithOccur> terms, QsOccur defaultOccur) {
if (ctx instanceof SearchParser.OrClauseContext) {
SearchParser.OrClauseContext orCtx = (SearchParser.OrClauseContext) ctx;
List<SearchParser.AndClauseContext> andClauses = orCtx.andClause();
for (int i = 0; i < andClauses.size(); i++) {
// Mark that this term is introduced by OR if not the first
boolean introducedByOr = (i > 0);
collectTermsFromAndClause(andClauses.get(i), terms, defaultOccur, introducedByOr);
}
}
}
private void collectTermsFromAndClause(SearchParser.AndClauseContext ctx, List<TermWithOccur> terms,
QsOccur defaultOccur, boolean introducedByOr) {
List<SearchParser.NotClauseContext> notClauses = ctx.notClause();
for (int i = 0; i < notClauses.size(); i++) {
boolean introducedByAnd = (i > 0);
collectTermsFromNotClause(notClauses.get(i), terms, defaultOccur, introducedByOr, introducedByAnd);
// After first term, all subsequent in same AND chain are introducedByOr=false
introducedByOr = false;
}
}
private void collectTermsFromNotClause(SearchParser.NotClauseContext ctx, List<TermWithOccur> terms,
QsOccur defaultOccur, boolean introducedByOr, boolean introducedByAnd) {
boolean isNegated = ctx.NOT() != null;
SearchParser.AtomClauseContext atomCtx = ctx.atomClause();
if (atomCtx.clause() != null) {
// Parenthesized clause - visit recursively
QsNode subNode = visit(atomCtx.clause());
TermWithOccur term = new TermWithOccur(subNode, defaultOccur);
term.introducedByOr = introducedByOr;
term.introducedByAnd = introducedByAnd;
term.isNegated = isNegated;
terms.add(term);
} else {
// Field query
QsNode node = visit(atomCtx.fieldQuery());
TermWithOccur term = new TermWithOccur(node, defaultOccur);
term.introducedByOr = introducedByOr;
term.introducedByAnd = introducedByAnd;
term.isNegated = isNegated;
terms.add(term);
}
}
/**
* Apply Lucene boolean logic to determine final MUST/SHOULD/MUST_NOT for each term.
* <p>
* Rules (processed left-to-right):
* 1. First term: MUST (due to default_operator=AND)
* 2. AND introduces: marks preceding and current as MUST
* 3. OR introduces: marks preceding and current as SHOULD
* 4. NOT modifier: marks current as MUST_NOT
* 5. AND after MUST_NOT: the MUST_NOT term is not affected, current becomes MUST
*/
private void applyLuceneBooleanLogic(List<TermWithOccur> terms) {
for (int i = 0; i < terms.size(); i++) {
TermWithOccur current = terms.get(i);
if (current.isNegated) {
// NOT modifier - mark as MUST_NOT
current.occur = QsOccur.MUST_NOT;
// OR + NOT: preceding becomes SHOULD (if not already MUST_NOT)
if (current.introducedByOr && i > 0) {
TermWithOccur prev = terms.get(i - 1);
if (prev.occur != QsOccur.MUST_NOT) {
prev.occur = QsOccur.SHOULD;
}
}
} else if (current.introducedByAnd) {
// AND introduces: both preceding and current are MUST
current.occur = QsOccur.MUST;
if (i > 0) {
TermWithOccur prev = terms.get(i - 1);
// Don't change MUST_NOT to MUST
if (prev.occur != QsOccur.MUST_NOT) {
prev.occur = QsOccur.MUST;
}
}
} else if (current.introducedByOr) {
// OR introduces: both preceding and current are SHOULD
current.occur = QsOccur.SHOULD;
if (i > 0) {
TermWithOccur prev = terms.get(i - 1);
// Don't change MUST_NOT to SHOULD
if (prev.occur != QsOccur.MUST_NOT) {
prev.occur = QsOccur.SHOULD;
}
}
} else {
// First term: MUST (default_operator=AND)
current.occur = QsOccur.MUST;
}
}
}
@Override
public QsNode visitAndClause(SearchParser.AndClauseContext ctx) {
// This is called for simple cases, delegate to parent's logic
if (ctx.notClause().size() == 1) {
return visit(ctx.notClause(0));
}
// Multiple AND terms - use processLuceneBooleanChain via parent
List<QsNode> children = new ArrayList<>();
for (SearchParser.NotClauseContext notCtx : ctx.notClause()) {
QsNode child = visit(notCtx);
if (child != null) {
children.add(child);
}
}
if (children.size() == 1) {
return children.get(0);
}
return new QsNode(QsClauseType.AND, children);
}
@Override
public QsNode visitNotClause(SearchParser.NotClauseContext ctx) {
if (ctx.NOT() != null) {
QsNode child = visit(ctx.atomClause());
if (child == null) {
throw new RuntimeException("Invalid NOT clause: missing operand");
}
List<QsNode> children = new ArrayList<>();
children.add(child);
return new QsNode(QsClauseType.NOT, children);
}
return visit(ctx.atomClause());
}
@Override
public QsNode visitAtomClause(SearchParser.AtomClauseContext ctx) {
if (ctx.clause() != null) {
return visit(ctx.clause());
}
return visit(ctx.fieldQuery());
}
@Override
public QsNode visitFieldQuery(SearchParser.FieldQueryContext ctx) {
// Build complete field path
StringBuilder fullPath = new StringBuilder();
List<SearchParser.FieldSegmentContext> segments = ctx.fieldPath().fieldSegment();
for (int i = 0; i < segments.size(); i++) {
if (i > 0) {
fullPath.append('.');
}
String segment = segments.get(i).getText();
if (segment.startsWith("\"") && segment.endsWith("\"")) {
segment = segment.substring(1, segment.length() - 1);
}
fullPath.append(segment);
}
String fieldPath = fullPath.toString();
fieldNames.add(fieldPath);
String previousFieldName = currentFieldName;
currentFieldName = fieldPath;
try {
return visit(ctx.searchValue());
} finally {
currentFieldName = previousFieldName;
}
}
@Override
public QsNode visitSearchValue(SearchParser.SearchValueContext ctx) {
String fieldName = currentFieldName != null ? currentFieldName : "_all";
if (ctx.TERM() != null) {
return new QsNode(QsClauseType.TERM, fieldName, unescapeTermValue(ctx.TERM().getText()));
}
if (ctx.PREFIX() != null) {
return new QsNode(QsClauseType.PREFIX, fieldName, unescapeTermValue(ctx.PREFIX().getText()));
}
if (ctx.WILDCARD() != null) {
return new QsNode(QsClauseType.WILDCARD, fieldName, unescapeTermValue(ctx.WILDCARD().getText()));
}
if (ctx.REGEXP() != null) {
String regexp = ctx.REGEXP().getText();
if (regexp.startsWith("/") && regexp.endsWith("/")) {
regexp = regexp.substring(1, regexp.length() - 1);
}
return new QsNode(QsClauseType.REGEXP, fieldName, regexp);
}
if (ctx.QUOTED() != null) {
String quoted = ctx.QUOTED().getText();
if (quoted.startsWith("\"") && quoted.endsWith("\"")) {
quoted = quoted.substring(1, quoted.length() - 1);
}
return new QsNode(QsClauseType.PHRASE, fieldName, quoted);
}
if (ctx.rangeValue() != null) {
SearchParser.RangeValueContext rangeCtx = ctx.rangeValue();
String rangeText;
if (rangeCtx.LBRACKET() != null) {
rangeText = "[" + rangeCtx.rangeEndpoint(0).getText() + " TO "
+ rangeCtx.rangeEndpoint(1).getText() + "]";
} else {
rangeText = "{" + rangeCtx.rangeEndpoint(0).getText() + " TO "
+ rangeCtx.rangeEndpoint(1).getText() + "}";
}
return new QsNode(QsClauseType.RANGE, fieldName, rangeText);
}
if (ctx.listValue() != null) {
StringBuilder listText = new StringBuilder("IN(");
for (int i = 0; i < ctx.listValue().LIST_TERM().size(); i++) {
if (i > 0) {
listText.append(" ");
}
listText.append(ctx.listValue().LIST_TERM(i).getText());
}
listText.append(")");
return new QsNode(QsClauseType.LIST, fieldName, listText.toString());
}
if (ctx.anyAllValue() != null) {
String text = ctx.anyAllValue().getText();
String innerContent = extractParenthesesContent(text);
String sanitizedContent = stripOuterQuotes(innerContent);
if (text.toUpperCase().startsWith("ANY(")) {
return new QsNode(QsClauseType.ANY, fieldName, sanitizedContent);
}
return new QsNode(QsClauseType.ALL, fieldName, sanitizedContent);
}
if (ctx.exactValue() != null) {
String innerContent = extractParenthesesContent(ctx.exactValue().getText());
return new QsNode(QsClauseType.EXACT, fieldName, innerContent);
}
return new QsNode(QsClauseType.TERM, fieldName, unescapeTermValue(ctx.getText()));
}
private String extractParenthesesContent(String text) {
int openParen = text.indexOf('(');
int closeParen = text.lastIndexOf(')');
if (openParen >= 0 && closeParen > openParen) {
return text.substring(openParen + 1, closeParen).trim();
}
return "";
}
private String stripOuterQuotes(String text) {
if (text == null || text.length() < 2) {
return text;
}
char first = text.charAt(0);
char last = text.charAt(text.length() - 1);
if ((first == '"' && last == '"') || (first == '\'' && last == '\'')) {
return text.substring(1, text.length() - 1);
}
return text;
}
}
/**
* Helper class to track term with its occur status during parsing
*/
private static class TermWithOccur {
QsNode node;
QsOccur occur;
boolean introducedByOr = false;
boolean introducedByAnd = false;
boolean isNegated = false;
TermWithOccur(QsNode node, QsOccur occur) {
this.node = node;
this.occur = occur;
}
}
/**
* Process escape sequences in a term value.
* Converts escape sequences to their literal characters:
* - \ (backslash space) -> space
* - \( -> (
* - \) -> )
* - \: -> :
* - \\ -> \
* - \* -> *
* - \? -> ?
* - etc.
*
* @param value The raw term value with escape sequences
* @return The unescaped value
*/
private static String unescapeTermValue(String value) {
if (value == null || value.isEmpty()) {
return value;
}
// Quick check: if no backslash, return as-is
if (value.indexOf('\\') < 0) {
return value;
}
StringBuilder result = new StringBuilder(value.length());
int i = 0;
while (i < value.length()) {
char c = value.charAt(i);
if (c == '\\' && i + 1 < value.length()) {
// Escape sequence - take the next character literally
result.append(value.charAt(i + 1));
i += 2;
} else {
result.append(c);
i++;
}
}
return result.toString();
}
}