Support of nested function calls in new engine

.github/workflows/sql-cli-release-workflow.yml

@@ -8,7 +8,7 @@ on:
 jobs:
   build:
 
-    runs-on: [ubuntu-16.04]
+    runs-on: [ubuntu-18.04]
     defaults:
       run:
         working-directory: sql-cli

.github/workflows/sql-cli-test-and-build-workflow.yml

@@ -5,7 +5,7 @@ on: [pull_request, push]
 jobs:
   build:
 
-    runs-on: [ubuntu-16.04]
+    runs-on: [ubuntu-18.04]
     defaults:
       run:
         working-directory: sql-cli

.github/workflows/sql-jdbc-push-jdbc-maven.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   upload-jdbc-jar:
-    runs-on: [ubuntu-16.04]
+    runs-on: [ubuntu-18.04]
     defaults:
       run:
         working-directory: sql-jdbc

.github/workflows/sql-release-workflow.yml

@@ -12,7 +12,7 @@ jobs:
         java: [14]
 
     name: Build and Release SQL Plugin
-    runs-on: [ubuntu-16.04]
+    runs-on: [ubuntu-18.04]
 
     steps:
       - name: Checkout SQL

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprDatetimeValue.java

@@ -17,6 +17,7 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
+import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters;
 import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
 import com.google.common.base.Objects;
 import java.time.Instant;
@@ -26,37 +27,21 @@
 import java.time.ZoneId;
 import java.time.ZonedDateTime;
 import java.time.format.DateTimeFormatter;
-import java.time.format.DateTimeFormatterBuilder;
 import java.time.format.DateTimeParseException;
-import java.time.temporal.ChronoField;
 import java.time.temporal.ChronoUnit;
 import lombok.RequiredArgsConstructor;
 
 @RequiredArgsConstructor
 public class ExprDatetimeValue extends AbstractExprValue {
   private final LocalDateTime datetime;
 
-  private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS;
-  private static final int MIN_FRACTION_SECONDS = 0;
-  private static final int MAX_FRACTION_SECONDS = 6;
-
-  static {
-    FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder()
-        .appendPattern("yyyy-MM-dd HH:mm:ss")
-        .appendFraction(
-            ChronoField.MICRO_OF_SECOND,
-            MIN_FRACTION_SECONDS,
-            MAX_FRACTION_SECONDS,
-            true)
-        .toFormatter();
-  }
-
   /**
    * Constructor with datetime string as input.
    */
   public ExprDatetimeValue(String datetime) {
     try {
-      this.datetime = LocalDateTime.parse(datetime, FORMATTER_VARIABLE_MICROS);
+      this.datetime = LocalDateTime.parse(datetime,
+          ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER);
     } catch (DateTimeParseException e) {
       throw new SemanticCheckException(String.format("datetime:%s in unsupported format, please "
           + "use yyyy-MM-dd HH:mm:ss[.SSSSSS]", datetime));

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprStringValue.java

@@ -18,6 +18,7 @@
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
 import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
+import java.time.Instant;
 import java.time.LocalDate;
 import java.time.LocalDateTime;
 import java.time.LocalTime;
@@ -80,6 +81,11 @@ public LocalTime timeValue() {
     }
   }
 
+  @Override
+  public Instant timestampValue() {
+    return new ExprTimestampValue(value).timestampValue();
+  }
+
   @Override
   public String toString() {
     return String.format("\"%s\"", value);

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/model/ExprTimestampValue.java

@@ -19,6 +19,7 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprType;
+import com.amazon.opendistroforelasticsearch.sql.data.utils.ExprDateFormatters;
 import com.amazon.opendistroforelasticsearch.sql.exception.SemanticCheckException;
 import java.time.Instant;
 import java.time.LocalDate;
@@ -45,14 +46,22 @@ public class ExprTimestampValue extends AbstractExprValue {
   /**
    * todo. only support timestamp in format yyyy-MM-dd HH:mm:ss.
    */
-  private static final DateTimeFormatter FORMATTER_WITNOUT_NANO = DateTimeFormatter
+  private static final DateTimeFormatter FORMATTER_WITHOUT_NANO = DateTimeFormatter
       .ofPattern("yyyy-MM-dd HH:mm:ss");
+  private static final DateTimeFormatter FORMATTER_WITHOUT_TIME = DateTimeFormatter
+          .ofPattern("yyyy-MM-dd");
   private final Instant timestamp;
 
+  private String datetimeFormat = ALWAYS_INCLUDE_TIME;
+
   private static final DateTimeFormatter FORMATTER_VARIABLE_MICROS;
   private static final int MIN_FRACTION_SECONDS = 0;
   private static final int MAX_FRACTION_SECONDS = 6;
 
+  public static final String ALWAYS_INCLUDE_TIME = "always_include_time";
+  public static final String NEVER_INCLUDE_TIME = "never_include_time";
+  public static final String INCLUDE_TIME_WHEN_NONZERO = "include_time_when_nonzero";
+
   static {
     FORMATTER_VARIABLE_MICROS = new DateTimeFormatterBuilder()
         .appendPattern("yyyy-MM-dd HH:mm:ss")
@@ -69,7 +78,8 @@ public class ExprTimestampValue extends AbstractExprValue {
    */
   public ExprTimestampValue(String timestamp) {
     try {
-      this.timestamp = LocalDateTime.parse(timestamp, FORMATTER_VARIABLE_MICROS)
+      this.timestamp = LocalDateTime.parse(timestamp,
+          ExprDateFormatters.TOLERANT_PARSER_DATE_TIME_FORMATTER)
           .atZone(ZONE)
           .toInstant();
     } catch (DateTimeParseException e) {
@@ -81,9 +91,29 @@ public ExprTimestampValue(String timestamp) {
 
   @Override
   public String value() {
-    return timestamp.getNano() == 0 ? FORMATTER_WITNOUT_NANO.withZone(ZONE)
-        .format(timestamp.truncatedTo(ChronoUnit.SECONDS))
-        : FORMATTER_VARIABLE_MICROS.withZone(ZONE).format(timestamp);
+    switch (datetimeFormat) {
+      case NEVER_INCLUDE_TIME:
+        return valueWithoutTime();
+
+      case INCLUDE_TIME_WHEN_NONZERO:
+        LocalTime time = timeValue();
+        return (time.getHour() + time.getMinute() + time.getSecond() == 0)
+                ? valueWithoutTime() : valueWithTime();
+
+      case ALWAYS_INCLUDE_TIME:
+      default:
+        return valueWithTime();
+    }
+  }
+
+  private String valueWithTime() {
+    return timestamp.getNano() == 0 ? FORMATTER_WITHOUT_NANO.withZone(ZONE)
+            .format(timestamp.truncatedTo(ChronoUnit.SECONDS))
+            : FORMATTER_VARIABLE_MICROS.withZone(ZONE).format(timestamp);
+  }
+
+  private String valueWithoutTime() {
+    return FORMATTER_WITHOUT_TIME.withZone(ZONE).format(timestamp);
   }
 
   @Override
@@ -130,4 +160,9 @@ public boolean equal(ExprValue other) {
   public int hashCode() {
     return Objects.hashCode(timestamp);
   }
+
+  public void setDatetimeFormat(String format) {
+    this.datetimeFormat = format;
+  }
+
 }

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/type/ExprCoreType.java

@@ -131,4 +131,8 @@ public static List<ExprCoreType> coreTypes() {
   public static List<ExprType> numberTypes() {
     return ImmutableList.of(INTEGER, LONG, FLOAT, DOUBLE);
   }
+
+  public static List<ExprType> dateTypes() {
+    return ImmutableList.of(DATE, DATETIME, TIMESTAMP, TIME);
+  }
 }

elasticsearch/src/main/java/com/amazon/opendistroforelasticsearch/sql/elasticsearch/data/value/ElasticsearchDateFormatters.java → core/src/main/java/com/amazon/opendistroforelasticsearch/sql/data/utils/ExprDateFormatters.java

@@ -15,7 +15,7 @@
  *
  */
 
-package com.amazon.opendistroforelasticsearch.sql.elasticsearch.data.value;
+package com.amazon.opendistroforelasticsearch.sql.data.utils;
 
 import static java.time.temporal.ChronoField.DAY_OF_MONTH;
 import static java.time.temporal.ChronoField.HOUR_OF_DAY;
@@ -37,7 +37,7 @@
  * Reference org.elasticsearch.common.time.DateFormatters.
  */
 @UtilityClass
-public class ElasticsearchDateFormatters {
+public class ExprDateFormatters {
 
   public static final DateTimeFormatter TIME_ZONE_FORMATTER_NO_COLON =
       new DateTimeFormatterBuilder()
@@ -73,7 +73,16 @@ public class ElasticsearchDateFormatters {
       new DateTimeFormatterBuilder()
           .append(STRICT_YEAR_MONTH_DAY_FORMATTER)
           .optionalStart()
+
+          // otherwise use space
+          .optionalStart()
+          .appendLiteral(' ')
+          .optionalEnd()
+          // optional T
+          .optionalStart()
           .appendLiteral('T')
+          .optionalEnd()
+
           .optionalStart()
           .appendValue(HOUR_OF_DAY, 2, 2, SignStyle.NOT_NEGATIVE)
           .optionalStart()
@@ -82,13 +91,18 @@ public class ElasticsearchDateFormatters {
           .optionalStart()
           .appendLiteral(':')
           .appendValue(SECOND_OF_MINUTE, 2, 2, SignStyle.NOT_NEGATIVE)
+
+          // optional millis with dot
           .optionalStart()
           .appendFraction(NANO_OF_SECOND, 1, 9, true)
           .optionalEnd()
+
+          // otherwise optional millis use with comma
           .optionalStart()
           .appendLiteral(',')
           .appendFraction(NANO_OF_SECOND, 1, 9, false)
           .optionalEnd()
+
           .optionalEnd()
           .optionalEnd()
           .optionalStart()
@@ -104,4 +118,11 @@ public class ElasticsearchDateFormatters {
 
   public static final DateTimeFormatter SQL_LITERAL_DATE_TIME_FORMAT = DateTimeFormatter
           .ofPattern("yyyy-MM-dd HH:mm:ss");
+
+  public static final DateTimeFormatter TOLERANT_PARSER_DATE_TIME_FORMATTER =
+      new DateTimeFormatterBuilder()
+          .appendOptional(STRICT_DATE_OPTIONAL_TIME_FORMATTER)
+          .appendOptional(SQL_LITERAL_DATE_TIME_FORMAT)
+          .appendOptional(STRICT_HOUR_MINUTE_SECOND_FORMATTER)
+          .toFormatter();
 }

core/src/main/java/com/amazon/opendistroforelasticsearch/sql/expression/operator/predicate/BinaryPredicateOperator.java

@@ -25,16 +25,26 @@
 
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprBooleanValue;
 import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValue;
+import com.amazon.opendistroforelasticsearch.sql.data.model.ExprValueUtils;
 import com.amazon.opendistroforelasticsearch.sql.data.type.ExprCoreType;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.BuiltinFunctionRepository;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionBuilder;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionDSL;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionName;
 import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionResolver;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.FunctionSignature;
+import com.amazon.opendistroforelasticsearch.sql.expression.function.SerializableFunction;
 import com.amazon.opendistroforelasticsearch.sql.utils.OperatorUtils;
 import com.google.common.collect.ImmutableTable;
 import com.google.common.collect.Table;
+import java.io.Serializable;
+import java.util.List;
+import java.util.function.Function;
 import java.util.stream.Collectors;
 import lombok.experimental.UtilityClass;
+import org.apache.commons.lang3.tuple.Pair;
+
 
 /**
  * The definition of binary predicate function
@@ -192,53 +202,44 @@ private static FunctionResolver notEqual() {
   }
 
   private static FunctionResolver less() {
-    return FunctionDSL
-        .define(BuiltinFunctionName.LESS.getName(), ExprCoreType.coreTypes().stream()
-            .map(type -> FunctionDSL
-                .impl(FunctionDSL
-                        .nullMissingHandling((v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) < 0)),
-                    BOOLEAN,
-                    type, type))
-            .collect(
-                Collectors.toList()));
+    return FunctionDSL.define(BuiltinFunctionName.LESS.getName(),
+        getComparisonFunctions((Function<Integer, Boolean> & Serializable) value -> value < 0));
   }
 
   private static FunctionResolver lte() {
-    return FunctionDSL
-        .define(BuiltinFunctionName.LTE.getName(), ExprCoreType.coreTypes().stream()
-            .map(type -> FunctionDSL
-                .impl(
-                    FunctionDSL
-                        .nullMissingHandling(
-                            (v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) <= 0)),
-                    BOOLEAN,
-                    type, type))
-            .collect(
-                Collectors.toList()));
+    return FunctionDSL.define(BuiltinFunctionName.LTE.getName(),
+        getComparisonFunctions((Function<Integer, Boolean> & Serializable) value -> value <= 0));
   }
 
   private static FunctionResolver greater() {
-    return FunctionDSL
-        .define(BuiltinFunctionName.GREATER.getName(), ExprCoreType.coreTypes().stream()
-            .map(type -> FunctionDSL
-                .impl(FunctionDSL
-                        .nullMissingHandling((v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) > 0)),
-                    BOOLEAN, type, type))
-            .collect(
-                Collectors.toList()));
+    return FunctionDSL.define(BuiltinFunctionName.GREATER.getName(),
+        getComparisonFunctions((Function<Integer, Boolean> & Serializable) value -> value > 0));
   }
 
   private static FunctionResolver gte() {
-    return FunctionDSL
-        .define(BuiltinFunctionName.GTE.getName(), ExprCoreType.coreTypes().stream()
-            .map(type -> FunctionDSL
-                .impl(
-                    FunctionDSL.nullMissingHandling(
-                        (v1, v2) -> ExprBooleanValue.of(v1.compareTo(v2) >= 0)),
-                    BOOLEAN,
-                    type, type))
-            .collect(
-                Collectors.toList()));
+    return FunctionDSL.define(BuiltinFunctionName.GTE.getName(),
+        getComparisonFunctions((Function<Integer, Boolean> & Serializable) value -> value >= 0));
+  }
+
+  private static List<SerializableFunction<FunctionName, Pair<FunctionSignature, FunctionBuilder>>>
+      getComparisonFunctions(Function<Integer, Boolean> f) {
+    List<SerializableFunction<FunctionName, Pair<FunctionSignature, FunctionBuilder>>>
+        comparisonFunctions = ExprCoreType.coreTypes()
+        .stream()
+        .map(type -> FunctionDSL.impl(FunctionDSL.nullMissingHandling(
+            (v1, v2) -> ExprBooleanValue.of(f.apply(v1.compareTo(v2)))), BOOLEAN, type, type))
+        .collect(Collectors.toList());
+
+    // all string to date conversions
+    comparisonFunctions.addAll(ExprCoreType.dateTypes()
+        .stream()
+        .map(type -> FunctionDSL.impl(FunctionDSL.nullMissingHandling((v1, v2) -> {
+          // casting V2 from string to whatever type v1 is
+          ExprValue v2Casted = ExprValueUtils.fromObjectValue(v2.value(), (ExprCoreType) v1.type());
+          return ExprBooleanValue.of(f.apply(v1.compareTo(v2Casted)));
+        }), BOOLEAN, type, STRING))
+        .collect(Collectors.toList()));
+    return comparisonFunctions;
   }
 
   private static FunctionResolver like() {