fix(#3430): filtering for numerical dimension properties in data expl…

…orer (#3436) * fix(#3430): Remove unused parameter in DataLakeUtils.ts * fix(#3430): Add method to validate amount of table results in data explorer * fix(#3430): Add e2e test to validate functionality of filter dimension property numbers * fix(#3430): Refactor filter selection panel row and filter operator according to data type * fix(#3430): Numerical values are now handled as expected by filters * fix(#3430): Numerical values are now handled as expected by filters * fix(#3430): Change directory name to pass windows CI builds * fix(#3430): Rename component to meet windows file name length restrictions
apache · Jan 23, 2025 · 1605f6b · 1605f6b
1 parent 8517968
commit 1605f6b
Show file tree

Hide file tree

Showing 28 changed files with 756 additions and 183 deletions.
diff --git a/...orer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java b/...orer/src/main/java/org/apache/streampipes/dataexplorer/param/model/WhereClauseParams.java
@@ -34,17 +34,21 @@ public class WhereClauseParams implements IQueryStatement {
 
   private final List<FilterCondition> filterConditions;
 
-  private WhereClauseParams(Long startTime,
-                            Long endTime,
-                            String whereConditions) {
+  private WhereClauseParams(
+      Long startTime,
+      Long endTime,
+      String whereConditions
+  ) {
     this(startTime, endTime);
     if (whereConditions != null) {
       buildConditions(whereConditions);
     }
   }
 
-  private WhereClauseParams(Long startTime,
-                            Long endTime) {
+  private WhereClauseParams(
+      Long startTime,
+      Long endTime
+  ) {
     this.filterConditions = new ArrayList<>();
     this.buildTimeConditions(startTime, endTime);
   }
@@ -56,23 +60,29 @@ private WhereClauseParams(String whereConditions) {
     }
   }
 
-  public static WhereClauseParams from(Long startTime,
-                                       Long endTime) {
+  public static WhereClauseParams from(
+      Long startTime,
+      Long endTime
+  ) {
     return new WhereClauseParams(startTime, endTime);
   }
 
   public static WhereClauseParams from(String whereConditions) {
     return new WhereClauseParams(whereConditions);
   }
 
-  public static WhereClauseParams from(Long startTime,
-                                       Long endTime,
-                                       String whereConditions) {
+  public static WhereClauseParams from(
+      Long startTime,
+      Long endTime,
+      String whereConditions
+  ) {
     return new WhereClauseParams(startTime, endTime, whereConditions);
   }
 
-  private void buildTimeConditions(Long startTime,
-                                   Long endTime) {
+  private void buildTimeConditions(
+      Long startTime,
+      Long endTime
+  ) {
     if (startTime == null) {
       this.filterConditions.add(buildTimeBoundary(endTime, LT));
     } else if (endTime == null) {
@@ -98,7 +108,9 @@ private void buildConditions(String whereConditions) {
   }
 
   private Object returnCondition(String inputCondition) {
-    if (NumberUtils.isParsable(inputCondition)) {
+    if (isQuotedString(inputCondition)) {
+      return removeQuotes(inputCondition);
+    } else if (NumberUtils.isParsable(inputCondition)) {
       return Double.parseDouble(inputCondition);
     } else if (isBoolean(inputCondition)) {
       return Boolean.parseBoolean(inputCondition);
@@ -107,6 +119,18 @@ private Object returnCondition(String inputCondition) {
     }
   }
 
+  private boolean isQuotedString(String input) {
+    if (input.startsWith("\"") && input.endsWith("\"")) {
+      String content = removeQuotes(input);
+      return NumberUtils.isParsable(content);
+    }
+    return false;
+  }
+
+  private String removeQuotes(String input) {
+    return input.substring(1, input.length() - 1);
+  }
+
   private boolean isBoolean(String input) {
     return "true".equalsIgnoreCase(input) || "false".equalsIgnoreCase(input);
   }

diff --git a/ui/angular.json b/ui/angular.json
@@ -189,7 +189,8 @@
   },
   "schematics": {
     "@schematics/angular:component": {
-      "style": "scss"
+      "style": "scss",
+      "standalone": false
     }
   },
   "cli": {

diff --git a/ui/cypress/fixtures/datalake/filterNumericalStringProperties.csv b/ui/cypress/fixtures/datalake/filterNumericalStringProperties.csv
@@ -0,0 +1,3 @@
+timestamp;dimensionKey;v1
+1737123058000;1.0;a
+1737123059000;2.0;20
diff --git a/ui/cypress/support/utils/connect/ConnectUtils.ts b/ui/cypress/support/utils/connect/ConnectUtils.ts
@@ -48,6 +48,8 @@ export class ConnectUtils {
             ConnectEventSchemaUtils.addTimestampProperty();
         }
 
+        ConnectUtils.configureDimensionProperties(adapterConfiguration);
+
         ConnectEventSchemaUtils.finishEventSchemaConfiguration();
 
         ConnectUtils.startAdapter(
@@ -66,6 +68,20 @@ export class ConnectUtils {
 
         ConnectUtils.configureAdapter(adapterConfiguration);
 
+        ConnectUtils.configureDimensionProperties(adapterConfiguration);
+
+        if (adapterConfiguration.timestampProperty) {
+            ConnectEventSchemaUtils.markPropertyAsTimestamp(
+                adapterConfiguration.timestampProperty,
+            );
+        }
+
+        ConnectEventSchemaUtils.finishEventSchemaConfiguration();
+    }
+
+    private static configureDimensionProperties(
+        adapterConfiguration: AdapterInput,
+    ) {
         if (adapterConfiguration.dimensionProperties.length > 0) {
             adapterConfiguration.dimensionProperties.forEach(
                 dimensionPropertyName => {
@@ -75,14 +91,6 @@ export class ConnectUtils {
                 },
             );
         }
-
-        if (adapterConfiguration.timestampProperty) {
-            ConnectEventSchemaUtils.markPropertyAsTimestamp(
-                adapterConfiguration.timestampProperty,
-            );
-        }
-
-        ConnectEventSchemaUtils.finishEventSchemaConfiguration();
     }
 
     public static addMachineDataSimulator(

diff --git a/ui/cypress/support/utils/datalake/DataLakeUtils.ts b/ui/cypress/support/utils/datalake/DataLakeUtils.ts
@@ -69,7 +69,6 @@ export class DataLakeUtils {
 
     public static loadDataIntoDataLake(
         dataSet: string,
-        wait = true,
         format: 'csv' | 'json_array' = 'csv',
     ) {
         // Create adapter with dataset
@@ -182,7 +181,9 @@ export class DataLakeUtils {
     }
 
     public static saveDataViewConfiguration() {
-        cy.dataCy('save-data-view-btn', { timeout: 10000 }).click();
+        cy.dataCy('save-data-view-btn', { timeout: 10000 }).click({
+            force: true,
+        });
     }
 
     public static saveDashboardConfiguration() {
@@ -279,6 +280,43 @@ export class DataLakeUtils {
         }
     }
 
+    /**
+     * This method validates that the defined filter options are available in the UI
+     * @param expectedFilterOptions
+     */
+    public static validateFilterOptions(
+        expectedFilterOptions: ('=' | '<' | '<=' | '>=' | '>' | '!=')[],
+    ) {
+        cy.dataCy('design-panel-data-settings-filter-operator')
+            .click()
+            .dataCy('operator-', {}, true)
+            .should('have.length', expectedFilterOptions.length);
+
+        expectedFilterOptions.forEach(option => {
+            const escapedOption = option.replace(/([=<>!])/g, '\\$1');
+            cy.dataCy('operator-' + escapedOption).should('be.visible');
+        });
+
+        cy.dataCy('design-panel-data-settings-filter-operator').click({
+            force: true,
+        });
+    }
+
+    public static validateAutoCompleteOptions(options: string[]) {
+        cy.dataCy('design-panel-data-settings-filter-value')
+            .click({ force: true })
+            .dataCy('autocomplete-value-', {}, true)
+            .should('have.length', options.length);
+
+        options.forEach(option => {
+            cy.dataCy('autocomplete-value-' + option).should('be.visible');
+        });
+
+        cy.dataCy('design-panel-data-settings-filter-value').click({
+            force: true,
+        });
+    }
+
     /**
      * In the data set panel select all property fields
      */
@@ -310,7 +348,9 @@ export class DataLakeUtils {
     }
 
     public static dataConfigRemoveFilter() {
-        cy.dataCy('design-panel-data-settings-remove-filter').first().click();
+        cy.dataCy('design-panel-data-settings-remove-filter')
+            .first()
+            .click({ force: true });
     }
 
     public static clickGroupBy(propertyName: string) {

diff --git a/ui/cypress/support/utils/datalake/DataLakeWidgetTableUtils.ts b/ui/cypress/support/utils/datalake/DataLakeWidgetTableUtils.ts
@@ -17,13 +17,17 @@
  */
 
 export class DataLakeWidgetTableUtils {
+    public static dataExplorerTableRowTimestamp() {
+        return cy.dataCy('data-explorer-table-row-timestamp', {
+            timeout: 10000,
+        });
+    }
+
     /**
      * Checks how many rows are visible within the table widget in the data explorer
      * @param amount of expected rows
      */
-    public static checkRows(amount: number) {
-        cy.dataCy('data-explorer-table-row-timestamp', {
-            timeout: 10000,
-        }).should('have.length', amount);
+    public static checkAmountOfRows(amount: number) {
+        this.dataExplorerTableRowTimestamp().should('have.length', amount);
     }
 }
diff --git a/ui/cypress/tests/datalake/deleteViewAndDashboard.spec.ts b/ui/cypress/tests/datalake/deleteViewAndDashboard.spec.ts
@@ -20,7 +20,7 @@ import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
 describe('Test Deletion of Data View and Dashboard', () => {
     beforeEach('Setup Test', () => {
         cy.initStreamPipesTest();
-        DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv', false);
+        DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv');
     });
 
     it('Perform Test', () => {

diff --git a/ui/cypress/tests/datalake/deleteWidget.ts b/ui/cypress/tests/datalake/deleteWidget.ts
@@ -20,7 +20,7 @@ import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
 describe('Test Table View in Data Explorer', () => {
     beforeEach('Setup Test', () => {
         cy.initStreamPipesTest();
-        DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv', false);
+        DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv');
     });
 
     it('Perform Test', () => {

diff --git a/ui/cypress/tests/datalake/filterNumericalStringProperties.spec.ts b/ui/cypress/tests/datalake/filterNumericalStringProperties.spec.ts
@@ -0,0 +1,89 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+import { DataLakeUtils } from '../../support/utils/datalake/DataLakeUtils';
+import { DataLakeWidgetTableUtils } from '../../support/utils/datalake/DataLakeWidgetTableUtils';
+import { DataLakeFilterConfig } from '../../support/model/DataLakeFilterConfig';
+import { AdapterBuilder } from '../../support/builder/AdapterBuilder';
+import { ConnectBtns } from '../../support/utils/connect/ConnectBtns';
+import { ConnectUtils } from '../../support/utils/connect/ConnectUtils';
+import { FileManagementUtils } from '../../support/utils/FileManagementUtils';
+
+describe('Validate that filter works for numerical dimension property', () => {
+    beforeEach('Setup Test', () => {
+        cy.initStreamPipesTest();
+
+        FileManagementUtils.addFile(
+            'datalake/filterNumericalStringProperties.csv',
+        );
+        const adapterInput = AdapterBuilder.create('File_Stream')
+            .setName('Test Adapter')
+            .setTimestampProperty('timestamp')
+            .addDimensionProperty('dimensionKey')
+            .setStoreInDataLake()
+            .setFormat('csv')
+            .addFormatInput('input', ConnectBtns.csvDelimiter(), ';')
+            .addFormatInput('checkbox', ConnectBtns.csvHeader(), 'check')
+            .build();
+        ConnectUtils.testAdapter(adapterInput);
+    });
+
+    it('Perform Test', () => {
+        DataLakeUtils.goToDatalake();
+        DataLakeUtils.createAndEditDataView();
+
+        // create table widget and select time range
+        const startDate = new Date(1737029442000);
+        const endDate = new Date(1742220659000);
+
+        DataLakeUtils.clickOrderBy('descending');
+
+        DataLakeUtils.openVisualizationConfig();
+        DataLakeUtils.selectVisualizationType('Table');
+        DataLakeUtils.selectTimeRange(startDate, endDate);
+        cy.wait(1000);
+
+        // validate data in table
+        DataLakeWidgetTableUtils.checkAmountOfRows(2);
+
+        // select filter for tag
+        DataLakeUtils.selectDataConfig();
+        var filterConfig = new DataLakeFilterConfig('dimensionKey', '1.0', '=');
+        DataLakeUtils.dataConfigAddFilter(filterConfig);
+
+        // validate data in table is filtered
+        DataLakeWidgetTableUtils.checkAmountOfRows(1);
+
+        // remove filter
+        DataLakeUtils.dataConfigRemoveFilter();
+
+        DataLakeUtils.selectDataConfig();
+
+        filterConfig = new DataLakeFilterConfig('v1', '20', '=');
+        DataLakeUtils.dataConfigAddFilter(filterConfig);
+
+        // validate data in table is filtered
+        DataLakeWidgetTableUtils.checkAmountOfRows(1);
+
+        // remove filter
+        DataLakeUtils.dataConfigRemoveFilter();
+
+        // validate data again
+        DataLakeWidgetTableUtils.checkAmountOfRows(2);
+    });
+});
diff --git a/ui/cypress/tests/datalake/missingDataInDataLake.spec.ts b/ui/cypress/tests/datalake/missingDataInDataLake.spec.ts
@@ -34,13 +34,13 @@ describe('Test missing properties in data lake', () => {
     it('Test table with missing properties', () => {
         DataLakeUtils.addDataViewAndTableWidget(dataViewName, 'Persist');
 
-        DataLakeWidgetTableUtils.checkRows(5);
+        DataLakeWidgetTableUtils.checkAmountOfRows(5);
 
         DataLakeUtils.selectDataConfig();
         cy.dataCy('data-explorer-ignore-missing-values-checkbox')
             .children()
             .click();
 
-        DataLakeWidgetTableUtils.checkRows(3);
+        DataLakeWidgetTableUtils.checkAmountOfRows(3);
     });
 });
diff --git a/ui/cypress/tests/datalake/timeOrderDataView.spec.ts b/ui/cypress/tests/datalake/timeOrderDataView.spec.ts
@@ -22,7 +22,7 @@ import { DataLakeBtns } from '../../support/utils/datalake/DataLakeBtns';
 describe('Test Time Order in Data Explorer', () => {
     beforeEach('Setup Test', () => {
         cy.initStreamPipesTest();
-        DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv', false);
+        DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv');
         DataLakeUtils.goToDatalake();
         DataLakeUtils.createAndEditDataView();
     });

diff --git a/ui/cypress/tests/datalake/timeRangeSelectors.spec.ts b/ui/cypress/tests/datalake/timeRangeSelectors.spec.ts
@@ -46,7 +46,7 @@ describe('Test Time Range Selectors in Data Explorer', () => {
 
     before('Setup Tests', () => {
         cy.initStreamPipesTest();
-        DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv', false);
+        DataLakeUtils.loadDataIntoDataLake('datalake/sample.csv');
     });
 
     it('Perform Test', () => {