diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java index 0ce628b034d..d3d62f58814 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java @@ -90,6 +90,7 @@ import java.sql.Date; import java.sql.DriverManager; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; @@ -108,6 +109,8 @@ import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -504,7 +507,42 @@ private Pair getRowKeyMatchersFromView(PhoenixConnection connect } - // Helper to get rowKeyMatcher from Metadata. + byte[] getRowKeyMatcherFromSyscatIndex(String tenantId, String schemaName, + String tableName, boolean useIndexTable) throws SQLException { + + final String + SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL = + "SELECT ROW_KEY_MATCHER FROM SYSTEM.CATALOG " + "WHERE %s AND TABLE_SCHEM <> 'SYSTEM' AND TABLE_NAME = '%s' AND ROW_KEY_MATCHER IS NOT NULL"; + final String SYS_CATALOG_IDX_ROW_KEY_MATCHER_HEADER_SQL = "SELECT \"0:ROW_KEY_MATCHER\" FROM SYSTEM.SYS_ROW_KEY_MATCHER_IDX " + "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'"; + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String + tenantClause = useIndexTable ? + (tenantId == null || tenantId.isEmpty() ? + "\":TENANT_ID\" IS NULL" : + String.format("\":TENANT_ID\" = '%s'", tenantId)) : + (tenantId == null || tenantId.isEmpty() ? + "TENANT_ID IS NULL" : + String.format("TENANT_ID = '%s'", tenantId)); + String + sql = useIndexTable ? + String.format(SYS_CATALOG_IDX_ROW_KEY_MATCHER_HEADER_SQL, tenantClause, schemaName, + tableName) : + String.format(SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL, tenantClause, + tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + byte[] matcherBytes = rs.next() ? rs.getBytes(1) : EMPTY_BYTE_ARRAY; + LOGGER.info("Row key matcher SQL: {}", sql); + LOGGER.info("Row key matcher: {}, {}", + Bytes.toStringBinary(matcherBytes), + Bytes.toStringBinary(PVarbinaryEncoded.INSTANCE.toBytes(matcherBytes))); + return PVarbinaryEncoded.INSTANCE.toBytes(matcherBytes); + } + } + + // Helper to get rowKeyMatcher from Metadata. private Pair getRowKeyMatchersFromView(PhoenixConnection connection, PTable view) throws SQLException { return getRowKeyMatchersFromView(connection, view.getName().getString()); @@ -560,6 +598,12 @@ private byte[] assertRowKeyMatcherForView(PhoenixConnection connection, PTable v PVarbinaryEncoded.INSTANCE.toBytes( WhereOptimizer.getRowKeyMatcher(connection, tableName, viewStatementTable, viewColumnConstantsToBe, isViewColumnReferencedToBe)); + byte[] + rowKeyMatcher3 = getRowKeyMatcherFromSyscatIndex(view.getTenantId() != null ? view.getTenantId().getString() : null, view.getSchemaName().getString(), view.getTableName().getString(), false); + + byte[] + rowKeyMatcher4 = getRowKeyMatcherFromSyscatIndex(view.getTenantId() != null ? view.getTenantId().getString() : null, view.getSchemaName().getString(), view.getTableName().getString(), true); + LOGGER.debug(String.format( "target-view-name = %s, physical = %s, stmt-table = %s\n, " + "row-matcher-0 = %s (syscat)\n, row-matcher-1 = %s\n, row-matcher-2 = %s\n", @@ -571,6 +615,10 @@ private byte[] assertRowKeyMatcherForView(PhoenixConnection connection, PTable v Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher1) == 0); assertTrue("RowKey matcher patterns do not match", Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher2) == 0); + assertTrue("RowKey matcher patterns do not match", + Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher3) == 0); + assertTrue("RowKey matcher patterns do not match", + Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher4) == 0); return rowKeyMatcher1; } @@ -808,6 +856,15 @@ public void testViewsWithExtendedPK() { List testCases = getTestCases(); SortOrder[][] sortOrders = getSortOrders(); + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) WHERE TABLE_SCHEM <> 'SYSTEM' AND ROW_KEY_MATCHER IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + conn.commit(); + } + String tableName = ""; tableName = createViewHierarchy( testCases, sortOrders, 500, 5000, 3, @@ -879,9 +936,20 @@ public void testViewsWithVariousTenantIdTypes() { @Test public void testViewsWithoutExtendedPK() { try { + List testCases = getTestCases(); SortOrder[][] sortOrders = getSortOrders(); String tableName = ""; + + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE IS NOT NULL AND LINK_TYPE IS NULL AND ROW_KEY_MATCHER IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + conn.commit(); + } + tableName = createViewHierarchy( testCases, sortOrders, 100, 1000, 3, diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java index ddf1823a45e..1448c2343f5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java @@ -13,12 +13,10 @@ import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PTable; -import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TableViewFinderResult; -import org.apache.phoenix.util.TestUtil; import org.apache.phoenix.util.ViewUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -52,7 +50,10 @@ public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT { static final int VIEW_TTL_300_SECS = 300; static final int VIEW_TTL_120_SECS = 120; - static final String SYS_CATALOG_VIEW_HEADER_SQL = "SELECT TTL FROM SYSTEM.CATALOG " + static final String SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL = "SELECT ROW_KEY_MATCHER FROM SYSTEM.CATALOG " + + "WHERE %s AND TABLE_SCHEM <> 'SYSTEM' AND TABLE_NAME = '%s' AND " + "ROW_KEY_MATCHER IS NOT NULL"; + + static final String SYS_CATALOG_VIEW_TTL_HEADER_SQL = "SELECT TTL FROM SYSTEM.CATALOG " + "WHERE %s AND TABLE_SCHEM = '%s' AND TABLE_NAME = '%s' AND TABLE_TYPE = 'v'"; static final String SYS_CATALOG_VIEW_INDEX_HEADER_SQL = "SELECT VIEW_INDEX_ID FROM SYSTEM.CATALOG " @@ -74,7 +75,7 @@ public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT { static final String SYS_CATALOG_IDX_VIEW_INDEX_HEADER_SQL = "SELECT \": DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID,VIEW_INDEX_ID_DATA_TYPE)\" FROM %s " + "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'" ; - private static RegionCoprocessorEnvironment TaskRegionEnvironment; + private static RegionCoprocessorEnvironment taskRegionEnvironment; @BeforeClass public static void doSetup() throws Exception { @@ -96,7 +97,7 @@ public static void doSetup() throws Exception { setUpTestDriver(new ReadOnlyProps(ReadOnlyProps.EMPTY_PROPS, DEFAULT_PROPERTIES.entrySet().iterator())); - TaskRegionEnvironment = + taskRegionEnvironment = getUtility() .getRSForFirstRegionInTable( PhoenixDatabaseMetaData.SYSTEM_TASK_HBASE_TABLE_NAME) @@ -105,7 +106,6 @@ public static void doSetup() throws Exception { .findCoprocessorEnvironment(TaskRegionObserver.class.getName()); - } @@ -161,7 +161,7 @@ void assertSystemCatalogHasViewHeaderRelatedColumns(String tenantId, String sche "TENANT_ID IS NULL" : String.format("TENANT_ID = '%s'", tenantId); String sql = String - .format(SYS_CATALOG_VIEW_HEADER_SQL, tenantClause, schemaName, tableName); + .format(SYS_CATALOG_VIEW_TTL_HEADER_SQL, tenantClause, schemaName, tableName); stmt.execute(sql); ResultSet rs = stmt.getResultSet(); if (exists) { @@ -177,6 +177,30 @@ void assertSystemCatalogHasViewHeaderRelatedColumns(String tenantId, String sche } } + void assertSystemCatalogHasRowKeyMatcherRelatedColumns(String tenantId, String schemaName, + String tableName, boolean exists) throws SQLException { + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String tenantClause = tenantId == null || tenantId.isEmpty() ? + "TENANT_ID IS NULL" : + String.format("TENANT_ID = '%s'", tenantId); + String sql = String + .format(SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL, tenantClause, tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + if (exists) { + byte[] matcherBytes = rs.next() ? rs.getBytes(1) : null; + assertNotNull(String.format("Expected rows do not match for schema = %s, table = %s", + schemaName, tableName), matcherBytes); + } else { + assertFalse(String.format("Rows do exists for schema = %s, table = %s", + schemaName, tableName), rs.next()); + + } + } + } + String stripQuotes(String name) { return name.replace("\"", ""); } @@ -291,9 +315,10 @@ void dropTableWithChildViews(String baseTable, int numTaskRuns) throws Exception // Run DropChildViewsTask to complete the tasks for dropping child views. The depth of the view tree is 2, // so we expect that this will be done in two task handling runs as each non-root level will be processed // in one run + TaskRegionObserver.SelfHealingTask task = new TaskRegionObserver.SelfHealingTask( - TaskRegionEnvironment, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS); + taskRegionEnvironment, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS); for (int i = 0; i < numTaskRuns; i++) { task.run(); } @@ -542,12 +567,15 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { // Assert index header rows (link_type IS NULL AND TABLE_TYPE = 'i') exists in SYSTEM. CATALOG assertSystemCatalogHasViewIndexHeaderRelatedColumns(tenantId, schemaName, indexOnTenantViewName,true); + assertSystemCatalogHasRowKeyMatcherRelatedColumns(tenantId, schemaName, tenantViewName,true); try (Connection conn = DriverManager.getConnection(getUrl()); Statement stmt = conn.createStatement()) { //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL"); stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + conn.commit(); } @@ -558,40 +586,50 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { // Assert System Catalog index table has been created assertSystemCatalogIndexTable("SYS_VIEW_HDR_IDX", true); assertSystemCatalogIndexTable("SYS_VIEW_INDEX_HDR_IDX", true); + assertSystemCatalogIndexTable("SYS_ROW_KEY_MATCHER_IDX", true); // Assert appropriate rows are inserted in the SYSTEM.CATALOG index tables assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_VIEW_HDR_IDX", tenantId, schemaName, tenantViewName, true); + assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_ROW_KEY_MATCHER_IDX", tenantId, schemaName, tenantViewName, true); assertSystemCatalogIndexHaveViewIndexHeaders("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", tenantId, schemaName, indexOnTenantViewName, true); /** * Testing explain plans */ - List plans = getExplain("select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'", new Properties()); + List plans = getExplain("select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v' ", new Properties()); assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_HDR_IDX", plans.get(0)); - plans = getExplain("select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL", new Properties()); + plans = getExplain("select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL", new Properties()); assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_INDEX_HDR_IDX", plans.get(0)); + plans = getExplain("select ROW_KEY_MATCHER, TTL, TABLE_NAME FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND ROW_KEY_MATCHER IS NOT NULL", new Properties()); + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER SYSTEM.SYS_ROW_KEY_MATCHER_IDX [not null]", plans.get(0)); + /** * Testing cleanup of SYS_INDEX rows after dropping tables and views */ LOGGER.info("Dropping base table " + fullBaseTableName); dropTableWithChildViews(fullBaseTableName, 1); - // Assert view header rows (link_type IS NULL AND TABLE_TYPE = 'v') does not exists in SYSTEM.CATALOG + // Assert view header rows (link_type IS NULL AND TABLE_TYPE = 'v') does not exist in SYSTEM.CATALOG assertSystemCatalogHasViewHeaderRelatedColumns(tenantId, schemaName, tenantViewName, false, VIEW_TTL_120_SECS); + // Assert view header rows (ROW_KEY_MATCHER IS NOT NULL does not exist in SYSTEM.CATALOG + assertSystemCatalogHasRowKeyMatcherRelatedColumns(tenantId, schemaName, tenantViewName,false); // Assert index header rows (link_type IS NULL AND TABLE_TYPE = 'i') does not exists in SYSTEM.CATALOG assertSystemCatalogHasViewIndexHeaderRelatedColumns(tenantId, schemaName, tenantViewName,false); // Assert appropriate rows are dropped/deleted in the SYSTEM.CATALOG index tables assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_VIEW_HDR_IDX", tenantId, schemaName, tenantViewName, false); + assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_ROW_KEY_MATCHER_IDX", tenantId, schemaName, tenantViewName, false); assertSystemCatalogIndexHaveViewIndexHeaders("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", tenantId, schemaName, tenantViewName, false); dropSystemCatalogIndex("SYS_VIEW_HDR_IDX"); + dropSystemCatalogIndex("SYS_ROW_KEY_MATCHER_IDX"); dropSystemCatalogIndex("SYS_VIEW_INDEX_HDR_IDX"); // Assert System Catalog index table dropped assertSystemCatalogIndexTable("SYSTEM.SYS_VIEW_HDR_IDX", false); + assertSystemCatalogIndexTable("SYSTEM.SYS_ROW_KEY_MATCHER_IDX", false); assertSystemCatalogIndexTable("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", false); }