From 78615c3fbe027a14e29b13546ec8aad5877f842e Mon Sep 17 00:00:00 2001 From: Jacob Isaac Date: Wed, 9 Oct 2024 18:42:31 -0700 Subject: [PATCH 1/8] Initial checkin for SYSTEM indexes --- .../phoenix/compile/CreateIndexCompiler.java | 34 + .../phoenix/exception/SQLExceptionCode.java | 3 + .../phoenix/expression/ExpressionType.java | 3 +- .../function/DecodeViewIndexIdFunction.java | 166 +++++ .../apache/phoenix/index/IndexMaintainer.java | 3 + .../index/IndexMetaDataCacheClient.java | 2 + .../phoenix/parse/CreateTableStatement.java | 7 +- .../parse/DecodeViewIndexIdParseNode.java | 79 +++ .../query/ConnectionQueryServicesImpl.java | 22 +- .../apache/phoenix/schema/MetaDataClient.java | 1 + .../ChildLinkMetaDataEndpoint.java | 7 +- .../coprocessor/MetaDataEndpointImpl.java | 114 +++- .../coprocessor/TaskMetaDataEndpoint.java | 7 +- .../index/PartialSystemCatalogIndexIT.java | 598 ++++++++++++++++++ .../parse/DecodeViewIndexIdFunctionTest.java | 59 ++ 15 files changed, 1069 insertions(+), 36 deletions(-) create mode 100644 phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DecodeViewIndexIdFunction.java create mode 100644 phoenix-core-client/src/main/java/org/apache/phoenix/parse/DecodeViewIndexIdParseNode.java create mode 100644 phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java create mode 100644 phoenix-core/src/test/java/org/apache/phoenix/parse/DecodeViewIndexIdFunctionTest.java diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java index c98f9a6a5c6..72a4173e836 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java @@ -19,6 +19,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CompareOperator; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Pair; @@ -31,7 +32,9 @@ import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; +import org.apache.phoenix.parse.ComparisonParseNode; import org.apache.phoenix.parse.CreateIndexStatement; +import org.apache.phoenix.parse.LiteralParseNode; import org.apache.phoenix.parse.ParseNode; import org.apache.phoenix.parse.StatelessTraverseAllParseNodeVisitor; import org.apache.phoenix.parse.SubqueryParseNode; @@ -87,12 +90,35 @@ public CreateIndexCompiler(PhoenixStatement statement, Operation operation) { */ private static class IndexWhereParseNodeVisitor extends StatelessTraverseAllParseNodeVisitor { private boolean hasSubquery = false; + private boolean hasExcludedSystemSchema = false; @Override public Void visit(SubqueryParseNode node) throws SQLException { hasSubquery = true; return null; } + + @Override + public boolean visitEnter(ComparisonParseNode node) throws SQLException { + if (node.getFilterOp() == CompareOperator.NOT_EQUAL) { + boolean hasTableSchemColumn = node.getLHS().toString().equals(PhoenixDatabaseMetaData.TABLE_SCHEM) || + node.getRHS().toString().equals(PhoenixDatabaseMetaData.TABLE_SCHEM); + boolean hasSystemLiteralNode = false; + if (node.getLHS().getClass().isAssignableFrom(LiteralParseNode.class)) { + hasSystemLiteralNode = + SchemaUtil.normalizeLiteral((LiteralParseNode) node.getLHS()) + .equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME); + } + if (node.getRHS().getClass().isAssignableFrom(LiteralParseNode.class)) { + hasSystemLiteralNode = + SchemaUtil.normalizeLiteral((LiteralParseNode) node.getRHS()) + .equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME); + } + hasExcludedSystemSchema = hasTableSchemColumn && hasSystemLiteralNode; + } + return true; + } + } private String getValue(PDataType type) { @@ -138,6 +164,14 @@ private void verifyIndexWhere(ParseNode indexWhere, StatementContext context, throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_INDEX_WHERE_WITH_SUBQUERY). build().buildException(); } + // Verify that index WHERE clause on SYSTEM.CATALOG table excludes the SYSTEM schema + // TABLE_SCHEM <> 'SYSTEM' + if (SchemaUtil.isMetaTable(dataTableName.getSchemaName(), dataTableName.getTableName()) + && !indexWhereParseNodeVisitor.hasExcludedSystemSchema) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_INDEX_WHERE_WITH_SYSTEM_CATALOG_ROWS_NOT_EXCLUDED). + build().buildException(); + } + // Verify that index WHERE clause can be evaluated on a single data table row diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index eb52fa0fba2..b7cecdcd873 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -113,6 +113,9 @@ public SQLException newException(SQLExceptionInfo info) { " Index where clause cannot include a subquery."), CANNOT_EVALUATE_INDEX_WHERE(304, "23102", "Invalid index where clause. It cannot be evaluated on a data table row."), + INVALID_INDEX_WHERE_WITH_SYSTEM_CATALOG_ROWS_NOT_EXCLUDED(305, "23103", + "Invalid index where clause. Index on SYSTEM.CATALOG should always exclude system catalog rows (TABLE_SCHEM <> 'SYSTEM')."), + /** * Invalid Cursor State (errorcode 04, sqlstate 24) */ diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/ExpressionType.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/ExpressionType.java index 1fb68d81ae6..453925ef5da 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/ExpressionType.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/ExpressionType.java @@ -202,7 +202,8 @@ public enum ExpressionType { BsonValueFunction(BsonValueFunction.class), PartitionIdFunction(PartitionIdFunction.class), DecodeBinaryFunction(DecodeBinaryFunction.class), - EncodeBinaryFunction(EncodeBinaryFunction.class); + EncodeBinaryFunction(EncodeBinaryFunction.class), + DecodeViewIdFunction(DecodeViewIndexIdFunction.class); ExpressionType(Class clazz) { this.clazz = clazz; diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DecodeViewIndexIdFunction.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DecodeViewIndexIdFunction.java new file mode 100644 index 00000000000..cf74d4be476 --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DecodeViewIndexIdFunction.java @@ -0,0 +1,166 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.expression.function; + +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.io.ImmutableBytesWritable; +import org.apache.phoenix.expression.Determinism; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.KeyValueColumnExpression; +import org.apache.phoenix.parse.FunctionParseNode; +import org.apache.phoenix.parse.FunctionParseNode.BuiltInFunction; +import org.apache.phoenix.parse.DecodeViewIndexIdParseNode; +import org.apache.phoenix.parse.PhoenixRowTimestampParseNode; +import org.apache.phoenix.schema.SortOrder; +import org.apache.phoenix.schema.tuple.Tuple; +import org.apache.phoenix.schema.types.PDataType; +import org.apache.phoenix.schema.types.PInteger; +import org.apache.phoenix.schema.types.PLong; +import org.apache.phoenix.schema.types.PSmallint; + +import java.sql.Types; +import java.util.List; + +import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.NULL_DATA_TYPE_VALUE; +import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN; + +/** + * Function to return the timestamp of the empty column which functions as the row timestamp. The + * result returned can be used for debugging(eg. using HBase shell), logging etc. + * Can also be used in sql predicates. + */ +@BuiltInFunction(name = DecodeViewIndexIdFunction.NAME, + nodeClass= DecodeViewIndexIdParseNode.class, + args = {@FunctionParseNode.Argument(allowedTypes = { PLong.class}), + @FunctionParseNode.Argument(allowedTypes = { PInteger.class}) + }) +public class DecodeViewIndexIdFunction extends ScalarFunction { + + public static final String NAME = "DECODE_VIEW_INDEX_ID"; + + public DecodeViewIndexIdFunction() { + } + + /** + * @param children An EMPTY_COLUMN key value expression injected thru + * {@link PhoenixRowTimestampParseNode#create create} + * will cause the empty column key value to be evaluated during scan filter processing. + */ + public DecodeViewIndexIdFunction(List children) { + super(children); + + // It takes 2 parameters - VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE. + if ((children.size() != 2) || !children.get(0).getClass().isAssignableFrom( + KeyValueColumnExpression.class) || !children.get(1).getClass().isAssignableFrom( + KeyValueColumnExpression.class)) { + throw new IllegalArgumentException( + "DecodeViewIndexIdFunction should only have a " + + "VIEW_INDEX_ID and a VIEW_INDEX_ID_DATA_TYPE key value expression." + ); + } + if (!(children.get(0).getDataType().equals(PLong.INSTANCE))) { + throw new IllegalArgumentException( + "DecodeViewIndexIdFunction should have an " + + "VIEW_INDEX_ID key value expression of type PLong" + ); + } + + if (!(children.get(1).getDataType().equals(PInteger.INSTANCE))) { + throw new IllegalArgumentException( + "DecodeViewIndexIdFunction should have an " + + "VIEW_INDEX_ID_DATA_TYPE key value expression of type PLong" + ); + } + } + + @Override + public String getName() { + return NAME; + } + + @Override + public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) { + if (tuple == null) { + return false; + } + + byte[] viewIndexIdCF = ((KeyValueColumnExpression) children.get(0)).getColumnFamily(); + byte[] viewIndexIdCQ = ((KeyValueColumnExpression) children.get(0)).getColumnQualifier(); + byte[] viewIndexIdTypeCF = ((KeyValueColumnExpression) children.get(1)).getColumnFamily(); + byte[] viewIndexIdTypeCQ = ((KeyValueColumnExpression) children.get(1)).getColumnQualifier(); + + Cell viewIndexIdCell = tuple.getValue(viewIndexIdCF, viewIndexIdCQ); + Cell viewIndexIdDataTypeCell = tuple.getValue(viewIndexIdTypeCF, viewIndexIdTypeCQ); + + + /* + This is combination of diff client created view index looks like: + client VIEW_INDEX_ID(Cell number of bytes) VIEW_INDEX_ID_DATA_TYPE + pre-4.15 2 bytes NULL + post-4.15[config smallint] 2 bytes 5(smallint) + post-4.15[config bigint] 8 bytes -5(bigint) + + VIEW_INDEX_ID_DATA_TYPE, VIEW_INDEX_ID(Cell representation of the data) + NULL, SMALLINT -> RETRIEVE AND CONVERT TO BIGINT + SMALLINT, SMALLINT -> RETRIEVE AND CONVERT TO BIGINT + BIGINT, BIGINT -> DO NOT CONVERT + + */ + + if (viewIndexIdCell != null) { + int type = NULL_DATA_TYPE_VALUE; + if (viewIndexIdDataTypeCell != null) { + type = (Integer) PInteger.INSTANCE.toObject( + viewIndexIdDataTypeCell.getValueArray(), + viewIndexIdDataTypeCell.getValueOffset(), + viewIndexIdDataTypeCell.getValueLength(), + PInteger.INSTANCE, + SortOrder.ASC); + } + + System.out.println("DecodeViewIndexIdFunction: Type: " + type); + ImmutableBytesWritable columnValue = + new ImmutableBytesWritable(CellUtil.cloneValue(viewIndexIdCell)); + if ((type == NULL_DATA_TYPE_VALUE || type == Types.SMALLINT) && (viewIndexIdCell.getValueLength() < + VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN)) { + byte[] newBytes = PLong.INSTANCE.toBytes(PSmallint.INSTANCE.toObject(columnValue.get())); + ptr.set(newBytes, 0, newBytes.length); + } else { + ptr.set(columnValue.get(), columnValue.getOffset(), columnValue.getLength()); + } + } + return true; + } + + @Override + public PDataType getDataType() { + return PLong.INSTANCE; + } + + @Override + public boolean isStateless() { + return false; + } + + @Override + public Determinism getDeterminism() { + return Determinism.PER_ROW; + } + +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java index 24645b29a93..380e1a0b8f4 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMaintainer.java @@ -1074,6 +1074,9 @@ public boolean shouldPrepareIndexMutations(Put dataRowState) { return true; } List cols = IndexUtil.readColumnsFromRow(dataRowState, getIndexWhereColumns()); + if (cols.isEmpty()) { + return false; + } // Cells should be sorted as they are searched using a binary search during expression // evaluation Collections.sort(cols, CellComparator.getInstance()); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java index 5eadc7d29d9..4b40eb96645 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/index/IndexMetaDataCacheClient.java @@ -39,6 +39,8 @@ import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.ScanUtil; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public class IndexMetaDataCacheClient { diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateTableStatement.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateTableStatement.java index 37376c985eb..8c08f0515d6 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateTableStatement.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/CreateTableStatement.java @@ -100,7 +100,12 @@ protected CreateTableStatement(TableName tableName, ListMultimap familyCounters, boolean noVerify) { this.tableName = tableName; this.props = props == null ? ImmutableListMultimap.>of() : props; - this.tableType = PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(tableName.getSchemaName()) ? PTableType.SYSTEM : tableType; + this.tableType = + (PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals( + tableName.getSchemaName()) && + (tableType == PTableType.TABLE || tableType == PTableType.SYSTEM) ? + PTableType.SYSTEM : + tableType); this.columns = columns == null ? ImmutableList.of() : ImmutableList.copyOf(columns); this.pkConstraint = pkConstraint == null ? PrimaryKeyConstraint.EMPTY : pkConstraint; this.splitNodes = splitNodes == null ? Collections.emptyList() : ImmutableList.copyOf(splitNodes); diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/parse/DecodeViewIndexIdParseNode.java b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/DecodeViewIndexIdParseNode.java new file mode 100644 index 00000000000..6b299de9aae --- /dev/null +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/parse/DecodeViewIndexIdParseNode.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.phoenix.parse; + +import org.apache.phoenix.compile.StatementContext; +import org.apache.phoenix.expression.Expression; +import org.apache.phoenix.expression.function.DecodeViewIndexIdFunction; +import org.apache.phoenix.expression.function.FunctionExpression; +import org.apache.phoenix.query.QueryConstants; +import org.apache.phoenix.util.IndexUtil; + +import java.sql.SQLException; +import java.util.List; + +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID; +import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE; + +public class DecodeViewIndexIdParseNode extends FunctionParseNode { + + DecodeViewIndexIdParseNode(String name, List children, + BuiltInFunctionInfo info) { + super(name, children, info); + // It takes 2 parameters - VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE. + if (children.size() != 2) { + throw new IllegalArgumentException( + "DecodeViewIndexIdParseNode should only have " + + "VIEW_INDEX_ID and VIEW_INDEX_ID_DATA_TYPE parse nodes." + ); + } + if (children.get(0).getClass().isAssignableFrom(ColumnParseNode.class) + && children.get(1).getClass().isAssignableFrom(ColumnParseNode.class) + && (!(((ColumnParseNode) children.get(0)).getName().equals(VIEW_INDEX_ID)) + || !(((ColumnParseNode) children.get(1)).getName().equals(VIEW_INDEX_ID_DATA_TYPE))) + ) { + throw new IllegalArgumentException( + "DecodeViewIndexIdParseNode should only have " + + "VIEW_INDEX_ID and VIEW_INDEX_ID_DATA_TYPE parse nodes." + ); + } + + // CastPastNode is generated during IndexStatement rewriting + if (children.get(0).getClass().isAssignableFrom(CastParseNode.class) + && children.get(1).getClass().isAssignableFrom(CastParseNode.class) + && (!((ColumnParseNode) (((CastParseNode) children.get(0)).getChildren().get(0))).getName().equals( + IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, VIEW_INDEX_ID)) + || !((ColumnParseNode) (((CastParseNode) children.get(1)).getChildren().get(0))).getName().equals( + IndexUtil.getIndexColumnName(QueryConstants.DEFAULT_COLUMN_FAMILY, VIEW_INDEX_ID_DATA_TYPE))) + ) { + throw new IllegalArgumentException( + "DecodeViewIndexIdParseNode should only have " + + "VIEW_INDEX_ID and VIEW_INDEX_ID_DATA_TYPE parse nodes." + ); + } + + } + + @Override + public FunctionExpression create(List children, StatementContext context) + throws SQLException { + return new DecodeViewIndexIdFunction(children); + } + +} diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index dba599f25cb..108eeb94607 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -1326,10 +1326,11 @@ private void addCoprocessors(byte[] tableName, TableDescriptorBuilder builder, // TODO: better encapsulation for this // Since indexes can't have indexes, don't install our indexing coprocessor for indexes. - // Also don't install on the SYSTEM.CATALOG and SYSTEM.STATS table because we use + // Also don't install on the SYSTEM.STATS table because we use // all-or-none mutate class which break when this coprocessor is installed (PHOENIX-1318). + // With PHOENIX-7107 which introduced indexes on SYSTEM.CATALOG we need to install the + // indexing coprocessor on SYSTEM.CATALOG if ((tableType != PTableType.INDEX && tableType != PTableType.VIEW && !isViewIndex) - && !SchemaUtil.isMetaTable(tableName) && !SchemaUtil.isStatsTable(tableName)) { if (isTransactional) { if (!newDesc.hasCoprocessor(QueryConstants.PHOENIX_TRANSACTIONAL_INDEXER_CLASSNAME)) { @@ -1759,8 +1760,23 @@ private TableDescriptor ensureTableCreated(byte[] physicalTableName, byte[] pare TableDescriptorBuilder newDesc = generateTableDescriptor(physicalTableName, parentPhysicalTableName, existingDesc, tableType, props, families, splits, isNamespaceMapped); + if (LOGGER.isInfoEnabled()) { + LOGGER.info(String.format("ensureTableCreated " + + "physicalTableName = %s, " + + "parentPhysicalTableName = %s, " + + "isUpgradeRequired = %s, " + + "isAutoUpgradeEnabled = %s, " + + "isDoNotUpgradePropSet = %s", + Bytes.toString(parentPhysicalTableName), + Bytes.toString(parentPhysicalTableName), + isUpgradeRequired(), + isAutoUpgradeEnabled, + isDoNotUpgradePropSet)); + } + + if (!tableExist) { - if (SchemaUtil.isSystemTable(physicalTableName) && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) { + if (SchemaUtil.isSystemTable(physicalTableName) && (tableType == PTableType.TABLE || tableType == PTableType.SYSTEM) && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) { // Disallow creating the SYSTEM.CATALOG or SYSTEM:CATALOG HBase table throw new UpgradeRequiredException(); } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java index 243b13ff78b..9180cf8d01a 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java @@ -32,6 +32,7 @@ import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME; import static org.apache.phoenix.query.QueryServices.INDEX_CREATE_DEFAULT_STATE; import static org.apache.phoenix.schema.PTableType.CDC; +import static org.apache.phoenix.schema.types.PDataType.NULL_BYTES; import static org.apache.phoenix.thirdparty.com.google.common.collect.Sets.newLinkedHashSet; import static org.apache.phoenix.thirdparty.com.google.common.collect.Sets.newLinkedHashSetWithExpectedSize; import static org.apache.phoenix.exception.SQLExceptionCode.INSUFFICIENT_MULTI_TENANT_COLUMNS; diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/ChildLinkMetaDataEndpoint.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/ChildLinkMetaDataEndpoint.java index 90f516c170d..8a86637b036 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/ChildLinkMetaDataEndpoint.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/ChildLinkMetaDataEndpoint.java @@ -95,8 +95,11 @@ public void createViewAddChildLink(RpcController controller, getCoprocessorHost().preCreateViewAddChildLink(fullparentTableName); // From 4.15 the parent->child links are stored in a separate table SYSTEM.CHILD_LINK - mutateRowsWithLocks(this.accessCheckEnabled, this.env.getRegion(), childLinkMutations, - Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE); + // Also check if SYSTEM indexes exists on SYSTEM.CHILD_LINK, if so set metadata attribs + // on child link mutations + mutateRowsWithLocks(this.accessCheckEnabled, env, this.env.getRegion(), childLinkMutations, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, + true, PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME); } catch (Throwable t) { LOGGER.error("Unable to write mutations to " + diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 49a55d78af1..1e17e0f62b4 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -84,6 +84,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA_TYPE_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT_BYTES; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE_BYTES; +import static org.apache.phoenix.query.QueryServices.INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB; import static org.apache.phoenix.query.QueryServices.SKIP_SYSTEM_TABLES_EXISTENCE_CHECK; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MAX_LOOKBACK_AGE_BYTES; import static org.apache.phoenix.schema.PTable.LinkType.PHYSICAL_TABLE; @@ -202,6 +203,7 @@ import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.hbase.index.util.KeyValueBuilder; import org.apache.phoenix.index.IndexMaintainer; +import org.apache.phoenix.index.IndexMetaDataCacheClient; import org.apache.phoenix.iterate.ResultIterator; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; @@ -269,6 +271,7 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixKeyValueUtil; +import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; @@ -605,6 +608,7 @@ public static PName newPName(byte[] keyBuffer, int keyOffset, int keyLength) { } private static boolean failConcurrentMutateAddColumnOneTimeForTesting = false; + private static final String FORCE_INDEX_MUTATE_METADATA_AS_ATTRIB = String.valueOf(Integer.MAX_VALUE); private RegionCoprocessorEnvironment env; private PhoenixMetaDataCoprocessorHost phoenixAccessCoprocessorHost; @@ -2367,6 +2371,7 @@ public void createTable(RpcController controller, CreateTableRequest request, schemaName = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX]; tableName = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX]; fullTableName = SchemaUtil.getTableName(schemaName, tableName); + boolean shouldCheckForSystemTableIndexes = !SchemaUtil.isSystemTable(Bytes.toBytes(fullTableName)); boolean isNamespaceMapped = MetaDataUtil.isNameSpaceMapped(tableMetadata, GenericKeyValueBuilder.INSTANCE, new ImmutableBytesWritable()); final IndexType indexType = MetaDataUtil.getIndexType(tableMetadata, GenericKeyValueBuilder.INSTANCE, @@ -2793,8 +2798,9 @@ public void createTable(RpcController controller, CreateTableRequest request, // primary and then index table locks are held, in that order). For now, we just don't support // indexing on the system table. This is an issue because of the way we manage batch mutation // in the Indexer. - mutateRowsWithLocks(this.accessCheckEnabled, region, localMutations, Collections.emptySet(), - HConstants.NO_NONCE, HConstants.NO_NONCE); + mutateRowsWithLocks(this.accessCheckEnabled, env, region, localMutations, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, + shouldCheckForSystemTableIndexes, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); // Invalidate the cache - the next getTable call will add it // TODO: consider loading the table that was just created here, patching up the parent table, and updating the cache @@ -3128,9 +3134,16 @@ public void dropTable(RpcController controller, DropTableRequest request, throw new IllegalStateException(msg); } + boolean + shouldCheckForSystemTableIndexes = + ((pTableType == INDEX) && (Bytes.toString(schemaName) + .equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA))) ? + false : + true; // drop rows from catalog on this region - mutateRowsWithLocks(this.accessCheckEnabled, region, localMutations, - Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE); + mutateRowsWithLocks(this.accessCheckEnabled, env, region, localMutations, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, + shouldCheckForSystemTableIndexes, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); long currentTime = MetaDataUtil.getClientTimeStamp(tableMetadata); for (ImmutableBytesPtr ckey : invalidateList) { @@ -3250,6 +3263,7 @@ private MetaDataResponse processRemoteRegionMutations(byte[] systemTableName, try (Table hTable = ServerUtil.getHTableForCoprocessorScan(env, SchemaUtil.getPhysicalTableName(systemTableName, env.getConfiguration()))) { + setMetaDataOnMutationsIfCatalogIndexExists(env, Bytes.toString(systemTableName), remoteMutations); hTable.batch(remoteMutations, null); } catch (Throwable t) { LOGGER.error("Unable to write mutations to " + Bytes.toString(systemTableName), t); @@ -3356,7 +3370,11 @@ private MetaDataMutationResult doDropTable(byte[] key, byte[] tenantId, byte[] s } Delete delete = new Delete(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), clientTimeStamp); - catalogMutations.add(delete); + if (Bytes.compareTo(key, 0, key.length, kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()) == 0) { + LOGGER.info("doDropTable:Found key {} ", Bytes.toStringBinary(key)); + } else { + catalogMutations.add(delete); + } results.clear(); scanner.next(results); } while (!results.isEmpty()); @@ -3735,8 +3753,9 @@ private MetaDataMutationResult mutateColumn( throw new IllegalStateException(msg); } } - mutateRowsWithLocks(this.accessCheckEnabled, region, localMutations, - Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE); + mutateRowsWithLocks(this.accessCheckEnabled, env, region, localMutations, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, + true, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); // Invalidate from cache for (ImmutableBytesPtr invalidateKey : invalidateList) { metaDataCache.invalidate(invalidateKey); @@ -4572,8 +4591,9 @@ public void updateIndexState(RpcController controller, UpdateIndexStateRequest r long serverTimestamp = EnvironmentEdgeManager.currentTimeMillis(); tableMetadata.add(MetaDataUtil.getLastDDLTimestampUpdate( key, clientTimeStamp, serverTimestamp)); - mutateRowsWithLocks(this.accessCheckEnabled, region, tableMetadata, Collections.emptySet(), - HConstants.NO_NONCE, HConstants.NO_NONCE); + mutateRowsWithLocks(this.accessCheckEnabled, env, region, tableMetadata, + Collections.emptySet(), HConstants.NO_NONCE, + HConstants.NO_NONCE, false, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); // Invalidate from cache Cache metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache(); @@ -4823,8 +4843,9 @@ public void createFunction(RpcController controller, CreateFunctionRequest reque } // Don't store function info for temporary functions. if (!temporaryFunction) { - mutateRowsWithLocks(this.accessCheckEnabled, region, functionMetaData, - Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE); + mutateRowsWithLocks(this.accessCheckEnabled, env, region, functionMetaData, + Collections.emptySet(), HConstants.NO_NONCE, + HConstants.NO_NONCE, false, PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME); } // Invalidate the cache - the next getFunction call will add it @@ -4880,8 +4901,9 @@ public void dropFunction(RpcController controller, DropFunctionRequest request, done.run(MetaDataMutationResult.toProto(result)); return; } - mutateRowsWithLocks(this.accessCheckEnabled, region, functionMetaData, Collections.emptySet(), - HConstants.NO_NONCE, HConstants.NO_NONCE); + mutateRowsWithLocks(this.accessCheckEnabled, env, region, functionMetaData, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, + false, PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME); Cache metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache(); long currentTime = MetaDataUtil.getClientTimeStamp(functionMetaData); @@ -5002,8 +5024,9 @@ public void createSchema(RpcController controller, CreateSchemaRequest request, return; } } - mutateRowsWithLocks(this.accessCheckEnabled, region, schemaMutations, Collections.emptySet(), - HConstants.NO_NONCE, HConstants.NO_NONCE); + mutateRowsWithLocks(this.accessCheckEnabled, env, region, schemaMutations, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, + false, null); // Invalidate the cache - the next getSchema call will add it Cache metaDataCache = @@ -5054,8 +5077,9 @@ public void dropSchema(RpcController controller, DropSchemaRequest request, RpcC done.run(MetaDataMutationResult.toProto(result)); return; } - mutateRowsWithLocks(this.accessCheckEnabled, region, schemaMetaData, Collections.emptySet(), - HConstants.NO_NONCE, HConstants.NO_NONCE); + mutateRowsWithLocks(this.accessCheckEnabled, env, region, schemaMetaData, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, + false, null); Cache metaDataCache = GlobalCache.getInstance(this.env) .getMetaDataCache(); long currentTime = MetaDataUtil.getClientTimeStamp(schemaMetaData); @@ -5124,18 +5148,30 @@ private MetaDataMutationResult doDropSchema(long clientTimeStamp, String schemaN /** * Perform atomic mutations on rows within a region - * * @param accessCheckEnabled Use the login user to mutate rows if enabled - * @param region Region containing rows to be mutated - * @param mutations List of mutations for rows that must be contained within the region - * @param rowsToLock Rows to lock - * @param nonceGroup Optional nonce group of the operation - * @param nonce Optional nonce of the operation + * @param env + * @param region Region containing rows to be mutated + * @param mutations List of mutations for rows that must be contained within the + * region + * @param rowsToLock Rows to lock + * @param nonceGroup Optional nonce group of the operation + * @param nonce Optional nonce of the operation + * @param checkIfSystemCatalogIndexesExists check if Catalog indexes exists + * @param systemTableName The SYSTEM table to apply index mutation * @throws IOException */ - static void mutateRowsWithLocks(final boolean accessCheckEnabled, final Region region, + static void mutateRowsWithLocks(final boolean accessCheckEnabled, + final RegionCoprocessorEnvironment env, final Region region, final List mutations, final Set rowsToLock, final long nonceGroup, - final long nonce) throws IOException { + final long nonce, boolean checkIfSystemCatalogIndexesExists, String systemTableName) throws IOException { + + try { + if ((checkIfSystemCatalogIndexesExists) && (systemTableName != null && !systemTableName.isEmpty())) { + setMetaDataOnMutationsIfCatalogIndexExists(env, systemTableName, mutations ); + } + } catch (SQLException e) { + throw new IOException(e); + } // We need to mutate SYSTEM.CATALOG or SYSTEM.CHILD_LINK with HBase/login user // if access is enabled. if (accessCheckEnabled) { @@ -5161,6 +5197,32 @@ public Void run() throws Exception { } } + private static void setMetaDataOnMutationsIfCatalogIndexExists( + final RegionCoprocessorEnvironment env, String forSystemTableName, final List mutations) + throws SQLException { + Properties metaConnectionProps = new Properties(); + metaConnectionProps.setProperty(INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, FORCE_INDEX_MUTATE_METADATA_AS_ATTRIB); + PhoenixConnection connection = getServerConnectionForMetaData(metaConnectionProps, env.getConfiguration()) + .unwrap(PhoenixConnection.class); + if (PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME.compareToIgnoreCase(forSystemTableName) == 0) { + PTable systemCatalogPTable = PhoenixRuntime.getTableNoCache(connection, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); + if ((systemCatalogPTable != null) && (!systemCatalogPTable.getIndexes().isEmpty())) { + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + IndexMaintainer.serialize(systemCatalogPTable, ptr, connection); + IndexMetaDataCacheClient.setMetaDataOnMutations(connection, systemCatalogPTable, mutations, ptr); + } + } + + if (PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME.compareToIgnoreCase(forSystemTableName) == 0) { + PTable childLinkCatalogPTable = PhoenixRuntime.getTableNoCache(connection, PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME); + if ((childLinkCatalogPTable != null) && (!childLinkCatalogPTable.getIndexes().isEmpty())) { + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + IndexMaintainer.serialize(childLinkCatalogPTable, ptr, connection); + IndexMetaDataCacheClient.setMetaDataOnMutations(connection, childLinkCatalogPTable, mutations, ptr); + } + } + } + private TableName getParentPhysicalTableName(PTable table) { return (table .getType() == PTableType.VIEW || (table.getType() == INDEX && table.getViewIndexId() != null)) @@ -5186,7 +5248,7 @@ private TableName getParentPhysicalTableName(PTable table) { * @return Connection object. * @throws SQLException If the Connection could not be retrieved. */ - private static Connection getServerConnectionForMetaData(final Configuration config) + private static Connection getServerConnectionForMetaData(final Configuration config) throws SQLException { Preconditions.checkNotNull(config, "The configs must not be null"); return getServerConnectionForMetaData(new Properties(), config); diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TaskMetaDataEndpoint.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TaskMetaDataEndpoint.java index bd6d0bd1c84..6fdbb49e543 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TaskMetaDataEndpoint.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TaskMetaDataEndpoint.java @@ -105,9 +105,10 @@ public void upsertTaskDetails(RpcController controller, phoenixAccessCoprocessorHost.preUpsertTaskDetails(fullTableName); - mutateRowsWithLocks(this.accessCheckEnabled, this.env.getRegion(), - taskMutations, Collections.emptySet(), HConstants.NO_NONCE, - HConstants.NO_NONCE); + // We do not enable indexes on SYSTEM.TASK yet, so no need to check for it. + mutateRowsWithLocks(this.accessCheckEnabled, env, this.env.getRegion(), taskMutations, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, + false, PhoenixDatabaseMetaData.SYSTEM_TASK_NAME); } catch (Throwable t) { LOGGER.error("Unable to write mutations to {}", PhoenixDatabaseMetaData.SYSTEM_TASK_NAME, t); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java new file mode 100644 index 00000000000..ddf1823a45e --- /dev/null +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java @@ -0,0 +1,598 @@ +package org.apache.phoenix.end2end.index; + +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.phoenix.coprocessor.TaskRegionObserver; +import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.end2end.LocalHBaseIT; +import org.apache.phoenix.end2end.ParallelStatsDisabledIT; +import org.apache.phoenix.end2end.ViewTTLIT; +import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.query.PhoenixTestBuilder; +import org.apache.phoenix.query.QueryServices; +import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.schema.PTable; +import org.apache.phoenix.schema.PTableType; +import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; +import org.apache.phoenix.util.ReadOnlyProps; +import org.apache.phoenix.util.SchemaUtil; +import org.apache.phoenix.util.TableViewFinderResult; +import org.apache.phoenix.util.TestUtil; +import org.apache.phoenix.util.ViewUtil; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.COLUMN_TYPES; +import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.TENANT_VIEW_COLUMNS; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT { + static final Logger LOGGER = LoggerFactory.getLogger(ViewTTLIT.class); + static final int VIEW_TTL_10_SECS = 10; + static final int VIEW_TTL_300_SECS = 300; + static final int VIEW_TTL_120_SECS = 120; + + static final String SYS_CATALOG_VIEW_HEADER_SQL = "SELECT TTL FROM SYSTEM.CATALOG " + + "WHERE %s AND TABLE_SCHEM = '%s' AND TABLE_NAME = '%s' AND TABLE_TYPE = 'v'"; + + static final String SYS_CATALOG_VIEW_INDEX_HEADER_SQL = "SELECT VIEW_INDEX_ID FROM SYSTEM.CATALOG " + + "WHERE %s AND TABLE_SCHEM = '%s' AND TABLE_NAME = '%s' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL"; + + static final String SYS_CATALOG_SYS_INDEX_TABLE_SQL = "SELECT count(*) FROM SYSTEM.CATALOG " + + "WHERE TABLE_SCHEM = 'SYSTEM' AND TABLE_NAME = '%s'"; + + static final String SYS_CATALOG_INDEX_TABLE_LINK_SQL = "SELECT count(*) FROM SYSTEM.CATALOG " + + "WHERE %s AND TABLE_SCHEM = '%s' AND TABLE_NAME = '%s' AND TABLE_TYPE = 'i'" + + " AND LINK_TYPE = 1"; + + static final String SYS_CATALOG_IDX_INDEX_TABLE_LINK_SQL = "SELECT \":COLUMN_FAMILY\" FROM %s " + + "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'" ; + + static final String SYS_CATALOG_IDX_VIEW_HEADER_SQL = "SELECT \"0:VIEW_STATEMENT\" FROM %s " + + "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'" ; + + static final String SYS_CATALOG_IDX_VIEW_INDEX_HEADER_SQL = "SELECT \": DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID,VIEW_INDEX_ID_DATA_TYPE)\" FROM %s " + + "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'" ; + + private static RegionCoprocessorEnvironment TaskRegionEnvironment; + + @BeforeClass + public static void doSetup() throws Exception { + // Turn on the View TTL feature + Map DEFAULT_PROPERTIES = new HashMap() {{ + put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true)); + put(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(true)); + put("hbase.procedure.remote.dispatcher.delay.msec", "0"); + // no max lookback + put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(0)); + put(QueryServices.PHOENIX_VIEW_TTL_ENABLED, Boolean.toString(true)); + put(QueryServices.PHOENIX_VIEW_TTL_TENANT_VIEWS_PER_SCAN_LIMIT, String.valueOf(1)); + put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, + Long.toString(Long.MAX_VALUE)); + put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, + Long.toString(Long.MAX_VALUE)); + }}; + + setUpTestDriver(new ReadOnlyProps(ReadOnlyProps.EMPTY_PROPS, + DEFAULT_PROPERTIES.entrySet().iterator())); + + TaskRegionEnvironment = + getUtility() + .getRSForFirstRegionInTable( + PhoenixDatabaseMetaData.SYSTEM_TASK_HBASE_TABLE_NAME) + .getRegions(PhoenixDatabaseMetaData.SYSTEM_TASK_HBASE_TABLE_NAME) + .get(0).getCoprocessorHost() + .findCoprocessorEnvironment(TaskRegionObserver.class.getName()); + + + + } + + + void assertSystemCatalogHasIndexTableLinks(String tenantId, String schemaName, + String tableName) throws SQLException { + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String tenantClause = tenantId == null || tenantId.isEmpty() ? + "TENANT_ID IS NULL" : + String.format("TENANT_ID = '%s'", tenantId); + String sql = String + .format(SYS_CATALOG_INDEX_TABLE_LINK_SQL, tenantClause, schemaName, tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + int numRows = rs.next() ? rs.getInt(1) : 0; + + assertEquals(String.format("Expected rows do not match for schema = %s, table = %s", + schemaName, tableName), 1, numRows); + } + } + + void assertSystemCatalogHasViewIndexHeaderRelatedColumns(String tenantId, String schemaName, + String tableName, boolean exists) throws SQLException { + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String tenantClause = tenantId == null || tenantId.isEmpty() ? + "TENANT_ID IS NULL" : + String.format("TENANT_ID = '%s'", tenantId); + String sql = String + .format(SYS_CATALOG_VIEW_INDEX_HEADER_SQL, tenantClause, schemaName, tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + String viewIndexId = rs.next() ? rs.getString(1) : null; + if (exists) { + assertNotNull(String.format("Expected rows do not match for schema = %s, table = %s", + schemaName, tableName), viewIndexId); + } else { + assertNull(String.format("Expected rows do not match for schema = %s, table = %s", + schemaName, tableName), viewIndexId); + } + + } + } + + void assertSystemCatalogHasViewHeaderRelatedColumns(String tenantId, String schemaName, + String tableName, boolean exists, long ttlValueExpected) throws SQLException { + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String tenantClause = tenantId == null || tenantId.isEmpty() ? + "TENANT_ID IS NULL" : + String.format("TENANT_ID = '%s'", tenantId); + String sql = String + .format(SYS_CATALOG_VIEW_HEADER_SQL, tenantClause, schemaName, tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + if (exists) { + String ttlStr = rs.next() ? rs.getString(1) : null; + long actualTTLValueReturned = ttlStr != null ? Integer.valueOf(ttlStr): 0; + assertEquals(String.format("Expected rows do not match for schema = %s, table = %s", + schemaName, tableName), ttlValueExpected, actualTTLValueReturned); + } else { + assertFalse(String.format("Rows do exists for schema = %s, table = %s", + schemaName, tableName), rs.next()); + + } + } + } + + String stripQuotes(String name) { + return name.replace("\"", ""); + } + + void assertSystemCatalogIndexTable(String systemCatalogIndexName, boolean exists) throws SQLException { + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String sql = String.format(SYS_CATALOG_SYS_INDEX_TABLE_SQL, systemCatalogIndexName, + systemCatalogIndexName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + rs.next(); + assertTrue(String.format("Expected rows do not match for index-table = SYSTEM.%s", + systemCatalogIndexName), exists ? rs.getInt(1) > 0 : rs.getInt(1) == 0 ); + } + } + + + void assertSystemCatalogIndexHaveIndexTableLinks(String systemCatalogIndexName, + String tenantId, String schemaName, + String tableName, boolean exists, String indexName) throws SQLException { + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String tenantClause = tenantId == null || tenantId.isEmpty() ? + "\":TENANT_ID\" IS NULL" : + String.format("\":TENANT_ID\" = '%s'", tenantId); + String sql = String.format(SYS_CATALOG_IDX_INDEX_TABLE_LINK_SQL, systemCatalogIndexName, + tenantClause, schemaName, tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + String colFamilyStr = rs.next() ? rs.getString(1) : null; + if (exists) { + assertEquals(String.format("Expected rows do not match for schema = %s, table = %s", + schemaName, tableName), indexName, colFamilyStr); + } else { + assertNull(String.format("Zero rows expected for schema = %s, table = %s", + schemaName, tableName), colFamilyStr); + } + } + } + + void assertSystemCatalogIndexHaveViewHeaders(String systemCatalogIndexName, + String tenantId, String schemaName, + String tableName, boolean exists) throws SQLException { + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String tenantClause = tenantId == null || tenantId.isEmpty() ? + "\":TENANT_ID\" IS NULL" : + String.format("\":TENANT_ID\" = '%s'", tenantId); + String sql = String.format(SYS_CATALOG_IDX_VIEW_HEADER_SQL, systemCatalogIndexName, + tenantClause, schemaName, tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + String viewStmt = rs.next() ? rs.getString(1) : null; + if (exists) { + assertNotNull(String.format("Expected rows do not match for schema = %s, table = %s", + schemaName, tableName), viewStmt); + } else { + assertNull(String.format("Zero rows expected for schema = %s, table = %s", + schemaName, tableName), viewStmt); + } + } + } + + + void assertSystemCatalogIndexHaveViewIndexHeaders(String systemCatalogIndexName, + String tenantId, String schemaName, + String tableName, boolean exists) throws SQLException { + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String tenantClause = tenantId == null || tenantId.isEmpty() ? + "\":TENANT_ID\" IS NULL" : + String.format("\":TENANT_ID\" = '%s'", tenantId); + String sql = String.format(SYS_CATALOG_IDX_VIEW_INDEX_HEADER_SQL, systemCatalogIndexName, + tenantClause, schemaName, tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + Integer viewIndexId = rs.next() ? rs.getInt(1) : null; + if (exists) { + assertNotNull(String.format("Expected rows do not match for schema = %s, table = %s", + schemaName, tableName), viewIndexId); + } else { + assertNull(String.format("Zero rows expected for schema = %s, table = %s", + schemaName, tableName), viewIndexId); + } + } + } + + void dropSystemCatalogIndex(String sysIndexName) throws SQLException { + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + stmt.execute(String.format("drop index %s ON SYSTEM.CATALOG", sysIndexName)); + conn.commit(); + } + } + + void dropTableWithChildViews(String baseTable, int numTaskRuns) throws Exception { + // Drop the base table + + try (Connection conn = DriverManager.getConnection(getUrl())) { + conn.setAutoCommit(true); + // Empty the task table first. + conn.createStatement() + .execute("DELETE " + " FROM " + PhoenixDatabaseMetaData.SYSTEM_TASK_NAME); + + String dropTableSQL = String.format("DROP TABLE IF EXISTS %s CASCADE", baseTable); + conn.createStatement().execute(dropTableSQL); + // Run DropChildViewsTask to complete the tasks for dropping child views. The depth of the view tree is 2, + // so we expect that this will be done in two task handling runs as each non-root level will be processed + // in one run + TaskRegionObserver.SelfHealingTask task = + new TaskRegionObserver.SelfHealingTask( + TaskRegionEnvironment, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS); + for (int i = 0; i < numTaskRuns; i++) { + task.run(); + } + + assertTaskColumns(conn, PTable.TaskStatus.COMPLETED.toString(), PTable.TaskType.DROP_CHILD_VIEWS, + null, null, null, null, null); + + // Views should be dropped by now + TableName linkTable = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES); + TableViewFinderResult childViewsResult = new TableViewFinderResult(); + ViewUtil.findAllRelatives(getUtility().getConnection().getTable(linkTable), + HConstants.EMPTY_BYTE_ARRAY, + SchemaUtil.getSchemaNameFromFullName(baseTable).getBytes(), + SchemaUtil.getTableNameFromFullName(baseTable).getBytes(), + PTable.LinkType.CHILD_TABLE, + childViewsResult); + assertEquals(0, childViewsResult.getLinks().size()); + } + + + } + + static void assertTaskColumns(Connection conn, String expectedStatus, PTable.TaskType taskType, + String expectedTableName, String expectedTenantId, String expectedSchema, Timestamp expectedTs, + String expectedIndexName) + throws SQLException { + ResultSet rs = conn.createStatement().executeQuery("SELECT * " + + " FROM " + PhoenixDatabaseMetaData.SYSTEM_TASK_NAME + + " WHERE " + PhoenixDatabaseMetaData.TASK_TYPE + " = " + + taskType.getSerializedValue()); + assertTrue(rs.next()); + String taskStatus = rs.getString(PhoenixDatabaseMetaData.TASK_STATUS); + assertEquals(expectedStatus, taskStatus); + + if (expectedTableName != null) { + String tableName = rs.getString(PhoenixDatabaseMetaData.TABLE_NAME); + assertEquals(expectedTableName, tableName); + } + + if (expectedTenantId != null) { + String tenantId = rs.getString(PhoenixDatabaseMetaData.TENANT_ID); + assertEquals(expectedTenantId, tenantId); + } + + if (expectedSchema != null) { + String schema = rs.getString(PhoenixDatabaseMetaData.TABLE_SCHEM); + assertEquals(expectedSchema, schema); + } + + if (expectedTs != null) { + Timestamp ts = rs.getTimestamp(PhoenixDatabaseMetaData.TASK_TS); + assertEquals(expectedTs, ts); + } + + if (expectedIndexName != null) { + String data = rs.getString(PhoenixDatabaseMetaData.TASK_DATA); + assertEquals(true, data.contains("\"IndexName\":\"" + expectedIndexName)); + } + } + + private List getExplain(String query, Properties props) throws SQLException { + List explainPlan = new ArrayList<>(); + try(Connection conn = DriverManager.getConnection(getUrl(), props); + PreparedStatement statement = conn.prepareStatement("EXPLAIN " + query); + ResultSet rs = statement.executeQuery()) { + while(rs.next()) { + String plan = rs.getString(1); + explainPlan.add(plan); + } + } + return explainPlan; + } + + + protected PhoenixTestBuilder.SchemaBuilder createLevel2TenantViewWithGlobalLevelTTL( + int globalTTL, + PhoenixTestBuilder.SchemaBuilder.TenantViewOptions tenantViewOptions, + PhoenixTestBuilder.SchemaBuilder.TenantViewIndexOptions tenantViewIndexOptions, + boolean allowIndex) throws Exception { + // Define the test schema. + // 1. Table with columns => (ORG_ID, KP, COL1, COL2, COL3), PK => (ORG_ID, KP) + // 2. GlobalView with columns => (ID, COL4, COL5, COL6), PK => (ID) + // 3. Tenant with columns => (ZID, COL7, COL8, COL9), PK => (ZID) + final PhoenixTestBuilder.SchemaBuilder schemaBuilder = new PhoenixTestBuilder.SchemaBuilder(getUrl()); + + PhoenixTestBuilder.SchemaBuilder.TableOptions + tableOptions = PhoenixTestBuilder.SchemaBuilder.TableOptions.withDefaults(); + tableOptions.setTableProps("COLUMN_ENCODED_BYTES=0,MULTI_TENANT=true"); + + PhoenixTestBuilder.SchemaBuilder.GlobalViewOptions + globalViewOptions = PhoenixTestBuilder.SchemaBuilder.GlobalViewOptions.withDefaults(); + // View TTL is set to 300s => 300000 ms + globalViewOptions.setTableProps(String.format("TTL=%d", globalTTL)); + + PhoenixTestBuilder.SchemaBuilder.GlobalViewIndexOptions globalViewIndexOptions + = PhoenixTestBuilder.SchemaBuilder.GlobalViewIndexOptions.withDefaults(); + globalViewIndexOptions.setLocal(false); + + PhoenixTestBuilder.SchemaBuilder.TenantViewOptions + tenantViewWithOverrideOptions = PhoenixTestBuilder.SchemaBuilder.TenantViewOptions.withDefaults(); + if (tenantViewOptions != null) { + tenantViewWithOverrideOptions = tenantViewOptions; + } + PhoenixTestBuilder.SchemaBuilder.TenantViewIndexOptions + tenantViewIndexOverrideOptions = PhoenixTestBuilder.SchemaBuilder.TenantViewIndexOptions.withDefaults(); + if (tenantViewIndexOptions != null) { + tenantViewIndexOverrideOptions = tenantViewIndexOptions; + } + if (allowIndex) { + schemaBuilder.withTableOptions(tableOptions) + .withGlobalViewOptions(globalViewOptions) + .withGlobalViewIndexOptions(globalViewIndexOptions) + .withTenantViewOptions(tenantViewWithOverrideOptions) + .withTenantViewIndexOptions(tenantViewIndexOverrideOptions) + .buildWithNewTenant(); + } else { + schemaBuilder.withTableOptions(tableOptions) + .withGlobalViewOptions(globalViewOptions) + .withTenantViewOptions(tenantViewWithOverrideOptions) + .buildWithNewTenant(); + } + return schemaBuilder; + } + + protected PhoenixTestBuilder.SchemaBuilder createLevel1TenantView( + PhoenixTestBuilder.SchemaBuilder.TenantViewOptions tenantViewOptions, + PhoenixTestBuilder.SchemaBuilder.TenantViewIndexOptions tenantViewIndexOptions) throws Exception { + // Define the test schema. + // 1. Table with default columns => (ORG_ID, KP, COL1, COL2, COL3), PK => (ORG_ID, KP) + // 2. Tenant with default columns => (ZID, COL7, COL8, COL9), PK => (ZID) + final PhoenixTestBuilder.SchemaBuilder schemaBuilder = new PhoenixTestBuilder.SchemaBuilder(getUrl()); + + PhoenixTestBuilder.SchemaBuilder.TableOptions + tableOptions = PhoenixTestBuilder.SchemaBuilder.TableOptions.withDefaults(); + tableOptions.setTableProps("COLUMN_ENCODED_BYTES=0,MULTI_TENANT=true"); + + PhoenixTestBuilder.SchemaBuilder.TenantViewOptions + tenantViewOverrideOptions = PhoenixTestBuilder.SchemaBuilder.TenantViewOptions.withDefaults(); + if (tenantViewOptions != null) { + tenantViewOverrideOptions = tenantViewOptions; + } + PhoenixTestBuilder.SchemaBuilder.TenantViewIndexOptions + tenantViewIndexOverrideOptions = PhoenixTestBuilder.SchemaBuilder.TenantViewIndexOptions.withDefaults(); + if (tenantViewIndexOptions != null) { + tenantViewIndexOverrideOptions = tenantViewIndexOptions; + } + + schemaBuilder.withTableOptions(tableOptions) + .withTenantViewOptions(tenantViewOverrideOptions) + .withTenantViewIndexOptions(tenantViewIndexOverrideOptions).buildNewView(); + return schemaBuilder; + } + + @Test + public void testIndexesOfIndexTableLinkTypeCondition() throws Exception { + + PhoenixTestBuilder.SchemaBuilder.TenantViewOptions + tenantViewOptions = new PhoenixTestBuilder.SchemaBuilder.TenantViewOptions(); + tenantViewOptions.setTenantViewColumns(Lists.newArrayList(TENANT_VIEW_COLUMNS)); + tenantViewOptions.setTenantViewColumnTypes(Lists.newArrayList(COLUMN_TYPES)); + + // Create 2 level view + final PhoenixTestBuilder.SchemaBuilder + schemaBuilder = createLevel2TenantViewWithGlobalLevelTTL(VIEW_TTL_300_SECS, tenantViewOptions, null, + true); + + String tenantId = schemaBuilder.getDataOptions().getTenantId(); + String fullBaseTableName = schemaBuilder.getEntityTableName(); + String schemaName = stripQuotes( + SchemaUtil.getSchemaNameFromFullName(schemaBuilder.getEntityTenantViewName())); + String globalViewName = stripQuotes( + SchemaUtil.getTableNameFromFullName(schemaBuilder.getEntityGlobalViewName())); + String tenantViewName = stripQuotes( + SchemaUtil.getTableNameFromFullName(schemaBuilder.getEntityTenantViewName())); + String globalIndexName = stripQuotes( + SchemaUtil.getTableNameFromFullName(schemaBuilder.getEntityGlobalViewIndexName())); + String tenantIndexName = stripQuotes( + SchemaUtil.getTableNameFromFullName(schemaBuilder.getEntityTenantViewIndexName())); + + // Assert View Header rows exists for global view + assertSystemCatalogHasViewHeaderRelatedColumns("", schemaName, globalViewName, true, VIEW_TTL_300_SECS); + // Assert View Header rows exists for tenant view + assertSystemCatalogHasViewHeaderRelatedColumns(tenantId, schemaName, tenantViewName, true, 0); + + // Assert index table link rows (link_type = 1) exists in SYSTEM. CATALOG + assertSystemCatalogHasIndexTableLinks(null, schemaName, globalViewName); + assertSystemCatalogHasIndexTableLinks(tenantId, schemaName, tenantViewName); + + //Create the SYSTEM.CATALOG index for Index Table links + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_INDEX_TABLE_LINK_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, TABLE_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE = 1"); + conn.commit(); + } + + // Assert System Catalog index table has been created + assertSystemCatalogIndexTable("SYS_INDEX_TABLE_LINK_IDX", true); + // Assert appropriate rows are inserted in the SYSTEM.CATALOG index tables + assertSystemCatalogIndexHaveIndexTableLinks("SYSTEM.SYS_INDEX_TABLE_LINK_IDX", null, schemaName, globalViewName, + true, globalIndexName); + assertSystemCatalogIndexHaveIndexTableLinks("SYSTEM.SYS_INDEX_TABLE_LINK_IDX", tenantId, schemaName, tenantViewName, + true, tenantIndexName); + + LOGGER.info("Dropping base table " + fullBaseTableName); + dropTableWithChildViews(fullBaseTableName, 2); + assertSystemCatalogHasViewHeaderRelatedColumns("", schemaName, globalViewName, + false, VIEW_TTL_300_SECS); + assertSystemCatalogHasViewHeaderRelatedColumns(tenantId, schemaName, tenantViewName, + false, 0); + + // Assert appropriate rows are dropped/deleted in the SYSTEM.CATALOG index tables + assertSystemCatalogIndexHaveIndexTableLinks("SYSTEM.SYS_INDEX_TABLE_LINK_IDX", null, schemaName, globalViewName, false, null); + assertSystemCatalogIndexHaveIndexTableLinks("SYSTEM.SYS_INDEX_TABLE_LINK_IDX", tenantId, schemaName, tenantViewName, false, null); + + dropSystemCatalogIndex("SYS_INDEX_TABLE_LINK_IDX"); + + // Assert System Catalog index table dropped + assertSystemCatalogIndexTable("SYSTEM.SYS_INDEX_TABLE_LINK_IDX", false); + } + + @Test + public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { + + PhoenixTestBuilder.SchemaBuilder.TenantViewOptions + tenantViewOptions = PhoenixTestBuilder.SchemaBuilder.TenantViewOptions.withDefaults(); + // View TTL is set to 120s => 120000 ms + tenantViewOptions.setTableProps(String.format("TTL=%d", VIEW_TTL_120_SECS)); + + final PhoenixTestBuilder.SchemaBuilder + schemaBuilder = createLevel1TenantView(tenantViewOptions, null); + String tenantId = schemaBuilder.getDataOptions().getTenantId(); + String fullBaseTableName = schemaBuilder.getEntityTableName(); + String schemaName = stripQuotes( + SchemaUtil.getSchemaNameFromFullName(schemaBuilder.getEntityTenantViewName())); + String tenantViewName = stripQuotes( + SchemaUtil.getTableNameFromFullName(schemaBuilder.getEntityTenantViewName())); + String indexOnTenantViewName = String + .format("IDX_%s", stripQuotes(schemaBuilder.getEntityKeyPrefix())); + + // TABLE_TYPE = 'v' + // Expected 1 rows - one for TenantView. + // Since the TTL property values are being set, + // we expect the view header columns to show up in regular queries + assertSystemCatalogHasViewHeaderRelatedColumns(tenantId, schemaName, tenantViewName, + true, VIEW_TTL_120_SECS); + // Assert index header rows (link_type IS NULL AND TABLE_TYPE = 'i') exists in SYSTEM. CATALOG + assertSystemCatalogHasViewIndexHeaderRelatedColumns(tenantId, schemaName, indexOnTenantViewName,true); + + + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + conn.commit(); + } + + /** + * Testing creation of SYS_INDEX rows + */ + + // Assert System Catalog index table has been created + assertSystemCatalogIndexTable("SYS_VIEW_HDR_IDX", true); + assertSystemCatalogIndexTable("SYS_VIEW_INDEX_HDR_IDX", true); + // Assert appropriate rows are inserted in the SYSTEM.CATALOG index tables + assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_VIEW_HDR_IDX", tenantId, schemaName, tenantViewName, true); + assertSystemCatalogIndexHaveViewIndexHeaders("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", tenantId, schemaName, indexOnTenantViewName, true); + + /** + * Testing explain plans + */ + + List plans = getExplain("select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'", new Properties()); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_HDR_IDX", plans.get(0)); + + plans = getExplain("select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL", new Properties()); + assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_INDEX_HDR_IDX", plans.get(0)); + + /** + * Testing cleanup of SYS_INDEX rows after dropping tables and views + */ + LOGGER.info("Dropping base table " + fullBaseTableName); + dropTableWithChildViews(fullBaseTableName, 1); + // Assert view header rows (link_type IS NULL AND TABLE_TYPE = 'v') does not exists in SYSTEM.CATALOG + assertSystemCatalogHasViewHeaderRelatedColumns(tenantId, schemaName, tenantViewName, + false, VIEW_TTL_120_SECS); + // Assert index header rows (link_type IS NULL AND TABLE_TYPE = 'i') does not exists in SYSTEM.CATALOG + assertSystemCatalogHasViewIndexHeaderRelatedColumns(tenantId, schemaName, tenantViewName,false); + + // Assert appropriate rows are dropped/deleted in the SYSTEM.CATALOG index tables + assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_VIEW_HDR_IDX", tenantId, schemaName, tenantViewName, false); + assertSystemCatalogIndexHaveViewIndexHeaders("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", tenantId, schemaName, tenantViewName, false); + + dropSystemCatalogIndex("SYS_VIEW_HDR_IDX"); + dropSystemCatalogIndex("SYS_VIEW_INDEX_HDR_IDX"); + + // Assert System Catalog index table dropped + assertSystemCatalogIndexTable("SYSTEM.SYS_VIEW_HDR_IDX", false); + assertSystemCatalogIndexTable("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", false); + } + +} diff --git a/phoenix-core/src/test/java/org/apache/phoenix/parse/DecodeViewIndexIdFunctionTest.java b/phoenix-core/src/test/java/org/apache/phoenix/parse/DecodeViewIndexIdFunctionTest.java new file mode 100644 index 00000000000..e661acad6e8 --- /dev/null +++ b/phoenix-core/src/test/java/org/apache/phoenix/parse/DecodeViewIndexIdFunctionTest.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.parse; + +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +public class DecodeViewIndexIdFunctionTest { + + @Test + public void testExpressionWithDecodeViewIndexIdFunction() throws Exception { + ParseNode parseNode = SQLParser.parseCondition("DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) = 32768"); + boolean hasGetViewIndexIdParseNode = false; + for (ParseNode childNode : parseNode.getChildren()) { + if (childNode.getClass().isAssignableFrom(DecodeViewIndexIdParseNode.class)) { + assertEquals(2, childNode.getChildren().size()); + hasGetViewIndexIdParseNode = true; + } + } + assertTrue(hasGetViewIndexIdParseNode); + } + + @Test + public void testValidationForDecodeViewIndexIdFunction() throws Exception { + boolean hasGetViewIndexIdParseNode = false; + try { + ParseNode parseNode = SQLParser.parseCondition("DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, b) = 32768"); + for (ParseNode childNode : parseNode.getChildren()) { + if (childNode.getClass().isAssignableFrom(DecodeViewIndexIdParseNode.class)) { + assertEquals(2, childNode.getChildren().size()); + hasGetViewIndexIdParseNode = true; + } + } + } catch (Exception e) { + hasGetViewIndexIdParseNode = false; + + } + assertFalse(hasGetViewIndexIdParseNode); + } + +} From 9adc731fe06f8a4adcf7ddab375bba470991c33e Mon Sep 17 00:00:00 2001 From: Jacob Isaac Date: Fri, 17 Jan 2025 17:04:17 -0800 Subject: [PATCH 2/8] Addresses review comments --- .../function/DecodeViewIndexIdFunction.java | 21 +++++++++++++------ .../coprocessor/MetaDataEndpointImpl.java | 4 +--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DecodeViewIndexIdFunction.java b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DecodeViewIndexIdFunction.java index cf74d4be476..2692c5414e0 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DecodeViewIndexIdFunction.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/expression/function/DecodeViewIndexIdFunction.java @@ -41,9 +41,21 @@ import static org.apache.phoenix.util.ViewIndexIdRetrieveUtil.VIEW_INDEX_ID_BIGINT_TYPE_PTR_LEN; /** - * Function to return the timestamp of the empty column which functions as the row timestamp. The - * result returned can be used for debugging(eg. using HBase shell), logging etc. + * Function to return the ViewIndexId value based on the ViewIndexIDDataType field. * Can also be used in sql predicates. + * THe ViewIndexId field value needs to be interpreted based on the type specified in the + * ViewIndexIdDataType field + This is how the various client created view index id's look like: + client VIEW_INDEX_ID(Cell number of bytes) VIEW_INDEX_ID_DATA_TYPE + pre-4.15 2 bytes NULL + post-4.15[config smallint] 2 bytes 5(smallint) + post-4.15[config bigint] 8 bytes -5(bigint) + + VIEW_INDEX_ID_DATA_TYPE, VIEW_INDEX_ID(Cell representation of the data) + NULL, SMALLINT -> RETRIEVE AND CONVERT TO BIGINT + SMALLINT, SMALLINT -> RETRIEVE AND CONVERT TO BIGINT + BIGINT, BIGINT -> DO NOT CONVERT + */ @BuiltInFunction(name = DecodeViewIndexIdFunction.NAME, nodeClass= DecodeViewIndexIdParseNode.class, @@ -58,9 +70,7 @@ public DecodeViewIndexIdFunction() { } /** - * @param children An EMPTY_COLUMN key value expression injected thru - * {@link PhoenixRowTimestampParseNode#create create} - * will cause the empty column key value to be evaluated during scan filter processing. + * @param children VIEW_INDEX_ID and VIEW_INDEX_ID_DATA_TYPE expressions */ public DecodeViewIndexIdFunction(List children) { super(children); @@ -134,7 +144,6 @@ VIEW_INDEX_ID_DATA_TYPE, VIEW_INDEX_ID(Cell representation of the data) SortOrder.ASC); } - System.out.println("DecodeViewIndexIdFunction: Type: " + type); ImmutableBytesWritable columnValue = new ImmutableBytesWritable(CellUtil.cloneValue(viewIndexIdCell)); if ((type == NULL_DATA_TYPE_VALUE || type == Types.SMALLINT) && (viewIndexIdCell.getValueLength() < diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 1e17e0f62b4..1d13ff23ef5 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -3370,9 +3370,7 @@ private MetaDataMutationResult doDropTable(byte[] key, byte[] tenantId, byte[] s } Delete delete = new Delete(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), clientTimeStamp); - if (Bytes.compareTo(key, 0, key.length, kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()) == 0) { - LOGGER.info("doDropTable:Found key {} ", Bytes.toStringBinary(key)); - } else { + if (Bytes.compareTo(key, 0, key.length, kv.getRowArray(), kv.getRowOffset(), kv.getRowLength()) != 0) { catalogMutations.add(delete); } results.clear(); From 715437597a2f87143ec8e153a74fed28edad1943 Mon Sep 17 00:00:00 2001 From: Jacob Isaac Date: Wed, 5 Feb 2025 09:58:24 -0800 Subject: [PATCH 3/8] Test improvements --- .../end2end/BaseRowKeyMatcherTestIT.java | 70 ++++++++++++++++++- .../index/PartialSystemCatalogIndexIT.java | 60 +++++++++++++--- 2 files changed, 118 insertions(+), 12 deletions(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java index 0ce628b034d..d3d62f58814 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java @@ -90,6 +90,7 @@ import java.sql.Date; import java.sql.DriverManager; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; @@ -108,6 +109,8 @@ import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY; import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @@ -504,7 +507,42 @@ private Pair getRowKeyMatchersFromView(PhoenixConnection connect } - // Helper to get rowKeyMatcher from Metadata. + byte[] getRowKeyMatcherFromSyscatIndex(String tenantId, String schemaName, + String tableName, boolean useIndexTable) throws SQLException { + + final String + SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL = + "SELECT ROW_KEY_MATCHER FROM SYSTEM.CATALOG " + "WHERE %s AND TABLE_SCHEM <> 'SYSTEM' AND TABLE_NAME = '%s' AND ROW_KEY_MATCHER IS NOT NULL"; + final String SYS_CATALOG_IDX_ROW_KEY_MATCHER_HEADER_SQL = "SELECT \"0:ROW_KEY_MATCHER\" FROM SYSTEM.SYS_ROW_KEY_MATCHER_IDX " + "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'"; + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String + tenantClause = useIndexTable ? + (tenantId == null || tenantId.isEmpty() ? + "\":TENANT_ID\" IS NULL" : + String.format("\":TENANT_ID\" = '%s'", tenantId)) : + (tenantId == null || tenantId.isEmpty() ? + "TENANT_ID IS NULL" : + String.format("TENANT_ID = '%s'", tenantId)); + String + sql = useIndexTable ? + String.format(SYS_CATALOG_IDX_ROW_KEY_MATCHER_HEADER_SQL, tenantClause, schemaName, + tableName) : + String.format(SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL, tenantClause, + tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + byte[] matcherBytes = rs.next() ? rs.getBytes(1) : EMPTY_BYTE_ARRAY; + LOGGER.info("Row key matcher SQL: {}", sql); + LOGGER.info("Row key matcher: {}, {}", + Bytes.toStringBinary(matcherBytes), + Bytes.toStringBinary(PVarbinaryEncoded.INSTANCE.toBytes(matcherBytes))); + return PVarbinaryEncoded.INSTANCE.toBytes(matcherBytes); + } + } + + // Helper to get rowKeyMatcher from Metadata. private Pair getRowKeyMatchersFromView(PhoenixConnection connection, PTable view) throws SQLException { return getRowKeyMatchersFromView(connection, view.getName().getString()); @@ -560,6 +598,12 @@ private byte[] assertRowKeyMatcherForView(PhoenixConnection connection, PTable v PVarbinaryEncoded.INSTANCE.toBytes( WhereOptimizer.getRowKeyMatcher(connection, tableName, viewStatementTable, viewColumnConstantsToBe, isViewColumnReferencedToBe)); + byte[] + rowKeyMatcher3 = getRowKeyMatcherFromSyscatIndex(view.getTenantId() != null ? view.getTenantId().getString() : null, view.getSchemaName().getString(), view.getTableName().getString(), false); + + byte[] + rowKeyMatcher4 = getRowKeyMatcherFromSyscatIndex(view.getTenantId() != null ? view.getTenantId().getString() : null, view.getSchemaName().getString(), view.getTableName().getString(), true); + LOGGER.debug(String.format( "target-view-name = %s, physical = %s, stmt-table = %s\n, " + "row-matcher-0 = %s (syscat)\n, row-matcher-1 = %s\n, row-matcher-2 = %s\n", @@ -571,6 +615,10 @@ private byte[] assertRowKeyMatcherForView(PhoenixConnection connection, PTable v Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher1) == 0); assertTrue("RowKey matcher patterns do not match", Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher2) == 0); + assertTrue("RowKey matcher patterns do not match", + Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher3) == 0); + assertTrue("RowKey matcher patterns do not match", + Bytes.compareTo(rowKeyInfo.getSecond(), rowKeyMatcher4) == 0); return rowKeyMatcher1; } @@ -808,6 +856,15 @@ public void testViewsWithExtendedPK() { List testCases = getTestCases(); SortOrder[][] sortOrders = getSortOrders(); + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) WHERE TABLE_SCHEM <> 'SYSTEM' AND ROW_KEY_MATCHER IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + conn.commit(); + } + String tableName = ""; tableName = createViewHierarchy( testCases, sortOrders, 500, 5000, 3, @@ -879,9 +936,20 @@ public void testViewsWithVariousTenantIdTypes() { @Test public void testViewsWithoutExtendedPK() { try { + List testCases = getTestCases(); SortOrder[][] sortOrders = getSortOrders(); String tableName = ""; + + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE IS NOT NULL AND LINK_TYPE IS NULL AND ROW_KEY_MATCHER IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + conn.commit(); + } + tableName = createViewHierarchy( testCases, sortOrders, 100, 1000, 3, diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java index ddf1823a45e..1448c2343f5 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java @@ -13,12 +13,10 @@ import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PTable; -import org.apache.phoenix.schema.PTableType; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TableViewFinderResult; -import org.apache.phoenix.util.TestUtil; import org.apache.phoenix.util.ViewUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -52,7 +50,10 @@ public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT { static final int VIEW_TTL_300_SECS = 300; static final int VIEW_TTL_120_SECS = 120; - static final String SYS_CATALOG_VIEW_HEADER_SQL = "SELECT TTL FROM SYSTEM.CATALOG " + static final String SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL = "SELECT ROW_KEY_MATCHER FROM SYSTEM.CATALOG " + + "WHERE %s AND TABLE_SCHEM <> 'SYSTEM' AND TABLE_NAME = '%s' AND " + "ROW_KEY_MATCHER IS NOT NULL"; + + static final String SYS_CATALOG_VIEW_TTL_HEADER_SQL = "SELECT TTL FROM SYSTEM.CATALOG " + "WHERE %s AND TABLE_SCHEM = '%s' AND TABLE_NAME = '%s' AND TABLE_TYPE = 'v'"; static final String SYS_CATALOG_VIEW_INDEX_HEADER_SQL = "SELECT VIEW_INDEX_ID FROM SYSTEM.CATALOG " @@ -74,7 +75,7 @@ public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT { static final String SYS_CATALOG_IDX_VIEW_INDEX_HEADER_SQL = "SELECT \": DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID,VIEW_INDEX_ID_DATA_TYPE)\" FROM %s " + "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'" ; - private static RegionCoprocessorEnvironment TaskRegionEnvironment; + private static RegionCoprocessorEnvironment taskRegionEnvironment; @BeforeClass public static void doSetup() throws Exception { @@ -96,7 +97,7 @@ public static void doSetup() throws Exception { setUpTestDriver(new ReadOnlyProps(ReadOnlyProps.EMPTY_PROPS, DEFAULT_PROPERTIES.entrySet().iterator())); - TaskRegionEnvironment = + taskRegionEnvironment = getUtility() .getRSForFirstRegionInTable( PhoenixDatabaseMetaData.SYSTEM_TASK_HBASE_TABLE_NAME) @@ -105,7 +106,6 @@ public static void doSetup() throws Exception { .findCoprocessorEnvironment(TaskRegionObserver.class.getName()); - } @@ -161,7 +161,7 @@ void assertSystemCatalogHasViewHeaderRelatedColumns(String tenantId, String sche "TENANT_ID IS NULL" : String.format("TENANT_ID = '%s'", tenantId); String sql = String - .format(SYS_CATALOG_VIEW_HEADER_SQL, tenantClause, schemaName, tableName); + .format(SYS_CATALOG_VIEW_TTL_HEADER_SQL, tenantClause, schemaName, tableName); stmt.execute(sql); ResultSet rs = stmt.getResultSet(); if (exists) { @@ -177,6 +177,30 @@ void assertSystemCatalogHasViewHeaderRelatedColumns(String tenantId, String sche } } + void assertSystemCatalogHasRowKeyMatcherRelatedColumns(String tenantId, String schemaName, + String tableName, boolean exists) throws SQLException { + + try (Connection connection = DriverManager.getConnection(getUrl())) { + Statement stmt = connection.createStatement(); + String tenantClause = tenantId == null || tenantId.isEmpty() ? + "TENANT_ID IS NULL" : + String.format("TENANT_ID = '%s'", tenantId); + String sql = String + .format(SYS_CATALOG_ROW_KEY_MATCHER_HEADER_SQL, tenantClause, tableName); + stmt.execute(sql); + ResultSet rs = stmt.getResultSet(); + if (exists) { + byte[] matcherBytes = rs.next() ? rs.getBytes(1) : null; + assertNotNull(String.format("Expected rows do not match for schema = %s, table = %s", + schemaName, tableName), matcherBytes); + } else { + assertFalse(String.format("Rows do exists for schema = %s, table = %s", + schemaName, tableName), rs.next()); + + } + } + } + String stripQuotes(String name) { return name.replace("\"", ""); } @@ -291,9 +315,10 @@ void dropTableWithChildViews(String baseTable, int numTaskRuns) throws Exception // Run DropChildViewsTask to complete the tasks for dropping child views. The depth of the view tree is 2, // so we expect that this will be done in two task handling runs as each non-root level will be processed // in one run + TaskRegionObserver.SelfHealingTask task = new TaskRegionObserver.SelfHealingTask( - TaskRegionEnvironment, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS); + taskRegionEnvironment, QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS); for (int i = 0; i < numTaskRuns; i++) { task.run(); } @@ -542,12 +567,15 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { // Assert index header rows (link_type IS NULL AND TABLE_TYPE = 'i') exists in SYSTEM. CATALOG assertSystemCatalogHasViewIndexHeaderRelatedColumns(tenantId, schemaName, indexOnTenantViewName,true); + assertSystemCatalogHasRowKeyMatcherRelatedColumns(tenantId, schemaName, tenantViewName,true); try (Connection conn = DriverManager.getConnection(getUrl()); Statement stmt = conn.createStatement()) { //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL"); stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + conn.commit(); } @@ -558,40 +586,50 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { // Assert System Catalog index table has been created assertSystemCatalogIndexTable("SYS_VIEW_HDR_IDX", true); assertSystemCatalogIndexTable("SYS_VIEW_INDEX_HDR_IDX", true); + assertSystemCatalogIndexTable("SYS_ROW_KEY_MATCHER_IDX", true); // Assert appropriate rows are inserted in the SYSTEM.CATALOG index tables assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_VIEW_HDR_IDX", tenantId, schemaName, tenantViewName, true); + assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_ROW_KEY_MATCHER_IDX", tenantId, schemaName, tenantViewName, true); assertSystemCatalogIndexHaveViewIndexHeaders("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", tenantId, schemaName, indexOnTenantViewName, true); /** * Testing explain plans */ - List plans = getExplain("select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'", new Properties()); + List plans = getExplain("select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v' ", new Properties()); assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_HDR_IDX", plans.get(0)); - plans = getExplain("select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL", new Properties()); + plans = getExplain("select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL", new Properties()); assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_INDEX_HDR_IDX", plans.get(0)); + plans = getExplain("select ROW_KEY_MATCHER, TTL, TABLE_NAME FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND ROW_KEY_MATCHER IS NOT NULL", new Properties()); + assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER SYSTEM.SYS_ROW_KEY_MATCHER_IDX [not null]", plans.get(0)); + /** * Testing cleanup of SYS_INDEX rows after dropping tables and views */ LOGGER.info("Dropping base table " + fullBaseTableName); dropTableWithChildViews(fullBaseTableName, 1); - // Assert view header rows (link_type IS NULL AND TABLE_TYPE = 'v') does not exists in SYSTEM.CATALOG + // Assert view header rows (link_type IS NULL AND TABLE_TYPE = 'v') does not exist in SYSTEM.CATALOG assertSystemCatalogHasViewHeaderRelatedColumns(tenantId, schemaName, tenantViewName, false, VIEW_TTL_120_SECS); + // Assert view header rows (ROW_KEY_MATCHER IS NOT NULL does not exist in SYSTEM.CATALOG + assertSystemCatalogHasRowKeyMatcherRelatedColumns(tenantId, schemaName, tenantViewName,false); // Assert index header rows (link_type IS NULL AND TABLE_TYPE = 'i') does not exists in SYSTEM.CATALOG assertSystemCatalogHasViewIndexHeaderRelatedColumns(tenantId, schemaName, tenantViewName,false); // Assert appropriate rows are dropped/deleted in the SYSTEM.CATALOG index tables assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_VIEW_HDR_IDX", tenantId, schemaName, tenantViewName, false); + assertSystemCatalogIndexHaveViewHeaders("SYSTEM.SYS_ROW_KEY_MATCHER_IDX", tenantId, schemaName, tenantViewName, false); assertSystemCatalogIndexHaveViewIndexHeaders("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", tenantId, schemaName, tenantViewName, false); dropSystemCatalogIndex("SYS_VIEW_HDR_IDX"); + dropSystemCatalogIndex("SYS_ROW_KEY_MATCHER_IDX"); dropSystemCatalogIndex("SYS_VIEW_INDEX_HDR_IDX"); // Assert System Catalog index table dropped assertSystemCatalogIndexTable("SYSTEM.SYS_VIEW_HDR_IDX", false); + assertSystemCatalogIndexTable("SYSTEM.SYS_ROW_KEY_MATCHER_IDX", false); assertSystemCatalogIndexTable("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", false); } From c47516568441257e109454351520a9ac9ee47f85 Mon Sep 17 00:00:00 2001 From: Jacob Isaac Date: Wed, 5 Feb 2025 13:41:53 -0800 Subject: [PATCH 4/8] Test refactoring and removed SYSTEM schema checks during index creation --- .../phoenix/compile/CreateIndexCompiler.java | 34 ------------------- .../phoenix/exception/SQLExceptionCode.java | 3 -- .../end2end/BaseRowKeyMatcherTestIT.java | 17 ++-------- .../index/PartialSystemCatalogIndexIT.java | 10 +++--- 4 files changed, 8 insertions(+), 56 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java index 72a4173e836..c98f9a6a5c6 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java @@ -19,7 +19,6 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CompareOperator; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Pair; @@ -32,9 +31,7 @@ import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; -import org.apache.phoenix.parse.ComparisonParseNode; import org.apache.phoenix.parse.CreateIndexStatement; -import org.apache.phoenix.parse.LiteralParseNode; import org.apache.phoenix.parse.ParseNode; import org.apache.phoenix.parse.StatelessTraverseAllParseNodeVisitor; import org.apache.phoenix.parse.SubqueryParseNode; @@ -90,35 +87,12 @@ public CreateIndexCompiler(PhoenixStatement statement, Operation operation) { */ private static class IndexWhereParseNodeVisitor extends StatelessTraverseAllParseNodeVisitor { private boolean hasSubquery = false; - private boolean hasExcludedSystemSchema = false; @Override public Void visit(SubqueryParseNode node) throws SQLException { hasSubquery = true; return null; } - - @Override - public boolean visitEnter(ComparisonParseNode node) throws SQLException { - if (node.getFilterOp() == CompareOperator.NOT_EQUAL) { - boolean hasTableSchemColumn = node.getLHS().toString().equals(PhoenixDatabaseMetaData.TABLE_SCHEM) || - node.getRHS().toString().equals(PhoenixDatabaseMetaData.TABLE_SCHEM); - boolean hasSystemLiteralNode = false; - if (node.getLHS().getClass().isAssignableFrom(LiteralParseNode.class)) { - hasSystemLiteralNode = - SchemaUtil.normalizeLiteral((LiteralParseNode) node.getLHS()) - .equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME); - } - if (node.getRHS().getClass().isAssignableFrom(LiteralParseNode.class)) { - hasSystemLiteralNode = - SchemaUtil.normalizeLiteral((LiteralParseNode) node.getRHS()) - .equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME); - } - hasExcludedSystemSchema = hasTableSchemColumn && hasSystemLiteralNode; - } - return true; - } - } private String getValue(PDataType type) { @@ -164,14 +138,6 @@ private void verifyIndexWhere(ParseNode indexWhere, StatementContext context, throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_INDEX_WHERE_WITH_SUBQUERY). build().buildException(); } - // Verify that index WHERE clause on SYSTEM.CATALOG table excludes the SYSTEM schema - // TABLE_SCHEM <> 'SYSTEM' - if (SchemaUtil.isMetaTable(dataTableName.getSchemaName(), dataTableName.getTableName()) - && !indexWhereParseNodeVisitor.hasExcludedSystemSchema) { - throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_INDEX_WHERE_WITH_SYSTEM_CATALOG_ROWS_NOT_EXCLUDED). - build().buildException(); - } - // Verify that index WHERE clause can be evaluated on a single data table row diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index b7cecdcd873..eb52fa0fba2 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -113,9 +113,6 @@ public SQLException newException(SQLExceptionInfo info) { " Index where clause cannot include a subquery."), CANNOT_EVALUATE_INDEX_WHERE(304, "23102", "Invalid index where clause. It cannot be evaluated on a data table row."), - INVALID_INDEX_WHERE_WITH_SYSTEM_CATALOG_ROWS_NOT_EXCLUDED(305, "23103", - "Invalid index where clause. Index on SYSTEM.CATALOG should always exclude system catalog rows (TABLE_SCHEM <> 'SYSTEM')."), - /** * Invalid Cursor State (errorcode 04, sqlstate 24) */ diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java index d3d62f58814..c7ba76a0d66 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java @@ -859,9 +859,9 @@ public void testViewsWithExtendedPK() { try (Connection conn = DriverManager.getConnection(getUrl()); Statement stmt = conn.createStatement()) { //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) WHERE TABLE_SCHEM <> 'SYSTEM' AND ROW_KEY_MATCHER IS NOT NULL"); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); conn.commit(); } @@ -936,20 +936,9 @@ public void testViewsWithVariousTenantIdTypes() { @Test public void testViewsWithoutExtendedPK() { try { - List testCases = getTestCases(); SortOrder[][] sortOrders = getSortOrders(); String tableName = ""; - - try (Connection conn = DriverManager.getConnection(getUrl()); - Statement stmt = conn.createStatement()) { - //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE IS NOT NULL AND LINK_TYPE IS NULL AND ROW_KEY_MATCHER IS NOT NULL"); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); - conn.commit(); - } - tableName = createViewHierarchy( testCases, sortOrders, 100, 1000, 3, diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java index 1448c2343f5..4a49b397e13 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java @@ -572,9 +572,9 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { try (Connection conn = DriverManager.getConnection(getUrl()); Statement stmt = conn.createStatement()) { //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES)); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v'"); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL"); - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_TYPE = 'v'"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); conn.commit(); } @@ -596,13 +596,13 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { * Testing explain plans */ - List plans = getExplain("select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'v' ", new Properties()); + List plans = getExplain("select TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, TABLE_TYPE FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'v' ", new Properties()); assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_HDR_IDX", plans.get(0)); plans = getExplain("select VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL", new Properties()); assertEquals("CLIENT PARALLEL 1-WAY FULL SCAN OVER SYSTEM.SYS_VIEW_INDEX_HDR_IDX", plans.get(0)); - plans = getExplain("select ROW_KEY_MATCHER, TTL, TABLE_NAME FROM SYSTEM.CATALOG WHERE TABLE_SCHEM <> 'SYSTEM' AND ROW_KEY_MATCHER IS NOT NULL", new Properties()); + plans = getExplain("select ROW_KEY_MATCHER, TTL, TABLE_NAME FROM SYSTEM.CATALOG WHERE TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL", new Properties()); assertEquals("CLIENT PARALLEL 1-WAY RANGE SCAN OVER SYSTEM.SYS_ROW_KEY_MATCHER_IDX [not null]", plans.get(0)); /** From caf7cbbb9682323af3eeafbe84b6a3e80681c718 Mon Sep 17 00:00:00 2001 From: Jacob Isaac Date: Wed, 5 Feb 2025 16:59:56 -0800 Subject: [PATCH 5/8] Fixed imports --- .../phoenix/end2end/index/PartialSystemCatalogIndexIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java index 4a49b397e13..4dc961912e4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java @@ -5,7 +5,6 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.phoenix.coprocessor.TaskRegionObserver; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; -import org.apache.phoenix.end2end.LocalHBaseIT; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.end2end.ViewTTLIT; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; From 9d9e1449cd0cd1fa5448c30d5c8060a9ec633c5c Mon Sep 17 00:00:00 2001 From: Jacob Isaac Date: Wed, 5 Feb 2025 17:01:47 -0800 Subject: [PATCH 6/8] Fixed license --- .../index/PartialSystemCatalogIndexIT.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java index 4dc961912e4..1b230f80f3e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java @@ -1,3 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.phoenix.end2end.index; import org.apache.hadoop.hbase.HConstants; From 18a7ecb0bd92105c724a7f78a8fdf339be5655cd Mon Sep 17 00:00:00 2001 From: Jacob Isaac Date: Wed, 19 Feb 2025 18:29:33 -0800 Subject: [PATCH 7/8] Added feature flag and refactored mutateRowsWithLocks --- .../phoenix/compile/CreateIndexCompiler.java | 38 +++++ .../phoenix/exception/SQLExceptionCode.java | 4 + .../apache/phoenix/query/QueryServices.java | 1 + .../phoenix/query/QueryServicesOptions.java | 2 + .../ChildLinkMetaDataEndpoint.java | 7 +- .../coprocessor/MetaDataEndpointImpl.java | 138 +++++++++++------- .../coprocessor/TaskMetaDataEndpoint.java | 8 +- .../index/PartialSystemCatalogIndexIT.java | 97 +++++++++++- 8 files changed, 230 insertions(+), 65 deletions(-) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java index c98f9a6a5c6..bcdae4013d2 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/compile/CreateIndexCompiler.java @@ -17,6 +17,7 @@ */ package org.apache.phoenix.compile; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.client.Scan; @@ -32,6 +33,7 @@ import org.apache.phoenix.jdbc.PhoenixStatement; import org.apache.phoenix.jdbc.PhoenixStatement.Operation; import org.apache.phoenix.parse.CreateIndexStatement; +import org.apache.phoenix.parse.NamedTableNode; import org.apache.phoenix.parse.ParseNode; import org.apache.phoenix.parse.StatelessTraverseAllParseNodeVisitor; import org.apache.phoenix.parse.SubqueryParseNode; @@ -73,6 +75,10 @@ import java.util.Iterator; import java.util.List; +import static org.apache.phoenix.query.QueryServices.DEFAULT_SYSTEM_KEEP_DELETED_CELLS_ATTRIB; +import static org.apache.phoenix.query.QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED; +import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_SYSTEM_CATALOG_INDEXES_ENABLED; + public class CreateIndexCompiler { private final PhoenixStatement statement; private final Operation operation; @@ -230,6 +236,7 @@ private void verifyIndexWhere(ParseNode indexWhere, StatementContext context, } public MutationPlan compile(final CreateIndexStatement create) throws SQLException { final PhoenixConnection connection = statement.getConnection(); + verifyDataTable(connection, create.getTable()); final ColumnResolver resolver = FromCompiler.getResolverForCreateIndex( create, connection, create.getUdfParseNodes()); @@ -278,4 +285,35 @@ public ExplainPlan getExplainPlan() throws SQLException { }; } + + /** + * Helper method to validate CREATE INDEX statements on SYSTEM tables. + * 1. Pass if scheme name not provided, assumption is - it not a SYSTEM table. + * 2. Fail if SYSTEM_CATALOG_INDEXES_ENABLED not enabled + * 3. Fail if table other than SYSTEM.CATALOG + * + * @param connection + * @param table + * @throws SQLException + */ + private void verifyDataTable(PhoenixConnection connection, NamedTableNode table) throws SQLException { + Configuration conf = connection.getQueryServices().getConfiguration(); + boolean catalogIndexesEnabled = conf.getBoolean(SYSTEM_CATALOG_INDEXES_ENABLED, DEFAULT_SYSTEM_CATALOG_INDEXES_ENABLED); + + TableName tableName = table.getName(); + if (tableName.getSchemaName() == null) { + return; + } + if (tableName.getSchemaName().equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME) && + !catalogIndexesEnabled) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.SYSTEM_TABLE_INDEXES_NOT_ENABLED). + build().buildException(); + } + + if (tableName.getSchemaName().equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME) && + !tableName.getTableName().equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE)) { + throw new SQLExceptionInfo.Builder(SQLExceptionCode.CANNOT_INDEX_SYSTEM_TABLE). + build().buildException(); + } + } } diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java index eb52fa0fba2..88fdfe98fae 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java @@ -113,6 +113,10 @@ public SQLException newException(SQLExceptionInfo info) { " Index where clause cannot include a subquery."), CANNOT_EVALUATE_INDEX_WHERE(304, "23102", "Invalid index where clause. It cannot be evaluated on a data table row."), + SYSTEM_TABLE_INDEXES_NOT_ENABLED(305, "23103", + "Invalid index on table. Indexes on SYSTEM tables are not enabled."), + CANNOT_INDEX_SYSTEM_TABLE(306, "23104", + "Invalid index on table. SYSTEM Indexes can only be on SYSTEM.CATALOG table."), /** * Invalid Cursor State (errorcode 04, sqlstate 24) */ diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java index 7035d17bc62..a9d1b928eb1 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java @@ -427,6 +427,7 @@ public interface QueryServices extends SQLCloseable { // As opposed to a copy and async (out of band) delete. public static final String MOVE_CHILD_LINKS_DURING_UPGRADE_ENABLED = "phoenix.move.child_link.during.upgrade"; + String SYSTEM_CATALOG_INDEXES_ENABLED = "phoenix.system.catalog.indexes.enabled"; /** * Parameter to indicate the source of operation attribute. * It can include metadata about the customer, service, etc. diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java index c4f3812b518..92130cb7350 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java @@ -449,6 +449,8 @@ public class QueryServicesOptions { public static final boolean DEFAULT_PHOENIX_GET_METADATA_READ_LOCK_ENABLED = true; public static final int DEFAULT_PHOENIX_STREAMS_GET_TABLE_REGIONS_TIMEOUT = 300000; // 5 minutes + public static final boolean DEFAULT_SYSTEM_CATALOG_INDEXES_ENABLED = true; + private final Configuration config; diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/ChildLinkMetaDataEndpoint.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/ChildLinkMetaDataEndpoint.java index 8a86637b036..11b2b4522dd 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/ChildLinkMetaDataEndpoint.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/ChildLinkMetaDataEndpoint.java @@ -95,11 +95,8 @@ public void createViewAddChildLink(RpcController controller, getCoprocessorHost().preCreateViewAddChildLink(fullparentTableName); // From 4.15 the parent->child links are stored in a separate table SYSTEM.CHILD_LINK - // Also check if SYSTEM indexes exists on SYSTEM.CHILD_LINK, if so set metadata attribs - // on child link mutations - mutateRowsWithLocks(this.accessCheckEnabled, env, this.env.getRegion(), childLinkMutations, - Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, - true, PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME); + mutateRowsWithLocks(this.accessCheckEnabled, this.env.getRegion(), childLinkMutations, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE); } catch (Throwable t) { LOGGER.error("Unable to write mutations to " + diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java index 1d13ff23ef5..498a548ef00 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java @@ -87,6 +87,8 @@ import static org.apache.phoenix.query.QueryServices.INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB; import static org.apache.phoenix.query.QueryServices.SKIP_SYSTEM_TABLES_EXISTENCE_CHECK; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MAX_LOOKBACK_AGE_BYTES; +import static org.apache.phoenix.query.QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED; +import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_SYSTEM_CATALOG_INDEXES_ENABLED; import static org.apache.phoenix.schema.PTable.LinkType.PHYSICAL_TABLE; import static org.apache.phoenix.schema.PTable.LinkType.VIEW_INDEX_PARENT_TABLE; import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CDC_INCLUDE_BYTES; @@ -271,7 +273,6 @@ import org.apache.phoenix.util.EnvironmentEdgeManager; import org.apache.phoenix.util.MetaDataUtil; import org.apache.phoenix.util.PhoenixKeyValueUtil; -import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.QueryUtil; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; @@ -2371,7 +2372,6 @@ public void createTable(RpcController controller, CreateTableRequest request, schemaName = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX]; tableName = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX]; fullTableName = SchemaUtil.getTableName(schemaName, tableName); - boolean shouldCheckForSystemTableIndexes = !SchemaUtil.isSystemTable(Bytes.toBytes(fullTableName)); boolean isNamespaceMapped = MetaDataUtil.isNameSpaceMapped(tableMetadata, GenericKeyValueBuilder.INSTANCE, new ImmutableBytesWritable()); final IndexType indexType = MetaDataUtil.getIndexType(tableMetadata, GenericKeyValueBuilder.INSTANCE, @@ -2792,15 +2792,23 @@ public void createTable(RpcController controller, CreateTableRequest request, } } + + // Not sure whether this TODO is relevant anymore. PHOENIX-7107 introduces indexes + // on system table. // TODO: Switch this to HRegion#batchMutate when we want to support indexes on the // system table. Basically, we get all the locks that we don't already hold for all the // tableMetadata rows. This ensures we don't have deadlock situations (ensuring // primary and then index table locks are held, in that order). For now, we just don't support // indexing on the system table. This is an issue because of the way we manage batch mutation // in the Indexer. + + // Update SYSTEM.CATALOG indexes only for + // 1. ordinary table/index mutations (create table/index). + // 2. When creating system indexes itself, no further index processing is required. + boolean updateCatalogIndexes = !SchemaUtil.isSystemTable(Bytes.toBytes(fullTableName)); mutateRowsWithLocks(this.accessCheckEnabled, env, region, localMutations, Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, - shouldCheckForSystemTableIndexes, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); + updateCatalogIndexes); // Invalidate the cache - the next getTable call will add it // TODO: consider loading the table that was just created here, patching up the parent table, and updating the cache @@ -3133,17 +3141,17 @@ public void dropTable(RpcController controller, DropTableRequest request, } throw new IllegalStateException(msg); } - + // Update SYSTEM.CATALOG indexes only for + // 1. ordinary table/index mutations (drop table/index). + // 2. When dropping system indexes itself, no further index processing is required. boolean - shouldCheckForSystemTableIndexes = - ((pTableType == INDEX) && (Bytes.toString(schemaName) - .equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA))) ? - false : - true; + updateCatalogIndexes = + (pTableType != INDEX) || (!Bytes.toString(schemaName) + .equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA)); // drop rows from catalog on this region mutateRowsWithLocks(this.accessCheckEnabled, env, region, localMutations, Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, - shouldCheckForSystemTableIndexes, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); + updateCatalogIndexes); long currentTime = MetaDataUtil.getClientTimeStamp(tableMetadata); for (ImmutableBytesPtr ckey : invalidateList) { @@ -3263,7 +3271,6 @@ private MetaDataResponse processRemoteRegionMutations(byte[] systemTableName, try (Table hTable = ServerUtil.getHTableForCoprocessorScan(env, SchemaUtil.getPhysicalTableName(systemTableName, env.getConfiguration()))) { - setMetaDataOnMutationsIfCatalogIndexExists(env, Bytes.toString(systemTableName), remoteMutations); hTable.batch(remoteMutations, null); } catch (Throwable t) { LOGGER.error("Unable to write mutations to " + Bytes.toString(systemTableName), t); @@ -3751,9 +3758,17 @@ private MetaDataMutationResult mutateColumn( throw new IllegalStateException(msg); } } + // Update SYSTEM.CATALOG indexes only for ordinary table column mutations. + // Column mutations of indexes are not allowed. See above + // Add column on SYSTEM.CATALOG should not be processed for index updates, + // since an index on a future column cannot exist. + boolean + updateCatalogIndexes = + !Bytes.toString(schemaName) + .equalsIgnoreCase(PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA); mutateRowsWithLocks(this.accessCheckEnabled, env, region, localMutations, Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, - true, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); + updateCatalogIndexes); // Invalidate from cache for (ImmutableBytesPtr invalidateKey : invalidateList) { metaDataCache.invalidate(invalidateKey); @@ -4589,15 +4604,18 @@ public void updateIndexState(RpcController controller, UpdateIndexStateRequest r long serverTimestamp = EnvironmentEdgeManager.currentTimeMillis(); tableMetadata.add(MetaDataUtil.getLastDDLTimestampUpdate( key, clientTimeStamp, serverTimestamp)); - mutateRowsWithLocks(this.accessCheckEnabled, env, region, tableMetadata, + mutateRowsWithLocks(this.accessCheckEnabled, region, tableMetadata, Collections.emptySet(), HConstants.NO_NONCE, - HConstants.NO_NONCE, false, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); + HConstants.NO_NONCE); // Invalidate from cache Cache metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache(); metaDataCache.invalidate(cacheKey); if (dataTableKey != null) { metaDataCache.invalidate(new ImmutableBytesPtr(dataTableKey)); + // Ensure that the data table is refreshed back in the cache + doGetTable(tenantId, schemaName, CellUtil.cloneValue(dataTableKV), + HConstants.LATEST_TIMESTAMP, null, request.getClientVersion()); } if (setRowKeyOrderOptimizableCell || disableTimeStampKVIndex != -1 || currentState.isDisabled() || newState == PIndexState.BUILDING) { @@ -4841,9 +4859,9 @@ public void createFunction(RpcController controller, CreateFunctionRequest reque } // Don't store function info for temporary functions. if (!temporaryFunction) { - mutateRowsWithLocks(this.accessCheckEnabled, env, region, functionMetaData, + mutateRowsWithLocks(this.accessCheckEnabled, region, functionMetaData, Collections.emptySet(), HConstants.NO_NONCE, - HConstants.NO_NONCE, false, PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME); + HConstants.NO_NONCE); } // Invalidate the cache - the next getFunction call will add it @@ -4899,9 +4917,8 @@ public void dropFunction(RpcController controller, DropFunctionRequest request, done.run(MetaDataMutationResult.toProto(result)); return; } - mutateRowsWithLocks(this.accessCheckEnabled, env, region, functionMetaData, - Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, - false, PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME); + mutateRowsWithLocks(this.accessCheckEnabled, region, functionMetaData, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE); Cache metaDataCache = GlobalCache.getInstance(this.env).getMetaDataCache(); long currentTime = MetaDataUtil.getClientTimeStamp(functionMetaData); @@ -5022,9 +5039,8 @@ public void createSchema(RpcController controller, CreateSchemaRequest request, return; } } - mutateRowsWithLocks(this.accessCheckEnabled, env, region, schemaMutations, - Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, - false, null); + mutateRowsWithLocks(this.accessCheckEnabled, region, schemaMutations, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE); // Invalidate the cache - the next getSchema call will add it Cache metaDataCache = @@ -5075,9 +5091,8 @@ public void dropSchema(RpcController controller, DropSchemaRequest request, RpcC done.run(MetaDataMutationResult.toProto(result)); return; } - mutateRowsWithLocks(this.accessCheckEnabled, env, region, schemaMetaData, - Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, - false, null); + mutateRowsWithLocks(this.accessCheckEnabled, region, schemaMetaData, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE); Cache metaDataCache = GlobalCache.getInstance(this.env) .getMetaDataCache(); long currentTime = MetaDataUtil.getClientTimeStamp(schemaMetaData); @@ -5146,30 +5161,53 @@ private MetaDataMutationResult doDropSchema(long clientTimeStamp, String schemaN /** * Perform atomic mutations on rows within a region + * additionally set metadata on mutations, if catalog indexes exists * @param accessCheckEnabled Use the login user to mutate rows if enabled - * @param env + * @param env The RegionCoprocessorEnvironment * @param region Region containing rows to be mutated * @param mutations List of mutations for rows that must be contained within the * region * @param rowsToLock Rows to lock * @param nonceGroup Optional nonce group of the operation * @param nonce Optional nonce of the operation - * @param checkIfSystemCatalogIndexesExists check if Catalog indexes exists - * @param systemTableName The SYSTEM table to apply index mutation + * @param updateCatalogIndexes check if Catalog indexes exists * @throws IOException */ + static void mutateRowsWithLocks(final boolean accessCheckEnabled, final RegionCoprocessorEnvironment env, final Region region, final List mutations, final Set rowsToLock, final long nonceGroup, - final long nonce, boolean checkIfSystemCatalogIndexesExists, String systemTableName) throws IOException { + final long nonce, boolean updateCatalogIndexes) throws IOException { try { - if ((checkIfSystemCatalogIndexesExists) && (systemTableName != null && !systemTableName.isEmpty())) { - setMetaDataOnMutationsIfCatalogIndexExists(env, systemTableName, mutations ); + Configuration conf = env.getConfiguration(); + boolean catalogIndexesEnabled = conf.getBoolean(SYSTEM_CATALOG_INDEXES_ENABLED, DEFAULT_SYSTEM_CATALOG_INDEXES_ENABLED); + if ((updateCatalogIndexes) && (catalogIndexesEnabled)) { + setMetaDataOnMutationsIfCatalogIndexExists(env, mutations ); } } catch (SQLException e) { throw new IOException(e); } + + mutateRowsWithLocks(accessCheckEnabled, region, mutations, rowsToLock, nonceGroup, nonce); + } + + + /** + * Perform atomic mutations on rows within a region + * @param accessCheckEnabled Use the login user to mutate rows if enabled + * @param region Region containing rows to be mutated + * @param mutations List of mutations for rows that must be contained within the + * region + * @param rowsToLock Rows to lock + * @param nonceGroup Optional nonce group of the operation + * @param nonce Optional nonce of the operation + * @throws IOException + */ + static void mutateRowsWithLocks(final boolean accessCheckEnabled, final Region region, + final List mutations, final Set rowsToLock, final long nonceGroup, + final long nonce) throws IOException { + // We need to mutate SYSTEM.CATALOG or SYSTEM.CHILD_LINK with HBase/login user // if access is enabled. if (accessCheckEnabled) { @@ -5196,28 +5234,28 @@ public Void run() throws Exception { } private static void setMetaDataOnMutationsIfCatalogIndexExists( - final RegionCoprocessorEnvironment env, String forSystemTableName, final List mutations) + final RegionCoprocessorEnvironment env, final List mutations) throws SQLException { - Properties metaConnectionProps = new Properties(); - metaConnectionProps.setProperty(INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, FORCE_INDEX_MUTATE_METADATA_AS_ATTRIB); - PhoenixConnection connection = getServerConnectionForMetaData(metaConnectionProps, env.getConfiguration()) - .unwrap(PhoenixConnection.class); - if (PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME.compareToIgnoreCase(forSystemTableName) == 0) { - PTable systemCatalogPTable = PhoenixRuntime.getTableNoCache(connection, PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME); - if ((systemCatalogPTable != null) && (!systemCatalogPTable.getIndexes().isEmpty())) { - ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - IndexMaintainer.serialize(systemCatalogPTable, ptr, connection); - IndexMetaDataCacheClient.setMetaDataOnMutations(connection, systemCatalogPTable, mutations, ptr); - } + final byte[] key = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE); + ImmutableBytesPtr cacheKey = new ImmutableBytesPtr(key); + Cache metaDataCache = GlobalCache.getInstance(env).getMetaDataCache(); + PTable systemCatalogPTable = (PTable) metaDataCache.getIfPresent(cacheKey); + if (systemCatalogPTable == null) { + LOGGER.error("PTable for SYSTEM.CATALOG was not found in GlobalCache: key = {}" , Bytes.toString(key)); + return; } - if (PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME.compareToIgnoreCase(forSystemTableName) == 0) { - PTable childLinkCatalogPTable = PhoenixRuntime.getTableNoCache(connection, PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME); - if ((childLinkCatalogPTable != null) && (!childLinkCatalogPTable.getIndexes().isEmpty())) { - ImmutableBytesWritable ptr = new ImmutableBytesWritable(); - IndexMaintainer.serialize(childLinkCatalogPTable, ptr, connection); - IndexMetaDataCacheClient.setMetaDataOnMutations(connection, childLinkCatalogPTable, mutations, ptr); - } + if ((systemCatalogPTable.getIndexes().isEmpty())) { + LOGGER.debug("No indexes found for SYSTEM.CATALOG: key = {}", Bytes.toString(key)); + return; + } + Properties metaConnectionProps = new Properties(); + metaConnectionProps.setProperty(INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, FORCE_INDEX_MUTATE_METADATA_AS_ATTRIB); + try (PhoenixConnection connection = getServerConnectionForMetaData(metaConnectionProps, env.getConfiguration()) + .unwrap(PhoenixConnection.class)) { + ImmutableBytesWritable ptr = new ImmutableBytesWritable(); + IndexMaintainer.serialize(systemCatalogPTable, ptr, connection); + IndexMetaDataCacheClient.setMetaDataOnMutations(connection, systemCatalogPTable, mutations, ptr); } } diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TaskMetaDataEndpoint.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TaskMetaDataEndpoint.java index 6fdbb49e543..d0ff2a44b34 100644 --- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TaskMetaDataEndpoint.java +++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TaskMetaDataEndpoint.java @@ -47,8 +47,7 @@ import java.util.Collections; import java.util.List; -import static org.apache.phoenix.coprocessor.MetaDataEndpointImpl - .mutateRowsWithLocks; +import static org.apache.phoenix.coprocessor.MetaDataEndpointImpl.mutateRowsWithLocks; /** * Phoenix metadata mutations for SYSTEM.TASK flows through this co-processor @@ -106,9 +105,8 @@ public void upsertTaskDetails(RpcController controller, phoenixAccessCoprocessorHost.preUpsertTaskDetails(fullTableName); // We do not enable indexes on SYSTEM.TASK yet, so no need to check for it. - mutateRowsWithLocks(this.accessCheckEnabled, env, this.env.getRegion(), taskMutations, - Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE, - false, PhoenixDatabaseMetaData.SYSTEM_TASK_NAME); + mutateRowsWithLocks(this.accessCheckEnabled, this.env.getRegion(), taskMutations, + Collections.emptySet(), HConstants.NO_NONCE, HConstants.NO_NONCE); } catch (Throwable t) { LOGGER.error("Unable to write mutations to {}", PhoenixDatabaseMetaData.SYSTEM_TASK_NAME, t); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java index 1b230f80f3e..a8343def9aa 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java @@ -19,25 +19,36 @@ package org.apache.phoenix.end2end.index; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.phoenix.coprocessor.TaskRegionObserver; import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants; +import org.apache.phoenix.end2end.NeedsOwnMiniClusterTest; import org.apache.phoenix.end2end.ParallelStatsDisabledIT; import org.apache.phoenix.end2end.ViewTTLIT; import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData; +import org.apache.phoenix.jdbc.PhoenixDriver; +import org.apache.phoenix.query.ConfigurationFactory; +import org.apache.phoenix.query.HBaseFactoryProvider; import org.apache.phoenix.query.PhoenixTestBuilder; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PTable; import org.apache.phoenix.thirdparty.com.google.common.collect.Lists; +import org.apache.phoenix.util.InstanceResolver; +import org.apache.phoenix.util.PhoenixRuntime; import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.apache.phoenix.util.TableViewFinderResult; import org.apache.phoenix.util.ViewUtil; +import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; +import org.junit.experimental.categories.Category; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,14 +65,22 @@ import java.util.Map; import java.util.Properties; +import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_INDEX_SYSTEM_TABLE; +import static org.apache.phoenix.exception.SQLExceptionCode.MISMATCHED_TOKEN; import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.COLUMN_TYPES; import static org.apache.phoenix.query.PhoenixTestBuilder.DDLDefaults.TENANT_VIEW_COLUMNS; +import static org.apache.phoenix.query.QueryServices.CLIENT_CONNECTION_MAX_ALLOWED_CONNECTIONS; +import static org.apache.phoenix.query.QueryServices.CONNECTION_QUERY_SERVICE_METRICS_ENABLED; +import static org.apache.phoenix.query.QueryServices.INTERNAL_CONNECTION_MAX_ALLOWED_CONNECTIONS; +import static org.apache.phoenix.query.QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +@Category(NeedsOwnMiniClusterTest.class) public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT { static final Logger LOGGER = LoggerFactory.getLogger(ViewTTLIT.class); static final int VIEW_TTL_10_SECS = 10; @@ -94,14 +113,44 @@ public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT { "WHERE %s AND \":TABLE_SCHEM\" = '%s' AND \":TABLE_NAME\" = '%s'" ; private static RegionCoprocessorEnvironment taskRegionEnvironment; + private static HBaseTestingUtility hbaseTestUtil; @BeforeClass public static void doSetup() throws Exception { + InstanceResolver.clearSingletons(); + // Override to get required config for static fields loaded that require HBase config + InstanceResolver.getSingleton(ConfigurationFactory.class, new ConfigurationFactory() { + + @Override public Configuration getConfiguration() { + Configuration conf = HBaseConfiguration.create(); + conf.set(SYSTEM_CATALOG_INDEXES_ENABLED, String.valueOf(true)); + return conf; + } + + @Override public Configuration getConfiguration(Configuration confToClone) { + Configuration conf = HBaseConfiguration.create(); + conf.set(SYSTEM_CATALOG_INDEXES_ENABLED, String.valueOf(true)); + Configuration copy = new Configuration(conf); + copy.addResource(confToClone); + return copy; + } + }); + Configuration conf = HBaseFactoryProvider.getConfigurationFactory().getConfiguration(); + conf.set(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, + Long.toString(Long.MAX_VALUE)); + conf.set(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, + Long.toString(Long.MAX_VALUE)); + hbaseTestUtil = new HBaseTestingUtility(conf); + setUpConfigForMiniCluster(conf); + conf.set(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, + QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS); + hbaseTestUtil.startMiniCluster(); + + // Turn on the View TTL feature Map DEFAULT_PROPERTIES = new HashMap() {{ + put(QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED, String.valueOf(true)); put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true)); - put(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(true)); - put("hbase.procedure.remote.dispatcher.delay.msec", "0"); // no max lookback put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(0)); put(QueryServices.PHOENIX_VIEW_TTL_ENABLED, Boolean.toString(true)); @@ -123,10 +172,10 @@ public static void doSetup() throws Exception { .get(0).getCoprocessorHost() .findCoprocessorEnvironment(TaskRegionObserver.class.getName()); - } + void assertSystemCatalogHasIndexTableLinks(String tenantId, String schemaName, String tableName) throws SQLException { @@ -528,17 +577,22 @@ public void testIndexesOfIndexTableLinkTypeCondition() throws Exception { //Create the SYSTEM.CATALOG index for Index Table links try (Connection conn = DriverManager.getConnection(getUrl()); Statement stmt = conn.createStatement()) { - stmt.execute("CREATE INDEX IF NOT EXISTS SYS_INDEX_TABLE_LINK_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, TABLE_TYPE) WHERE TABLE_SCHEM <> 'SYSTEM' AND TABLE_TYPE = 'i' AND LINK_TYPE = 1"); + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_INDEX_TABLE_LINK_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, TABLE_TYPE) WHERE TABLE_TYPE = 'i' AND LINK_TYPE = 1"); conn.commit(); } + LOGGER.info("Finished creating index: " + "CREATE INDEX IF NOT EXISTS SYS_INDEX_TABLE_LINK_IDX"); // Assert System Catalog index table has been created + LOGGER.info("Begin assertSystemCatalogIndexTable: " + "SYS_INDEX_TABLE_LINK_IDX"); assertSystemCatalogIndexTable("SYS_INDEX_TABLE_LINK_IDX", true); + LOGGER.info("assertSystemCatalogIndexTable: " + "SYS_INDEX_TABLE_LINK_IDX"); // Assert appropriate rows are inserted in the SYSTEM.CATALOG index tables assertSystemCatalogIndexHaveIndexTableLinks("SYSTEM.SYS_INDEX_TABLE_LINK_IDX", null, schemaName, globalViewName, true, globalIndexName); + LOGGER.info("assertSystemCatalogIndexHaveIndexTableLinks: " + "SYSTEM.SYS_INDEX_TABLE_LINK_IDX"); assertSystemCatalogIndexHaveIndexTableLinks("SYSTEM.SYS_INDEX_TABLE_LINK_IDX", tenantId, schemaName, tenantViewName, true, tenantIndexName); + LOGGER.info("assertSystemCatalogIndexHaveIndexTableLinks: , " + tenantId + ", SYSTEM.SYS_INDEX_TABLE_LINK_IDX"); LOGGER.info("Dropping base table " + fullBaseTableName); dropTableWithChildViews(fullBaseTableName, 2); @@ -627,7 +681,7 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { * Testing cleanup of SYS_INDEX rows after dropping tables and views */ LOGGER.info("Dropping base table " + fullBaseTableName); - dropTableWithChildViews(fullBaseTableName, 1); + dropTableWithChildViews(fullBaseTableName, 2); // Assert view header rows (link_type IS NULL AND TABLE_TYPE = 'v') does not exist in SYSTEM.CATALOG assertSystemCatalogHasViewHeaderRelatedColumns(tenantId, schemaName, tenantViewName, false, VIEW_TTL_120_SECS); @@ -651,4 +705,37 @@ public void testIndexesOfViewAndIndexHeadersCondition() throws Exception { assertSystemCatalogIndexTable("SYSTEM.SYS_VIEW_INDEX_HDR_IDX", false); } + @Test + public void testIndexesOnOtherSystemTables() throws Exception { + try (Connection conn = DriverManager.getConnection(getUrl()); + Statement stmt = conn.createStatement()) { + try { + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_INDEX_LINK_4_IDX ON SYSTEM.CHILD_LINK(TENANT_ID, TABLE_SCHEM, TABLE_NAME, LINK_TYPE) WHERE LINK_TYPE = 4"); + fail(); + } catch (SQLException sqle) { + Assert.assertEquals(CANNOT_INDEX_SYSTEM_TABLE.getErrorCode(), sqle.getErrorCode()); + } + try { + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_INDEX_STATS_IDX ON SYSTEM.STATS(PHYSICAL_NAME, COLUMN_FAMILY, GUIDE_POST_WIDTH, GUIDE_POSTS_ROW_COUNT) WHERE COLUMN_FAMILY = '4'"); + fail(); + } catch (SQLException sqle) { + Assert.assertEquals(CANNOT_INDEX_SYSTEM_TABLE.getErrorCode(), sqle.getErrorCode()); + } + try { + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_INDEX_LOG_IDX ON SYSTEM.LOG(USER, CLIENT_IP, QUERY) WHERE QUERY_ID = '4'"); + fail(); + } catch (SQLException sqle) { + Assert.assertEquals(CANNOT_INDEX_SYSTEM_TABLE.getErrorCode(), sqle.getErrorCode()); + } + + try { + stmt.execute("CREATE INDEX IF NOT EXISTS SYS_INDEX_FUNCTION_IDX ON SYSTEM.FUNCTION(CLASS_NAME,JAR_PATH) WHERE FUNCTION_NAME = '4'"); + fail(); + } catch (SQLException sqle) { + Assert.assertEquals(MISMATCHED_TOKEN.getErrorCode(), sqle.getErrorCode()); + } + + } + } + } From a0cf0ac9c9f7354062b75da12c827b1ddefb8ff3 Mon Sep 17 00:00:00 2001 From: Jacob Isaac Date: Thu, 20 Feb 2025 09:14:58 -0800 Subject: [PATCH 8/8] Revertible changes for running IT tests with SYSTEM indexes --- .../query/ConnectionQueryServicesImpl.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java index 108eeb94607..c46745c52c5 100644 --- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java +++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java @@ -4041,6 +4041,9 @@ private void createOtherSystemTables(PhoenixConnection metaConnection) throws SQ try { metaConnection.createStatement().executeUpdate(getCDCStreamDDL()); } catch (TableAlreadyExistsException ignore) {} + try { + upgradeSystemCatalogIndexes(metaConnection); + } catch (TableAlreadyExistsException ignore) {} } /** @@ -5033,6 +5036,20 @@ private PhoenixConnection upgradeSystemCDCStream(PhoenixConnection metaConnectio return metaConnection; } + private PhoenixConnection upgradeSystemCatalogIndexes(PhoenixConnection metaConnection) + throws SQLException { + Properties p = PropertiesUtil.deepCopy(metaConnection.getClientInfo()); + p.remove(PhoenixRuntime.CURRENT_SCN_ATTRIB); + + try (PhoenixConnection conn = new PhoenixConnection( + ConnectionQueryServicesImpl.this, metaConnection.getURL(), p)) { + conn.createStatement().execute("CREATE INDEX IF NOT EXISTS SYS_INDEX_TABLE_LINK_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, TABLE_TYPE) WHERE TABLE_TYPE = 'i' AND LINK_TYPE = 1"); + conn.createStatement().execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_HDR_IDX ON SYSTEM.CATALOG(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY) INCLUDE (TABLE_TYPE, VIEW_STATEMENT, TTL, ROW_KEY_MATCHER) WHERE TABLE_TYPE = 'v'"); + conn.createStatement().execute("CREATE INDEX IF NOT EXISTS SYS_ROW_KEY_MATCHER_IDX ON SYSTEM.CATALOG(ROW_KEY_MATCHER, TTL, TABLE_TYPE, TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE (VIEW_STATEMENT) WHERE TABLE_TYPE = 'v' AND ROW_KEY_MATCHER IS NOT NULL"); + conn.createStatement().execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL"); + } catch (TableAlreadyExistsException ignore) {} + return metaConnection; + } // Special method for adding the column qualifier column for 4.10. private PhoenixConnection addColumnQualifierColumn(PhoenixConnection oldMetaConnection, Long timestamp) throws SQLException {