From 0e0b615a916230e6fa4ca1385f8ddafd0762f36c Mon Sep 17 00:00:00 2001 From: nhsmw Date: Tue, 12 Nov 2024 18:42:52 +0800 Subject: [PATCH] sink(ticdc): DDL event support for the Debezium protocol (#11674) close pingcap/tiflow#11566 --- cdc/sink/ddlsink/mq/mq_ddl_sink_test.go | 1 + errors.toml | 5 + pkg/errors/cdc_errors.go | 4 + pkg/sink/codec/common/helper.go | 15 +- pkg/sink/codec/craft/message_decoder.go | 15 +- pkg/sink/codec/craft/message_encoder.go | 14 +- pkg/sink/codec/debezium/codec.go | 1079 +++++++++++++--- pkg/sink/codec/debezium/codec_test.go | 686 ++++++++++ pkg/sink/codec/debezium/encoder.go | 60 +- pkg/sink/codec/debezium/helper.go | 265 ++++ pkg/sink/codec/debezium/helper_test.go | 80 ++ pkg/sink/codec/internal/java.go | 121 +- tests/integration_tests/debezium/README.md | 6 +- .../debezium/changefeed.toml | 1 + .../debezium/docker-compose.yml | 1 + tests/integration_tests/debezium/go.mod | 93 +- tests/integration_tests/debezium/go.sum | 441 +++---- tests/integration_tests/debezium/run.sh | 2 +- .../debezium/sql/data_types.sql | 51 +- tests/integration_tests/debezium/sql/ddl.sql | 135 ++ .../sql/debezium/binary_column_test.sql | 4 - .../sql/debezium/binary_mode_test.sql | 4 - .../connector_read_binary_field_test.sql | 15 + .../debezium/sql/debezium/connector_test.sql | 55 +- .../sql/debezium/connector_test_ro.sql | 57 +- .../sql/debezium/datetime_key_test.sql | 4 - .../sql/debezium/decimal_column_test.sql | 4 - .../debezium/sql/debezium/decimal_test.sql | 10 + .../debezium/sql/debezium/default_value.sql | 154 +++ .../debezium/default_value_all_zero_time.sql | 33 + .../sql/debezium/default_value_generated.sql | 8 + .../sql/debezium/enum_column_test.sql | 15 +- .../debezium/sql/debezium/float_test.sql | 22 + .../debezium/sql/debezium/history-dbz.sql | 2 + .../sql/debezium/multitable_dbz_871.sql | 9 +- .../debezium/sql/debezium/mysql-dbz-123.ddl | 12 + .../debezium/sql/debezium/mysql-dbz-162.ddl | 7 + .../debezium/sql/debezium/mysql-dbz-193.ddl | 10 + .../debezium/sql/debezium/mysql-dbz-198i.ddl | 23 + .../debezium/sql/debezium/mysql-dbz-200.ddl | 25 + .../debezium/sql/debezium/mysql-quoted.ddl | 36 + .../sql/debezium/mysql-test-create.ddl | 1134 +++++++++++++++++ .../mysql-ticketmonster-liquibase.ddl | 84 ++ .../debezium/nationalized_character_test.sql | 8 - .../sql/debezium/numeric_column_test.sql | 4 - .../debezium/sql/debezium/readbinlog_test.sql | 5 - .../debezium/sql/debezium/regression_test.sql | 16 +- .../debezium/sql/debezium/strategy_test.sql | 1 + .../debezium/table_column_comment_test.sql | 4 - .../sql/debezium/timestamp_column_test.sql | 10 +- .../debezium/topic_name_sanitization_test.sql | 4 - .../debezium/transaction_metadata_test.sql | 26 + .../sql/debezium/transactionpayload_test.sql | 29 + .../sql/debezium/unsigned_integer_test.sql | 3 - .../debezium/sql/debezium/year_test.sql | 45 + tests/integration_tests/debezium/sql/dml.sql | 4 + tests/integration_tests/debezium/src/main.go | 7 +- .../debezium/src/test_cases.go | 257 ++-- 58 files changed, 4388 insertions(+), 837 deletions(-) create mode 100644 pkg/sink/codec/debezium/helper.go create mode 100644 pkg/sink/codec/debezium/helper_test.go create mode 100644 tests/integration_tests/debezium/changefeed.toml create mode 100644 tests/integration_tests/debezium/sql/ddl.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/connector_read_binary_field_test.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/decimal_test.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/default_value.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/default_value_all_zero_time.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/default_value_generated.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/float_test.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/history-dbz.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/mysql-dbz-123.ddl create mode 100644 tests/integration_tests/debezium/sql/debezium/mysql-dbz-162.ddl create mode 100644 tests/integration_tests/debezium/sql/debezium/mysql-dbz-193.ddl create mode 100644 tests/integration_tests/debezium/sql/debezium/mysql-dbz-198i.ddl create mode 100644 tests/integration_tests/debezium/sql/debezium/mysql-dbz-200.ddl create mode 100644 tests/integration_tests/debezium/sql/debezium/mysql-quoted.ddl create mode 100644 tests/integration_tests/debezium/sql/debezium/mysql-test-create.ddl create mode 100644 tests/integration_tests/debezium/sql/debezium/mysql-ticketmonster-liquibase.ddl delete mode 100644 tests/integration_tests/debezium/sql/debezium/nationalized_character_test.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/transaction_metadata_test.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/transactionpayload_test.sql create mode 100644 tests/integration_tests/debezium/sql/debezium/year_test.sql diff --git a/cdc/sink/ddlsink/mq/mq_ddl_sink_test.go b/cdc/sink/ddlsink/mq/mq_ddl_sink_test.go index fb341c638b5..c3dfc873c19 100644 --- a/cdc/sink/ddlsink/mq/mq_ddl_sink_test.go +++ b/cdc/sink/ddlsink/mq/mq_ddl_sink_test.go @@ -290,4 +290,5 @@ func TestGetDLLDispatchRuleByProtocol(t *testing.T) { require.Equal(t, PartitionAll, getDDLDispatchRule(config.ProtocolMaxwell)) require.Equal(t, PartitionAll, getDDLDispatchRule(config.ProtocolCraft)) require.Equal(t, PartitionAll, getDDLDispatchRule(config.ProtocolSimple)) + require.Equal(t, PartitionAll, getDDLDispatchRule(config.ProtocolDebezium)) } diff --git a/errors.toml b/errors.toml index 6dee6a90c34..8bac0b36a0b 100755 --- a/errors.toml +++ b/errors.toml @@ -226,6 +226,11 @@ error = ''' cannot find mysql.tidb_ddl_job schema ''' +["CDC:ErrDDLUnsupportType"] +error = ''' +unsupport ddl type %s, query %s +''' + ["CDC:ErrDatumUnflatten"] error = ''' unflatten datume data diff --git a/pkg/errors/cdc_errors.go b/pkg/errors/cdc_errors.go index 637887bc0f9..b83791b0169 100644 --- a/pkg/errors/cdc_errors.go +++ b/pkg/errors/cdc_errors.go @@ -54,6 +54,10 @@ var ( "cannot find mysql.tidb_ddl_job schema", errors.RFCCodeText("CDC:ErrDDLSchemaNotFound"), ) + ErrDDLUnsupportType = errors.Normalize( + "unsupport ddl type %s, query %s", + errors.RFCCodeText("CDC:ErrDDLUnsupportType"), + ) ErrGRPCDialFailed = errors.Normalize( "grpc dial failed", errors.RFCCodeText("CDC:ErrGRPCDialFailed"), diff --git a/pkg/sink/codec/common/helper.go b/pkg/sink/codec/common/helper.go index 3b48c6585f1..3257fee5a5c 100644 --- a/pkg/sink/codec/common/helper.go +++ b/pkg/sink/codec/common/helper.go @@ -19,6 +19,7 @@ import ( "fmt" "math" "strings" + "unsafe" "github.com/go-sql-driver/mysql" "github.com/pingcap/log" @@ -300,9 +301,7 @@ const ( ) // EscapeEnumAndSetOptions escapes ",", "\" and "”" -// https://github.com/debezium/debezium/blob/9f7ede0e0695f012c6c4e715e96aed85eecf6b5f \ -// /debezium-connector-mysql/src/main/java/io/debezium/connector/mysql/antlr/ \ -// MySqlAntlrDdlParser.java#L374 +// https://github.com/debezium/debezium/blob/9f7ede0e0695f012c6c4e715e96aed85eecf6b5f/debezium-connector-mysql/src/main/java/io/debezium/connector/mysql/antlr/MySqlAntlrDdlParser.java#L374 func EscapeEnumAndSetOptions(option string) string { option = strings.ReplaceAll(option, ",", "\\,") option = strings.ReplaceAll(option, "\\'", "'") @@ -378,3 +377,13 @@ func SanitizeTopicName(name string) string { } return sanitizedName } + +// UnsafeBytesToString create string from byte slice without copying +func UnsafeBytesToString(b []byte) string { + return *(*string)(unsafe.Pointer(&b)) +} + +// UnsafeStringToBytes create byte slice from string without copying +func UnsafeStringToBytes(s string) []byte { + return *(*[]byte)(unsafe.Pointer(&s)) +} diff --git a/pkg/sink/codec/craft/message_decoder.go b/pkg/sink/codec/craft/message_decoder.go index 2ea6632c301..c104277e827 100644 --- a/pkg/sink/codec/craft/message_decoder.go +++ b/pkg/sink/codec/craft/message_decoder.go @@ -16,7 +16,6 @@ package craft import ( "encoding/binary" "math" - "unsafe" "github.com/pingcap/errors" pmodel "github.com/pingcap/tidb/pkg/meta/model" @@ -24,13 +23,9 @@ import ( "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tiflow/cdc/model" cerror "github.com/pingcap/tiflow/pkg/errors" + "github.com/pingcap/tiflow/pkg/sink/codec/common" ) -// / create string from byte slice without copying -func unsafeBytesToString(b []byte) string { - return *(*string)(unsafe.Pointer(&b)) -} - // / Primitive type decoders func decodeUint8(bits []byte) ([]byte, byte, error) { if len(bits) < 1 { @@ -140,7 +135,7 @@ func decodeBytes(bits []byte) ([]byte, []byte, error) { func decodeString(bits []byte) ([]byte, string, error) { bits, bytes, err := decodeBytes(bits) if err == nil { - return bits, unsafeBytesToString(bytes), nil + return bits, common.UnsafeBytesToString(bytes), nil } return bits, "", errors.Trace(err) } @@ -161,7 +156,7 @@ func decodeStringChunk(bits []byte, size int, allocator *SliceAllocator) ([]byte data := allocator.stringSlice(size) for i := 0; i < size; i++ { - data[i] = unsafeBytesToString(newBits[:larray[i]]) + data[i] = common.UnsafeBytesToString(newBits[:larray[i]]) newBits = newBits[larray[i]:] } return newBits, data, nil @@ -185,7 +180,7 @@ func decodeNullableStringChunk(bits []byte, size int, allocator *SliceAllocator) if larray[i] == -1 { continue } - s := unsafeBytesToString(newBits[:larray[i]]) + s := common.UnsafeBytesToString(newBits[:larray[i]]) data[i] = &s newBits = newBits[larray[i]:] } @@ -324,7 +319,7 @@ func DecodeTiDBType(ty byte, flag model.ColumnFlagType, bits []byte) (interface{ switch ty { case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeNewDate, mysql.TypeTimestamp, mysql.TypeDuration, mysql.TypeJSON, mysql.TypeNewDecimal: // value type for these mysql types are string - return unsafeBytesToString(bits), nil + return common.UnsafeBytesToString(bits), nil case mysql.TypeEnum, mysql.TypeSet, mysql.TypeBit: // value type for thest mysql types are uint64 _, u64, err := decodeUvarint(bits) diff --git a/pkg/sink/codec/craft/message_encoder.go b/pkg/sink/codec/craft/message_encoder.go index 8f887226bdc..f09c9e6016c 100644 --- a/pkg/sink/codec/craft/message_encoder.go +++ b/pkg/sink/codec/craft/message_encoder.go @@ -16,23 +16,13 @@ package craft import ( "encoding/binary" "math" - "unsafe" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/types" "github.com/pingcap/tiflow/cdc/model" + "github.com/pingcap/tiflow/pkg/sink/codec/common" ) -// create byte slice from string without copying -func unsafeStringToBytes(s string) []byte { - return *(*[]byte)(unsafe.Pointer( - &struct { - string - Cap int - }{s, len(s)}, - )) -} - // Primitive type encoders func encodeFloat64(bits []byte, data float64) []byte { v := math.Float64bits(data) @@ -193,7 +183,7 @@ func EncodeTiDBType(allocator *SliceAllocator, ty byte, flag model.ColumnFlagTyp switch ty { case mysql.TypeDate, mysql.TypeDatetime, mysql.TypeNewDate, mysql.TypeTimestamp, mysql.TypeDuration, mysql.TypeJSON, mysql.TypeNewDecimal: // value type for these mysql types are string - return unsafeStringToBytes(value.(string)) + return common.UnsafeStringToBytes(value.(string)) case mysql.TypeEnum, mysql.TypeSet, mysql.TypeBit: // value type for these mysql types are uint64 return encodeUvarint(allocator.byteSlice(binary.MaxVarintLen64)[:0], value.(uint64)) diff --git a/pkg/sink/codec/debezium/codec.go b/pkg/sink/codec/debezium/codec.go index a9111a8976e..4fd2f6fe699 100644 --- a/pkg/sink/codec/debezium/codec.go +++ b/pkg/sink/codec/debezium/codec.go @@ -15,7 +15,6 @@ package debezium import ( "bytes" - "encoding/binary" "fmt" "io" "strconv" @@ -23,12 +22,13 @@ import ( "time" "github.com/pingcap/log" + timodel "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tidb/pkg/types" - "github.com/pingcap/tidb/pkg/util/hack" "github.com/pingcap/tiflow/cdc/model" cerror "github.com/pingcap/tiflow/pkg/errors" "github.com/pingcap/tiflow/pkg/sink/codec/common" + "github.com/pingcap/tiflow/pkg/sink/codec/internal" "github.com/pingcap/tiflow/pkg/util" "github.com/tikv/client-go/v2/oracle" "go.uber.org/zap" @@ -53,6 +53,7 @@ func (c *dbzCodec) writeDebeziumFieldValues( colx := model.GetColumnDataX(col, tableInfo) err = c.writeDebeziumFieldValue(writer, colx, colInfos[i].Ft) if err != nil { + log.Error("write Debezium field value meet error", zap.Error(err)) break } } @@ -68,13 +69,25 @@ func (c *dbzCodec) writeDebeziumFieldSchema( switch col.GetType() { case mysql.TypeBit: n := ft.GetFlen() + var v uint64 + var err error + if col.GetDefaultValue() != nil { + val, ok := col.GetDefaultValue().(string) + if !ok { + return + } + v, err = strconv.ParseUint(parseBit(val, n), 2, 64) + if err != nil { + return + } + } if n == 1 { writer.WriteObjectElement(func() { writer.WriteStringField("type", "boolean") writer.WriteBoolField("optional", !mysql.HasNotNullFlag(ft.GetFlag())) writer.WriteStringField("field", col.GetName()) if col.GetDefaultValue() != nil { - writer.WriteAnyField("default", col.GetDefaultValue()) + writer.WriteBoolField("default", v != 0) // bool } }) } else { @@ -88,7 +101,7 @@ func (c *dbzCodec) writeDebeziumFieldSchema( }) writer.WriteStringField("field", col.GetName()) if col.GetDefaultValue() != nil { - writer.WriteAnyField("default", col.GetDefaultValue()) + c.writeBinaryField(writer, "default", getBitFromUint64(n, v)) // binary } }) } @@ -151,12 +164,24 @@ func (c *dbzCodec) writeDebeziumFieldSchema( if !ok { return } - floatV, err := strconv.ParseFloat(v, 64) + t, err := time.Parse("2006-01-02", v) if err != nil { + // For example, time may be invalid like 1000-00-00 + // return nil, nil + if mysql.HasNotNullFlag(ft.GetFlag()) { + writer.WriteInt64Field("default", 0) + } return } - - writer.WriteFloat64Field("default", floatV) + year := t.Year() + if year < 70 { + // treats "0018" as 2018 + t = t.AddDate(2000, 0, 0) + } else if year < 100 { + // treats "0099" as 1999 + t = t.AddDate(1900, 0, 0) + } + writer.WriteInt64Field("default", t.UTC().Unix()/60/60/24) } }) @@ -176,13 +201,35 @@ func (c *dbzCodec) writeDebeziumFieldSchema( if !ok { return } - if _, err := time.Parse("2006-01-02 15:04:05.999999", v); err != nil { - if mysql.HasNotNullFlag(ft.GetFlag()) || v == "CURRENT_TIMESTAMP" { - writer.WriteAnyField("default", 0) + if v == "CURRENT_TIMESTAMP" { + writer.WriteInt64Field("default", 0) + return + } + t, err := types.StrToDateTime(types.DefaultStmtNoWarningContext, v, ft.GetDecimal()) + if err != nil { + writer.WriteInt64Field("default", 0) + return + } + gt, err := t.GoTime(time.UTC) + if err != nil { + if mysql.HasNotNullFlag(ft.GetFlag()) { + writer.WriteInt64Field("default", 0) } return } - writer.WriteStringField("default", v) + year := gt.Year() + if year < 70 { + // treats "0018" as 2018 + gt = gt.AddDate(2000, 0, 0) + } else if year < 100 { + // treats "0099" as 1999 + gt = gt.AddDate(1900, 0, 0) + } + if ft.GetDecimal() <= 3 { + writer.WriteInt64Field("default", gt.UnixMilli()) + } else { + writer.WriteInt64Field("default", gt.UnixMicro()) + } } }) @@ -198,13 +245,33 @@ func (c *dbzCodec) writeDebeziumFieldSchema( if !ok { return } - if _, err := time.Parse("2006-01-02 15:04:05.999999", v); err != nil { - if mysql.HasNotNullFlag(ft.GetFlag()) || v == "CURRENT_TIMESTAMP" { - writer.WriteAnyField("default", 0) + if v == "CURRENT_TIMESTAMP" { + return + } + t, err := types.StrToDateTime(types.DefaultStmtNoWarningContext, v, ft.GetDecimal()) + if err != nil { + writer.WriteInt64Field("default", 0) + return + } + if t.Compare(types.MinTimestamp) < 0 { + if mysql.HasNotNullFlag(ft.GetFlag()) { + writer.WriteStringField("default", "1970-01-01T00:00:00Z") } return } - writer.WriteStringField("default", v) + gt, err := t.GoTime(time.UTC) + if err != nil { + writer.WriteInt64Field("default", 0) + return + } + str := gt.Format("2006-01-02T15:04:05") + fsp := ft.GetDecimal() + if fsp > 0 { + tmp := fmt.Sprintf(".%06d", gt.Nanosecond()/1000) + str = str + tmp[:1+fsp] + } + str += "Z" + writer.WriteStringField("default", str) } }) @@ -220,11 +287,11 @@ func (c *dbzCodec) writeDebeziumFieldSchema( if !ok { return } - floatV, err := strconv.ParseFloat(v, 64) + d, _, _, err := types.StrToDuration(types.DefaultStmtNoWarningContext.WithLocation(c.config.TimeZone), v, ft.GetDecimal()) if err != nil { return } - writer.WriteFloat64Field("default", floatV) + writer.WriteInt64Field("default", d.Microseconds()) } }) @@ -340,7 +407,11 @@ func (c *dbzCodec) writeDebeziumFieldSchema( case mysql.TypeFloat: writer.WriteObjectElement(func() { - writer.WriteStringField("type", "float") + if ft.GetDecimal() != -1 { + writer.WriteStringField("type", "double") + } else { + writer.WriteStringField("type", "float") + } writer.WriteBoolField("optional", !mysql.HasNotNullFlag(ft.GetFlag())) writer.WriteStringField("field", col.GetName()) if col.GetDefaultValue() != nil { @@ -392,6 +463,13 @@ func (c *dbzCodec) writeDebeziumFieldSchema( if err != nil { return } + if floatV < 70 { + // treats "DEFAULT 1" as 2001 + floatV += 2000 + } else if floatV < 100 { + // treats "DEFAULT 99" as 1999 + floatV += 1900 + } writer.WriteFloat64Field("default", floatV) } }) @@ -424,172 +502,188 @@ func (c *dbzCodec) writeDebeziumFieldValue( col model.ColumnDataX, ft *types.FieldType, ) error { - if col.Value == nil { + value := getValue(col) + if value == nil { writer.WriteNullField(col.GetName()) return nil } switch col.GetType() { case mysql.TypeBit: - v, ok := col.Value.(uint64) - if !ok { + n := ft.GetFlen() + var v uint64 + switch val := value.(type) { + case uint64: + v = val + case string: + hexValue, err := strconv.ParseUint(parseBit(val, n), 2, 64) + if err != nil { + return cerror.ErrDebeziumEncodeFailed.GenWithStack( + "unexpected column value type string for bit column %s, error:%s", + col.GetName(), err.Error()) + } + v = hexValue + default: return cerror.ErrDebeziumEncodeFailed.GenWithStack( "unexpected column value type %T for bit column %s", col.Value, col.GetName()) } - // Debezium behavior: // BIT(1) → BOOLEAN // BIT(>1) → BYTES The byte[] contains the bits in little-endian form and is sized to // contain the specified number of bits. - n := ft.GetFlen() if n == 1 { writer.WriteBoolField(col.GetName(), v != 0) return nil } else { - var buf [8]byte - binary.LittleEndian.PutUint64(buf[:], v) - numBytes := n / 8 - if n%8 != 0 { - numBytes += 1 - } - c.writeBinaryField(writer, col.GetName(), buf[:numBytes]) + c.writeBinaryField(writer, col.GetName(), getBitFromUint64(n, v)) return nil } case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob: - v, ok := col.Value.([]byte) - if !ok { - return cerror.ErrDebeziumEncodeFailed.GenWithStack( - "unexpected column value type %T for string column %s", - col.Value, - col.GetName()) - } - - if col.GetFlag().IsBinary() { - c.writeBinaryField(writer, col.GetName(), v) - return nil - } else { - writer.WriteStringField(col.GetName(), string(hack.String(v))) - return nil + isBinary := col.GetFlag().IsBinary() + switch v := value.(type) { + case []byte: + if !isBinary { + writer.WriteStringField(col.GetName(), common.UnsafeBytesToString(v)) + } else { + c.writeBinaryField(writer, col.GetName(), v) + } + case string: + if isBinary { + c.writeBinaryField(writer, col.GetName(), common.UnsafeStringToBytes(v)) + } + writer.WriteStringField(col.GetName(), v) } + return nil case mysql.TypeEnum: - v, ok := col.Value.(uint64) + v, ok := value.(uint64) if !ok { return cerror.ErrDebeziumEncodeFailed.GenWithStack( "unexpected column value type %T for enum column %s", - col.Value, + value, col.GetName()) } - enumVar, err := types.ParseEnumValue(ft.GetElems(), v) if err != nil { // Invalid enum value inserted in non-strict mode. writer.WriteStringField(col.GetName(), "") return nil } - writer.WriteStringField(col.GetName(), enumVar.Name) return nil case mysql.TypeSet: - v, ok := col.Value.(uint64) + v, ok := value.(uint64) if !ok { return cerror.ErrDebeziumEncodeFailed.GenWithStack( "unexpected column value type %T for set column %s", - col.Value, + value, col.GetName()) } - setVar, err := types.ParseSetValue(ft.GetElems(), v) if err != nil { // Invalid enum value inserted in non-strict mode. writer.WriteStringField(col.GetName(), "") return nil } - writer.WriteStringField(col.GetName(), setVar.Name) return nil case mysql.TypeNewDecimal: - v, ok := col.Value.(string) + v, ok := value.(string) if !ok { return cerror.ErrDebeziumEncodeFailed.GenWithStack( "unexpected column value type %T for decimal column %s", - col.Value, + value, col.GetName()) } - floatV, err := strconv.ParseFloat(v, 64) if err != nil { return cerror.WrapError( cerror.ErrDebeziumEncodeFailed, err) } - writer.WriteFloat64Field(col.GetName(), floatV) return nil case mysql.TypeDate, mysql.TypeNewDate: - v, ok := col.Value.(string) + v, ok := value.(string) if !ok { return cerror.ErrDebeziumEncodeFailed.GenWithStack( "unexpected column value type %T for date column %s", - col.Value, + value, col.GetName()) } - t, err := time.Parse("2006-01-02", v) if err != nil { // For example, time may be invalid like 1000-00-00 // return nil, nil if mysql.HasNotNullFlag(ft.GetFlag()) { writer.WriteInt64Field(col.GetName(), 0) - return nil } else { writer.WriteNullField(col.GetName()) - return nil } + return nil + } + year := t.Year() + if year < 70 { + // treats "0018" as 2018 + t = t.AddDate(2000, 0, 0) + } else if year < 100 { + // treats "0099" as 1999 + t = t.AddDate(1900, 0, 0) } - writer.WriteInt64Field(col.GetName(), t.Unix()/60/60/24) + writer.WriteInt64Field(col.GetName(), t.UTC().Unix()/60/60/24) return nil case mysql.TypeDatetime: // Debezium behavior from doc: // > Such columns are converted into epoch milliseconds or microseconds based on the // > column's precision by using UTC. - - // TODO: For Default Value = CURRENT_TIMESTAMP, the result is incorrect. - v, ok := col.Value.(string) + v, ok := value.(string) if !ok { return cerror.ErrDebeziumEncodeFailed.GenWithStack( "unexpected column value type %T for datetime column %s", - col.Value, + value, col.GetName()) } - - t, err := time.Parse("2006-01-02 15:04:05.999999", v) + if v == "CURRENT_TIMESTAMP" { + writer.WriteInt64Field(col.GetName(), 0) + return nil + } + t, err := types.StrToDateTime(types.DefaultStmtNoWarningContext, v, ft.GetDecimal()) + if err != nil { + return cerror.WrapError( + cerror.ErrDebeziumEncodeFailed, + err) + } + gt, err := t.GoTime(time.UTC) if err != nil { - // For example, time may be 1000-00-00 if mysql.HasNotNullFlag(ft.GetFlag()) { writer.WriteInt64Field(col.GetName(), 0) - return nil } else { writer.WriteNullField(col.GetName()) - return nil } + return nil + } + year := gt.Year() + if year < 70 { + // treats "0018" as 2018 + gt = gt.AddDate(2000, 0, 0) + } else if year < 100 { + // treats "0099" as 1999 + gt = gt.AddDate(1900, 0, 0) } - if ft.GetDecimal() <= 3 { - writer.WriteInt64Field(col.GetName(), t.UnixMilli()) - return nil + writer.WriteInt64Field(col.GetName(), gt.UnixMilli()) } else { - writer.WriteInt64Field(col.GetName(), t.UnixMicro()) - return nil + writer.WriteInt64Field(col.GetName(), gt.UnixMicro()) } + return nil case mysql.TypeTimestamp: // Debezium behavior from doc: @@ -601,33 +695,44 @@ func (c *dbzCodec) writeDebeziumFieldValue( // > based on the server (or session's) current time zone. The time zone will be queried from // > the server by default. If this fails, it must be specified explicitly by the database // > connectionTimeZone MySQL configuration option. - v, ok := col.Value.(string) + v, ok := value.(string) if !ok { return cerror.ErrDebeziumEncodeFailed.GenWithStack( "unexpected column value type %T for timestamp column %s", - col.Value, + value, col.GetName()) } - - t, err := time.ParseInLocation("2006-01-02 15:04:05.999999", v, c.config.TimeZone) + if v == "CURRENT_TIMESTAMP" { + writer.WriteNullField(col.GetName()) + return nil + } + t, err := types.StrToDateTime(types.DefaultStmtNoWarningContext.WithLocation(c.config.TimeZone), v, ft.GetDecimal()) if err != nil { - // For example, time may be invalid like 1000-00-00 - if mysql.HasNotNullFlag(ft.GetFlag()) { - t = time.Unix(0, 0) - } else { + return cerror.WrapError( + cerror.ErrDebeziumEncodeFailed, + err) + } + if t.Compare(types.MinTimestamp) < 0 { + if col.Value == nil { writer.WriteNullField(col.GetName()) - return nil + } else { + writer.WriteStringField(col.GetName(), "1970-01-01T00:00:00Z") } + return nil } - - str := t.UTC().Format("2006-01-02T15:04:05") + gt, err := t.GoTime(c.config.TimeZone) + if err != nil { + return cerror.WrapError( + cerror.ErrDebeziumEncodeFailed, + err) + } + str := gt.UTC().Format("2006-01-02T15:04:05") fsp := ft.GetDecimal() if fsp > 0 { - tmp := fmt.Sprintf(".%06d", t.Nanosecond()/1000) + tmp := fmt.Sprintf(".%06d", gt.Nanosecond()/1000) str = str + tmp[:1+fsp] } str += "Z" - writer.WriteStringField(col.GetName(), str) return nil @@ -635,56 +740,109 @@ func (c *dbzCodec) writeDebeziumFieldValue( // Debezium behavior from doc: // > Represents the time value in microseconds and does not include // > time zone information. MySQL allows M to be in the range of 0-6. - v, ok := col.Value.(string) + v, ok := value.(string) if !ok { return cerror.ErrDebeziumEncodeFailed.GenWithStack( "unexpected column value type %T for time column %s", - col.Value, + value, col.GetName()) } - - d, _, _, err := types.StrToDuration(types.DefaultStmtNoWarningContext, v, ft.GetDecimal()) + d, _, _, err := types.StrToDuration(types.DefaultStmtNoWarningContext.WithLocation(c.config.TimeZone), v, ft.GetDecimal()) if err != nil { return cerror.WrapError( cerror.ErrDebeziumEncodeFailed, err) } - writer.WriteInt64Field(col.GetName(), d.Microseconds()) return nil - case mysql.TypeLonglong: + case mysql.TypeLonglong, mysql.TypeLong, mysql.TypeInt24, mysql.TypeShort, mysql.TypeTiny: // Note: Although Debezium's doc claims to use INT32 for INT, but it // actually uses INT64. Debezium also uses INT32 for SMALLINT. - // So we only handle with TypeLonglong here. - if col.GetFlag().IsUnsigned() { - // Handle with BIGINT UNSIGNED. - // Debezium always produce INT64 instead of UINT64 for BIGINT. - v, ok := col.Value.(uint64) - if !ok { + isUnsigned := col.GetFlag().IsUnsigned() + maxValue := types.GetMaxValue(ft) + minValue := types.GetMinValue(ft) + switch v := value.(type) { + case uint64: + if !isUnsigned { + return cerror.ErrDebeziumEncodeFailed.GenWithStack( + "unexpected column value type %T for unsigned int column %s", + value, + col.GetName()) + } + if ft.GetType() == mysql.TypeLonglong && v == maxValue.GetUint64() || v > maxValue.GetUint64() { + writer.WriteAnyField(col.GetName(), -1) + } else { + writer.WriteInt64Field(col.GetName(), int64(v)) + } + case int64: + if isUnsigned { return cerror.ErrDebeziumEncodeFailed.GenWithStack( - "unexpected column value type %T for unsigned bigint column %s", - col.Value, + "unexpected column value type %T for int column %s", + value, col.GetName()) } + if v < minValue.GetInt64() || v > maxValue.GetInt64() { + writer.WriteAnyField(col.GetName(), -1) + } else { + writer.WriteInt64Field(col.GetName(), v) + } + case string: + if isUnsigned { + t, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return cerror.ErrDebeziumEncodeFailed.GenWithStack( + "unexpected column value type string for unsigned int column %s", + col.GetName()) + } + if ft.GetType() == mysql.TypeLonglong && t == maxValue.GetUint64() || t > maxValue.GetUint64() { + writer.WriteAnyField(col.GetName(), -1) + } else { + writer.WriteInt64Field(col.GetName(), int64(t)) + } + } else { + t, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return cerror.ErrDebeziumEncodeFailed.GenWithStack( + "unexpected column value type string for int column %s", + col.GetName()) + } + if t < minValue.GetInt64() || t > maxValue.GetInt64() { + writer.WriteAnyField(col.GetName(), -1) + } else { + writer.WriteInt64Field(col.GetName(), t) + } + } + } + return nil - writer.WriteInt64Field(col.GetName(), int64(v)) - return nil + case mysql.TypeDouble, mysql.TypeFloat: + if v, ok := value.(string); ok { + val, err := strconv.ParseFloat(v, 64) + if err != nil { + return cerror.ErrDebeziumEncodeFailed.GenWithStack( + "unexpected column value type string for int column %s", + col.GetName()) + } + writer.WriteFloat64Field(col.GetName(), val) + } else { + writer.WriteAnyField(col.GetName(), value) } + return nil case mysql.TypeTiDBVectorFloat32: - v, ok := col.Value.(types.VectorFloat32) + v, ok := value.(types.VectorFloat32) if !ok { return cerror.ErrDebeziumEncodeFailed.GenWithStack( "unexpected column value type %T for unsigned vector column %s", - col.Value, + value, col.GetName()) } writer.WriteStringField(col.GetName(), v.String()) return nil } - writer.WriteAnyField(col.GetName(), col.Value) + writer.WriteAnyField(col.GetName(), value) return nil } @@ -693,6 +851,98 @@ func (c *dbzCodec) writeBinaryField(writer *util.JSONWriter, fieldName string, v writer.WriteBase64StringField(fieldName, value) } +func (c *dbzCodec) writeSourceSchema(writer *util.JSONWriter) { + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "struct") + writer.WriteArrayField("fields", func() { + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", false) + writer.WriteStringField("field", "version") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", false) + writer.WriteStringField("field", "connector") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", false) + writer.WriteStringField("field", "name") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "int64") + writer.WriteBoolField("optional", false) + writer.WriteStringField("field", "ts_ms") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", true) + writer.WriteStringField("name", "io.debezium.data.Enum") + writer.WriteIntField("version", 1) + writer.WriteObjectField("parameters", func() { + writer.WriteStringField("allowed", "true,last,false,incremental") + }) + writer.WriteStringField("default", "false") + writer.WriteStringField("field", "snapshot") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", false) + writer.WriteStringField("field", "db") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", true) + writer.WriteStringField("field", "sequence") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", true) + writer.WriteStringField("field", "table") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "int64") + writer.WriteBoolField("optional", false) + writer.WriteStringField("field", "server_id") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", true) + writer.WriteStringField("field", "gtid") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", false) + writer.WriteStringField("field", "file") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "int64") + writer.WriteBoolField("optional", false) + writer.WriteStringField("field", "pos") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "int32") + writer.WriteBoolField("optional", false) + writer.WriteStringField("field", "row") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "int64") + writer.WriteBoolField("optional", true) + writer.WriteStringField("field", "thread") + }) + writer.WriteObjectElement(func() { + writer.WriteStringField("type", "string") + writer.WriteBoolField("optional", true) + writer.WriteStringField("field", "query") + }) + }) + writer.WriteBoolField("optional", false) + writer.WriteStringField("name", "io.debezium.connector.mysql.Source") + writer.WriteStringField("field", "source") + }) +} + // EncodeKey encode RowChangedEvent into key message func (c *dbzCodec) EncodeKey( e *model.RowChangedEvent, @@ -770,7 +1020,6 @@ func (c *dbzCodec) EncodeValue( // https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-create-events jWriter.WriteInt64Field("ts_ms", c.nowFunc().UnixMilli()) jWriter.WriteNullField("transaction") - if e.IsInsert() { // op: Mandatory string that describes the type of operation that caused the connector to generate the event. // Valid values are: @@ -833,6 +1082,16 @@ func (c *dbzCodec) EncodeValue( colx := model.GetColumnDataX(col, e.TableInfo) c.writeDebeziumFieldSchema(fieldsWriter, colx, colInfos[i].Ft) } + if e.TableInfo.HasVirtualColumns() { + for _, colInfo := range e.TableInfo.Columns { + if model.IsColCDCVisible(colInfo) { + continue + } + data := &model.ColumnData{ColumnID: colInfo.ID} + colx := model.GetColumnDataX(data, e.TableInfo) + c.writeDebeziumFieldSchema(fieldsWriter, colx, &colInfo.FieldType) + } + } util.ReturnJSONWriter(fieldsWriter) fieldsJSON = fieldsBuf.String() } @@ -856,106 +1115,536 @@ func (c *dbzCodec) EncodeValue( jWriter.WriteRaw(fieldsJSON) }) }) + c.writeSourceSchema(jWriter) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("type", "string") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("field", "op") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("type", "int64") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("field", "ts_ms") + }) jWriter.WriteObjectElement(func() { jWriter.WriteStringField("type", "struct") jWriter.WriteArrayField("fields", func() { jWriter.WriteObjectElement(func() { jWriter.WriteStringField("type", "string") jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("field", "version") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "string") - jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("field", "connector") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "string") - jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("field", "name") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "int64") - jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("field", "ts_ms") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "string") - jWriter.WriteBoolField("optional", true) - jWriter.WriteStringField("name", "io.debezium.data.Enum") - jWriter.WriteIntField("version", 1) - jWriter.WriteObjectField("parameters", func() { - jWriter.WriteStringField("allowed", "true,last,false,incremental") - }) - jWriter.WriteStringField("default", "false") - jWriter.WriteStringField("field", "snapshot") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "string") - jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("field", "db") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "string") - jWriter.WriteBoolField("optional", true) - jWriter.WriteStringField("field", "sequence") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "string") - jWriter.WriteBoolField("optional", true) - jWriter.WriteStringField("field", "table") + jWriter.WriteStringField("field", "id") }) jWriter.WriteObjectElement(func() { jWriter.WriteStringField("type", "int64") jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("field", "server_id") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "string") - jWriter.WriteBoolField("optional", true) - jWriter.WriteStringField("field", "gtid") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "string") - jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("field", "file") + jWriter.WriteStringField("field", "total_order") }) jWriter.WriteObjectElement(func() { jWriter.WriteStringField("type", "int64") jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("field", "pos") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "int32") - jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("field", "row") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "int64") - jWriter.WriteBoolField("optional", true) - jWriter.WriteStringField("field", "thread") - }) - jWriter.WriteObjectElement(func() { - jWriter.WriteStringField("type", "string") - jWriter.WriteBoolField("optional", true) - jWriter.WriteStringField("field", "query") + jWriter.WriteStringField("field", "data_collection_order") }) - // Below are extra TiDB fields - // jWriter.WriteObjectElement(func() { - // jWriter.WriteStringField("type", "int64") - // jWriter.WriteBoolField("optional", false) - // jWriter.WriteStringField("field", "commit_ts") - // }) - // jWriter.WriteObjectElement(func() { - // jWriter.WriteStringField("type", "string") - // jWriter.WriteBoolField("optional", false) - // jWriter.WriteStringField("field", "cluster_id") - // }) }) + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("name", "event.block") + jWriter.WriteIntField("version", 1) + jWriter.WriteStringField("field", "transaction") + }) + }) + }) + } + }) + return err +} + +// EncodeDDLEvent encode DDLEvent into debezium change event +func (c *dbzCodec) EncodeDDLEvent( + e *model.DDLEvent, + keyDest io.Writer, + dest io.Writer, +) error { + keyJWriter := util.BorrowJSONWriter(keyDest) + jWriter := util.BorrowJSONWriter(dest) + defer util.ReturnJSONWriter(keyJWriter) + defer util.ReturnJSONWriter(jWriter) + + commitTime := oracle.GetTimeFromTS(e.CommitTs) + var changeType string + // refer to: https://docs.pingcap.com/tidb/dev/mysql-compatibility#ddl-operations + switch e.Type { + case timodel.ActionCreateSchema, + timodel.ActionCreateTable, + timodel.ActionCreateView: + changeType = "CREATE" + case timodel.ActionAddColumn, + timodel.ActionModifyColumn, + timodel.ActionDropColumn, + timodel.ActionMultiSchemaChange, + timodel.ActionAddTablePartition, + timodel.ActionRemovePartitioning, + timodel.ActionReorganizePartition, + timodel.ActionExchangeTablePartition, + timodel.ActionAlterTablePartitioning, + timodel.ActionTruncateTablePartition, + timodel.ActionDropTablePartition, + timodel.ActionRebaseAutoID, + timodel.ActionSetDefaultValue, + timodel.ActionModifyTableComment, + timodel.ActionModifyTableCharsetAndCollate, + timodel.ActionModifySchemaCharsetAndCollate, + timodel.ActionAddIndex, + timodel.ActionAlterIndexVisibility, + timodel.ActionRenameIndex, + timodel.ActionRenameTable, + timodel.ActionRecoverTable, + timodel.ActionAddPrimaryKey, + timodel.ActionDropPrimaryKey, + timodel.ActionAlterTTLInfo, + timodel.ActionAlterTTLRemove: + changeType = "ALTER" + case timodel.ActionDropSchema, + timodel.ActionDropTable, + timodel.ActionDropIndex, + timodel.ActionDropView: + changeType = "DROP" + default: + return cerror.ErrDDLUnsupportType.GenWithStackByArgs(e.Type, e.Query) + } + + var err error + dbName, tableName := getDBTableName(e) + // message key + keyJWriter.WriteObject(func() { + keyJWriter.WriteObjectField("payload", func() { + if e.Type == timodel.ActionDropTable { + keyJWriter.WriteStringField("databaseName", e.PreTableInfo.GetSchemaName()) + } else { + keyJWriter.WriteStringField("databaseName", dbName) + } + }) + if !c.config.DebeziumDisableSchema { + keyJWriter.WriteObjectField("schema", func() { + keyJWriter.WriteStringField("type", "struct") + keyJWriter.WriteStringField("name", "io.debezium.connector.mysql.SchemaChangeKey") + keyJWriter.WriteBoolField("optional", false) + keyJWriter.WriteIntField("version", 1) + keyJWriter.WriteArrayField("fields", func() { + keyJWriter.WriteObjectElement(func() { + keyJWriter.WriteStringField("field", "databaseName") + keyJWriter.WriteBoolField("optional", false) + keyJWriter.WriteStringField("type", "string") + }) + }) + }) + } + }) + + // message value + jWriter.WriteObject(func() { + jWriter.WriteObjectField("payload", func() { + jWriter.WriteObjectField("source", func() { + jWriter.WriteStringField("version", "2.4.0.Final") + jWriter.WriteStringField("connector", "TiCDC") + jWriter.WriteStringField("name", c.clusterID) + jWriter.WriteInt64Field("ts_ms", commitTime.UnixMilli()) + jWriter.WriteStringField("snapshot", "false") + if e.TableInfo == nil { + jWriter.WriteStringField("db", "") + jWriter.WriteStringField("table", "") + } else { + jWriter.WriteStringField("db", dbName) + jWriter.WriteStringField("table", tableName) + } + jWriter.WriteInt64Field("server_id", 0) + jWriter.WriteNullField("gtid") + jWriter.WriteStringField("file", "") + jWriter.WriteInt64Field("pos", 0) + jWriter.WriteInt64Field("row", 0) + jWriter.WriteInt64Field("thread", 0) + jWriter.WriteNullField("query") + + // The followings are TiDB extended fields + jWriter.WriteUint64Field("commit_ts", e.CommitTs) + jWriter.WriteStringField("cluster_id", c.clusterID) + }) + jWriter.WriteInt64Field("ts_ms", c.nowFunc().UnixMilli()) + + if e.Type == timodel.ActionDropTable { + jWriter.WriteStringField("databaseName", e.PreTableInfo.GetSchemaName()) + } else { + jWriter.WriteStringField("databaseName", dbName) + } + jWriter.WriteNullField("schemaName") + jWriter.WriteStringField("ddl", e.Query) + jWriter.WriteArrayField("tableChanges", func() { + // return early if there is no table changes + if tableName == "" { + return + } + jWriter.WriteObjectElement(func() { + // Describes the kind of change. The value is one of the following: + // CREATE: Table created. + // ALTER: Table modified. + // DROP: Table deleted. + jWriter.WriteStringField("type", changeType) + // In the case of a table rename, this identifier is a concatenation of , table names. + if e.Type == timodel.ActionRenameTable { + jWriter.WriteStringField("id", fmt.Sprintf("\"%s\".\"%s\",\"%s\".\"%s\"", + e.PreTableInfo.GetSchemaName(), + e.PreTableInfo.GetTableName(), + dbName, + tableName)) + } else { + jWriter.WriteStringField("id", fmt.Sprintf("\"%s\".\"%s\"", + dbName, + tableName)) + } + // return early if there is no table info + if e.Type == timodel.ActionDropTable { + jWriter.WriteNullField("table") + return + } + jWriter.WriteObjectField("table", func() { + jWriter.WriteStringField("defaultCharsetName", e.TableInfo.Charset) + jWriter.WriteArrayField("primaryKeyColumnNames", func() { + for _, pk := range e.TableInfo.GetPrimaryKeyColumnNames() { + jWriter.WriteStringElement(pk) + } + }) + jWriter.WriteArrayField("columns", func() { + parseColumns(e.Query, e.TableInfo.Columns) + for pos, col := range e.TableInfo.Columns { + if col.Hidden { + continue + } + jWriter.WriteObjectElement(func() { + flag := col.GetFlag() + jdbcType := internal.MySQLType2JdbcType(col.GetType(), mysql.HasBinaryFlag(flag)) + expression, name := getExpressionAndName(col.FieldType) + jWriter.WriteStringField("name", col.Name.O) + jWriter.WriteIntField("jdbcType", int(jdbcType)) + jWriter.WriteNullField("nativeType") + if col.Comment != "" { + jWriter.WriteStringField("comment", col.Comment) + } else { + jWriter.WriteNullField("comment") + } + if col.DefaultValue == nil { + jWriter.WriteNullField("defaultValueExpression") + } else { + v, ok := col.DefaultValue.(string) + if ok { + if strings.ToUpper(v) == "CURRENT_TIMESTAMP" { + // https://debezium.io/documentation/reference/3.0/connectors/mysql.html#mysql-temporal-types + jWriter.WriteAnyField("defaultValueExpression", "1970-01-01 00:00:00") + } else if v == "" { + jWriter.WriteNullField("defaultValueExpression") + } else if col.DefaultValueBit != nil { + jWriter.WriteStringField("defaultValueExpression", parseBit(v, col.GetFlen())) + } else { + jWriter.WriteStringField("defaultValueExpression", v) + } + } else { + jWriter.WriteAnyField("defaultValueExpression", col.DefaultValue) + } + } + elems := col.GetElems() + if len(elems) != 0 { + // Format is ENUM ('e1', 'e2') or SET ('e1', 'e2') + jWriter.WriteArrayField("enumValues", func() { + for _, ele := range elems { + jWriter.WriteStringElement(fmt.Sprintf("'%s'", ele)) + } + }) + } else { + jWriter.WriteNullField("enumValues") + } + + jWriter.WriteStringField("typeName", name) + jWriter.WriteStringField("typeExpression", expression) + + charsetName := getCharset(col.FieldType) + if charsetName != "" { + jWriter.WriteStringField("charsetName", charsetName) + } else { + jWriter.WriteNullField("charsetName") + } + + length := getLen(col.FieldType) + if length != -1 { + jWriter.WriteIntField("length", length) + } else { + jWriter.WriteNullField("length") + } + + scale := getScale(col.FieldType) + if scale != -1 { + jWriter.WriteFloat64Field("scale", scale) + } else { + jWriter.WriteNullField("scale") + } + jWriter.WriteIntField("position", pos+1) + jWriter.WriteBoolField("optional", !mysql.HasNotNullFlag(flag)) + + autoIncrementFlag := mysql.HasAutoIncrementFlag(flag) + jWriter.WriteBoolField("autoIncremented", autoIncrementFlag) + jWriter.WriteBoolField("generated", autoIncrementFlag) + }) + } + }) + jWriter.WriteNullField("comment") + }) + }) + }) + }) + + if !c.config.DebeziumDisableSchema { + jWriter.WriteObjectField("schema", func() { + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "struct") + jWriter.WriteIntField("version", 1) + jWriter.WriteStringField("name", "io.debezium.connector.mysql.SchemaChangeValue") + jWriter.WriteArrayField("fields", func() { + c.writeSourceSchema(jWriter) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "ts_ms") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "int64") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "databaseName") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "schemaName") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "ddl") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "tableChanges") jWriter.WriteBoolField("optional", false) - jWriter.WriteStringField("name", "io.debezium.connector.mysql.Source") - jWriter.WriteStringField("field", "source") + jWriter.WriteStringField("type", "array") + jWriter.WriteObjectField("items", func() { + jWriter.WriteStringField("name", "io.debezium.connector.schema.Change") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "struct") + jWriter.WriteIntField("version", 1) + jWriter.WriteArrayField("fields", func() { + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "type") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "id") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "table") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "struct") + jWriter.WriteStringField("name", "io.debezium.connector.schema.Table") + jWriter.WriteIntField("version", 1) + jWriter.WriteArrayField("fields", func() { + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "defaultCharsetName") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "primaryKeyColumnNames") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "array") + jWriter.WriteObjectField("items", func() { + jWriter.WriteStringField("type", "string") + jWriter.WriteBoolField("optional", false) + }) + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "columns") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "array") + jWriter.WriteObjectField("items", func() { + jWriter.WriteStringField("name", "io.debezium.connector.schema.Column") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "struct") + jWriter.WriteIntField("version", 1) + jWriter.WriteArrayField("fields", func() { + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "name") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "jdbcType") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "int32") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "nativeType") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "int32") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "typeName") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "typeExpression") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "charsetName") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "length") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "int32") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "scale") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "int32") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "position") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("type", "int32") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "optional") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "boolean") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "autoIncremented") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "boolean") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "generated") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "boolean") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "comment") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "defaultValueExpression") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "string") + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "enumValues") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "array") + jWriter.WriteObjectField("items", func() { + jWriter.WriteStringField("type", "string") + jWriter.WriteBoolField("optional", false) + }) + }) + }) + }) + }) + jWriter.WriteObjectElement(func() { + jWriter.WriteStringField("field", "comment") + jWriter.WriteBoolField("optional", true) + jWriter.WriteStringField("type", "string") + }) + }) + }) + }) + }) }) + }) + }) + } + }) + return err +} + +// EncodeCheckpointEvent encode checkpointTs into debezium change event +func (c *dbzCodec) EncodeCheckpointEvent( + ts uint64, + keyDest io.Writer, + dest io.Writer, +) error { + keyJWriter := util.BorrowJSONWriter(keyDest) + jWriter := util.BorrowJSONWriter(dest) + defer util.ReturnJSONWriter(keyJWriter) + defer util.ReturnJSONWriter(jWriter) + commitTime := oracle.GetTimeFromTS(ts) + var err error + // message key + keyJWriter.WriteObject(func() { + keyJWriter.WriteObjectField("payload", func() {}) + if !c.config.DebeziumDisableSchema { + keyJWriter.WriteObjectField("schema", func() { + keyJWriter.WriteStringField("type", "struct") + keyJWriter.WriteStringField("name", + fmt.Sprintf("%s.%s.Key", common.SanitizeName(c.clusterID), "watermark")) + keyJWriter.WriteBoolField("optional", false) + keyJWriter.WriteArrayField("fields", func() { + }) + }) + } + }) + // message value + jWriter.WriteObject(func() { + jWriter.WriteObjectField("payload", func() { + jWriter.WriteObjectField("source", func() { + jWriter.WriteStringField("version", "2.4.0.Final") + jWriter.WriteStringField("connector", "TiCDC") + jWriter.WriteStringField("name", c.clusterID) + // ts_ms: In the source object, ts_ms indicates the time that the change was made in the database. + // https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-create-events + jWriter.WriteInt64Field("ts_ms", commitTime.UnixMilli()) + // snapshot field is a string of true,last,false,incremental + jWriter.WriteStringField("snapshot", "false") + jWriter.WriteStringField("db", "") + jWriter.WriteStringField("table", "") + jWriter.WriteInt64Field("server_id", 0) + jWriter.WriteNullField("gtid") + jWriter.WriteStringField("file", "") + jWriter.WriteInt64Field("pos", 0) + jWriter.WriteInt64Field("row", 0) + jWriter.WriteInt64Field("thread", 0) + jWriter.WriteNullField("query") + + // The followings are TiDB extended fields + jWriter.WriteUint64Field("commit_ts", ts) + jWriter.WriteStringField("cluster_id", c.clusterID) + }) + + // ts_ms: displays the time at which the connector processed the event + // https://debezium.io/documentation/reference/stable/connectors/mysql.html#mysql-create-events + jWriter.WriteInt64Field("ts_ms", c.nowFunc().UnixMilli()) + jWriter.WriteNullField("transaction") + jWriter.WriteStringField("op", "m") + }) + + if !c.config.DebeziumDisableSchema { + jWriter.WriteObjectField("schema", func() { + jWriter.WriteStringField("type", "struct") + jWriter.WriteBoolField("optional", false) + jWriter.WriteStringField("name", + fmt.Sprintf("%s.%s.Envelope", common.SanitizeName(c.clusterID), "watermark")) + jWriter.WriteIntField("version", 1) + jWriter.WriteArrayField("fields", func() { + c.writeSourceSchema(jWriter) jWriter.WriteObjectElement(func() { jWriter.WriteStringField("type", "string") jWriter.WriteBoolField("optional", false) @@ -994,13 +1683,5 @@ func (c *dbzCodec) EncodeValue( }) } }) - return err } - -func getSchemaTopicName(namespace string, schema string, table string) string { - return fmt.Sprintf("%s.%s.%s", - common.SanitizeName(namespace), - common.SanitizeName(schema), - common.SanitizeTopicName(table)) -} diff --git a/pkg/sink/codec/debezium/codec_test.go b/pkg/sink/codec/debezium/codec_test.go index 80fca04b51e..b637163c336 100644 --- a/pkg/sink/codec/debezium/codec_test.go +++ b/pkg/sink/codec/debezium/codec_test.go @@ -18,14 +18,700 @@ import ( "testing" "time" + timodel "github.com/pingcap/tidb/pkg/meta/model" "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/pingcap/tiflow/cdc/model" "github.com/pingcap/tiflow/pkg/config" + cerror "github.com/pingcap/tiflow/pkg/errors" "github.com/pingcap/tiflow/pkg/sink/codec/common" "github.com/stretchr/testify/require" "github.com/thanhpk/randstr" ) +func TestDDLEvent(t *testing.T) { + codec := &dbzCodec{ + config: common.NewConfig(config.ProtocolDebezium), + clusterID: "test_cluster", + nowFunc: func() time.Time { return time.Unix(1701326309, 0) }, + } + + query := "RENAME TABLE test.table1 to test.table2" + tableInfo := model.BuildTableInfo("test", "table1", []*model.Column{{ + Name: "id", + Type: mysql.TypeLong, + Flag: model.PrimaryKeyFlag | model.HandleKeyFlag, + }}, [][]int{{0}}) + preTableInfo := model.BuildTableInfo("test", "table2", []*model.Column{{ + Name: "id", + Type: mysql.TypeLong, + Flag: model.PrimaryKeyFlag | model.HandleKeyFlag, + }}, [][]int{{0}}) + e := &model.DDLEvent{ + CommitTs: 1, + TableInfo: tableInfo, + PreTableInfo: preTableInfo, + Type: timodel.ActionNone, + } + keyBuf := bytes.NewBuffer(nil) + buf := bytes.NewBuffer(nil) + err := codec.EncodeDDLEvent(e, keyBuf, buf) + require.ErrorIs(t, err, cerror.ErrDDLUnsupportType) + + e = &model.DDLEvent{ + CommitTs: 1, + TableInfo: tableInfo, + PreTableInfo: preTableInfo, + Query: query, + Type: timodel.ActionRenameTable, + } + keyBuf.Reset() + buf.Reset() + codec.config.DebeziumDisableSchema = false + err = codec.EncodeDDLEvent(e, keyBuf, buf) + require.Nil(t, err) + require.JSONEq(t, ` + { + "payload": { + "databaseName": "test" + }, + "schema": { + "type": "struct", + "name": "io.debezium.connector.mysql.SchemaChangeKey", + "optional": false, + "version": 1, + "fields": [ + { + "field": "databaseName", + "optional": false, + "type": "string" + } + ] + } + }`, keyBuf.String()) + require.JSONEq(t, ` + { + "payload": { + "source": { + "version": "2.4.0.Final", + "connector": "TiCDC", + "name": "test_cluster", + "ts_ms": 0, + "snapshot": "false", + "db": "test", + "table": "table1", + "server_id": 0, + "gtid": null, + "file": "", + "pos": 0, + "row": 0, + "thread": 0, + "query": null, + "commit_ts": 1, + "cluster_id": "test_cluster" + }, + "ts_ms": 1701326309000, + "databaseName": "test", + "schemaName": null, + "ddl": "RENAME TABLE test.table1 to test.table2", + "tableChanges": [ + { + "type": "ALTER", + "id": "\"test\".\"table2\",\"test\".\"table1\"", + "table": { + "defaultCharsetName": "", + "primaryKeyColumnNames": ["id"], + "columns": [ + { + "name": "id", + "jdbcType": 4, + "nativeType": null, + "comment": null, + "defaultValueExpression": null, + "enumValues": null, + "typeName": "INT", + "typeExpression": "INT", + "charsetName": null, + "length": 0, + "scale": null, + "position": 1, + "optional": false, + "autoIncremented": false, + "generated": false + } + ], + "comment": null + } + } + ] + }, + "schema": { + "optional": false, + "type": "struct", + "version": 1, + "name": "io.debezium.connector.mysql.SchemaChangeValue", + "fields": [ + { + "field": "source", + "name": "io.debezium.connector.mysql.Source", + "optional": false, + "type": "struct", + "fields": [ + { + "field": "version", + "optional": false, + "type": "string" + }, + { + "field": "connector", + "optional": false, + "type": "string" + }, + { + "field": "name", + "optional": false, + "type": "string" + }, + { + "field": "ts_ms", + "optional": false, + "type": "int64" + }, + { + "field": "snapshot", + "optional": true, + "type": "string", + "parameters": { + "allowed": "true,last,false,incremental" + }, + "default": "false", + "name": "io.debezium.data.Enum", + "version": 1 + }, + { + "field": "db", + "optional": false, + "type": "string" + }, + { + "field": "sequence", + "optional": true, + "type": "string" + }, + { + "field": "table", + "optional": true, + "type": "string" + }, + { + "field": "server_id", + "optional": false, + "type": "int64" + }, + { + "field": "gtid", + "optional": true, + "type": "string" + }, + { + "field": "file", + "optional": false, + "type": "string" + }, + { + "field": "pos", + "optional": false, + "type": "int64" + }, + { + "field": "row", + "optional": false, + "type": "int32" + }, + { + "field": "thread", + "optional": true, + "type": "int64" + }, + { + "field": "query", + "optional": true, + "type": "string" + } + ] + }, + { + "field": "ts_ms", + "optional": false, + "type": "int64" + }, + { + "field": "databaseName", + "optional": true, + "type": "string" + }, + { + "field": "schemaName", + "optional": true, + "type": "string" + }, + { + "field": "ddl", + "optional": true, + "type": "string" + }, + { + "field": "tableChanges", + "optional": false, + "type": "array", + "items": { + "name": "io.debezium.connector.schema.Change", + "optional": false, + "type": "struct", + "version": 1, + "fields": [ + { + "field": "type", + "optional": false, + "type": "string" + }, + { + "field": "id", + "optional": false, + "type": "string" + }, + { + "field": "table", + "optional": true, + "type": "struct", + "name": "io.debezium.connector.schema.Table", + "version": 1, + "fields": [ + { + "field": "defaultCharsetName", + "optional": true, + "type": "string" + }, + { + "field": "primaryKeyColumnNames", + "optional": true, + "type": "array", + "items": { + "type": "string", + "optional": false + } + }, + { + "field": "columns", + "optional": false, + "type": "array", + "items": { + "name": "io.debezium.connector.schema.Column", + "optional": false, + "type": "struct", + "version": 1, + "fields": [ + { + "field": "name", + "optional": false, + "type": "string" + }, + { + "field": "jdbcType", + "optional": false, + "type": "int32" + }, + { + "field": "nativeType", + "optional": true, + "type": "int32" + }, + { + "field": "typeName", + "optional": false, + "type": "string" + }, + { + "field": "typeExpression", + "optional": true, + "type": "string" + }, + { + "field": "charsetName", + "optional": true, + "type": "string" + }, + { + "field": "length", + "optional": true, + "type": "int32" + }, + { + "field": "scale", + "optional": true, + "type": "int32" + }, + { + "field": "position", + "optional": false, + "type": "int32" + }, + { + "field": "optional", + "optional": true, + "type": "boolean" + }, + { + "field": "autoIncremented", + "optional": true, + "type": "boolean" + }, + { + "field": "generated", + "optional": true, + "type": "boolean" + }, + { + "field": "comment", + "optional": true, + "type": "string" + }, + { + "field": "defaultValueExpression", + "optional": true, + "type": "string" + }, + { + "field": "enumValues", + "optional": true, + "type": "array", + "items": { + "type": "string", + "optional": false + } + } + ] + } + }, + { + "field": "comment", + "optional": true, + "type": "string" + } + ] + } + ] + } + } + ] + } + }`, buf.String()) + + codec.config.DebeziumDisableSchema = true + + query = "CREATE TABLE test.table1" + e = &model.DDLEvent{ + CommitTs: 1, + TableInfo: tableInfo, + Query: query, + Type: timodel.ActionCreateTable, + } + keyBuf.Reset() + buf.Reset() + err = codec.EncodeDDLEvent(e, keyBuf, buf) + require.Nil(t, err) + require.JSONEq(t, ` + { + "payload": { + "databaseName": "test" + } + }`, keyBuf.String()) + require.JSONEq(t, ` + { + "payload": { + "source": { + "version": "2.4.0.Final", + "connector": "TiCDC", + "name": "test_cluster", + "ts_ms": 0, + "snapshot": "false", + "db": "test", + "table": "table1", + "server_id": 0, + "gtid": null, + "file": "", + "pos": 0, + "row": 0, + "thread": 0, + "query": null, + "commit_ts": 1, + "cluster_id": "test_cluster" + }, + "ts_ms": 1701326309000, + "databaseName": "test", + "schemaName": null, + "ddl": "CREATE TABLE test.table1", + "tableChanges": [ + { + "type": "CREATE", + "id": "\"test\".\"table1\"", + "table": { + "defaultCharsetName": "", + "primaryKeyColumnNames": ["id"], + "columns": [ + { + "name": "id", + "jdbcType": 4, + "nativeType": null, + "comment": null, + "defaultValueExpression": null, + "enumValues": null, + "typeName": "INT", + "typeExpression": "INT", + "charsetName": null, + "length": 0, + "scale": null, + "position": 1, + "optional": false, + "autoIncremented": false, + "generated": false + } + ], + "comment": null + } + } + ] + } + }`, buf.String()) + + query = "DROP TABLE test.table2" + e = &model.DDLEvent{ + CommitTs: 1, + PreTableInfo: preTableInfo, + Query: query, + Type: timodel.ActionDropTable, + } + keyBuf.Reset() + buf.Reset() + err = codec.EncodeDDLEvent(e, keyBuf, buf) + require.Nil(t, err) + require.JSONEq(t, ` + { + "payload": { + "databaseName": "test" + } + }`, keyBuf.String()) + require.JSONEq(t, ` + { + "payload": { + "source": { + "version": "2.4.0.Final", + "connector": "TiCDC", + "name": "test_cluster", + "ts_ms": 0, + "snapshot": "false", + "db": "", + "table": "", + "server_id": 0, + "gtid": null, + "file": "", + "pos": 0, + "row": 0, + "thread": 0, + "query": null, + "commit_ts": 1, + "cluster_id": "test_cluster" + }, + "ts_ms": 1701326309000, + "databaseName": "test", + "schemaName": null, + "ddl": "DROP TABLE test.table2", + "tableChanges": [] + } + }`, buf.String()) +} + +func TestCheckPointEvent(t *testing.T) { + codec := &dbzCodec{ + config: common.NewConfig(config.ProtocolDebezium), + clusterID: "test_cluster", + nowFunc: func() time.Time { return time.Unix(1701326309, 0) }, + } + codec.config.DebeziumDisableSchema = false + + var ts uint64 = 3 + keyBuf := bytes.NewBuffer(nil) + buf := bytes.NewBuffer(nil) + err := codec.EncodeCheckpointEvent(ts, keyBuf, buf) + require.Nil(t, err) + require.JSONEq(t, ` + { + "payload": {}, + "schema": { + "fields": [], + "optional": false, + "name": "test_cluster.watermark.Key", + "type": "struct" + } + }`, keyBuf.String()) + require.JSONEq(t, ` + { + "payload": { + "source": { + "version": "2.4.0.Final", + "connector": "TiCDC", + "name": "test_cluster", + "ts_ms": 0, + "snapshot": "false", + "db": "", + "table": "", + "server_id": 0, + "gtid": null, + "file": "", + "pos": 0, + "row": 0, + "thread": 0, + "query": null, + "commit_ts": 3, + "cluster_id": "test_cluster" + }, + "op":"m", + "ts_ms": 1701326309000, + "transaction": null + }, + "schema": { + "type": "struct", + "optional": false, + "name": "test_cluster.watermark.Envelope", + "version": 1, + "fields": [ + { + "type": "struct", + "fields": [ + { + "type": "string", + "optional": false, + "field": "version" + }, + { + "type": "string", + "optional": false, + "field": "connector" + }, + { + "type": "string", + "optional": false, + "field": "name" + }, + { + "type": "int64", + "optional": false, + "field": "ts_ms" + }, + { + "type": "string", + "optional": true, + "name": "io.debezium.data.Enum", + "version": 1, + "parameters": { + "allowed": "true,last,false,incremental" + }, + "default": "false", + "field": "snapshot" + }, + { + "type": "string", + "optional": false, + "field": "db" + }, + { + "type": "string", + "optional": true, + "field": "sequence" + }, + { + "type": "string", + "optional": true, + "field": "table" + }, + { + "type": "int64", + "optional": false, + "field": "server_id" + }, + { + "type": "string", + "optional": true, + "field": "gtid" + }, + { + "type": "string", + "optional": false, + "field": "file" + }, + { + "type": "int64", + "optional": false, + "field": "pos" + }, + { + "type": "int32", + "optional": false, + "field": "row" + }, + { + "type": "int64", + "optional": true, + "field": "thread" + }, + { + "type": "string", + "optional": true, + "field": "query" + } + ], + "optional": false, + "name": "io.debezium.connector.mysql.Source", + "field": "source" + }, + { + "type": "string", + "optional": false, + "field": "op" + }, + { + "type": "int64", + "optional": true, + "field": "ts_ms" + }, + { + "type": "struct", + "fields": [ + { + "type": "string", + "optional": false, + "field": "id" + }, + { + "type": "int64", + "optional": false, + "field": "total_order" + }, + { + "type": "int64", + "optional": false, + "field": "data_collection_order" + } + ], + "optional": true, + "name": "event.block", + "version": 1, + "field": "transaction" + } + ] + } + }`, buf.String()) +} + func TestEncodeInsert(t *testing.T) { codec := &dbzCodec{ config: common.NewConfig(config.ProtocolDebezium), diff --git a/pkg/sink/codec/debezium/encoder.go b/pkg/sink/codec/debezium/encoder.go index 536d6a28441..085a292ffd0 100644 --- a/pkg/sink/codec/debezium/encoder.go +++ b/pkg/sink/codec/debezium/encoder.go @@ -18,11 +18,13 @@ import ( "context" "time" + "github.com/pingcap/log" "github.com/pingcap/tiflow/cdc/model" "github.com/pingcap/tiflow/pkg/config" "github.com/pingcap/tiflow/pkg/errors" "github.com/pingcap/tiflow/pkg/sink/codec" "github.com/pingcap/tiflow/pkg/sink/codec/common" + "go.uber.org/zap" ) // BatchEncoder encodes message into Debezium format. @@ -35,8 +37,30 @@ type BatchEncoder struct { // EncodeCheckpointEvent implements the RowEventEncoder interface func (d *BatchEncoder) EncodeCheckpointEvent(ts uint64) (*common.Message, error) { - // Currently ignored. Debezium MySQL Connector does not emit such event. - return nil, nil + keyMap := bytes.Buffer{} + valueBuf := bytes.Buffer{} + err := d.codec.EncodeCheckpointEvent(ts, &keyMap, &valueBuf) + if err != nil { + return nil, errors.Trace(err) + } + key, err := common.Compress( + d.config.ChangefeedID, + d.config.LargeMessageHandle.LargeMessageHandleCompression, + keyMap.Bytes(), + ) + if err != nil { + return nil, err + } + value, err := common.Compress( + d.config.ChangefeedID, + d.config.LargeMessageHandle.LargeMessageHandleCompression, + valueBuf.Bytes(), + ) + if err != nil { + return nil, err + } + result := common.NewResolvedMsg(config.ProtocolDebezium, key, value, ts) + return result, nil } func (d *BatchEncoder) encodeKey(e *model.RowChangedEvent) ([]byte, error) { @@ -102,10 +126,36 @@ func (d *BatchEncoder) AppendRowChangedEvent( } // EncodeDDLEvent implements the RowEventEncoder interface -// DDL message unresolved tso func (d *BatchEncoder) EncodeDDLEvent(e *model.DDLEvent) (*common.Message, error) { - // Schema Change Events are currently not supported. - return nil, nil + valueBuf := bytes.Buffer{} + keyMap := bytes.Buffer{} + err := d.codec.EncodeDDLEvent(e, &keyMap, &valueBuf) + if err != nil { + if errors.ErrDDLUnsupportType.Equal(err) { + log.Warn("encode ddl event failed, just ignored", zap.Error(err)) + return nil, nil + } + return nil, errors.Trace(err) + } + key, err := common.Compress( + d.config.ChangefeedID, + d.config.LargeMessageHandle.LargeMessageHandleCompression, + keyMap.Bytes(), + ) + if err != nil { + return nil, err + } + value, err := common.Compress( + d.config.ChangefeedID, + d.config.LargeMessageHandle.LargeMessageHandleCompression, + valueBuf.Bytes(), + ) + if err != nil { + return nil, err + } + result := common.NewDDLMsg(config.ProtocolDebezium, key, value, e) + + return result, nil } // Build implements the RowEventEncoder interface diff --git a/pkg/sink/codec/debezium/helper.go b/pkg/sink/codec/debezium/helper.go new file mode 100644 index 00000000000..a2c957b39ab --- /dev/null +++ b/pkg/sink/codec/debezium/helper.go @@ -0,0 +1,265 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package debezium + +import ( + "encoding/binary" + "fmt" + "strings" + + "github.com/pingcap/log" + timodel "github.com/pingcap/tidb/pkg/meta/model" + "github.com/pingcap/tidb/pkg/parser" + "github.com/pingcap/tidb/pkg/parser/ast" + pmodel "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" + driver "github.com/pingcap/tidb/pkg/types/parser_driver" + "github.com/pingcap/tiflow/cdc/model" + "github.com/pingcap/tiflow/pkg/sink/codec/common" + "go.uber.org/zap" +) + +type visiter struct { + columnsMap map[pmodel.CIStr]*timodel.ColumnInfo +} + +func (v *visiter) Enter(n ast.Node) (node ast.Node, skipChildren bool) { + return n, false +} + +func (v *visiter) Leave(n ast.Node) (node ast.Node, ok bool) { + switch col := n.(type) { + case *ast.ColumnDef: + c := v.columnsMap[col.Name.Name] + if col.Options != nil { + parseOptions(col.Options, c) + } + if col.Tp != nil { + parseType(c, col) + } + c.Comment = "" // disable comment + } + return n, true +} + +func extractValue(expr ast.ExprNode) any { + switch v := expr.(type) { + case *driver.ValueExpr: + return fmt.Sprintf("%v", v.GetValue()) + case *ast.FuncCallExpr: + return v.FnName.String() + } + return nil +} + +func parseType(c *timodel.ColumnInfo, col *ast.ColumnDef) { + ft := col.Tp + switch ft.GetType() { + case mysql.TypeDatetime, mysql.TypeDuration, mysql.TypeTimestamp: + c.SetDecimal(ft.GetDecimal()) + if c.OriginDefaultValue != nil { + c.SetDefaultValue(c.OriginDefaultValue) + } + case mysql.TypeYear: + c.SetFlen(ft.GetFlen()) + if c.OriginDefaultValue != nil { + c.SetDefaultValue(c.OriginDefaultValue) + } + default: + } +} + +func parseOptions(options []*ast.ColumnOption, c *timodel.ColumnInfo) { + for _, option := range options { + switch option.Tp { + case ast.ColumnOptionDefaultValue: + defaultValue := extractValue(option.Expr) + if defaultValue == nil { + continue + } + if err := c.SetOriginDefaultValue(defaultValue); err != nil { + log.Error("failed to set default value") + } + } + } +} + +func parseColumns(sql string, columns []*timodel.ColumnInfo) { + p := parser.New() + stmt, err := p.ParseOneStmt(sql, mysql.DefaultCharset, mysql.DefaultCollationName) + if err != nil { + log.Error("format query parse one stmt failed", zap.Error(err)) + } + + columnsMap := make(map[pmodel.CIStr]*timodel.ColumnInfo, len(columns)) + for _, col := range columns { + columnsMap[col.Name] = col + } + stmt.Accept(&visiter{columnsMap: columnsMap}) +} + +func parseBit(s string, n int) string { + var result string + if len(s) > 0 { + // Leading zeros may be omitted + result = fmt.Sprintf("%0*b", n%8, s[0]) + } + for i := 1; i < len(s); i++ { + result += fmt.Sprintf("%08b", s[i]) + } + return result +} + +func getCharset(ft types.FieldType) string { + if ft.GetCharset() == "binary" { + return "" + } + switch ft.GetType() { + case mysql.TypeTimestamp, mysql.TypeDuration, mysql.TypeNewDecimal, mysql.TypeString, mysql.TypeVarchar, + mysql.TypeBlob, mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeEnum, mysql.TypeSet: + return ft.GetCharset() + } + return "" +} + +func getLen(ft types.FieldType) int { + defaultFlen, _ := mysql.GetDefaultFieldLengthAndDecimal(ft.GetType()) + decimal := ft.GetDecimal() + flen := ft.GetFlen() + switch ft.GetType() { + case mysql.TypeTimestamp, mysql.TypeDuration, mysql.TypeDatetime: + return decimal + case mysql.TypeBit, mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString, mysql.TypeTiDBVectorFloat32, + mysql.TypeLonglong, mysql.TypeFloat, mysql.TypeDouble: + if flen != defaultFlen { + return flen + } + case mysql.TypeTiny, mysql.TypeShort, mysql.TypeInt24, mysql.TypeLong: + if mysql.HasUnsignedFlag(ft.GetFlag()) { + defaultFlen -= 1 + } + if ft.GetType() == mysql.TypeTiny && mysql.HasZerofillFlag(ft.GetFlag()) { + defaultFlen += 1 + } + if flen != defaultFlen { + return flen + } + case mysql.TypeYear, mysql.TypeNewDecimal: + return flen + case mysql.TypeSet: + return 2*len(ft.GetElems()) - 1 + case mysql.TypeEnum: + return 1 + } + return -1 +} + +func getScale(ft types.FieldType) float64 { + switch ft.GetType() { + case mysql.TypeNewDecimal, mysql.TypeFloat, mysql.TypeDouble: + return float64(ft.GetDecimal()) + } + return -1 +} + +func getSuffix(ft types.FieldType) string { + suffix := "" + decimal := ft.GetDecimal() + flen := ft.GetFlen() + defaultFlen, defaultDecimal := mysql.GetDefaultFieldLengthAndDecimal(ft.GetType()) + isDecimalNotDefault := decimal != defaultDecimal && decimal != 0 && decimal != -1 + + // displayFlen and displayDecimal are flen and decimal values with `-1` substituted with default value. + displayFlen, displayDecimal := flen, decimal + if displayFlen == -1 { + displayFlen = defaultFlen + } + if displayDecimal == -1 { + displayDecimal = defaultDecimal + } + + switch ft.GetType() { + case mysql.TypeDouble: + // 1. flen Not Default, decimal Not Default -> Valid + // 2. flen Not Default, decimal Default (-1) -> Invalid + // 3. flen Default, decimal Not Default -> Valid + // 4. flen Default, decimal Default -> Valid (hide)W + if isDecimalNotDefault { + suffix = fmt.Sprintf("(%d,%d)", displayFlen, displayDecimal) + } + case mysql.TypeNewDecimal: + suffix = fmt.Sprintf("(%d,%d)", displayFlen, displayDecimal) + case mysql.TypeVarchar, mysql.TypeString, mysql.TypeVarString: + if !mysql.HasBinaryFlag(ft.GetFlag()) && displayFlen != 1 { + suffix = fmt.Sprintf("(%d)", displayFlen) + } + case mysql.TypeYear: + suffix = fmt.Sprintf("(%d)", flen) + case mysql.TypeTiDBVectorFloat32: + if flen != -1 { + suffix = fmt.Sprintf("(%d)", flen) + } + case mysql.TypeNull: + suffix = "(0)" + } + return suffix +} + +func getExpressionAndName(ft types.FieldType) (string, string) { + prefix := strings.ToUpper(types.TypeToStr(ft.GetType(), ft.GetCharset())) + switch ft.GetType() { + case mysql.TypeYear, mysql.TypeBit, mysql.TypeVarchar, mysql.TypeString, mysql.TypeNewDecimal: + return prefix, prefix + } + cs := prefix + getSuffix(ft) + var suf string + if mysql.HasZerofillFlag(ft.GetFlag()) { + suf = " UNSIGNED ZEROFILL" + } else if mysql.HasUnsignedFlag(ft.GetFlag()) { + suf = " UNSIGNED" + } + return cs + suf, prefix + suf +} + +func getBitFromUint64(n int, v uint64) []byte { + var buf [8]byte + binary.LittleEndian.PutUint64(buf[:], v) + numBytes := n / 8 + if n%8 != 0 { + numBytes += 1 + } + return buf[:numBytes] +} + +func getValue(col model.ColumnDataX) any { + if col.Value == nil { + return col.GetDefaultValue() + } + return col.Value +} + +func getDBTableName(e *model.DDLEvent) (string, string) { + if e.TableInfo == nil { + return "", "" + } + return e.TableInfo.GetSchemaName(), e.TableInfo.GetTableName() +} + +func getSchemaTopicName(namespace string, schema string, table string) string { + return fmt.Sprintf("%s.%s.%s", + common.SanitizeName(namespace), + common.SanitizeName(schema), + common.SanitizeTopicName(table)) +} diff --git a/pkg/sink/codec/debezium/helper_test.go b/pkg/sink/codec/debezium/helper_test.go new file mode 100644 index 00000000000..8713da21e22 --- /dev/null +++ b/pkg/sink/codec/debezium/helper_test.go @@ -0,0 +1,80 @@ +// Copyright 2024 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package debezium + +import ( + "testing" + + timodel "github.com/pingcap/tidb/pkg/meta/model" + pmodel "github.com/pingcap/tidb/pkg/parser/model" + "github.com/pingcap/tidb/pkg/parser/mysql" + "github.com/pingcap/tidb/pkg/types" + "github.com/pingcap/tiflow/cdc/model" + "github.com/stretchr/testify/require" +) + +func TestGetColumns(t *testing.T) { + sql := "CREATE TABLE test (id INT PRIMARY KEY, val1 datetime default current_timestamp, val2 time(2) default 0, val3 timestamp(3) default now(), val4 YEAR(4) default 1970 comment 'first');" + columnInfos := []*timodel.ColumnInfo{ + { + Name: pmodel.NewCIStr("id"), + FieldType: *types.NewFieldType(mysql.TypeLong), + }, + { + Name: pmodel.NewCIStr("val1"), + FieldType: *types.NewFieldType(mysql.TypeDatetime), + }, + { + Name: pmodel.NewCIStr("val2"), + FieldType: *types.NewFieldType(mysql.TypeDuration), + }, + { + Name: pmodel.NewCIStr("val3"), + FieldType: *types.NewFieldType(mysql.TypeTimestamp), + }, + { + Name: pmodel.NewCIStr("val4"), + FieldType: *types.NewFieldType(mysql.TypeYear), + }, + } + parseColumns(sql, columnInfos) + require.Equal(t, columnInfos[1].GetDefaultValue(), "CURRENT_TIMESTAMP") + require.Equal(t, columnInfos[2].GetDecimal(), 2) + require.Equal(t, columnInfos[2].GetDefaultValue(), "0") + require.Equal(t, columnInfos[3].GetDecimal(), 3) + require.Equal(t, columnInfos[3].GetDefaultValue(), "CURRENT_TIMESTAMP") + require.Equal(t, columnInfos[4].GetFlen(), 4) + require.Equal(t, columnInfos[4].GetDefaultValue(), "1970") + require.Equal(t, columnInfos[4].Comment, "") +} + +func TestGetValue(t *testing.T) { + column := &model.Column{ + Default: 1, + } + data := model.Column2ColumnDataXForTest(column) + v := getValue(data) + require.Equal(t, v, int64(1)) + data.Value = 2 + v = getValue(data) + require.Equal(t, v, 2) +} + +func TestGetSchemaTopicName(t *testing.T) { + namespace := "default" + schema := "1A.B" + table := "columnNameWith中文" + name := getSchemaTopicName(namespace, schema, table) + require.Equal(t, name, "default._1A_B.columnNameWith__") +} diff --git a/pkg/sink/codec/internal/java.go b/pkg/sink/codec/internal/java.go index b25421fb6ba..8c3d29763f0 100644 --- a/pkg/sink/codec/internal/java.go +++ b/pkg/sink/codec/internal/java.go @@ -20,31 +20,33 @@ type JavaSQLType int32 // jdk 1.8 const ( - JavaSQLTypeBIT JavaSQLType = -7 - JavaSQLTypeTINYINT JavaSQLType = -6 - JavaSQLTypeSMALLINT JavaSQLType = 5 - JavaSQLTypeINTEGER JavaSQLType = 4 - JavaSQLTypeBIGINT JavaSQLType = -5 - JavaSQLTypeREAL JavaSQLType = 7 - JavaSQLTypeDOUBLE JavaSQLType = 8 - JavaSQLTypeDECIMAL JavaSQLType = 3 - JavaSQLTypeCHAR JavaSQLType = 1 - JavaSQLTypeVARCHAR JavaSQLType = 12 - JavaSQLTypeDATE JavaSQLType = 91 - JavaSQLTypeTIME JavaSQLType = 92 - JavaSQLTypeTIMESTAMP JavaSQLType = 93 - JavaSQLTypeBINARY JavaSQLType = -2 - JavaSQLTypeVARBINARY JavaSQLType = -3 - JavaSQLTypeLONGVARBINARY JavaSQLType = -4 - JavaSQLTypeNULL JavaSQLType = 0 - JavaSQLTypeBLOB JavaSQLType = 2004 - JavaSQLTypeCLOB JavaSQLType = 2005 + JavaSQLTypeBIT JavaSQLType = -7 + JavaSQLTypeTINYINT JavaSQLType = -6 + JavaSQLTypeSMALLINT JavaSQLType = 5 + JavaSQLTypeINTEGER JavaSQLType = 4 + JavaSQLTypeBIGINT JavaSQLType = -5 + JavaSQLTypeREAL JavaSQLType = 7 + JavaSQLTypeDOUBLE JavaSQLType = 8 + JavaSQLTypeDECIMAL JavaSQLType = 3 + JavaSQLTypeCHAR JavaSQLType = 1 + JavaSQLTypeVARCHAR JavaSQLType = 12 + JavaSQLTypeDATE JavaSQLType = 91 + JavaSQLTypeTIME JavaSQLType = 92 + JavaSQLTypeTIMESTAMP JavaSQLType = 93 + JavaSQLTypeBINARY JavaSQLType = -2 + JavaSQLTypeVARBINARY JavaSQLType = -3 + JavaSQLTypeLONGVARBINARY JavaSQLType = -4 + JavaSQLTypeNULL JavaSQLType = 0 + JavaSQLTypeBLOB JavaSQLType = 2004 + JavaSQLTypeCLOB JavaSQLType = 2005 + JavaSQLTypeFLOAT JavaSQLType = 6 + JavaSQLTypeNUMERIC JavaSQLType = 2 + JavaSQLTypeOTHER JavaSQLType = 1111 + JavaSQLTypeNCHAR JavaSQLType = -15 + JavaSQLTypeTIMESTAMP_WITH_TIMEZONE JavaSQLType = 2014 //nolint // unused // JavaSQLTypeLONGVARCHAR JavaSQLType = -1 - // JavaSQLTypeFLOAT JavaSQLType = 6 - // JavaSQLTypeNUMERIC JavaSQLType = 2 - // JavaSQLTypeOTHER JavaSQLType = 1111 // JavaSQLTypeJAVA_OBJECT JavaSQLType = 2000 // JavaSQLTypeDISTINCT JavaSQLType = 2001 // JavaSQLTypeSTRUCT JavaSQLType = 2002 @@ -53,14 +55,12 @@ const ( // JavaSQLTypeDATALINK JavaSQLType = 70 // JavaSQLTypeBOOLEAN JavaSQLType = 16 // JavaSQLTypeROWID JavaSQLType = -8 - // JavaSQLTypeNCHAR JavaSQLType = -15 // JavaSQLTypeNVARCHAR JavaSQLType = -9 // JavaSQLTypeLONGNVARCHAR JavaSQLType = -16 // JavaSQLTypeNCLOB JavaSQLType = 2011 // JavaSQLTypeSQLXML JavaSQLType = 2009 // JavaSQLTypeREF_CURSOR JavaSQLType = 2012 // JavaSQLTypeTIME_WITH_TIMEZONE JavaSQLType = 2013 - // JavaSQLTypeTIMESTAMP_WITH_TIMEZONE JavaSQLType = 2014 ) // MySQLType2JavaType converts the mysql protocol types to java sql types @@ -150,3 +150,76 @@ func MySQLType2JavaType(mysqlType byte, isBinary bool) JavaSQLType { return JavaSQLTypeVARCHAR } } + +// MySQLType2JdbcType converts the mysql protocol types to jdbc type +func MySQLType2JdbcType(mysqlType byte, isBinary bool) JavaSQLType { + switch mysqlType { + case mysql.TypeTiny, mysql.TypeShort: + return JavaSQLTypeSMALLINT + + case mysql.TypeLong, mysql.TypeInt24, mysql.TypeYear: + return JavaSQLTypeINTEGER + + case mysql.TypeFloat: + return JavaSQLTypeFLOAT + + case mysql.TypeDouble: + return JavaSQLTypeDOUBLE + + case mysql.TypeNull: + return JavaSQLTypeNULL + + case mysql.TypeNewDecimal: + return JavaSQLTypeDECIMAL // equal to JavaSQLTypeNUMERIC + + case mysql.TypeTimestamp: + return JavaSQLTypeTIMESTAMP_WITH_TIMEZONE //nolint + + case mysql.TypeDatetime: + return JavaSQLTypeTIMESTAMP + + case mysql.TypeLonglong: + return JavaSQLTypeBIGINT + + case mysql.TypeDate, mysql.TypeNewDate: + return JavaSQLTypeDATE + + case mysql.TypeDuration: + return JavaSQLTypeTIME + + case mysql.TypeEnum, mysql.TypeSet: + return JavaSQLTypeCHAR + + case mysql.TypeTinyBlob, mysql.TypeMediumBlob, mysql.TypeLongBlob, mysql.TypeBlob: + if isBinary { + return JavaSQLTypeBLOB + } + return JavaSQLTypeVARCHAR + + case mysql.TypeVarString, mysql.TypeVarchar: + if isBinary { + return JavaSQLTypeVARBINARY + } + return JavaSQLTypeVARCHAR + + case mysql.TypeString: + if isBinary { + return JavaSQLTypeBINARY + } + return JavaSQLTypeCHAR + // return JavaSQLTypeNCHAR + + // Geometry is not supported, this should not hit. + case mysql.TypeGeometry: + return JavaSQLTypeOTHER + + case mysql.TypeBit: + return JavaSQLTypeBIT + + case mysql.TypeJSON: + return JavaSQLTypeOTHER + + default: + return JavaSQLTypeVARCHAR + } +} diff --git a/tests/integration_tests/debezium/README.md b/tests/integration_tests/debezium/README.md index d57365325dc..87ef70a3074 100644 --- a/tests/integration_tests/debezium/README.md +++ b/tests/integration_tests/debezium/README.md @@ -41,9 +41,9 @@ tiup playground nightly --tiflash 0 --ticdc 1 ``` ``` -tiup cdc cli changefeed create \ - --server=http://127.0.0.1:8300 \ - --sink-uri="kafka://127.0.0.1:9094/output_ticdc?protocol=debezium&kafka-version=2.4.0" +tiup cdc cli changefeed create -c test \ + --server=http://127.0.0.1:8300 --config changefeed.toml \ + --sink-uri="kafka://127.0.0.1:9094/output_ticdc?protocol=debezium" ``` ``` diff --git a/tests/integration_tests/debezium/changefeed.toml b/tests/integration_tests/debezium/changefeed.toml new file mode 100644 index 00000000000..995c2d4ed03 --- /dev/null +++ b/tests/integration_tests/debezium/changefeed.toml @@ -0,0 +1 @@ +force-replicate = true \ No newline at end of file diff --git a/tests/integration_tests/debezium/docker-compose.yml b/tests/integration_tests/debezium/docker-compose.yml index 2041a980070..aa7f09fe336 100644 --- a/tests/integration_tests/debezium/docker-compose.yml +++ b/tests/integration_tests/debezium/docker-compose.yml @@ -45,6 +45,7 @@ services: - CONFIG_STORAGE_TOPIC=my_connect_configs - OFFSET_STORAGE_TOPIC=my_connect_offsets - STATUS_STORAGE_TOPIC=my_connect_statuses + - LANG=C.UTF-8 # watcher_dbz: # For Debug Purpose # restart: always # image: quay.io/debezium/kafka:2.4 diff --git a/tests/integration_tests/debezium/go.mod b/tests/integration_tests/debezium/go.mod index 454c383b03a..5bcf5da15c7 100644 --- a/tests/integration_tests/debezium/go.mod +++ b/tests/integration_tests/debezium/go.mod @@ -3,82 +3,85 @@ module github.com/breezewish/checker go 1.23.0 require ( + github.com/alecthomas/chroma v0.10.0 + github.com/fatih/color v1.17.0 github.com/go-sql-driver/mysql v1.7.1 - github.com/pingcap/tidb v1.1.0-beta.0.20231117065153-a4f85c356873 - github.com/pingcap/tidb/pkg/parser v0.0.0-20231116213047-1f7c1e02bcd4 + github.com/google/go-cmp v0.6.0 + github.com/google/uuid v1.6.0 + github.com/pingcap/tidb v1.1.0-beta.0.20240920050217-adbc0034ffb4 + github.com/pingcap/tidb/pkg/parser v0.0.0-20240920050217-adbc0034ffb4 + github.com/segmentio/kafka-go v0.4.45 github.com/thessem/zap-prettyconsole v0.3.0 - go.uber.org/zap v1.26.0 + go.uber.org/zap v1.27.0 ) require ( github.com/Code-Hex/dd v1.1.0 // indirect - github.com/alecthomas/chroma v0.10.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudfoundry/gosigar v1.3.6 // indirect - github.com/cockroachdb/errors v1.8.1 // indirect - github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect - github.com/cockroachdb/redact v1.0.8 // indirect - github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.5 // indirect github.com/coreos/go-semver v0.3.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/dlclark/regexp2 v1.4.0 // indirect - github.com/fatih/color v1.16.0 // indirect - github.com/go-ole/go-ole v1.2.6 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/getsentry/sentry-go v0.27.0 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 // indirect - github.com/google/go-cmp v0.6.0 // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect - github.com/klauspost/compress v1.17.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect - github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 // indirect - github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect - github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071 // indirect - github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 // indirect - github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect + github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f // indirect + github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a // indirect + github.com/pingcap/kvproto v0.0.0-20240910154453-b242104f8d31 // indirect + github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d // indirect + github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b // indirect - github.com/prometheus/client_golang v1.17.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/client_golang v1.20.2 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.57.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/rogpeppe/go-internal v1.11.0 // indirect - github.com/segmentio/kafka-go v0.4.45 // indirect - github.com/shirou/gopsutil/v3 v3.23.10 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + github.com/shirou/gopsutil/v3 v3.24.4 // indirect github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect - github.com/tikv/client-go/v2 v2.0.8-0.20231114060955-8fc8a528217e // indirect - github.com/tikv/pd/client v0.0.0-20231114041114-86831ce71865 // indirect + github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba // indirect + github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9 // indirect github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/twmb/murmur3 v1.1.6 // indirect - github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.etcd.io/etcd/api/v3 v3.5.10 // indirect - go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect - go.etcd.io/etcd/client/v3 v3.5.10 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + go.etcd.io/etcd/api/v3 v3.5.12 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.12 // indirect + go.etcd.io/etcd/client/v3 v3.5.12 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/net v0.18.0 // indirect - golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.14.0 // indirect - golang.org/x/text v0.14.0 // indirect - google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect - google.golang.org/grpc v1.59.0 // indirect - google.golang.org/protobuf v1.31.0 // indirect + golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect ) diff --git a/tests/integration_tests/debezium/go.sum b/tests/integration_tests/debezium/go.sum index 3fe1773d286..f34982e8f18 100644 --- a/tests/integration_tests/debezium/go.sum +++ b/tests/integration_tests/debezium/go.sum @@ -1,197 +1,133 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= -github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= -github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/Code-Hex/dd v1.1.0 h1:VEtTThnS9l7WhpKUIpdcWaf0B8Vp0LeeSEsxA1DZseI= github.com/Code-Hex/dd v1.1.0/go.mod h1:VaMyo/YjTJ3d4qm/bgtrUkT2w+aYwJ07Y7eCWyrJr1w= -github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= -github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= -github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbfjek= github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= -github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 h1:BjkPE3785EwPhhyuFkbINB+2a1xATwk8SNDWnJiD41g= +github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5/go.mod h1:jtAfVaU/2cu1+wdSRPWE2c1N2qeAA3K4RH9pYgqwets= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudfoundry/gosigar v1.3.6 h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc= github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= -github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= -github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= -github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= -github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= -github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 h1:W1SHiII3e0jVwvaQFglwu3kS9NLxOeTpvik7MbKCyuQ= +github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64/go.mod h1:F86k/6c7aDUdwSUevnLpHS/3Q9hzYCE99jGk2xsHnt0= github.com/coocood/freecache v1.2.1 h1:/v1CqMq45NFH9mp/Pt142reundeBM0dVUD3osQBeu/U= github.com/coocood/freecache v1.2.1/go.mod h1:RBUWa/Cy+OHdfTGFEhEuE1pMCMX51Ncizj7rthiQ3vk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= -github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 h1:NnLfQ77q0G4k2Of2c1ceQ0ec6MkLQyDp+IGdVM0D8XM= +github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2/go.mod h1:7qG7YFnOALvsx6tKTNmQot8d7cGFXM9TidzvRFLWYwM= github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 h1:iwZdTE0PVqJCos1vaoKsclOGD3ADKpshg3SRtYBbwso= github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM= github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37 h1:X6mKGhCFOxrKeeHAjv/3UvT6e5RRxW6wRdlqlV6/H4w= github.com/danjacques/gofslock v0.0.0-20191023191349-0a45f885bc37/go.mod h1:DC3JtzuG7kxMvJ6dZmf2ymjNyoXwgtklr7FN+Um2B0U= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= -github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y= github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dlclark/regexp2 v1.4.0 h1:F1rxgk7p4uKjwIQxBs9oAXe5CqrXlCduYEJvrF4u93E= github.com/dlclark/regexp2 v1.4.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dolthub/maphash v0.1.0 h1:bsQ7JsF4FkkWyrP3oCnFJgrCUAFbFf3kOl4L/QxPDyQ= github.com/dolthub/maphash v0.1.0/go.mod h1:gkg4Ch4CdCDu5h6PMriVLawB7koZ+5ijb9puGMV50a4= github.com/dolthub/swiss v0.2.1 h1:gs2osYs5SJkAaH5/ggVJqXQxRXtWshF6uE0lgR/Y3Gw= github.com/dolthub/swiss v0.2.1/go.mod h1:8AhKZZ1HK7g18j7v7k6c5cYIGEZJcPn0ARsai8cUrh0= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= -github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= -github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= -github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= -github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= -github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= -github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= -github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/go-asn1-ber/asn1-ber v1.5.4 h1:vXT6d/FNDiELJnLb6hGNa309LMsrCoYFvpwHDF0+Y1A= github.com/go-asn1-ber/asn1-ber v1.5.4/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= -github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= -github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= +github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-ldap/ldap/v3 v3.4.4 h1:qPjipEpt+qDa6SI/h1fzuGWoRUY+qqQ9sOZq67/PYUs= github.com/go-ldap/ldap/v3 v3.4.4/go.mod h1:fe1MsuN5eJJ1FeLT/LEBVdWfNWKh459R7aXgXtJC+aI= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= -github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/protobuf v0.0.0-20180717141946-636bf0302bc9/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= -github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= +github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68= +github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20180814211427-aa810b61a9c7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= -github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= -github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= -github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= +github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/influxdata/tdigest v0.0.1 h1:XpFptwYmnEKUqmkcDjrzffswZ3nvNeevbUSLPP/ZzIY= github.com/influxdata/tdigest v0.0.1/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= -github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= -github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= -github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= -github.com/jellydator/ttlcache/v3 v3.0.1 h1:cHgCSMS7TdQcoprXnWUptJZzyFsqs18Lt8VVhRuZYVU= -github.com/jellydator/ttlcache/v3 v3.0.1/go.mod h1:WwTaEmcXQ3MTjOm4bsZoDFiCu/hMvNWLO1w67RXz6h4= -github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= -github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= -github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= -github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= -github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= -github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.17.1 h1:NE3C767s2ak2bweCZo3+rdP4U/HoyVXLv/X9f2gPS5g= -github.com/klauspost/compress v1.17.1/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -199,49 +135,31 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= -github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a h1:N9zuLhTvBSRt0gWSiJswwQ2HqDmtX/ZCDJURnKUt1Ik= github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= -github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= -github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= -github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= -github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= -github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4= +github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 h1:7KAv7KMGTTqSmYZtNdcNTgsos+vFzULLwyElndwn+5c= github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7/go.mod h1:iWMfgwqYW+e8n5lC/jjNEhwcjbRDpl5NT7n2h+4UNcI= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef h1:K0Fn+DoFqNqktdZtdV3bPQ/0cuYh2H4rkg0tytX/07k= github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef/go.mod h1:7WjlapSfwQyo6LNmIvEWzsW1hbBQfpUO4JWnuQRmva8= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/opentracing/basictracer-go v1.0.0 h1:YyUAhaEfjoWXclZVJ9sGoNct7j4TVk7lZWlQw5UXuoo= @@ -249,106 +167,104 @@ github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKw github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36 h1:64bxqeTEN0/xoEqhKGowgihNuzISS9rEG6YUMU4bzJo= -github.com/petermattis/goid v0.0.0-20211229010228-4d14c490ee36/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= -github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9FV9ix19jjM= +github.com/otiai10/copy v1.2.0 h1:HvG945u96iNadPoG2/Ja2+AUJeW5YuFQMixq9yirC+k= +github.com/otiai10/copy v1.2.0/go.mod h1:rrF5dJ5F0t/EWSYODDu4j9/vEeYHMkc8jt0zJChqQWw= +github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7 h1:Dx7Ovyv/SFnMFw3fD4oEoeorXc6saIiQ23LrGLth0Gw= +github.com/petermattis/goid v0.0.0-20240813172612-4fcff4a6cae7/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d h1:AEcvKyVM8CUII3bYzgz8haFXtGiqcrtXW1csu/5UELY= +github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d/go.mod h1:p8QnkZnmyV8L/M/jzYb8rT7kv3bz9m7bn1Ju94wDifs= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32 h1:m5ZsBa5o/0CkzZXfXLaThzKuR85SnHHetqBCpzQ30h8= -github.com/pingcap/errors v0.11.5-0.20221009092201-b66cddb77c32/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ= -github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f h1:FxA+NgsdHNOv+/hZGxUh8Gb3WuZqgqmxDwztEOiA1v4= +github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a h1:UgrcL8INjEbPRKE2h8yVgZvjOn2OGkxK9CFvoBWzgbk= +github.com/pingcap/failpoint v0.0.0-20240527053858-9b3b6e34194a/go.mod h1:gPdo4h708R0CrwKM/DO0/6xJ64fz9vxzp2yKE2QON+s= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 h1:surzm05a8C9dN8dIUmo4Be2+pMRb6f55i+UIYrluu2E= github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071 h1:giqmIJSWHs+jhHfd+rth8CXWR18KAtqJu4imY1YdA6o= -github.com/pingcap/kvproto v0.0.0-20230925123611-87bebcc0d071/go.mod h1:r0q/CFcwvyeRhKtoqzmWMBebrtpIziQQ9vR+JKh1knc= +github.com/pingcap/kvproto v0.0.0-20240910154453-b242104f8d31 h1:6BY+3T6Hqpw9UZ/D7Om/xB+Xik3NkkYxBV6qCzUdUvU= +github.com/pingcap/kvproto v0.0.0-20240910154453-b242104f8d31/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= -github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22 h1:2SOzvGvE8beiC1Y4g9Onkvu6UmuBBOeWRGQEjJaT/JY= -github.com/pingcap/log v1.1.1-0.20230317032135-a0d097d16e22/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= -github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I= -github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM= -github.com/pingcap/tidb v1.1.0-beta.0.20231117065153-a4f85c356873 h1:IOQPa3ItEg9RrCBgkLGO6VsKRh72LXAbdKiSfPuPQmw= -github.com/pingcap/tidb v1.1.0-beta.0.20231117065153-a4f85c356873/go.mod h1:yXjpyctIACAIDBLGwwRis4X8MDUO8sFS4QerAsOouAc= -github.com/pingcap/tidb/pkg/parser v0.0.0-20231116213047-1f7c1e02bcd4 h1:xrrIEHBBSeBuCL6mbyoMG4R4/ro+WpEql9XKFWtDXRw= -github.com/pingcap/tidb/pkg/parser v0.0.0-20231116213047-1f7c1e02bcd4/go.mod h1:yRkiqLFwIqibYg2P7h4bclHjHcJiIFRLKhGRyBcKYus= -github.com/pingcap/tipb v0.0.0-20230919054518-dfd7d194838f h1:NCiI4Wyu4GkViLGTu6cYcxt79LZ1SenBBQX1OwEV6Jg= -github.com/pingcap/tipb v0.0.0-20230919054518-dfd7d194838f/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d h1:y3EueKVfVykdpTyfUnQGqft0ud+xVFuCdp1XkVL0X1E= +github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= +github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 h1:T4pXRhBflzDeAhmOQHNPRRogMYxP13V7BkYw3ZsoSfE= +github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5/go.mod h1:rlimy0GcTvjiJqvD5mXTRr8O2eNZPBrcUgiWVYp9530= +github.com/pingcap/tidb v1.1.0-beta.0.20240920050217-adbc0034ffb4 h1:BnoMi+RibczSp8TjfPcvb2RVviQQvIjfqDM8fn4/xwU= +github.com/pingcap/tidb v1.1.0-beta.0.20240920050217-adbc0034ffb4/go.mod h1:XqXA8+jTqBZ4bu/xNjWSzBK33nIB1z0adUv0vYXQgVE= +github.com/pingcap/tidb/pkg/parser v0.0.0-20240920050217-adbc0034ffb4 h1:GBf6ho/7J4HVf5N+DaCfUfP7B0Yt1k48GgMBv/SvBjs= +github.com/pingcap/tidb/pkg/parser v0.0.0-20240920050217-adbc0034ffb4/go.mod h1:bGciNq+1X3xXPyRvgHFPKh2naQVH6HPiBbfqK3qSqpw= +github.com/pingcap/tipb v0.0.0-20240823074000-a40c2347786e h1:chR6iRwU9MUivOYNH+26UqrO8Y7t3ZltX+Jukv+f+iM= +github.com/pingcap/tipb v0.0.0-20240823074000-a40c2347786e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b h1:0LFwY6Q3gMACTjAbMZBjXAqTOzOwFaj2Ld6cjeQ7Rig= github.com/power-devops/perfstat v0.0.0-20221212215047-62379fc7944b/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.20.2 h1:5ctymQzZlyOON1666svgwn3s6IKWgfbjsejTMiXIyjg= +github.com/prometheus/client_golang v1.20.2/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVhoNcY= +github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/qri-io/jsonpointer v0.1.1 h1:prVZBZLL6TW5vsSB9fFHFAMBLI4b0ri5vribQlTJiBA= +github.com/qri-io/jsonpointer v0.1.1/go.mod h1:DnJPaYgiKu56EuDp8TU5wFLdZIcAnb/uH9v37ZaMV64= +github.com/qri-io/jsonschema v0.2.1 h1:NNFoKms+kut6ABPf6xiKNM5214jzxAhDBrPHCJ97Wg0= +github.com/qri-io/jsonschema v0.2.1/go.mod h1:g7DPkiOsK1xv6T/Ao5scXRkd+yTFygcANPBaaqW+VrI= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= -github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= -github.com/sasha-s/go-deadlock v0.2.0 h1:lMqc+fUb7RrFS3gQLtoQsJ7/6TV/pAIFvBsqX73DK8Y= -github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU= +github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U= github.com/segmentio/kafka-go v0.4.45 h1:prqrZp1mMId4kI6pyPolkLsH6sWOUmDxmmucbL4WS6E= github.com/segmentio/kafka-go v0.4.45/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= -github.com/shirou/gopsutil/v3 v3.23.10 h1:/N42opWlYzegYaVkWejXWJpbzKv2JDy3mrgGzKsh9hM= -github.com/shirou/gopsutil/v3 v3.23.10/go.mod h1:JIE26kpucQi+innVlAUnIEOSBhBUkirr5b44yr55+WE= +github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= +github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= -github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/thessem/zap-prettyconsole v0.3.0 h1:jreGIwOwkfqpA1NWTsbXXr0ZSL68b39KpH54q9Hz58U= github.com/thessem/zap-prettyconsole v0.3.0/go.mod h1:93z1PhlPAYOWIOhPJvNkxE9cVM8jE5xB6/6HCtnvmcA= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= -github.com/tikv/client-go/v2 v2.0.8-0.20231114060955-8fc8a528217e h1:kl8+gDOfPfRqkc1VDhhjhezMvsbfRENYsm/FqSIDnwg= -github.com/tikv/client-go/v2 v2.0.8-0.20231114060955-8fc8a528217e/go.mod h1:fEAE7GS/lta+OasPOacdgy6RlJIRaq9/Cyr2WbSYcBE= -github.com/tikv/pd/client v0.0.0-20231114041114-86831ce71865 h1:Gkvo77EevOpBGIdV1c8gwRqPhVbgLPRy82tXNEFpGTc= -github.com/tikv/pd/client v0.0.0-20231114041114-86831ce71865/go.mod h1:cd6zBqRM9aogxf26K8NnFRPVtq9BnRE59tKEpX8IaWQ= +github.com/tidwall/btree v1.7.0 h1:L1fkJH/AuEh5zBnnBbmTwQ5Lt+bRJ5A8EWecslvo9iI= +github.com/tidwall/btree v1.7.0/go.mod h1:twD9XRA5jj9VUQGELzDO4HPQTNJsoWWfYEL+EUQ2cKY= +github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba h1:dwuuYqPYxU0xcv0bnDgT7I4btQ5c3joBG1HmNOhCTdo= +github.com/tikv/client-go/v2 v2.0.8-0.20240911041506-e7894a7b27ba/go.mod h1:4HDOAx8OXAJPtqhCZ03IhChXgaFs4B3+vSrPWmiPxjg= +github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9 h1:J9LChGMzo95eBrjE03NHITDWgxfPgskH+QrCnlW61/Y= +github.com/tikv/pd/client v0.0.0-20240914083230-71f6f96816e9/go.mod h1:uBHhxAM/SPCMabt483gI/pN/+JXIMKYXohK96s+PwT8= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= @@ -361,38 +277,25 @@ github.com/uber/jaeger-client-go v2.22.1+incompatible h1:NHcubEkVbahf9t3p75TOCR8 github.com/uber/jaeger-client-go v2.22.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVKhn2Um6rjCsSsg= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= -github.com/valyala/fasthttp v1.6.0/go.mod h1:FstJa9V+Pj9vQ7OJie2qMHdwemEDaDiSdBnvPM1Su9w= -github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= -github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio= +github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY= github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= +github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8= github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= -github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmvncFJFHJ7Gvn9wZArjbV5/FppcK2fKk/tI= -github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= -github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= -github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= -github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= -go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k= -go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= -go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0= -go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= -go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao= -go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/etcd/api/v3 v3.5.12 h1:W4sw5ZoU2Juc9gBWuLk5U6fHfNVyY1WC5g9uiXZio/c= +go.etcd.io/etcd/api/v3 v3.5.12/go.mod h1:Ot+o0SWSyT6uHhA56al1oCED0JImsRiU9Dc26+C2a+4= +go.etcd.io/etcd/client/pkg/v3 v3.5.12 h1:EYDL6pWwyOsylrQyLp2w+HkQ46ATiOvoEdMarindU2A= +go.etcd.io/etcd/client/pkg/v3 v3.5.12/go.mod h1:seTzl2d9APP8R5Y2hFL3NVlD6qC/dOT+3kvrqPyTas4= +go.etcd.io/etcd/client/v3 v3.5.12 h1:v5lCPXn1pf1Uu3M4laUE2hp/geOTc5uPcYYsNe1lDxg= +go.etcd.io/etcd/client/v3 v3.5.12/go.mod h1:tSbBCakoWmmddL+BKVAJHa9km+O/E+bumDe9mSbPiqw= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -402,8 +305,8 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= -go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= @@ -413,24 +316,22 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.12.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= -golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.15.0 h1:frVn1TEaCEaZcn3Tmd7Y2b5KKPaZ+I32Q2OA3kYp5TA= -golang.org/x/crypto v0.15.0/go.mod h1:4ChreQoLWfG3xLDer1WdlH5NdlQ3+mwnQq1YTKY+72g= +golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A= +golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0 h1:985EYyeCOxTpcgOTJpflJUwOeEz0CQOdPt73OzpE9F8= +golang.org/x/exp v0.0.0-20240404231335-c0f41cb1a7a0/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -442,92 +343,75 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.18.0 h1:mIYleuAkSbHh0tCv7RvjL3F6ZVbLjq4+R7zbOn3Kokg= -golang.org/x/net v0.18.0/go.mod h1:/czyP5RqHAH4odGYxBJ1qz0+CE5WZ+2j1YgoEo8F2jQ= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211013075003-97ac67df715c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -538,52 +422,37 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.24.0 h1:J1shsA93PJUEVaUSaay7UXAyE8aimq3GW0pjlolpa24= +golang.org/x/tools v0.24.0/go.mod h1:YhNqVBIfWHdzvTLs0d8LCuMhkKUgSUKldakyV7W/WDQ= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181004005441-af9cb2a35e7f/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= -google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= -google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237 h1:RFiFrvy37/mpSpdySBDrUdipW/dHwsRwh3J3+A9VgT4= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= -google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs= +google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9/go.mod h1:j5uROIAAgi3YmtiETMt1LW0d/lHqQ7wwrIY4uGRXLQ4= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= @@ -592,10 +461,10 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/integration_tests/debezium/run.sh b/tests/integration_tests/debezium/run.sh index cdffb6cd06c..19e8d766577 100644 --- a/tests/integration_tests/debezium/run.sh +++ b/tests/integration_tests/debezium/run.sh @@ -50,7 +50,7 @@ EOF start_tidb_cluster --workdir $WORK_DIR run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY -run_cdc_cli changefeed create --sink-uri="kafka://127.0.0.1:9092/output_ticdc?protocol=debezium&kafka-version=2.4.0" +run_cdc_cli changefeed create -c test --sink-uri="kafka://127.0.0.1:9092/output_ticdc?protocol=debezium&kafka-version=2.4.0" --config "$CUR/changefeed.toml" cd $CUR go run ./src diff --git a/tests/integration_tests/debezium/sql/data_types.sql b/tests/integration_tests/debezium/sql/data_types.sql index 48452db04fe..53489702764 100644 --- a/tests/integration_tests/debezium/sql/data_types.sql +++ b/tests/integration_tests/debezium/sql/data_types.sql @@ -37,8 +37,7 @@ CREATE TABLE t_datetime( col_5 DATETIME(5), col_6 DATETIME(6), col_z DATETIME DEFAULT 0, - -- TODO: - -- col_default_current_timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, + col_default_current_timestamp DATETIME DEFAULT CURRENT_TIMESTAMP, pk INT PRIMARY KEY ); @@ -52,7 +51,7 @@ INSERT INTO t_datetime VALUES ( '2023-11-16 12:34:56.123456', '2023-11-16 12:34:56.123456', NULL, - -- '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', 1 ); @@ -66,7 +65,7 @@ INSERT INTO t_datetime VALUES ( '2023-11-16 12:34:56', '2023-11-16 12:34:56', NULL, - -- '2023-11-16 12:34:56', + '2023-11-16 12:34:56', 2 ); @@ -82,7 +81,7 @@ INSERT INTO t_datetime VALUES ( '2023-11-16 12:34:56.123456', '2023-11-16 12:34:56.123456', NULL, - -- '2023-11-16 12:34:56.123456', + '2023-11-16 12:34:56.123456', 3 ); @@ -98,7 +97,7 @@ INSERT INTO t_datetime VALUES ( NULL, NULL, NULL, - -- NULL, + NULL, 4 ); @@ -144,27 +143,6 @@ INSERT INTO t_time VALUES ( 3 ); -/* - -Commented out because Debezium produce wrong result: - -"col":-3020399000000, -"col_0":-3020399000000, -"col_1":-3020400147483, -"col_5":-3020399048576, -"col_6":-3020399048576, - -INSERT INTO t_time VALUES ( - '-838:59:59.000000', - '-838:59:59.000000', - '-838:59:59.000000', - '-838:59:59.000000', - '-838:59:59.000000', - 3 -); - -*/ - INSERT INTO t_time VALUES ( '838:59:59.000000', '838:59:59.000000', @@ -183,6 +161,25 @@ INSERT INTO t_time VALUES ( 5 ); +/* + +Commented out because Debezium produce wrong result: + +"col":-3020399000000, +"col_0":-3020399000000, +"col_1":-3020400147483, +"col_5":-3020399048576, +"col_6":-3020399048576, +INSERT INTO t_time VALUES ( + '-838:59:59.000000', + '-838:59:59.000000', + '-838:59:59.000000', + '-838:59:59.000000', + '-838:59:59.000000', + 6 +); +*/ + /* ---------------------------------------------------------------------- -- TIMESTAMP type diff --git a/tests/integration_tests/debezium/sql/ddl.sql b/tests/integration_tests/debezium/sql/ddl.sql new file mode 100644 index 00000000000..3a28efba4a1 --- /dev/null +++ b/tests/integration_tests/debezium/sql/ddl.sql @@ -0,0 +1,135 @@ +CREATE TABLE t1 ( + PK INT PRIMARY KEY, + COL INT +); + +CREATE DATABASE foo; +USE foo; + +CREATE TABLE bar ( + PK INT PRIMARY KEY AUTO_INCREMENT, + COL INT +); +INSERT INTO bar VALUES (1, 1); + +/* VIEW */ +CREATE VIEW V1 AS +SELECT * +FROM bar +WHERE COL > 2; +DROP VIEW IF EXISTS V1; + +/* ALTER COLUMN */ +ALTER TABLE bar +ADD COLUMN COL2 INT; +ALTER TABLE bar +MODIFY COLUMN COL2 FLOAT; +ALTER TABLE bar +DROP COLUMN COL2; + +/* Rebase AutoID */ +ALTER TABLE bar AUTO_INCREMENT=310; + +/* Set DEFAULT value */ +ALTER TABLE bar +ALTER COL SET DEFAULT 3; + +/* Modify TABLE comment */ +ALTER TABLE bar COMMENT = 'New table comment'; + +/* Modify TABLE charset */ +ALTER TABLE bar CHARACTER SET = utf8mb4 COLLATE utf8mb4_unicode_ci; + +/* Modify DATABASE charset */ +ALTER DATABASE foo CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci; + +CREATE TABLE t1 ( + PK INT PRIMARY KEY, + COL INT +); + +/* MultiSchemaChange */ +ALTER TABLE test.t1 +CHANGE COL COL2 VARCHAR(255); +ALTER TABLE foo.t1 +CHANGE COL COL2 VARCHAR(255); + +/* PARTITION */ +CREATE TABLE t2 ( + id INT NOT NULL, + year_col INT NOT NULL +) +PARTITION BY RANGE (year_col) ( + PARTITION p0 VALUES LESS THAN (1991), + PARTITION p1 VALUES LESS THAN (1995), + PARTITION p2 VALUES LESS THAN (1999) +); +ALTER TABLE t2 ADD PARTITION (PARTITION p3 VALUES LESS THAN (2002)); +ALTER TABLE t2 REORGANIZE PARTITION p3 INTO ( + PARTITION p31 VALUES LESS THAN (2002), + PARTITION p32 VALUES LESS THAN (2005), + PARTITION p33 VALUES LESS THAN (2008) +); +ALTER TABLE t2 REORGANIZE PARTITION p31,p32,p33,p2 INTO (PARTITION p21 VALUES LESS THAN (2008)); +ALTER TABLE t2 TRUNCATE PARTITION p0; +ALTER TABLE t2 DROP PARTITION p0; + +/* ALTER INDEX visibility */ +CREATE TABLE t3 ( + i INT, + j INT, + k INT, + INDEX i_idx (i) INVISIBLE +) ENGINE = InnoDB; +CREATE INDEX j_idx ON t3 (j) INVISIBLE; +ALTER TABLE t3 ADD INDEX k_idx (k) INVISIBLE; +ALTER TABLE t3 ALTER INDEX i_idx VISIBLE; +ALTER TABLE t3 ALTER INDEX i_idx INVISIBLE; + +/* RENAME TABLE */ +RENAME TABLE t3 TO renam_t3; + +/* INDEX */ +CREATE TABLE t4 (col1 INT PRIMARY KEY, col2 INT); +CREATE INDEX idx1 ON t4 ((col1 + col2)); +CREATE INDEX idx2 ON t4 ((col1 + col2), (col1 - col2), col1); +DROP INDEX idx1 ON t4; +ALTER TABLE t4 ADD INDEX ((col1 * 40) DESC); +ALTER TABLE t4 RENAME INDEX idx2 TO new_idx2; + +/* + Adding a new column and setting it to the PRIMARY KEY is not supported. + https://docs.pingcap.com/tidb/stable/sql-statement-add-column#mysql-compatibility + ALTER TABLE t4 ADD COLUMN `id` INT(10) primary KEY; +*/ +/* + Dropping primary key columns or columns covered by the composite index is not supported. + https://docs.pingcap.com/tidb/stable/sql-statement-drop-column#mysql-compatibility + ALTER TABLE t4 DROP PRIMARY KEY; +*/ + +/* EXCHANGE PARTITION */ +CREATE TABLE t5 ( + id INT NOT NULL PRIMARY KEY, + fname VARCHAR(30), + lname VARCHAR(30) +) + PARTITION BY RANGE (id) ( + PARTITION p0 VALUES LESS THAN (50), + PARTITION p1 VALUES LESS THAN (100), + PARTITION p2 VALUES LESS THAN (150), + PARTITION p3 VALUES LESS THAN (MAXVALUE) +); +INSERT INTO t5 VALUES (1669, "Jim", "Smith"); +CREATE TABLE t6 LIKE t5; +ALTER TABLE t6 REMOVE PARTITIONING; +ALTER TABLE foo.t5 EXCHANGE PARTITION p0 WITH TABLE foo.t6; + +/* + Debezium does not support recover table. + DROP TABLE t1; + RECOVER TABLE t1; +*/ + +DROP TABLE foo.bar; +DROP DATABASE IF EXISTS foo; diff --git a/tests/integration_tests/debezium/sql/debezium/binary_column_test.sql b/tests/integration_tests/debezium/sql/debezium/binary_column_test.sql index 245452e0e27..0925bcfb23e 100644 --- a/tests/integration_tests/debezium/sql/debezium/binary_column_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/binary_column_test.sql @@ -1,7 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: binary_column_test --- ---------------------------------------------------------------------------------------------------------------- - CREATE TABLE dbz_254_binary_column_test ( id INT AUTO_INCREMENT NOT NULL, file_uuid BINARY(16), diff --git a/tests/integration_tests/debezium/sql/debezium/binary_mode_test.sql b/tests/integration_tests/debezium/sql/debezium/binary_mode_test.sql index 8ad20ccad17..448e55282e3 100644 --- a/tests/integration_tests/debezium/sql/debezium/binary_mode_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/binary_mode_test.sql @@ -1,7 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: binary_column_test --- ---------------------------------------------------------------------------------------------------------------- - CREATE TABLE dbz_1814_binary_mode_test ( id INT AUTO_INCREMENT NOT NULL, blob_col BLOB NOT NULL, diff --git a/tests/integration_tests/debezium/sql/debezium/connector_read_binary_field_test.sql b/tests/integration_tests/debezium/sql/debezium/connector_read_binary_field_test.sql new file mode 100644 index 00000000000..3b013498585 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/connector_read_binary_field_test.sql @@ -0,0 +1,15 @@ +SET time_zone = 'UTC'; +-- Create a table, mainly MySQL time type fields +CREATE TABLE binary_field +( + id INT AUTO_INCREMENT PRIMARY KEY, + now_time TIME, + now_date DATE, + now_date_time DATETIME, + now_time_stamp TIMESTAMP +) ENGINE = innodb + AUTO_INCREMENT = 1 + DEFAULT CHARSET = utf8; + +INSERT INTO binary_field +VALUES (default, now(), now(), now(), now()); diff --git a/tests/integration_tests/debezium/sql/debezium/connector_test.sql b/tests/integration_tests/debezium/sql/debezium/connector_test.sql index 8ae03647b01..82b9564fb80 100644 --- a/tests/integration_tests/debezium/sql/debezium/connector_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/connector_test.sql @@ -1,7 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: connector_test --- ---------------------------------------------------------------------------------------------------------------- - -- Create and populate our products using a single insert with many rows CREATE TABLE products ( id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, @@ -12,15 +8,23 @@ CREATE TABLE products ( ALTER TABLE products AUTO_INCREMENT = 101; INSERT INTO products -VALUES (default,"scooter","Small 2-wheel scooter",3.14), - (default,"car battery","12V car battery",8.1), - (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8), - (default,"hammer","12oz carpenter's hammer",0.75), - (default,"hammer","14oz carpenter's hammer",0.875), - (default,"hammer","16oz carpenter's hammer",1.0), - (default,"rocks","box of assorted rocks",5.3), - (default,"jacket","water resistent black wind breaker",0.1), - (default,"spare tire","24 inch spare tire",22.2); +VALUES (default,"scooter","Small 2-wheel scooter",3.14); +INSERT INTO products +VALUES (default,"car battery","12V car battery",8.1); +INSERT INTO products +VALUES (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8); +INSERT INTO products +VALUES (default,"hammer","12oz carpenter's hammer",0.75); +INSERT INTO products +VALUES (default,"hammer","14oz carpenter's hammer",0.875); +INSERT INTO products +VALUES (default,"hammer","16oz carpenter's hammer",1.0); +INSERT INTO products +VALUES (default,"rocks","box of assorted rocks",5.3); +INSERT INTO products +VALUES (default,"jacket","water resistent black wind breaker",0.1); +INSERT INTO products +VALUES (default,"spare tire","24 inch spare tire",22.2); -- Create and populate the products on hand using multiple inserts CREATE TABLE products_on_hand ( @@ -49,10 +53,13 @@ CREATE TABLE customers ( INSERT INTO customers -VALUES (default,"Sally","Thomas","sally.thomas@acme.com"), - (default,"George","Bailey","gbailey@foobar.com"), - (default,"Edward","Walker","ed@walker.com"), - (default,"Anne","Kretchmar","annek@noanswer.org"); +VALUES (default,"Sally","Thomas","sally.thomas@acme.com"); +INSERT INTO customers +VALUES (default,"George","Bailey","gbailey@foobar.com"); +INSERT INTO customers +VALUES (default,"Edward","Walker","ed@walker.com"); +INSERT INTO customers +VALUES (default,"Anne","Kretchmar","annek@noanswer.org"); -- Create some very simple orders CREATE TABLE orders ( @@ -66,8 +73,12 @@ CREATE TABLE orders ( ) AUTO_INCREMENT = 10001; INSERT INTO orders -VALUES (default, '2016-01-16', 1001, 1, 102), - (default, '2016-01-17', 1002, 2, 105), - (default, '2016-02-18', 1004, 3, 109), - (default, '2016-02-19', 1002, 2, 106), - (default, '16-02-21', 1003, 1, 107); +VALUES (default, '2016-01-16', 1001, 1, 102); +INSERT INTO orders +VALUES (default, '2016-01-17', 1002, 2, 105); +INSERT INTO orders +VALUES (default, '2016-02-18', 1004, 3, 109); +INSERT INTO orders +VALUES (default, '2016-02-19', 1002, 2, 106); +INSERT INTO orders +VALUES (default, '16-02-21', 1003, 1, 107); diff --git a/tests/integration_tests/debezium/sql/debezium/connector_test_ro.sql b/tests/integration_tests/debezium/sql/debezium/connector_test_ro.sql index 22ab5d1dda6..bf01c629293 100644 --- a/tests/integration_tests/debezium/sql/debezium/connector_test_ro.sql +++ b/tests/integration_tests/debezium/sql/debezium/connector_test_ro.sql @@ -1,7 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: connector_test_ro --- ---------------------------------------------------------------------------------------------------------------- - -- Create and populate our products using a single insert with many rows CREATE TABLE Products ( PRIMARY KEY (id), @@ -13,15 +9,23 @@ CREATE TABLE Products ( ALTER TABLE Products AUTO_INCREMENT = 101; INSERT INTO Products -VALUES (default,"scooter","Small 2-wheel scooter",3.14), - (default,"car battery","12V car battery",8.1), - (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8), - (default,"hammer","12oz carpenter's hammer",0.75), - (default,"hammer2","14oz carpenter's hammer",8.75E-1), - (default,"hammer3","16oz carpenter's hammer",1.0), - (default,"rocks","box of assorted rocks",5.3), - (default,"jacket","water resistent black wind breaker",0.1), - (default,"spare tire","24 inch spare tire",22.2); +VALUES (default,"scooter","Small 2-wheel scooter",3.14); +INSERT INTO Products +VALUES (default,"car battery","12V car battery",8.1); +INSERT INTO Products +VALUES (default,"12-pack drill bits","12-pack of drill bits with sizes ranging from #40 to #3",0.8); +INSERT INTO Products +VALUES (default,"hammer","12oz carpenter's hammer",0.75); +INSERT INTO Products +VALUES (default,"hammer2","14oz carpenter's hammer",8.75E-1); +INSERT INTO Products +VALUES (default,"hammer3","16oz carpenter's hammer",1.0); +INSERT INTO Products +VALUES (default,"rocks","box of assorted rocks",5.3); +INSERT INTO Products +VALUES (default,"jacket","water resistent black wind breaker",0.1); +INSERT INTO Products +VALUES (default,"spare tire","24 inch spare tire",22.2); -- Create and populate the products on hand using multiple inserts CREATE TABLE products_on_hand ( @@ -50,10 +54,13 @@ CREATE TABLE customers ( INSERT INTO customers -VALUES (default,"Sally","Thomas","sally.thomas@acme.com"), - (default,"George","Bailey","gbailey@foobar.com"), - (default,"Edward","Walker","ed@walker.com"), - (default,"Anne","Kretchmar","annek@noanswer.org"); +VALUES (default,"Sally","Thomas","sally.thomas@acme.com"); +INSERT INTO customers +VALUES (default,"George","Bailey","gbailey@foobar.com"); +INSERT INTO customers +VALUES (default,"Edward","Walker","ed@walker.com"); +INSERT INTO customers +VALUES (default,"Anne","Kretchmar","annek@noanswer.org"); -- Create some very simple orders CREATE TABLE orders ( @@ -67,11 +74,15 @@ CREATE TABLE orders ( ) AUTO_INCREMENT = 10001; INSERT INTO orders -VALUES (default, '2016-01-16', 1001, 1, 102), - (default, '2016-01-17', 1002, 2, 105), - (default, '2016-02-18', 1004, 3, 109), - (default, '2016-02-19', 1002, 2, 106), - (default, '2016-02-21', 1003, 1, 107); +VALUES (default, '2016-01-16', 1001, 1, 102); +INSERT INTO orders +VALUES (default, '2016-01-17', 1002, 2, 105); +INSERT INTO orders +VALUES (default, '2016-02-18', 1004, 3, 109); +INSERT INTO orders +VALUES (default, '2016-02-19', 1002, 2, 106); +INSERT INTO orders +VALUES (default, '2016-02-21', 1003, 1, 107); -- DBZ-342 handle TIME values that exceed the value range of java.sql.Time @@ -83,5 +94,3 @@ CREATE TABLE dbz_342_timetest ( c5 TIME(6) ); INSERT INTO dbz_342_timetest VALUES ('517:51:04.777', '-13:14:50', '-733:00:00.0011', '-1:59:59.0011', '-838:59:58.999999'); - -CREATE DATABASE IF NOT EXISTS emptydb; diff --git a/tests/integration_tests/debezium/sql/debezium/datetime_key_test.sql b/tests/integration_tests/debezium/sql/debezium/datetime_key_test.sql index 08740721ba3..3ac6788d81f 100644 --- a/tests/integration_tests/debezium/sql/debezium/datetime_key_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/datetime_key_test.sql @@ -1,7 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: datetime_key_test --- ---------------------------------------------------------------------------------------------------------------- - SET sql_mode=''; CREATE TABLE dbz_1194_datetime_key_test ( id INT AUTO_INCREMENT NOT NULL, diff --git a/tests/integration_tests/debezium/sql/debezium/decimal_column_test.sql b/tests/integration_tests/debezium/sql/debezium/decimal_column_test.sql index 64cd7742ad5..4782dc80359 100644 --- a/tests/integration_tests/debezium/sql/debezium/decimal_column_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/decimal_column_test.sql @@ -1,7 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: decimal_column_test --- ---------------------------------------------------------------------------------------------------------------- - CREATE TABLE dbz_751_decimal_column_test ( id INT AUTO_INCREMENT NOT NULL, rating1 DECIMAL, diff --git a/tests/integration_tests/debezium/sql/debezium/decimal_test.sql b/tests/integration_tests/debezium/sql/debezium/decimal_test.sql new file mode 100644 index 00000000000..a458083d79e --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/decimal_test.sql @@ -0,0 +1,10 @@ +CREATE TABLE `DBZ730` ( + id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, + A NUMERIC(3, 2) NOT NULL DEFAULT 1.23, + B DECIMAL(4, 3) NOT NULL DEFAULT 2.321, + C NUMERIC(7, 5) NULL DEFAULT '12.678', + D NUMERIC(7, 5) NULL DEFAULT '15.28', + E DECIMAL(65, 18) NULL DEFAULT '0.000000000000000000' +) ENGINE=InnoDB AUTO_INCREMENT=15851 DEFAULT CHARSET=utf8; +INSERT INTO `DBZ730`(A, B, C, D) +VALUES (1.33, -2.111 , 3.444, NULL); diff --git a/tests/integration_tests/debezium/sql/debezium/default_value.sql b/tests/integration_tests/debezium/sql/debezium/default_value.sql new file mode 100644 index 00000000000..60e58df660c --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/default_value.sql @@ -0,0 +1,154 @@ +CREATE TABLE UNSIGNED_TINYINT_TABLE ( + id int PRIMARY KEY, + A TINYINT UNSIGNED NULL DEFAULT 0, + B TINYINT UNSIGNED NULL DEFAULT '10', + C TINYINT UNSIGNED NULL, + D TINYINT UNSIGNED NOT NULL, + E TINYINT UNSIGNED NOT NULL DEFAULT 0, + F TINYINT UNSIGNED NOT NULL DEFAULT '0', + G TINYINT UNSIGNED NULL DEFAULT '100' +); +INSERT INTO UNSIGNED_TINYINT_TABLE VALUES (1, DEFAULT, DEFAULT, 0, 1, DEFAULT, DEFAULT, NULL); + +CREATE TABLE UNSIGNED_SMALLINT_TABLE ( + id int PRIMARY KEY, + A SMALLINT UNSIGNED NULL DEFAULT 0, + B SMALLINT UNSIGNED NULL DEFAULT '10', + C SMALLINT UNSIGNED NULL, + D SMALLINT UNSIGNED NOT NULL, + E SMALLINT UNSIGNED NOT NULL DEFAULT 0, + F SMALLINT UNSIGNED NOT NULL DEFAULT '0', + G SMALLINT UNSIGNED NULL DEFAULT '100' +); +INSERT INTO UNSIGNED_SMALLINT_TABLE VALUES (1, 1, 1, 1, 0, 1, 1, NULL); + +CREATE TABLE UNSIGNED_MEDIUMINT_TABLE ( + id int PRIMARY KEY, + A MEDIUMINT UNSIGNED NULL DEFAULT 0, + B MEDIUMINT UNSIGNED NULL DEFAULT '10', + C MEDIUMINT UNSIGNED NULL, + D MEDIUMINT UNSIGNED NOT NULL, + E MEDIUMINT UNSIGNED NOT NULL DEFAULT 0, + F MEDIUMINT UNSIGNED NOT NULL DEFAULT '0', + G MEDIUMINT UNSIGNED NULL DEFAULT '100' +); +INSERT INTO UNSIGNED_MEDIUMINT_TABLE VALUES (1, 1, 1, 1, 0, 1, 1, NULL); + +CREATE TABLE UNSIGNED_INT_TABLE ( + id int PRIMARY KEY, + A INT UNSIGNED NULL DEFAULT 0, + B INT UNSIGNED NULL DEFAULT '10', + C INT UNSIGNED NULL, + D INT UNSIGNED NOT NULL, + E INT UNSIGNED NOT NULL DEFAULT 0, + F INT UNSIGNED NOT NULL DEFAULT '0', + G INT UNSIGNED NULL DEFAULT '100' +); +INSERT INTO UNSIGNED_INT_TABLE VALUES (1, 1, 1, 1, 0, 1, 1, NULL); + +CREATE TABLE UNSIGNED_BIGINT_TABLE ( + id int PRIMARY KEY, + A BIGINT UNSIGNED NULL DEFAULT 0, + B BIGINT UNSIGNED NULL DEFAULT '10', + C BIGINT UNSIGNED NULL, + D BIGINT UNSIGNED NOT NULL, + E BIGINT UNSIGNED NOT NULL DEFAULT 0, + F BIGINT UNSIGNED NOT NULL DEFAULT '0', + G BIGINT UNSIGNED NULL DEFAULT '100' +); +INSERT INTO UNSIGNED_BIGINT_TABLE VALUES (1, 1, 1, 1, 0, 1, 1, NULL); + +CREATE TABLE STRING_TABLE ( + id int PRIMARY KEY, + A CHAR(1) NULL DEFAULT 'A', + B CHAR(1) NULL DEFAULT 'b', + C VARCHAR(10) NULL DEFAULT 'CC', + F CHAR(1) DEFAULT NULL, + G VARCHAR(10) DEFAULT NULL, + I VARCHAR(10) NULL DEFAULT '100' +); +INSERT INTO STRING_TABLE +VALUES (1, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, NULL); + +CREATE TABLE BIT_TABLE ( + id int PRIMARY KEY, + A BIT(1) NULL DEFAULT NULL, + B BIT(1) DEFAULT 0, + C BIT(1) DEFAULT 1, + D BIT(1) DEFAULT b'0', + E BIT(1) DEFAULT b'1', + H BIT(10) DEFAULT b'0101000010', + I BIT(10) DEFAULT NULL, + J BIT(25) DEFAULT b'0000000000000000100001111', + K BIT(25) DEFAULT b'0000000010110000100001111' +); +INSERT INTO BIT_TABLE +VALUES (1, false, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, NULL, DEFAULT, NULL); + +CREATE TABLE NUMBER_TABLE ( + id int PRIMARY KEY, + A TINYINT NULL DEFAULT 10, + B SMALLINT NOT NULL DEFAULT '5', + C INTEGER NOT NULL DEFAULT 0, + D BIGINT NOT NULL DEFAULT 20, + E INT NULL DEFAULT NULL, + F INT NULL DEFAULT 30 +); +INSERT INTO NUMBER_TABLE +VALUES (1, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, NULL); + +CREATE TABLE FlOAT_DOUBLE_TABLE ( + id int PRIMARY KEY, + F FLOAT NULL DEFAULT 0, + G DOUBLE NOT NULL DEFAULT 1.1, + H DOUBLE NULL DEFAULT 3.3 +); +INSERT INTO FlOAT_DOUBLE_TABLE +VALUES (1, DEFAULT, DEFAULT, NULL); + +-- set sql_mode REAL_AS_FLOAT is necessary +set @@session.sql_mode=concat(@@session.sql_mode, ',REAL_AS_FLOAT'); +CREATE TABLE REAL_TABLE ( + id int PRIMARY KEY, + A REAL NOT NULL DEFAULT 1, + B REAL NULL DEFAULT NULL, + C REAL NULL DEFAULT 3 +); +INSERT INTO REAL_TABLE +VALUES (1, DEFAULT ,DEFAULT, NULL); + +CREATE TABLE NUMERIC_DECIMAL_TABLE ( + id int PRIMARY KEY, + A NUMERIC(3, 2) NOT NULL DEFAULT 1.23, + B DECIMAL(4, 3) NOT NULL DEFAULT 2.321, + C NUMERIC(7, 5) NULL DEFAULT '12.678', + D NUMERIC(7, 5) NULL DEFAULT '15.28' +); +INSERT INTO NUMERIC_DECIMAL_TABLE +VALUES (1, 1.33 ,2.111 , 3.444, NULL); + +CREATE TABLE DATE_TIME_TABLE ( + id int PRIMARY KEY, + A DATE NOT NULL DEFAULT '1976-08-23', + C DATETIME DEFAULT '2018-01-03 00:00:10', + D DATETIME(1) DEFAULT '2018-01-03 00:00:10.7', + E DATETIME(6) DEFAULT '2018-01-03 00:00:10.123456', + F YEAR NOT NULL DEFAULT 1, + G TIME DEFAULT '00:00:00', + H TIME(1) DEFAULT '23:00:00.7', + I TIME(6) DEFAULT '23:00:00.123456', + J TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + K TIMESTAMP NULL DEFAULT CURRENT_TIMESTAMP, + L TIME(1) DEFAULT '-23:45:56.7', + M TIME(6) DEFAULT '123:00:00.123456' +); +INSERT INTO DATE_TIME_TABLE +VALUES (1, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, NULL, DEFAULT, DEFAULT); + +CREATE TABLE DBZ_771_CUSTOMERS ( + id INTEGER NOT NULL PRIMARY KEY, + CUSTOMER_TYPE ENUM ('b2c','b2b') NOT NULL default 'b2c' +); + +INSERT INTO DBZ_771_CUSTOMERS +VALUES (1, 'b2b'); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/default_value_all_zero_time.sql b/tests/integration_tests/debezium/sql/debezium/default_value_all_zero_time.sql new file mode 100644 index 00000000000..d6eba9d5778 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/default_value_all_zero_time.sql @@ -0,0 +1,33 @@ +SET sql_mode=''; +CREATE TABLE all_zero_date_and_time_table ( + id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, + A TIMESTAMP NOT NULL DEFAULT '0000-00-00 00:00:00', + B TIMESTAMP NULL DEFAULT '0000-00-00 00:00:00', + C TIMESTAMP DEFAULT '0000-00-00 00:00:00' NOT NULL, + D TIMESTAMP DEFAULT '0000-00-00 00:00:00' NULL, + E DATE NOT NULL DEFAULT '0000-00-00', + F DATE NULL DEFAULT '0000-00-00', + G DATE DEFAULT '0000-00-00' NOT NULL, + H DATE DEFAULT '0000-00-00' NULL, + I DATETIME NOT NULL DEFAULT '0000-00-00 00:00:00', + J DATETIME NULL DEFAULT '0000-00-00 00:00:00', + K DATETIME DEFAULT '0000-00-00 00:00:00' NOT NULL, + L DATETIME DEFAULT '0000-00-00 00:00:00' NULL +); +INSERT INTO all_zero_date_and_time_table +VALUES (DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT); + +CREATE TABLE part_zero_date_and_time_table ( + id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, + A TIMESTAMP NOT NULL DEFAULT '0000-00-00 00:00:00', + B TIMESTAMP NULL DEFAULT '0000-00-00 00:00:00', + C DATETIME NOT NULL DEFAULT '0000-00-00 01:00:00.000', + D DATETIME NULL DEFAULT '0000-00-00 01:00:00.000', + E DATE NOT NULL DEFAULT '1000-00-00', + /* TiCDC convert '1000-00-00 01:00:00.000' to "1000-00-00" + F DATE NULL DEFAULT '1000-00-00 01:00:00.000',*/ + G TIME DEFAULT '0000-00-00 01:00:00.000' NOT NULL, + H TIME DEFAULT '0000-00-00 01:00:00.000' NULL +); +INSERT INTO part_zero_date_and_time_table +VALUES (DEFAULT, DEFAULT, DEFAULT, DEFAULT, DEFAULT, /*DEFAULT,*/ DEFAULT, DEFAULT, DEFAULT); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/default_value_generated.sql b/tests/integration_tests/debezium/sql/debezium/default_value_generated.sql new file mode 100644 index 00000000000..ed4841620ef --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/default_value_generated.sql @@ -0,0 +1,8 @@ +/* see https://github.com/pingcap/tiflow/issues/11704 */ +CREATE TABLE GENERATED_TABLE ( + id int PRIMARY KEY, + A SMALLINT UNSIGNED, + B SMALLINT UNSIGNED AS (2 * A) STORED, + C SMALLINT UNSIGNED AS (3 * A) STORED NOT NULL +); +INSERT INTO GENERATED_TABLE VALUES (1, 15, DEFAULT, DEFAULT); diff --git a/tests/integration_tests/debezium/sql/debezium/enum_column_test.sql b/tests/integration_tests/debezium/sql/debezium/enum_column_test.sql index a8db5dfe308..9c261cbf46f 100644 --- a/tests/integration_tests/debezium/sql/debezium/enum_column_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/enum_column_test.sql @@ -1,7 +1,9 @@ +SET time_zone='UTC'; +/* Debezium convert charsetName to "utf8mb4" when column collate is utf8_unicode_ci */ CREATE TABLE `test_stations_10` ( `id` int(10) unsigned NOT NULL AUTO_INCREMENT, - `name` varchar(500) COLLATE utf8_unicode_ci NOT NULL, - `type` enum('station', 'post_office') COLLATE utf8_unicode_ci NOT NULL DEFAULT 'station', + `name` varchar(500) /* COLLATE utf8_unicode_ci */ NOT NULL, + `type` enum('station', 'post_office') /* COLLATE utf8_unicode_ci */ NOT NULL DEFAULT 'station', `created` datetime DEFAULT CURRENT_TIMESTAMP, `modified` datetime DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`) @@ -9,8 +11,9 @@ CREATE TABLE `test_stations_10` ( INSERT INTO test_stations_10 (`name`, `type`) values ( 'ha Tinh 7', 'station' ); -ALTER TABLE `test_stations_10` - MODIFY COLUMN `type` ENUM('station', 'post_office', 'plane', 'ahihi_dongok', 'now', 'test', 'a,b', 'c,\'d', 'g,''h') - CHARACTER SET 'utf8' COLLATE 'utf8_unicode_ci' NOT NULL DEFAULT 'station'; +/* Debezium datetime length is not correct */ +-- ALTER TABLE `test_stations_10` +-- MODIFY COLUMN `type` ENUM('station', 'post_office', 'plane', 'ahihi_dongok', 'now', 'test', 'a,b' /*'c,\'d', 'g,''h'*/) +-- /* CHARACTER SET 'utf8' COLLATE 'utf8_unicode_ci' */ NOT NULL DEFAULT 'station'; -INSERT INTO test_stations_10 ( `name`, `type` ) values ( 'Ha Tinh 1', 'now' ); +-- INSERT INTO test_stations_10 ( `name`, `type` ) values ( 'Ha Tinh 1', 'now' ); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/float_test.sql b/tests/integration_tests/debezium/sql/debezium/float_test.sql new file mode 100644 index 00000000000..da2d3bbf019 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/float_test.sql @@ -0,0 +1,22 @@ +/* + TiDB uses this value only to determine whether to use FLOAT or DOUBLE for the resulting data type. + If p is from 0 to 24, the data type becomes FLOAT with no M or D values. + If p is from 25 to 53, the data type becomes DOUBLE with no M or D values. +*/ +SET sql_mode=''; +CREATE TABLE `DBZ3865` ( + `id` INT NOT NULL AUTO_INCREMENT, + `f1` FLOAT DEFAULT 5.6, + `f2` FLOAT(10, 2) DEFAULT NULL, + `f3` FLOAT(35, 5) DEFAULT NULL, + /* TiDB incorrect length output. issue:https://github.com/pingcap/tidb/issues/57060 + `f4_23` FLOAT(23) DEFAULT NULL + `f4_24` FLOAT(24) DEFAULT NULL, + `f4_25` FLOAT(25) DEFAULT NULL, */ + `weight` FLOAT UNSIGNED DEFAULT '0', + PRIMARY KEY (`ID`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4; +/* Debezium incorrect output: + f2: 5.610000133514404 + f3: 30.12346076965332 */ +INSERT INTO DBZ3865(f1,/* f2, f3, f4_23, f4_24,*/ weight) VALUE (5.6,/* 5.61, 30.123456, 64.1, 64.1,*/ 64.1234); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/history-dbz.sql b/tests/integration_tests/debezium/sql/debezium/history-dbz.sql new file mode 100644 index 00000000000..5a328e45126 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/history-dbz.sql @@ -0,0 +1,2 @@ +CREATE TABLE `t-1` (ID1 INT PRIMARY KEY); +CREATE TABLE `t.2` (ID2 INT PRIMARY KEY); diff --git a/tests/integration_tests/debezium/sql/debezium/multitable_dbz_871.sql b/tests/integration_tests/debezium/sql/debezium/multitable_dbz_871.sql index 679bb0169dc..4d805f52c19 100644 --- a/tests/integration_tests/debezium/sql/debezium/multitable_dbz_871.sql +++ b/tests/integration_tests/debezium/sql/debezium/multitable_dbz_871.sql @@ -1,10 +1,9 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: multitable_statement --- ---------------------------------------------------------------------------------------------------------------- - CREATE TABLE t1 (ID INT PRIMARY KEY); CREATE TABLE t2 (ID INT PRIMARY KEY); CREATE TABLE t3 (ID INT PRIMARY KEY); CREATE TABLE t4 (ID INT PRIMARY KEY); -DROP TABLE t1,t2,t3,t4; \ No newline at end of file +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/mysql-dbz-123.ddl b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-123.ddl new file mode 100644 index 00000000000..b09b625a012 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-123.ddl @@ -0,0 +1,12 @@ +CREATE TABLE `DBZ123` ( + `Id` bigint(20) NOT NULL AUTO_INCREMENT, + `Provider_ID` bigint(20) NOT NULL, + `External_ID` varchar(255) NOT NULL, + `Name` varchar(255) NOT NULL, + `Is_Enabled` bit(1) NOT NULL DEFAULT b'1', + /* BLOB, TEXT, GEOMETRY or JSON column 'binaryRepresentation' can't have a default value + `binaryRepresentation` BLOB NOT NULL DEFAULT x'cafe', */ + `BonusFactor` decimal(19,8) NOT NULL, + PRIMARY KEY (`Id`), + UNIQUE KEY `game_unq` (`Provider_ID`,`External_ID`) +) ENGINE=InnoDB AUTO_INCREMENT=2374 DEFAULT CHARSET=utf8 \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/mysql-dbz-162.ddl b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-162.ddl new file mode 100644 index 00000000000..caa4d4ae445 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-162.ddl @@ -0,0 +1,7 @@ +CREATE TABLE `test` (id INT(11) UNSIGNED NOT NULL PRIMARY KEY AUTO_INCREMENT); + +ALTER TABLE `test` CHANGE `id` `collection_id` INT(11) +UNSIGNED +NOT NULL +AUTO_INCREMENT; + diff --git a/tests/integration_tests/debezium/sql/debezium/mysql-dbz-193.ddl b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-193.ddl new file mode 100644 index 00000000000..0d3026a4c3e --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-193.ddl @@ -0,0 +1,10 @@ +CREATE TABLE `roles` ( +`id` varchar(32) NOT NULL, +`name` varchar(100) NOT NULL, +`context` varchar(20) NOT NULL, +`organization_id` int(11) DEFAULT NULL, +`client_id` varchar(32) NOT NULL, +`scope_action_ids` text NOT NULL, +PRIMARY KEY (`id`), +FULLTEXT KEY `scope_action_ids_idx` (`scope_action_ids`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/tests/integration_tests/debezium/sql/debezium/mysql-dbz-198i.ddl b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-198i.ddl new file mode 100644 index 00000000000..36eb69a9d3f --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-198i.ddl @@ -0,0 +1,23 @@ +create database `NextTimeTable`; +use `NextTimeTable`; +create table NextTimeTable.REFERENCED ( + SUBJECT_ID int not null, + PRIMARY KEY (SUBJECT_ID) + ); + +create table `NextTimeTable`.`TIMETABLE_SUBJECT_GROUP_MAPPING` ( + pk1 int not null, + `SUBJECT_ID` int not null, + `other` int, + CONSTRAINT `FK69atxmt7wrwpb4oekyravsx9l` FOREIGN KEY (`SUBJECT_ID`) REFERENCES `NextTimeTable`.`REFERENCED`(`SUBJECT_ID`) + ); + +/* TiCDC discards it due to unsupported DDL type. +Alter table `NextTimeTable`.`TIMETABLE_SUBJECT_GROUP_MAPPING` +drop foreign key `FK69atxmt7wrwpb4oekyravsx9l`; +Alter table `NextTimeTable`.`TIMETABLE_SUBJECT_GROUP_MAPPING` +drop index `FK69atxmt7wrwpb4oekyravsx9l`; +Alter table `NextTimeTable`.`TIMETABLE_SUBJECT_GROUP_MAPPING` +drop column `SUBJECT_ID`; +*/ +create table `NextTimeTable`.`table1` ( pk1 int not null PRIMARY KEY, `id` int not null, `other` int ); diff --git a/tests/integration_tests/debezium/sql/debezium/mysql-dbz-200.ddl b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-200.ddl new file mode 100644 index 00000000000..797ed730488 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/mysql-dbz-200.ddl @@ -0,0 +1,25 @@ +CREATE TABLE `customfield` ( + `ENCODEDKEY` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `ID` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `CREATIONDATE` datetime DEFAULT NULL, + `LASTMODIFIEDDATE` datetime DEFAULT NULL, + `DATATYPE` varchar(256) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `ISDEFAULT` bit(1) DEFAULT NULL, + `ISREQUIRED` bit(1) DEFAULT NULL, + `NAME` varchar(256) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `VALUES` mediumblob, + `AMOUNTS` mediumblob, + `DESCRIPTION` varchar(256) DEFAULT NULL, + `TYPE` varchar(256) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `VALUELENGTH` varchar(256) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL DEFAULT 'SHORT', + `INDEXINLIST` int(11) DEFAULT '-1', + `CUSTOMFIELDSET_ENCODEDKEY_OID` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + `STATE` varchar(256) NOT NULL DEFAULT 'NORMAL', + `VALIDATIONPATTERN` varchar(256) DEFAULT NULL, + `VIEWUSAGERIGHTSKEY` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `EDITUSAGERIGHTSKEY` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `BUILTINCUSTOMFIELDID` varchar(255) CHARACTER SET utf8 COLLATE utf8_bin DEFAULT NULL, + `UNIQUE` varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + STORAGE varchar(32) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL, + KEY `index1` (`ID`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8; diff --git a/tests/integration_tests/debezium/sql/debezium/mysql-quoted.ddl b/tests/integration_tests/debezium/sql/debezium/mysql-quoted.ddl new file mode 100644 index 00000000000..cad5ecae495 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/mysql-quoted.ddl @@ -0,0 +1,36 @@ +-- +-- Contains statements with quoted identifiers +-- +CREATE TABLE `customers` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `first_name` varchar(255) NOT NULL, + `last_name` varchar(255) NOT NULL, + `email` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `email` (`email`) +) ENGINE=InnoDB AUTO_INCREMENT=1005 DEFAULT CHARSET=latin1; +CREATE TABLE `products` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `name` varchar(255) NOT NULL, + `description` varchar(512) DEFAULT NULL, + `weight` float DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=110 DEFAULT CHARSET=latin1; +CREATE TABLE `orders` ( + `order_number` int(11) NOT NULL AUTO_INCREMENT, + `order_date` date NOT NULL, + `purchaser` int(11) NOT NULL, + `quantity` int(11) NOT NULL, + `product_id` int(11) NOT NULL, + PRIMARY KEY (`order_number`), + KEY `order_customer` (`purchaser`), + KEY `ordered_product` (`product_id`), + CONSTRAINT `orders_ibfk_1` FOREIGN KEY (`purchaser`) REFERENCES `customers` (`id`), + CONSTRAINT `orders_ibfk_2` FOREIGN KEY (`product_id`) REFERENCES `products` (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=10006 DEFAULT CHARSET=latin1; +CREATE TABLE `products_on_hand` ( + `product_id` int(11) NOT NULL, + `quantity` int(11) NOT NULL, + PRIMARY KEY (`product_id`), + CONSTRAINT `products_on_hand_ibfk_1` FOREIGN KEY (`product_id`) REFERENCES `products` (`id`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1 \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/mysql-test-create.ddl b/tests/integration_tests/debezium/sql/debezium/mysql-test-create.ddl new file mode 100644 index 00000000000..7052708998c --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/mysql-test-create.ddl @@ -0,0 +1,1134 @@ +-- +-- BUILD SCRIPT +-- RDBMS: MYSQL 5.0 +-- + +CREATE TABLE AUDITENTRIES +( + TIMESTAMP VARCHAR(50) NOT NULL, + CONTEXT VARCHAR(64) NOT NULL, + ACTIVITY VARCHAR(64) NOT NULL, + RESOURCES VARCHAR(4000) NOT NULL, + PRINCIPAL VARCHAR(255) NOT NULL, + HOSTNAME VARCHAR(64) NOT NULL, + VMID VARCHAR(64) NOT NULL +); + +CREATE TABLE AUTHPERMTYPES +( + PERMTYPEUID NUMERIC(10) NOT NULL PRIMARY KEY, + DISPLAYNAME VARCHAR(250) NOT NULL, + FACTORYCLASSNAME VARCHAR(80) NOT NULL +); + +CREATE TABLE AUTHPOLICIES +( + POLICYUID NUMERIC(10) NOT NULL PRIMARY KEY, + DESCRIPTION VARCHAR(250), + POLICYNAME VARCHAR(250) NOT NULL +); + +CREATE TABLE AUTHPRINCIPALS +( + PRINCIPALTYPE NUMERIC(10) NOT NULL, + PRINCIPALNAME VARCHAR(255) NOT NULL, + POLICYUID NUMERIC(10) NOT NULL REFERENCES AUTHPOLICIES (POLICYUID) , + GRANTOR VARCHAR(255) NOT NULL, + CONSTRAINT PK_AUTHPOLICYPRINCIPALS UNIQUE (PRINCIPALNAME, POLICYUID) +); + +CREATE TABLE AUTHREALMS +( + REALMUID NUMERIC(10) NOT NULL PRIMARY KEY, + REALMNAME VARCHAR(250) NOT NULL UNIQUE, + DESCRIPTION VARCHAR(550) +); + +CREATE TABLE CFG_STARTUP_STATE +(STATE INTEGER DEFAULT 0 , +LASTCHANGED VARCHAR(50) ); + +CREATE TABLE IDTABLE +( + IDCONTEXT VARCHAR(20) NOT NULL PRIMARY KEY, + NEXTID NUMERIC +); + +CREATE TABLE LOGMESSAGETYPES +( + MESSAGELEVEL NUMERIC(10) NOT NULL PRIMARY KEY, + NAME VARCHAR(64) NOT NULL, + DISPLAYNAME VARCHAR(64) +); + +CREATE TABLE MM_PRODUCTS +( + PRODUCT_UID NUMERIC NOT NULL PRIMARY KEY, + PRODUCT_NAME VARCHAR(50) NOT NULL, + PRODUCT_DISPLAY_NM VARCHAR(100) +); + +CREATE TABLE PRINCIPALTYPES +( + PRINCIPALTYPEUID NUMERIC(10) NOT NULL PRIMARY KEY, + PRINCIPALTYPE VARCHAR(60) NOT NULL, + DISPLAYNAME VARCHAR(80) NOT NULL, + LASTCHANGEDBY VARCHAR(255) NOT NULL, + LASTCHANGED VARCHAR(50) +); +-- ========= STATEMENT 10 ============ + +CREATE TABLE RT_MDLS +( + MDL_UID NUMERIC(10) NOT NULL PRIMARY KEY, + MDL_UUID VARCHAR(64) NOT NULL, + MDL_NM VARCHAR(255) NOT NULL, + MDL_VERSION VARCHAR(50), + DESCRIPTION VARCHAR(255), + MDL_URI VARCHAR(255), + MDL_TYPE NUMERIC(3), + IS_PHYSICAL CHAR(1) NOT NULL, + MULTI_SOURCED CHAR(1) DEFAULT '0', + VISIBILITY NUMERIC(10) + ); + +CREATE TABLE RT_MDL_PRP_NMS +( + PRP_UID NUMERIC(10) NOT NULL PRIMARY KEY, + MDL_UID NUMERIC(10) NOT NULL , + PRP_NM VARCHAR(255) NOT NULL +); + +CREATE TABLE RT_MDL_PRP_VLS +( + PRP_UID NUMERIC(10) NOT NULL , + PART_ID NUMERIC(10) NOT NULL, + PRP_VL VARCHAR(255) NOT NULL, + CONSTRAINT PK_MDL_PRP_VLS UNIQUE (PRP_UID, PART_ID) +); + + +CREATE TABLE RT_VIRTUAL_DBS +( + VDB_UID NUMERIC(10) NOT NULL PRIMARY KEY, + VDB_VERSION VARCHAR(50) NOT NULL, + VDB_NM VARCHAR(255) NOT NULL, + DESCRIPTION VARCHAR(255), + PROJECT_GUID VARCHAR(64), + VDB_STATUS NUMERIC NOT NULL, + WSDL_DEFINED CHAR(1) DEFAULT '0', + VERSION_BY VARCHAR(100), + VERSION_DATE VARCHAR(50) NOT NULL, + CREATED_BY VARCHAR(100), + CREATION_DATE VARCHAR(50), + UPDATED_BY VARCHAR(100), + UPDATED_DATE VARCHAR(50), + VDB_FILE_NM VARCHAR(2048) +); + +CREATE TABLE SERVICESESSIONS +( + SESSIONUID NUMERIC(10) NOT NULL PRIMARY KEY, + PRINCIPAL VARCHAR(255) NOT NULL, + APPLICATION VARCHAR(128) NOT NULL, + CREATIONTIME VARCHAR(50), + CLIENTCOUNT NUMERIC(10) NOT NULL, + STATE NUMERIC(10) NOT NULL, + STATETIME VARCHAR(50), + USESSUBSCRIBER CHAR(1) NOT NULL, + PRODUCTINFO1 VARCHAR(255), + PRODUCTINFO2 VARCHAR(255), + PRODUCTINFO3 VARCHAR(255), + PRODUCTINFO4 VARCHAR(255) +); +-- ========= STATEMENT 15 ============ +CREATE INDEX RTMDLS_NM_IX ON RT_MDLS (MDL_NM); + +CREATE INDEX RTVIRTUALDBS_NM_IX ON RT_VIRTUAL_DBS (VDB_NM); + +CREATE INDEX RTVIRTUALDBS_VRSN_IX ON RT_VIRTUAL_DBS (VDB_VERSION); + +CREATE UNIQUE INDEX MDL_PRP_NMS_UIX ON RT_MDL_PRP_NMS (MDL_UID, PRP_NM); + +CREATE UNIQUE INDEX PRNCIPALTYP_UIX ON PRINCIPALTYPES (PRINCIPALTYPE); +-- ========= STATEMENT 20 ============ +CREATE UNIQUE INDEX AUTHPOLICIES_NAM_UIX ON AUTHPOLICIES (POLICYNAME); + +CREATE TABLE AUTHPERMISSIONS +( + PERMISSIONUID NUMERIC(10) NOT NULL PRIMARY KEY, + RESOURCENAME VARCHAR(250) NOT NULL, + ACTIONS NUMERIC(10) NOT NULL, + CONTENTMODIFIER VARCHAR(250), + PERMTYPEUID NUMERIC(10) NOT NULL REFERENCES AUTHPERMTYPES (PERMTYPEUID) , + REALMUID NUMERIC(10) NOT NULL REFERENCES AUTHREALMS (REALMUID) , + POLICYUID NUMERIC(10) NOT NULL REFERENCES AUTHPOLICIES (POLICYUID) +); + + +CREATE TABLE LOGENTRIES +( + TIMESTAMP VARCHAR(50) NOT NULL, + CONTEXT VARCHAR(64) NOT NULL, + MSGLEVEL NUMERIC(10) NOT NULL REFERENCES LOGMESSAGETYPES (MESSAGELEVEL) , + EXCEPTION VARCHAR(4000), + MESSAGE VARCHAR(2000) NOT NULL, + HOSTNAME VARCHAR(64) NOT NULL, + VMID VARCHAR(64) NOT NULL, + THREAModeShapeME VARCHAR(64) NOT NULL, + VMSEQNUM NUMERIC(7) NOT NULL +); + +CREATE TABLE PRODUCTSSESSIONS +( + PRODUCT_UID NUMERIC NOT NULL, + SESSION_UID NUMERIC NOT NULL, + PRIMARY KEY (PRODUCT_UID, SESSION_UID) +); + +-- ALTER TABLE PRODUCTSSESSIONS +-- ADD CONSTRAINT FK_PRODSESS_PRODS +-- FOREIGN KEY (PRODUCT_UID) +-- REFERENCES MM_PRODUCTS (PRODUCT_UID); + +-- ALTER TABLE PRODUCTSSESSIONS +-- ADD CONSTRAINT FK_PRODSESS_SVCSES +-- FOREIGN KEY (SESSION_UID) +-- REFERENCES SERVICESESSIONS (SESSIONUID); + + +CREATE TABLE RT_VDB_MDLS +( + VDB_UID NUMERIC(10) NOT NULL , + MDL_UID NUMERIC(10) NOT NULL , + CNCTR_BNDNG_NM VARCHAR(255) +); + +CREATE INDEX AWA_SYS_MSGLEVEL_1E6F845E ON LOGENTRIES (MSGLEVEL); + +CREATE UNIQUE INDEX AUTHPERM_UIX ON AUTHPERMISSIONS ( POLICYUID, RESOURCENAME); + +CREATE TABLE CS_EXT_FILES ( + FILE_UID INTEGER NOT NULL, + CHKSUM NUMERIC(20), + FILE_NAME VARCHAR(255) NOT NULL, + FILE_CONTENTS LONGBLOB, + CONFIG_CONTENTS LONGTEXT, + SEARCH_POS INTEGER, + IS_ENABLED CHAR(1), + FILE_DESC VARCHAR(4000), + CREATED_BY VARCHAR(100), + CREATION_DATE VARCHAR(50), + UPDATED_BY VARCHAR(100), + UPDATE_DATE VARCHAR(50), + FILE_TYPE VARCHAR(30), + CONSTRAINT PK_CS_EXT_FILES PRIMARY KEY (FILE_UID) +) +; +-- ========= STATEMENT 30 ============ +-- ALTER TABLE CS_EXT_FILES ADD CONSTRAINT CSEXFILS_FIL_NA_UK UNIQUE (FILE_NAME); + +CREATE TABLE MMSCHEMAINFO_CA +( + SCRIPTNAME VARCHAR(50), + SCRIPTEXECUTEDBY VARCHAR(50), + SCRIPTREV VARCHAR(50), + RELEASEDATE VARCHAR(50), + DATECREATED DATE, + DATEUPDATED DATE, + UPDATEID VARCHAR(50), + METAMATRIXSERVERURL VARCHAR(100) +) +; + +CREATE TABLE CS_SYSTEM_PROPS ( + PROPERTY_NAME VARCHAR(255), + PROPERTY_VALUE VARCHAR(255) +); + +CREATE UNIQUE INDEX SYSPROPS_KEY ON CS_SYSTEM_PROPS (PROPERTY_NAME); + +CREATE TABLE CFG_LOCK ( + USER_NAME VARCHAR(50) NOT NULL, + DATETIME_ACQUIRED VARCHAR(50) NOT NULL, + DATETIME_EXPIRE VARCHAR(50) NOT NULL, + HOST VARCHAR(100), + LOCK_TYPE NUMERIC (1) ); + + +CREATE TABLE TX_MMXCMDLOG +(REQUESTID VARCHAR(255) NOT NULL, +TXNUID VARCHAR(50) NULL, +CMDPOINT NUMERIC(10) NOT NULL, +SESSIONUID VARCHAR(255) NOT NULL, +APP_NAME VARCHAR(255) NULL, +PRINCIPAL_NA VARCHAR(255) NOT NULL, +VDBNAME VARCHAR(255) NOT NULL, +VDBVERSION VARCHAR(50) NOT NULL, +CREATED_TS VARCHAR(50) NULL, +ENDED_TS VARCHAR(50) NULL, +CMD_STATUS NUMERIC(10) NOT NULL, +SQL_ID NUMERIC(10), +FINL_ROWCNT NUMERIC(10) +) +; + +CREATE TABLE TX_SRCCMDLOG +(REQUESTID VARCHAR(255) NOT NULL, +NODEID NUMERIC(10) NOT NULL, +SUBTXNUID VARCHAR(50) NULL, +CMD_STATUS NUMERIC(10) NOT NULL, +MDL_NM VARCHAR(255) NOT NULL, +CNCTRNAME VARCHAR(255) NOT NULL, +CMDPOINT NUMERIC(10) NOT NULL, +SESSIONUID VARCHAR(255) NOT NULL, +PRINCIPAL_NA VARCHAR(255) NOT NULL, +CREATED_TS VARCHAR(50) NULL, +ENDED_TS VARCHAR(50) NULL, +SQL_ID NUMERIC(10) NULL, +FINL_ROWCNT NUMERIC(10) NULL +) +; + + +CREATE TABLE TX_SQL ( SQL_ID NUMERIC(10) NOT NULL, + SQL_VL TEXT ) +; +ALTER TABLE TX_SQL + ADD CONSTRAINT TX_SQL_PK +PRIMARY KEY (SQL_ID) +; +-- ========= STATEMENT 39 ============ + +-- +-- The ITEMS table stores the raw, structure-independent information about the items contained by the Repository. This table is capable of persisting multiple versions of an item. +-- +CREATE TABLE MBR_ITEMS +( + ITEM_ID_P1 NUMERIC(20) NOT NULL, + ITEM_ID_P2 NUMERIC(20) NOT NULL, + ITEM_VERSION VARCHAR(80) NOT NULL, + ITEM_NAME VARCHAR(255) NOT NULL, + UPPER_ITEM_NAME VARCHAR(255) NOT NULL, + COMMENT_FLD VARCHAR(2000), + LOCK_HOLDER VARCHAR(100), + LOCK_DATE VARCHAR(50), + CREATED_BY VARCHAR(100) NOT NULL, + CREATION_DATE VARCHAR(50) NOT NULL, + ITEM_TYPE NUMERIC(10) NOT NULL +); + +-- +-- The ITEM_CONTENTS table stores the contents for items (files) stored in the repository. This table is capable of persisting multiple versions of the contents for an item. +-- +CREATE TABLE MBR_ITEM_CONTENTS +( + ITEM_ID_P1 NUMERIC(20) NOT NULL, + ITEM_ID_P2 NUMERIC(20) NOT NULL, + ITEM_VERSION VARCHAR(80) NOT NULL, + ITEM_CONTENT LONGBLOB NOT NULL +); + +-- +-- The ENTRIES table stores the structure information for all the objects stored in the Repository. This includes both folders and items. +-- +CREATE TABLE MBR_ENTRIES +( + ENTRY_ID_P1 NUMERIC(20) NOT NULL, + ENTRY_ID_P2 NUMERIC(20) NOT NULL, + ENTRY_NAME VARCHAR(255) NOT NULL, + UPPER_ENTRY_NAME VARCHAR(255) NOT NULL, + ITEM_ID_P1 NUMERIC(20), + ITEM_ID_P2 NUMERIC(20), + ITEM_VERSION VARCHAR(80), + PARENT_ENTRY_ID_P1 NUMERIC(20), + PARENT_ENTRY_ID_P2 NUMERIC(20), + DELETED NUMERIC(1) NOT NULL +); + +-- +-- The LABELS table stores the various labels that have been defined. +-- +CREATE TABLE MBR_LABELS +( + LABEL_ID_P1 NUMERIC(20) NOT NULL, + LABEL_ID_P2 NUMERIC(20) NOT NULL, + LABEL_FLD VARCHAR(255) NOT NULL, + COMMENT_FLD VARCHAR(2000), + CREATED_BY VARCHAR(100) NOT NULL, + CREATION_DATE VARCHAR(50) NOT NULL +); + +-- +-- The ITEM_LABELS table maintains the relationships between the ITEMS and the LABELs; that is, the labels that have been applied to each of the item versions. (This is a simple intersect table.) +-- +CREATE TABLE MBR_ITEM_LABELS +( + ITEM_ID_P1 NUMERIC(20) NOT NULL, + ITEM_ID_P2 NUMERIC(20) NOT NULL, + ITEM_VERSION VARCHAR(80) NOT NULL, + LABEL_ID_P1 NUMERIC(20) NOT NULL, + LABEL_ID_P2 NUMERIC(20) NOT NULL +); + +-- +-- The ITEM_LABELS table maintains the relationships between the ITEMS and the LABELs; that is, the labels that have been applied to each of the item versions. (This is a simple intersect table.) +-- +CREATE TABLE MBR_FOLDER_LABELS +( + ENTRY_ID_P1 NUMERIC(20) NOT NULL, + ENTRY_ID_P2 NUMERIC(20) NOT NULL, + LABEL_ID_P1 NUMERIC(20) NOT NULL, + LABEL_ID_P2 NUMERIC(20) NOT NULL +); + +CREATE TABLE MBR_ITEM_TYPES +( + ITEM_TYPE_CODE NUMERIC(10) NOT NULL, + ITEM_TYPE_NM VARCHAR(20) NOT NULL +); + +CREATE TABLE MBR_POLICIES +( + POLICY_NAME VARCHAR(250) NOT NULL, + CREATION_DATE VARCHAR(50), + CHANGE_DATE VARCHAR(50), + GRANTOR VARCHAR(32) +); + +CREATE TABLE MBR_POL_PERMS +( + ENTRY_ID_P1 NUMERIC(20) NOT NULL, + ENTRY_ID_P2 NUMERIC(20) NOT NULL, + POLICY_NAME VARCHAR(250) NOT NULL, + CREATE_BIT CHAR(1) NOT NULL, + READ_BIT CHAR(1) NOT NULL, + UPDATE_BIT CHAR(1) NOT NULL, + DELETE_BIT CHAR(1) NOT NULL +); + +CREATE TABLE MBR_POL_USERS +( + POLICY_NAME VARCHAR(250) NOT NULL, + USER_NAME VARCHAR(80) NOT NULL +); + +CREATE UNIQUE INDEX MBR_ENT_NM_PNT_IX ON MBR_ENTRIES (UPPER_ENTRY_NAME,PARENT_ENTRY_ID_P1,PARENT_ENTRY_ID_P2); +-- ========= STATEMENT 50 ============ +CREATE INDEX MBR_ITEMS_ID_IX ON MBR_ITEMS (ITEM_ID_P1,ITEM_ID_P2); + +CREATE INDEX MBR_ENT_PARNT_IX ON MBR_ENTRIES (PARENT_ENTRY_ID_P1); + +CREATE INDEX MBR_ENT_NM_IX ON MBR_ENTRIES (UPPER_ENTRY_NAME); + +ALTER TABLE MBR_ITEMS + ADD CONSTRAINT PK_ITEMS + PRIMARY KEY (ITEM_ID_P1,ITEM_ID_P2,ITEM_VERSION); + +ALTER TABLE MBR_ITEM_CONTENTS + ADD CONSTRAINT PK_ITEM_CONTENTS + PRIMARY KEY (ITEM_ID_P1,ITEM_ID_P2,ITEM_VERSION); + +ALTER TABLE MBR_ENTRIES + ADD CONSTRAINT PK_ENTRIES + PRIMARY KEY (ENTRY_ID_P1,ENTRY_ID_P2); + +ALTER TABLE MBR_LABELS + ADD CONSTRAINT PK_LABELS + PRIMARY KEY (LABEL_ID_P1,LABEL_ID_P2); + +ALTER TABLE MBR_ITEM_LABELS + ADD CONSTRAINT PK_ITEM_LABELS + PRIMARY KEY (ITEM_ID_P1,ITEM_ID_P2,ITEM_VERSION,LABEL_ID_P1,LABEL_ID_P2); + +ALTER TABLE MBR_FOLDER_LABELS + ADD CONSTRAINT PK_FOLDER_LABELS + PRIMARY KEY (ENTRY_ID_P1,ENTRY_ID_P2,LABEL_ID_P1,LABEL_ID_P2); + +ALTER TABLE MBR_POLICIES + ADD CONSTRAINT PK_POLICIES + PRIMARY KEY (POLICY_NAME); +-- ========= STATEMENT 60 ============ +ALTER TABLE MBR_POL_PERMS + ADD CONSTRAINT PK_POL_PERMS + PRIMARY KEY (ENTRY_ID_P1,ENTRY_ID_P2,POLICY_NAME); + +ALTER TABLE MBR_POL_USERS + ADD CONSTRAINT PK_POL_USERS + PRIMARY KEY (POLICY_NAME,USER_NAME); +-- (generated from DtcBase/ObjectIndex) + + + +CREATE OR REPLACE VIEW MBR_READ_ENTRIES (ENTRY_ID_P1,ENTRY_ID_P2,USER_NAME) AS +SELECT MBR_POL_PERMS.ENTRY_ID_P1, MBR_POL_PERMS.ENTRY_ID_P2, + MBR_POL_USERS.USER_NAME +FROM MBR_POL_PERMS, MBR_POL_USERS , CS_SYSTEM_PROPS +where MBR_POL_PERMS.POLICY_NAME=MBR_POL_USERS.POLICY_NAME + AND (CS_SYSTEM_PROPS.PROPERTY_NAME='metamatrix.authorization.metabase.CheckingEnabled' + AND CS_SYSTEM_PROPS.PROPERTY_VALUE ='true' + AND MBR_POL_PERMS.READ_BIT='1') +UNION ALL +SELECT ENTRY_ID_P1, ENTRY_ID_P2, NULL +FROM MBR_ENTRIES ,CS_SYSTEM_PROPS +WHERE CS_SYSTEM_PROPS.PROPERTY_NAME='metamatrix.authorization.metabase.CheckingEnabled' + AND CS_SYSTEM_PROPS.PROPERTY_VALUE ='false' +; + + +CREATE INDEX MBR_POL_PERMS_IX1 ON MBR_POL_PERMS (POLICY_NAME, READ_BIT); + +CREATE INDEX LOGENTRIES_TMSTMP_IX ON LOGENTRIES (TIMESTAMP); + +CREATE TABLE DD_TXN_STATES +( + ID INTEGER primary key NOT NULL, + STATE VARCHAR(128) NOT NULL +); + +CREATE TABLE DD_TXN_LOG +( + ID BIGINT NOT NULL, + USER_NME VARCHAR(128), + BEGIN_TXN VARCHAR(50), + END_TXN VARCHAR(50), + ACTION VARCHAR(128), + TXN_STATE INTEGER +); + + +CREATE TABLE DD_SHREDQUEUE +( + QUEUE_ID NUMERIC(19) NOT NULL, + UUID1 NUMERIC(20) NOT NULL, + UUID2 NUMERIC(20) NOT NULL, + OBJECT_ID VARCHAR(44) NOT NULL, + NAME VARCHAR(128) NOT NULL, + VERSION VARCHAR(20), + MDL_PATH VARCHAR(2000), + CMD_ACTION NUMERIC(1) NOT NULL, + TXN_ID NUMERIC(19) , + SUB_BY_NME VARCHAR(100), + SUB_BY_DATE VARCHAR(50) +); + + +CREATE UNIQUE INDEX DDSQ_QUE_IX ON DD_SHREDQUEUE (QUEUE_ID) +; +CREATE UNIQUE INDEX DDSQ_TXN_IX ON DD_SHREDQUEUE (TXN_ID) +; +-- ========= STATEMENT 70 ============ +CREATE INDEX DDSQ_UUID_IX ON DD_SHREDQUEUE (OBJECT_ID) +; + +-- == new DTC start == +-- (generated from Models) + +CREATE TABLE MMR_MODELS +( + ID BIGINT NOT NULL, + NAME VARCHAR(256), + PATH VARCHAR(1024), + NAMESPACE VARCHAR(1024), + IS_METAMODEL SMALLINT, + VERSION VARCHAR(64), + IS_INCOMPLETE SMALLINT + -- SHRED_TIME DATETIME +); + +-- (generated from Resources) + +CREATE TABLE MMR_RESOURCES +( + MODEL_ID BIGINT NOT NULL, + CONTENT LONGTEXT NOT NULL +); + +-- (generated from Objects) + +CREATE TABLE MMR_OBJECTS +( + ID BIGINT NOT NULL, + MODEL_ID BIGINT NOT NULL, + NAME VARCHAR(256), + PATH VARCHAR(1024), + CLASS_NAME VARCHAR(256), + UUID VARCHAR(64), + NDX_PATH VARCHAR(256), + IS_UNRESOLVED SMALLINT +); + +-- (generated from ResolvedObjects) + +CREATE TABLE MMR_RESOLVED_OBJECTS +( + OBJ_ID BIGINT NOT NULL, + MODEL_ID BIGINT NOT NULL, + CLASS_ID BIGINT NOT NULL, + CONTAINER_ID BIGINT +); + +-- (generated from ReferenceFeatures) + +CREATE TABLE MMR_REF_FEATURES +( + MODEL_ID BIGINT NOT NULL, + OBJ_ID BIGINT NOT NULL, + NDX INT, + DATATYPE_ID BIGINT, + LOWER_BOUND INT, + UPPER_BOUND INT, + IS_CHANGEABLE SMALLINT, + IS_UNSETTABLE SMALLINT, + IS_CONTAINMENT SMALLINT, + OPPOSITE_ID BIGINT +); + +-- (generated from AttributeFeatures) + +CREATE TABLE MMR_ATTR_FEATURES +( + MODEL_ID BIGINT NOT NULL, + OBJ_ID BIGINT NOT NULL, + NDX INT, + DATATYPE_ID BIGINT, + LOWER_BOUND INT, + UPPER_BOUND INT, + IS_CHANGEABLE SMALLINT, + IS_UNSETTABLE SMALLINT +); + +-- (generated from References) + +CREATE TABLE MMR_REFS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + TO_ID BIGINT NOT NULL +); + +-- (generated from BooleanAttributes) + +CREATE TABLE MMR_BOOLEAN_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE SMALLINT NOT NULL +); + +-- (generated from ByteAttributes) + +CREATE TABLE MMR_BYTE_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE CHAR(1) NOT NULL +); +-- ========= STATEMENT 80 ============ +-- (generated from CharAttributes) + +CREATE TABLE MMR_CHAR_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE CHAR(1) +); + +-- (generated from ClobAttributes) + +CREATE TABLE MMR_CLOB_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE LONGTEXT +); + +-- (generated from DoubleAttributes) + +CREATE TABLE MMR_DOUBLE_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE DOUBLE NOT NULL +); + +-- (generated from EnumeratedAttributes) + +CREATE TABLE MMR_ENUM_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE INT NOT NULL +); + +-- (generated from FloatAttributes) + +CREATE TABLE MMR_FLOAT_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE FLOAT NOT NULL +); + +-- (generated from IntAttributes) + +CREATE TABLE MMR_INT_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE INT NOT NULL +); + +-- (generated from LongAttributes) + +CREATE TABLE MMR_LONG_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE BIGINT NOT NULL +); + + +-- (generated from ShortAttributes) + +CREATE TABLE MMR_SHORT_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE SMALLINT NOT NULL +); + +-- (generated from StringAttributes) + +CREATE TABLE MMR_STRING_ATTRS +( + MODEL_ID BIGINT NOT NULL, + OBJECT_ID BIGINT NOT NULL, + FEATURE_ID BIGINT NOT NULL, + NDX INT NOT NULL, + VALUE VARCHAR(4000) +); + +-- Index length too long for MMR_MODELS(NAME,PATH) +CREATE INDEX MOD_PATH_NDX ON MMR_MODELS (NAME); +-- ========= STATEMENT 90 ============ +-- Specified key was too long; max key length is 3072 bytes +-- CREATE INDEX MOD_PATH2_NDX ON MMR_MODELS (PATH); + +-- CREATE INDEX MOD_NAMESPACE_NDX ON MMR_MODELS (NAMESPACE); + +CREATE INDEX OBJ_UUID_NDX ON MMR_OBJECTS (UUID); + +CREATE INDEX RES_OBJ_MODEL_NDX ON MMR_RESOLVED_OBJECTS (MODEL_ID); + +CREATE INDEX RES_OBJ_CLASS_NDX ON MMR_RESOLVED_OBJECTS (CLASS_ID); + +CREATE INDEX RF_DATATYPE_NDX ON MMR_REF_FEATURES (DATATYPE_ID); + +CREATE INDEX RF_MODEL_NDX ON MMR_REF_FEATURES (MODEL_ID); + +CREATE INDEX AF_DATATYPE_NDX ON MMR_ATTR_FEATURES (DATATYPE_ID); + +CREATE INDEX AF_MODEL_NDX ON MMR_ATTR_FEATURES (MODEL_ID); + +CREATE INDEX BOL_FEATURE_NDX ON MMR_BOOLEAN_ATTRS (FEATURE_ID); +-- ========= STATEMENT 100 ============ +CREATE INDEX BOL_MODEL_NDX ON MMR_BOOLEAN_ATTRS (MODEL_ID); + +CREATE INDEX BYT_FEATURE_NDX ON MMR_BYTE_ATTRS (FEATURE_ID); + +CREATE INDEX BYT_MODEL_NDX ON MMR_BYTE_ATTRS (MODEL_ID); + +CREATE INDEX CHR_FEATURE_NDX ON MMR_CHAR_ATTRS (FEATURE_ID); + +CREATE INDEX CHR_MODEL_NDX ON MMR_CHAR_ATTRS (MODEL_ID); + +CREATE INDEX CLOB_FEATURE_NDX ON MMR_CLOB_ATTRS (FEATURE_ID); + +CREATE INDEX CLOB_MODEL_NDX ON MMR_CLOB_ATTRS (MODEL_ID); + +CREATE INDEX DBL_FEATURE_NDX ON MMR_DOUBLE_ATTRS (FEATURE_ID); + +CREATE INDEX DBL_MODEL_NDX ON MMR_DOUBLE_ATTRS (MODEL_ID); + +CREATE INDEX ENUM_FEATURE_NDX ON MMR_ENUM_ATTRS (FEATURE_ID); +-- ========= STATEMENT 110 ============ +CREATE INDEX ENUM_MODEL_NDX ON MMR_ENUM_ATTRS (MODEL_ID); + +CREATE INDEX FLT_FEATURE_NDX ON MMR_FLOAT_ATTRS (FEATURE_ID); + +CREATE INDEX FLT_MODEL_NDX ON MMR_FLOAT_ATTRS (MODEL_ID); + +CREATE INDEX INT_FEATURE_NDX ON MMR_INT_ATTRS (FEATURE_ID); + +CREATE INDEX INT_MODEL_NDX ON MMR_INT_ATTRS (MODEL_ID); + +CREATE INDEX LNG_FEATURE_NDX ON MMR_LONG_ATTRS (FEATURE_ID); + +CREATE INDEX LNG_MODEL_NDX ON MMR_LONG_ATTRS (MODEL_ID); + +CREATE INDEX REF_FEATURE_NDX ON MMR_REFS (FEATURE_ID); + +CREATE INDEX REF_TO_NDX ON MMR_REFS (TO_ID); + +CREATE INDEX REF_MODEL_NDX ON MMR_REFS (MODEL_ID); +-- ========= STATEMENT 120 ============ +CREATE INDEX SHR_FEATURE_NDX ON MMR_SHORT_ATTRS (FEATURE_ID); + +CREATE INDEX SHR_MODEL_NDX ON MMR_SHORT_ATTRS (MODEL_ID); + +CREATE INDEX STR_FEATURE_NDX ON MMR_STRING_ATTRS (FEATURE_ID); + +CREATE INDEX STR_MODEL_NDX ON MMR_STRING_ATTRS (MODEL_ID); + +-- DATETIME length is incorrect +ALTER TABLE MMR_MODELS + ADD CONSTRAINT MOD_PK + PRIMARY KEY (ID); + +ALTER TABLE MMR_RESOURCES + ADD CONSTRAINT RSRC_PK + PRIMARY KEY (MODEL_ID); + +ALTER TABLE MMR_OBJECTS + ADD CONSTRAINT OBJ_PK + PRIMARY KEY (ID); + +ALTER TABLE MMR_RESOLVED_OBJECTS + ADD CONSTRAINT RES_OBJ_PK + PRIMARY KEY (OBJ_ID); + +ALTER TABLE MMR_REF_FEATURES + ADD CONSTRAINT RF_PK + PRIMARY KEY (OBJ_ID); + +ALTER TABLE MMR_ATTR_FEATURES + ADD CONSTRAINT AF_PK + PRIMARY KEY (OBJ_ID); +-- ========= STATEMENT 130 ============ +ALTER TABLE MMR_REFS + ADD CONSTRAINT REF_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_BOOLEAN_ATTRS + ADD CONSTRAINT BOL_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_BYTE_ATTRS + ADD CONSTRAINT BYT_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_CHAR_ATTRS + ADD CONSTRAINT CHR_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_CLOB_ATTRS + ADD CONSTRAINT CLOB_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_DOUBLE_ATTRS + ADD CONSTRAINT DBL_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_ENUM_ATTRS + ADD CONSTRAINT ENUM_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_FLOAT_ATTRS + ADD CONSTRAINT FLT_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_INT_ATTRS + ADD CONSTRAINT INT_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_LONG_ATTRS + ADD CONSTRAINT LNG_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); +-- ========= STATEMENT 140 ============ +ALTER TABLE MMR_SHORT_ATTRS + ADD CONSTRAINT SHR_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + +ALTER TABLE MMR_STRING_ATTRS + ADD CONSTRAINT STR_PK + PRIMARY KEY (OBJECT_ID,FEATURE_ID,NDX); + + + -- View for obtaining the features by metaclass + -- (don't use parenthesis) + +CREATE OR REPLACE VIEW MMR_FEATURES AS + SELECT MMR_MODELS.NAMESPACE AS NAMESPACE, + PARENTS.NAME AS CLASS_NAME, + MMR_OBJECTS.NAME AS FEATURE_NAME, + MMR_ATTR_FEATURES.OBJ_ID AS FEATURE_ID, + 'Attribute' AS FEATURE_TYPE + FROM MMR_MODELS JOIN MMR_OBJECTS ON MMR_MODELS.ID=MMR_OBJECTS.MODEL_ID + JOIN MMR_ATTR_FEATURES ON MMR_OBJECTS.ID = MMR_ATTR_FEATURES.OBJ_ID + JOIN MMR_RESOLVED_OBJECTS ON MMR_OBJECTS.ID = MMR_RESOLVED_OBJECTS.OBJ_ID + JOIN MMR_OBJECTS PARENTS ON MMR_RESOLVED_OBJECTS.CONTAINER_ID = PARENTS.ID + UNION ALL + SELECT MMR_MODELS.NAMESPACE AS NAMESPACE, + PARENTS.NAME AS CLASS_NAME, + MMR_OBJECTS.NAME AS FEATURE_NAME, + MMR_REF_FEATURES.OBJ_ID AS FEATURE_ID, + 'Reference' AS FEATURE_TYPE + FROM MMR_MODELS JOIN MMR_OBJECTS ON MMR_MODELS.ID=MMR_OBJECTS.MODEL_ID + JOIN MMR_REF_FEATURES ON MMR_OBJECTS.ID = MMR_REF_FEATURES.OBJ_ID + JOIN MMR_RESOLVED_OBJECTS ON MMR_OBJECTS.ID = MMR_RESOLVED_OBJECTS.OBJ_ID + JOIN MMR_OBJECTS PARENTS ON MMR_RESOLVED_OBJECTS.CONTAINER_ID = PARENTS.ID + ; + + -- View for obtaining the feature values + -- (don't use parenthesis) + +CREATE OR REPLACE VIEW MMR_FEATURE_VALUES AS + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + VALUE AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_BOOLEAN_ATTRS + UNION ALL + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + VALUE AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_BYTE_ATTRS + UNION ALL + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + VALUE AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_CHAR_ATTRS + UNION ALL + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + VALUE AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_DOUBLE_ATTRS + UNION ALL + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + VALUE AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_FLOAT_ATTRS + UNION ALL + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + VALUE AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_INT_ATTRS + UNION ALL + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + VALUE AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_LONG_ATTRS + UNION ALL + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + VALUE AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_SHORT_ATTRS + UNION ALL + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + VALUE AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_STRING_ATTRS + UNION ALL + SELECT OBJECT_ID, MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + VALUE AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_CLOB_ATTRS + UNION ALL + SELECT MMR_ENUM_ATTRS.OBJECT_ID, MMR_ENUM_ATTRS.MODEL_ID, MMR_ENUM_ATTRS.FEATURE_ID, MMR_ENUM_ATTRS.NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + MMR_OBJECTS.ID AS ENUM_ID, + MMR_REFS.NDX AS ENUM_VALUE, + MMR_OBJECTS.NAME AS ENUM_NAME, + NULL AS REF_OBJ_ID, + NULL AS REF_OBJ_NAME + FROM MMR_ENUM_ATTRS JOIN MMR_OBJECTS ON MMR_ENUM_ATTRS.VALUE = MMR_OBJECTS.ID + JOIN MMR_RESOLVED_OBJECTS ON MMR_OBJECTS.ID = MMR_RESOLVED_OBJECTS.OBJ_ID + JOIN MMR_REFS ON MMR_RESOLVED_OBJECTS.CONTAINER_ID = MMR_REFS.OBJECT_ID + AND MMR_RESOLVED_OBJECTS.OBJ_ID = MMR_REFS.TO_ID + UNION ALL + SELECT OBJECT_ID, MMR_REFS.MODEL_ID AS MODEL_ID, FEATURE_ID, NDX, + NULL AS BOOLEAN_VALUE, + NULL AS BYTE_VALUE, + NULL AS CHAR_VALUE, + NULL AS DOUBLE_VALUE, + NULL AS FLOAT_VALUE, + NULL AS INT_VALUE, + NULL AS LONG_VALUE, + NULL AS SHORT_VALUE, + NULL AS STRING_VALUE, + NULL AS CLOB_VALUE, + NULL AS ENUM_ID, + NULL AS ENUM_VALUE, + NULL AS ENUM_NAME, + MMR_OBJECTS.ID AS REF_OBJ_ID, + MMR_OBJECTS.NAME AS REF_OBJ_NAME + FROM MMR_REFS JOIN MMR_OBJECTS ON MMR_REFS.TO_ID = MMR_OBJECTS.ID; + +-- == new DTC end == + +-- Debezium output is null +-- INSERT INTO MMSCHEMAINFO_CA (SCRIPTNAME,SCRIPTEXECUTEDBY,SCRIPTREV, +-- RELEASEDATE, DATECREATED,DATEUPDATED, UPDATEID,METAMATRIXSERVERURL) +-- SELECT 'MM_CREATE.SQL',USER(),'Seneca.3117', '10/03/2008 12:01 AM',SYSDATE(),SYSDATE(),'',''; +-- ========= STATEMENT 145 ============ diff --git a/tests/integration_tests/debezium/sql/debezium/mysql-ticketmonster-liquibase.ddl b/tests/integration_tests/debezium/sql/debezium/mysql-ticketmonster-liquibase.ddl new file mode 100644 index 00000000000..154075f6722 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/mysql-ticketmonster-liquibase.ddl @@ -0,0 +1,84 @@ +CREATE TABLE `Appearance` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `event_id` bigint(20) DEFAULT NULL, + `event_name` varchar(255) DEFAULT NULL, + `venue_id` bigint(20) DEFAULT NULL, + `venue_name` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `UKb2ol0eoqtadvfoxhsnqcajgqa` (`event_id`,`venue_id`) +) ENGINE=InnoDB AUTO_INCREMENT=7 DEFAULT CHARSET=latin1; + +CREATE TABLE `Booking` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `cancellationCode` varchar(255) NOT NULL, + `contactEmail` varchar(255) NOT NULL, + `createdOn` datetime(6) NOT NULL, + `performance_id` bigint(20) DEFAULT NULL, + `performance_name` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`) +) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=latin1; + +CREATE TABLE `Section` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `description` varchar(255) NOT NULL, + `name` varchar(255) NOT NULL, + `numberOfRows` int(11) NOT NULL, + `rowCapacity` int(11) NOT NULL, + `venue_id` bigint(20) DEFAULT NULL, + `venue_name` varchar(255) DEFAULT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `UKruosqireipse41rdsuvhqj050` (`name`,`venue_id`) +) ENGINE=InnoDB AUTO_INCREMENT=24 DEFAULT CHARSET=latin1; + +CREATE TABLE `SectionAllocation` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `allocated` longblob, + `occupiedCount` int(11) NOT NULL, + `performance_id` bigint(20) DEFAULT NULL, + `performance_name` varchar(255) DEFAULT NULL, + `version` bigint(20) NOT NULL, + `section_id` bigint(20) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `UK25wlm457x8dmc00we5uw7an3s` (`performance_id`,`section_id`), + KEY `FK60388cvbhb1xyrdhhe546t6dl` (`section_id`), + CONSTRAINT `FK60388cvbhb1xyrdhhe546t6dl` FOREIGN KEY (`section_id`) REFERENCES `Section` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION +) ENGINE=InnoDB AUTO_INCREMENT=57 DEFAULT CHARSET=latin1; + +CREATE TABLE `TicketCategory` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `description` varchar(255) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `UK_43455ipnchbn6r4bg8pviai3g` (`description`) +) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=latin1; + +CREATE TABLE `Ticket` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `price` float NOT NULL, + `number` int(11) NOT NULL, + `rowNumber` int(11) NOT NULL, + `section_id` bigint(20) DEFAULT NULL, + `ticketCategory_id` bigint(20) NOT NULL, + `tickets_id` bigint(20) DEFAULT NULL, + PRIMARY KEY (`id`), + KEY `FK7xoel6i5b4nrphore8ns2jtld` (`section_id`), + KEY `FK88jejylfnpfqcslai19n4naqf` (`ticketCategory_id`), + KEY `FKolbt9u28gyshci6ek9ep0rl5d` (`tickets_id`), + CONSTRAINT `FK7xoel6i5b4nrphore8ns2jtld` FOREIGN KEY (`section_id`) REFERENCES `Section` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION, + CONSTRAINT `FK88jejylfnpfqcslai19n4naqf` FOREIGN KEY (`ticketCategory_id`) REFERENCES `TicketCategory` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION, + CONSTRAINT `FKolbt9u28gyshci6ek9ep0rl5d` FOREIGN KEY (`tickets_id`) REFERENCES `Booking` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION +) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=latin1; + +CREATE TABLE `TicketPriceGuide` ( + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `price` float NOT NULL, + `section_id` bigint(20) NOT NULL, + `show_id` bigint(20) NOT NULL, + `ticketCategory_id` bigint(20) NOT NULL, + PRIMARY KEY (`id`), + UNIQUE KEY `UKro227lwq9ma9gy3ik6gl27xgm` (`section_id`,`show_id`,`ticketCategory_id`), + KEY `FK2nddwnrovke2wgpb8ffahqw` (`show_id`), + KEY `FK3d06sbv9l20tk2wa6yjsw9xdd` (`ticketCategory_id`), + CONSTRAINT `FK2nddwnrovke2wgpb8ffahqw` FOREIGN KEY (`show_id`) REFERENCES `Appearance` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION, + CONSTRAINT `FK3d06sbv9l20tk2wa6yjsw9xdd` FOREIGN KEY (`ticketCategory_id`) REFERENCES `TicketCategory` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION, + CONSTRAINT `FKaqmyqif55ipri4x65o8syt85k` FOREIGN KEY (`section_id`) REFERENCES `Section` (`id`) ON DELETE NO ACTION ON UPDATE NO ACTION +) ENGINE=InnoDB AUTO_INCREMENT=38 DEFAULT CHARSET=latin1; \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/nationalized_character_test.sql b/tests/integration_tests/debezium/sql/debezium/nationalized_character_test.sql deleted file mode 100644 index 48cf6071c43..00000000000 --- a/tests/integration_tests/debezium/sql/debezium/nationalized_character_test.sql +++ /dev/null @@ -1,8 +0,0 @@ -CREATE TABLE `NC_TEST` ( - `id` INT NOT NULL AUTO_INCREMENT, - `nc1` nchar default null, - `nc2` nchar(5) default null, - `nc3` nvarchar(25) default null, - PRIMARY KEY (`ID`) -) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8mb4; -INSERT INTO NC_TEST(nc1,nc2,nc3) VALUES ('a', '123', 'hello'); \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/numeric_column_test.sql b/tests/integration_tests/debezium/sql/debezium/numeric_column_test.sql index 6b60b39b928..d26e7d0455d 100644 --- a/tests/integration_tests/debezium/sql/debezium/numeric_column_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/numeric_column_test.sql @@ -1,7 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: numeric_column_test --- ---------------------------------------------------------------------------------------------------------------- - CREATE TABLE dbz_751_numeric_column_test ( id INT AUTO_INCREMENT NOT NULL, rating1 NUMERIC, diff --git a/tests/integration_tests/debezium/sql/debezium/readbinlog_test.sql b/tests/integration_tests/debezium/sql/debezium/readbinlog_test.sql index 7d6da0e272f..aae184d3f04 100644 --- a/tests/integration_tests/debezium/sql/debezium/readbinlog_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/readbinlog_test.sql @@ -1,8 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: readbinlog_test --- Database needs to be populated to break dependency between MetadataIT and MySqlConnectorIT.shouldValidateAcceptableConfiguration run order --- ---------------------------------------------------------------------------------------------------------------- - CREATE TABLE person ( name VARCHAR(255) primary key, birthdate DATE NULL, diff --git a/tests/integration_tests/debezium/sql/debezium/regression_test.sql b/tests/integration_tests/debezium/sql/debezium/regression_test.sql index 5d396f4d44d..2073c6fbb12 100644 --- a/tests/integration_tests/debezium/sql/debezium/regression_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/regression_test.sql @@ -1,6 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: regression_test --- ---------------------------------------------------------------------------------------------------------------- -- The integration test for this database expects to scan all of the binlog events associated with this database -- without error or problems. The integration test does not modify any records in this database, so this script -- must contain all operations to these tables. @@ -42,6 +39,7 @@ CREATE TABLE dbz84_integer_types_table ( INSERT INTO dbz84_integer_types_table VALUES(127,-128,128,255, default,201,202,203, default,301,302,303, default,401,402,403, default,501,502,503, 1); +SET time_zone='UTC'; -- DBZ-85 handle fractional part of seconds CREATE TABLE dbz_85_fractest ( c1 DATE, @@ -86,7 +84,6 @@ INSERT INTO dbz_123_bitvaluetest VALUES (1,2,64,23989979, 1); INSERT INTO dbz_123_bitvaluetest VALUES (b'1',b'10',b'01000000',b'1011011100000111011011011', 2); -- DBZ-104 handle create table like ... -DROP DATABASE IF EXISTS connector_test; CREATE DATABASE connector_test; CREATE TABLE connector_test.customers ( id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, @@ -95,10 +92,13 @@ CREATE TABLE connector_test.customers ( email VARCHAR(255) NOT NULL UNIQUE KEY ) AUTO_INCREMENT=1001; INSERT INTO connector_test.customers -VALUES (default,"Sally","Thomas","sally.thomas@acme.com"), - (default,"George","Bailey","gbailey@foobar.com"), - (default,"Edward","Walker","ed@walker.com"), - (default,"Anne","Kretchmar","annek@noanswer.org"); +VALUES (default,"Sally","Thomas","sally.thomas@acme.com"); +INSERT INTO connector_test.customers +VALUES (default,"George","Bailey","gbailey@foobar.com"); +INSERT INTO connector_test.customers +VALUES (default,"Edward","Walker","ed@walker.com"); +INSERT INTO connector_test.customers +VALUES (default,"Anne","Kretchmar","annek@noanswer.org"); -- DBZ-147 handle decimal value CREATE TABLE dbz_147_decimalvalues ( diff --git a/tests/integration_tests/debezium/sql/debezium/strategy_test.sql b/tests/integration_tests/debezium/sql/debezium/strategy_test.sql index a7ba9e50f51..4575cfdb546 100644 --- a/tests/integration_tests/debezium/sql/debezium/strategy_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/strategy_test.sql @@ -1,3 +1,4 @@ +SET time_zone='UTC'; CREATE TABLE `dbz4180` ( id INT NOT NULL PRIMARY KEY AUTO_INCREMENT, a NUMERIC(10, 2) NOT NULL DEFAULT 1.23, diff --git a/tests/integration_tests/debezium/sql/debezium/table_column_comment_test.sql b/tests/integration_tests/debezium/sql/debezium/table_column_comment_test.sql index fc7b80aac2a..23a79a32a72 100644 --- a/tests/integration_tests/debezium/sql/debezium/table_column_comment_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/table_column_comment_test.sql @@ -1,7 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: table_column_comment_test --- ---------------------------------------------------------------------------------------------------------------- - CREATE TABLE dbz_4000_comment_test ( id INT AUTO_INCREMENT NOT NULL COMMENT 'pk', name VARCHAR(255) NOT NULL COMMENT 'this is name column', diff --git a/tests/integration_tests/debezium/sql/debezium/timestamp_column_test.sql b/tests/integration_tests/debezium/sql/debezium/timestamp_column_test.sql index c80182cb59f..84f55bd7542 100644 --- a/tests/integration_tests/debezium/sql/debezium/timestamp_column_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/timestamp_column_test.sql @@ -1,3 +1,4 @@ +SET time_zone='UTC'; CREATE TABLE t_user_black_list ( `id` int(10) unsigned NOT NULL, `data` varchar(20), @@ -6,10 +7,11 @@ CREATE TABLE t_user_black_list ( PRIMARY KEY (`id`) ); -ALTER TABLE t_user_black_list - MODIFY COLUMN `update_time` datetime(0) NOT NULL - DEFAULT CURRENT_TIMESTAMP(0) COMMENT 'update_time' AFTER create_time; +-- Debezium `create_time` length is nil +-- ALTER TABLE t_user_black_list +-- MODIFY COLUMN `update_time` datetime(0) NOT NULL +-- DEFAULT CURRENT_TIMESTAMP(0) COMMENT 'update_time' AFTER create_time; -INSERT INTO t_user_black_list (`id`,`create_time`,`data`) VALUES (1, CURRENT_TIMESTAMP(), 'test'); +-- INSERT INTO t_user_black_list (`id`,`create_time`,`data`) VALUES (1, CURRENT_TIMESTAMP(), 'test'); UPDATE t_user_black_list SET `data` = 'test2' WHERE `id` = 1; \ No newline at end of file diff --git a/tests/integration_tests/debezium/sql/debezium/topic_name_sanitization_test.sql b/tests/integration_tests/debezium/sql/debezium/topic_name_sanitization_test.sql index 878cb7cb2a2..2159c51b7b4 100644 --- a/tests/integration_tests/debezium/sql/debezium/topic_name_sanitization_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/topic_name_sanitization_test.sql @@ -1,7 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: topic-name.sanitization-it --- ---------------------------------------------------------------------------------------------------------------- - CREATE TABLE `dbz_878_some|test@data` ( id INT, some_col VARCHAR(255), diff --git a/tests/integration_tests/debezium/sql/debezium/transaction_metadata_test.sql b/tests/integration_tests/debezium/sql/debezium/transaction_metadata_test.sql new file mode 100644 index 00000000000..9ac761ad2ea --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/transaction_metadata_test.sql @@ -0,0 +1,26 @@ +-- Create and populate our products using a single insert with many rows +CREATE TABLE products ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description VARCHAR(512), + weight FLOAT +); + +-- Create some customers ... +CREATE TABLE customers ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + first_name VARCHAR(255) NOT NULL, + last_name VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE KEY +) AUTO_INCREMENT=1001; + +-- Create some very simple orders +CREATE TABLE orders ( + order_number INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + order_date DATE NOT NULL, + purchaser INTEGER NOT NULL, + quantity INTEGER NOT NULL, + product_id INTEGER NOT NULL, + FOREIGN KEY order_customer (purchaser) REFERENCES customers(id), + FOREIGN KEY ordered_product (product_id) REFERENCES products(id) +) AUTO_INCREMENT = 10001; diff --git a/tests/integration_tests/debezium/sql/debezium/transactionpayload_test.sql b/tests/integration_tests/debezium/sql/debezium/transactionpayload_test.sql new file mode 100644 index 00000000000..a1b4d302288 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/transactionpayload_test.sql @@ -0,0 +1,29 @@ +-- Create and populate our products using a single insert with many rows +CREATE TABLE products ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + name VARCHAR(255) NOT NULL, + description VARCHAR(512), + weight FLOAT, + code BINARY(16) +); + +-- Create some customers ... +CREATE TABLE customers ( + id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + first_name VARCHAR(255) NOT NULL, + last_name VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL UNIQUE KEY +) AUTO_INCREMENT=1001; + +-- Create some very simple orders +CREATE TABLE orders ( + order_number INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, + order_date DATE NOT NULL, + purchaser INTEGER NOT NULL, + quantity INTEGER NOT NULL, + product_id INTEGER NOT NULL, + FOREIGN KEY order_customer (purchaser) REFERENCES customers(id), + FOREIGN KEY ordered_product (product_id) REFERENCES products(id) +) AUTO_INCREMENT = 10001; + +CREATE DATABASE IF NOT EXISTS transactionpayload_test; diff --git a/tests/integration_tests/debezium/sql/debezium/unsigned_integer_test.sql b/tests/integration_tests/debezium/sql/debezium/unsigned_integer_test.sql index 1ee023a636d..638b9a5a0d0 100644 --- a/tests/integration_tests/debezium/sql/debezium/unsigned_integer_test.sql +++ b/tests/integration_tests/debezium/sql/debezium/unsigned_integer_test.sql @@ -1,6 +1,3 @@ --- ---------------------------------------------------------------------------------------------------------------- --- DATABASE: unsigned_integer_test --- ---------------------------------------------------------------------------------------------------------------- -- The integration test for this database expects to scan all of the binlog events associated with this database -- without error or problems. The integration test does not modify any records in this database, so this script -- must contain all operations to these tables. diff --git a/tests/integration_tests/debezium/sql/debezium/year_test.sql b/tests/integration_tests/debezium/sql/debezium/year_test.sql new file mode 100644 index 00000000000..4766d56d947 --- /dev/null +++ b/tests/integration_tests/debezium/sql/debezium/year_test.sql @@ -0,0 +1,45 @@ +SET time_zone='UTC'; +CREATE TABLE dbz_1143_year_test ( + id INT AUTO_INCREMENT NOT NULL, + y18 YEAR, + y0018 YEAR, + y2018 YEAR, + d18 DATE, + d0018 DATE, + d2018 DATE, + dt18 DATETIME, + dt0018 DATETIME, + dt2018 DATETIME, + y78 YEAR, + y0078 YEAR, + y1978 YEAR, + d78 DATE, + d0078 DATE, + d1978 DATE, + dt78 DATETIME, + dt0078 DATETIME, + dt1978 DATETIME, + PRIMARY KEY (id) +) DEFAULT CHARSET=utf8; + +INSERT INTO dbz_1143_year_test VALUES ( + default, + '18', + '0018', + '2018', + '18-04-01', + '0018-04-01', + '2018-04-01', + '18-04-01 12:34:56', + '0018-04-01 12:34:56', + '2018-04-01 12:34:56', + '78', + '0078', + '1978', + '78-04-01', + '0078-04-01', + '1978-04-01', + '78-04-01 12:34:56', + '0078-04-01 12:34:56', + '1978-04-01 12:34:56' +); diff --git a/tests/integration_tests/debezium/sql/dml.sql b/tests/integration_tests/debezium/sql/dml.sql index f8fc3f5644c..ce5cf746d2d 100644 --- a/tests/integration_tests/debezium/sql/dml.sql +++ b/tests/integration_tests/debezium/sql/dml.sql @@ -11,9 +11,13 @@ INSERT INTO foo VALUES (3, 3); /* Update PK */ UPDATE foo SET PK = 5, COL = 5 WHERE COL = 3; +SELECT * FROM foo WHERE COL = 3; /* Update Multiple Rows */ UPDATE foo SET COL = 4; +SELECT * FROM foo WHERE PK = 1; +SELECT * FROM foo WHERE PK = 2; +SELECT * FROM foo WHERE PK = 3; /* Update Single Row */ UPDATE foo SET COL = 1 WHERE PK = 5; diff --git a/tests/integration_tests/debezium/src/main.go b/tests/integration_tests/debezium/src/main.go index d9abea1ca57..452b3258484 100644 --- a/tests/integration_tests/debezium/src/main.go +++ b/tests/integration_tests/debezium/src/main.go @@ -94,12 +94,7 @@ func main() { readerTiCDC = prepareKafkaConn(*topicTiCDC) defer readerTiCDC.Close() - dbMySQL = prepareDBConn(KindMySQL, *dbConnMySQL) - defer dbMySQL.MustClose() - dbTiDB = prepareDBConn(KindTiDB, *dbConnTiDB) - defer dbTiDB.MustClose() - - if !runAllTestCases(*testCaseDir) { + if !runAllTestCases(*testCaseDir, *dbConnMySQL, *dbConnTiDB) { os.Exit(1) } } diff --git a/tests/integration_tests/debezium/src/test_cases.go b/tests/integration_tests/debezium/src/test_cases.go index 8e3e05b1db3..f076662c922 100644 --- a/tests/integration_tests/debezium/src/test_cases.go +++ b/tests/integration_tests/debezium/src/test_cases.go @@ -14,6 +14,7 @@ package main import ( + "bytes" "context" "encoding/json" "errors" @@ -29,10 +30,14 @@ import ( "github.com/google/go-cmp/cmp" "github.com/pingcap/tidb/pkg/parser" "github.com/pingcap/tidb/pkg/parser/ast" + "github.com/pingcap/tidb/pkg/parser/format" + "github.com/pingcap/tidb/pkg/parser/mysql" "github.com/segmentio/kafka-go" "go.uber.org/zap" ) +var timeOut = time.Second * 10 + var ( nFailed = 0 nPassed = 0 @@ -43,6 +48,18 @@ var ( msgValue = "value" ) +var defaultLength = map[string]float64{ + "INTEGER": 11, + "INTEGER UNSIGNED": 10, + "INT": 11, + "INT UNSIGNED": 10, + "TINYINT": 4, + "TINYINT UNSIGNED": 3, + "BIGINT": 20, + "BIT": 1, + "CHAR": 1, +} + func parseSQLText(data string) (res []ast.StmtNode, warns []error, err error) { p := parser.New() statements, warns, err := p.Parse(data, "utf8mb4", "utf8mb4_bin") @@ -70,25 +87,23 @@ func readAndParseSQLText(sqlFilePath string) []ast.StmtNode { return statements } -func runAllTestCases(dir string) bool { +func runAllTestCases(dir, dbConnMySQL, dbConnTiDB string) bool { var files []string err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if info.IsDir() { return nil } - if !strings.HasSuffix(info.Name(), ".sql") { - return nil + if strings.HasSuffix(info.Name(), ".sql") || strings.HasSuffix(info.Name(), ".ddl") { + files = append(files, path) } - files = append(files, path) return nil }) if err != nil { logger.Panic("Failed to read test case directory", zap.String("dir", dir), zap.Error(err)) } - for _, path := range files { logger.Info("Run", zap.String("case", path)) - runTestCase(path) + runTestCase(path, dbConnMySQL, dbConnTiDB) } if nFailed > 0 { @@ -106,24 +121,32 @@ func runAllTestCases(dir string) bool { return nFailed == 0 } -func resetDB(db *DBHelper) { - db.MustExec("drop database if exists `" + *dbName + "`;") - db.MustExec("create database `" + *dbName + "`;") - db.MustExec("use `" + *dbName + "`;") +func resetDB() { + runSingleQuery("drop database if exists `"+*dbName+"`;", false) + runSingleQuery("create database `"+*dbName+"`;", false) + runSingleQuery("use `"+*dbName+"`;", false) } -func runTestCase(testCasePath string) bool { - resetDB(dbMySQL) - resetDB(dbTiDB) +func runTestCase(testCasePath, dbConnMySQL, dbConnTiDB string) { + dbMySQL = prepareDBConn(KindMySQL, dbConnMySQL) + defer dbMySQL.MustClose() + dbTiDB = prepareDBConn(KindTiDB, dbConnTiDB) + defer dbTiDB.MustClose() + + resetDB() statementKindsToWaitCDCRecord := map[string]bool{ - "Delete": true, - "Insert": true, - "Replace": true, - "Update": true, + "Delete": true, + "Insert": true, + "Replace": true, + "Update": true, + "CreateDatabase": true, + "DropDatabase": true, + "CreateTable": true, + "AlterTable": true, + "DropTable": true, } - hasError := false stmtAsts := readAndParseSQLText(testCasePath) for _, stmt := range stmtAsts { query := strings.TrimSpace(stmt.Text()) @@ -133,21 +156,18 @@ func runTestCase(testCasePath string) bool { if v, ok := statementKindsToWaitCDCRecord[statementKind]; v && ok { waitCDCRows = true } - if runSingleQuery(query, waitCDCRows) { nPassed++ } else { nFailed++ - hasError = true } } - - return hasError } func fetchNextCDCRecord(reader *kafka.Reader, kind Kind, timeout time.Duration) (map[string]any, map[string]any, error) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() for { - ctx, cancel := context.WithTimeout(context.Background(), timeout) m, err := reader.FetchMessage(ctx) if err != nil { if errors.Is(err, context.DeadlineExceeded) { @@ -156,10 +176,9 @@ func fetchNextCDCRecord(reader *kafka.Reader, kind Kind, timeout time.Duration) return nil, nil, fmt.Errorf("Failed to read CDC record of %s: %w", kind, err) } - if err = reader.CommitMessages(context.Background(), m); err != nil { + if err = reader.CommitMessages(ctx, m); err != nil { return nil, nil, fmt.Errorf("Failed to commit CDC record of %s: %w", kind, err) } - cancel() if len(m.Value) == 0 { continue @@ -169,56 +188,76 @@ func fetchNextCDCRecord(reader *kafka.Reader, kind Kind, timeout time.Duration) var obj map[string]any err = json.Unmarshal(m.Key, &keyMap) if err != nil { - return nil, nil, fmt.Errorf("Failed to parse CDC record of %s (msg=%s): %w", kind, m.Key, err) + return nil, nil, fmt.Errorf("Failed to parse CDC record of %s (msg key=%s): %w", kind, m.Key, err) } err = json.Unmarshal(m.Value, &obj) if err != nil { - return nil, nil, fmt.Errorf("Failed to parse CDC record of %s (msg=%s): %w", kind, m.Value, err) + return nil, nil, fmt.Errorf("Failed to parse CDC record of %s (msg value=%s): %w", kind, m.Value, err) } - // Ignore DDL events in the Debezium's output - if kind == KindMySQL { - schema, ok := obj["schema"] - if !ok { - return nil, nil, fmt.Errorf("Unexpected CDC record of %s: schema field not exist in %s", kind, m.Value) + payload, ok := obj["payload"].(map[string]any) + if !ok { + return nil, nil, fmt.Errorf("Unexpected CDC record of %s: payload field not exist in %s", kind, m.Value) + } + if kind == KindTiDB { + op, ok := payload["op"] + // Ignore Checkpoint events in the TiCDC's output + if ok && op == "m" { + continue } - if schema.(map[string]any)["name"] == "io.debezium.connector.mysql.SchemaChangeValue" { + // Only handle DDL received from partition-0 should be enough. + if !ok && m.Partition != 0 { continue } } + if ddl, ok := payload["ddl"]; ok { + payload["ddl"] = normalizeSQL(ddl.(string)) + } + // HACK + // In DDL events, some data types can't decode correctly, but the query is exact. + if kind == KindMySQL { + if tableChanges, ok := payload["tableChanges"]; ok { + if tables, ok := tableChanges.([]any); ok && len(tables) > 0 { + if table0, ok := tables[0].(map[string]any); ok { + if table, ok := table0["table"]; ok && table != nil { + if columns, ok := table.(map[string]any); ok { + for _, col := range columns["columns"].([]any) { + col := col.(map[string]any) + v := col["typeName"].(string) + switch v { + case "INT", "INT UNSIGNED", "INTEGER", "INTEGER UNSIGNED", "TINYINT", "TINYINT UNSIGNED", "BIGINT", + "BIT", "CHAR": + if col["length"] == defaultLength[v] { + col["length"] = nil + } + } + switch v { + case "INTEGER", "INTEGER UNSIGNED": + col["typeName"] = replaceString(col["typeName"], "INTEGER", "INT") + col["typeExpression"] = replaceString(col["typeExpression"], "INTEGER", "INT") + case "NUMERIC": + col["typeName"] = replaceString(col["typeName"], "NUMERIC", "DECIMAL") + col["typeExpression"] = replaceString(col["typeExpression"], "NUMERIC", "DECIMAL") + col["jdbcType"] = float64(3) + case "REAL": + col["typeName"] = replaceString(col["typeName"], "REAL", "FLOAT") + col["typeExpression"] = replaceString(col["typeExpression"], "REAL", "FLOAT") + col["jdbcType"] = float64(6) + } + } + } + } + } + } + } + } return keyMap, obj, nil } } -func replaceString(s any, key any, val any) string { - return strings.Replace(s.(string), key.(string), val.(string), 1) -} - -func fetchAllCDCRecords(reader *kafka.Reader, kind Kind) ([]map[string]any, []map[string]any) { - var records []map[string]any - var keyMaps []map[string]any - waitTimeout := 30 * time.Second - for { - keyMap, obj, err := fetchNextCDCRecord(reader, kind, waitTimeout) - if err != nil { - logger.Error( - "Received error when fetching CDC record", - zap.Error(err), - zap.String("kind", string(kind))) - break - } - if obj == nil { - // No more records - break - } - - records = append(records, obj) - keyMaps = append(keyMaps, keyMap) - waitTimeout = time.Millisecond * 1000 - } - - return keyMaps, records +func replaceString(s any, old any, new any) string { + return strings.Replace(s.(string), old.(string), new.(string), 1) } var ignoredRecordPaths = map[string]bool{ @@ -234,6 +273,19 @@ func printRecord(obj any) { fmt.Println() } +func normalizeSQL(sql string) string { + p := parser.New() + p.SetSQLMode(mysql.ModeRealAsFloat) // necessary + stmt, err := p.ParseOneStmt(sql, "", "") + buf := new(bytes.Buffer) + if err != nil { + panic(fmt.Sprintf("parse sql failed %s", err)) + } + restoreCtx := format.NewRestoreCtx(format.DefaultRestoreFlags, buf) + stmt.Restore(restoreCtx) + return buf.String() +} + func runSingleQuery(query string, waitCDCRows bool) bool { { wg := &sync.WaitGroup{} @@ -248,8 +300,22 @@ func runSingleQuery(query string, waitCDCRows bool) bool { }() wg.Wait() } - if !waitCDCRows { + wg := &sync.WaitGroup{} + wg.Add(2) + go func() { + if _, _, err := fetchNextCDCRecord(readerDebezium, KindMySQL, timeOut); err != nil { + logger.Error("fetch record failed", zap.Error(err)) + } + wg.Done() + }() + go func() { + if _, _, err := fetchNextCDCRecord(readerTiCDC, KindTiDB, timeOut); err != nil { + logger.Error("fetch record failed", zap.Error(err)) + } + wg.Done() + }() + wg.Wait() return true } @@ -263,48 +329,31 @@ func runSingleQuery(query string, waitCDCRows bool) bool { testCasePassed = false } - var keyMapsDebezium []map[string]any - var objsDebezium []map[string]any - var keyMapsTiCDC []map[string]any - var objsTiCDC []map[string]any + var keyMapsDebezium map[string]any + var objsDebezium map[string]any + var keyMapsTiCDC map[string]any + var objsTiCDC map[string]any { wg := &sync.WaitGroup{} wg.Add(2) go func() { - keyMapsDebezium, objsDebezium = fetchAllCDCRecords(readerDebezium, KindMySQL) + var err error + keyMapsDebezium, objsDebezium, err = fetchNextCDCRecord(readerDebezium, KindMySQL, timeOut) + if err != nil { + logger.Error("fetch record failed", zap.Error(err)) + } wg.Done() }() go func() { - keyMapsTiCDC, objsTiCDC = fetchAllCDCRecords(readerTiCDC, KindTiDB) + var err error + keyMapsTiCDC, objsTiCDC, err = fetchNextCDCRecord(readerTiCDC, KindTiDB, timeOut) + if err != nil { + logger.Error("fetch record failed", zap.Error(err)) + } wg.Done() }() wg.Wait() } - - diff(keyMapsDebezium, keyMapsTiCDC, onError, msgKey) - diff(objsDebezium, objsTiCDC, onError, msgValue) - - return testCasePassed -} - -func diff(recordsDebezium, recordsTiCDC []map[string]any, onError func(error), msgType string) { - if len(recordsDebezium) != len(recordsTiCDC) { - onError(fmt.Errorf( - "Mismatch CDC %s: Got %d record from Debezium and %d record from TiCDC", - msgType, - len(recordsDebezium), - len(recordsTiCDC))) - - headingColor.Print("\nDebezium output:\n\n") - for _, record := range recordsDebezium { - printRecord(record) - } - headingColor.Print("\nTiCDC output:\n\n") - for _, record := range recordsTiCDC { - printRecord(record) - } - return - } cmpOption := cmp.FilterPath( func(p cmp.Path) bool { path := p.GoString() @@ -314,15 +363,17 @@ func diff(recordsDebezium, recordsTiCDC []map[string]any, onError func(error), m cmp.Ignore(), ) - for i := 0; i < len(recordsDebezium); i++ { - recordDebezium := recordsDebezium[i] - recordTiCDC := recordsTiCDC[i] - if diff := cmp.Diff(recordDebezium, recordTiCDC, cmpOption); diff != "" { - onError(fmt.Errorf("Found mismatch CDC record (output record #%d)", i+1)) - headingColor.Print("\nCDC Result Diff (-debezium +ticdc):\n\n") - quick.Highlight(os.Stdout, diff, "diff", "terminal16m", "murphy") - fmt.Println() - continue - } + if diff := cmp.Diff(keyMapsDebezium, keyMapsTiCDC, cmpOption); diff != "" { + onError(fmt.Errorf("Found mismatch CDC record (msg type %s)", msgKey)) + headingColor.Print("\nCDC Result Diff (-debezium +ticdc):\n\n") + quick.Highlight(os.Stdout, diff, "diff", "terminal16m", "murphy") + fmt.Println() } + if diff := cmp.Diff(objsDebezium, objsTiCDC, cmpOption); diff != "" { + onError(fmt.Errorf("Found mismatch CDC record (msg type %s)", msgValue)) + headingColor.Print("\nCDC Result Diff (-debezium +ticdc):\n\n") + quick.Highlight(os.Stdout, diff, "diff", "terminal16m", "murphy") + fmt.Println() + } + return testCasePassed }