-
Notifications
You must be signed in to change notification settings - Fork 287
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into release-4.0
- Loading branch information
Showing
43 changed files
with
1,070 additions
and
16 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| .DS_Store | ||
| */.DS_Store | ||
| target | ||
| dist | ||
| .idea | ||
| *.iml | ||
| .classpath | ||
| .project | ||
| .settings/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,52 @@ | ||
| # How to use | ||
|
|
||
| The following code shows how to parse ticdc data([ticdc open protocol](https://docs.pingcap.com/tidb/stable/ticdc-open-protocol)) which consumed from kafka. | ||
|
|
||
| ``` | ||
| TicdcEventFilter filter = new TicdcEventFilter(); | ||
| for (KafkaMessage kafkaMessage : kafkaMessages) { | ||
| TicdcEventDecoder ticdcEventDecoder = new TicdcEventDecoder(kafkaMessage); | ||
| while (ticdcEventDecoder.hasNext()) { | ||
| TicdcEventData data = ticdcEventDecoder.next(); | ||
| if (data.getTicdcEventValue() instanceof TicdcEventRowChange) { | ||
| boolean ok = filter.check(data.getTicdcEventKey().getTbl(), data.getTicdcEventValue().getKafkaPartition(), data.getTicdcEventKey().getTs()); | ||
| if (ok) { | ||
| // deal with row change event | ||
| } else { | ||
| // ignore duplicated messages | ||
| } | ||
| } else if (data.getTicdcEventValue() instanceof TicdcEventDDL) { | ||
| // deal with ddl event | ||
| } else if (data.getTicdcEventValue() instanceof TicdcEventResolve) { | ||
| filter.resolveEvent(data.getTicdcEventValue().getKafkaPartition(), data.getTicdcEventKey().getTs()); | ||
| // deal with resolve event | ||
| } | ||
| System.out.println(JSON.toJSONString(data, true)); | ||
| } | ||
| } | ||
| ``` | ||
| [See com.pingcap.ticdc.cdc.TicdcEventDecoderTest.](src/test/java/com/pingcap/ticdc/cdc/TicdcEventDecoderTest.java). | ||
|
|
||
| # How to install | ||
| Prerequisites for building: | ||
|
|
||
| * Git | ||
| * Maven (we recommend version 3.2.5) | ||
| * Java 8 | ||
|
|
||
| ``` | ||
| git clone [email protected]:pingcap/ticdc.git | ||
| cd ticdc/demo/java | ||
| mvn install | ||
| ``` | ||
|
|
||
| Now ticdc-decoder is installed. To add a dependency : | ||
|
|
||
| ```xml | ||
| <dependency> | ||
| <groupId>com.pingcap.ticdc.cdc</groupId> | ||
| <artifactId>ticdc-decoder</artifactId> | ||
| <version>4.0.6-SNAPSHOT</version> | ||
| </dependency> | ||
| ``` |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| <?xml version="1.0" encoding="UTF-8"?> | ||
| <project xmlns="http://maven.apache.org/POM/4.0.0" | ||
| xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" | ||
| xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> | ||
| <modelVersion>4.0.0</modelVersion> | ||
|
|
||
| <groupId>com.pingcap.ticdc.cdc</groupId> | ||
| <artifactId>ticdc-decoder</artifactId> | ||
| <version>4.0.6-SNAPSHOT</version> | ||
|
|
||
| <properties> | ||
| <java.version>1.8</java.version> | ||
| <fastjson.version>1.2.70</fastjson.version> | ||
| </properties> | ||
|
|
||
| <dependencies> | ||
| <dependency> | ||
| <groupId>com.alibaba</groupId> | ||
| <artifactId>fastjson</artifactId> | ||
| <version>${fastjson.version}</version> | ||
| </dependency> | ||
|
|
||
| <dependency> | ||
| <groupId>junit</groupId> | ||
| <artifactId>junit</artifactId> | ||
| <version>4.12</version> | ||
| <scope>test</scope> | ||
| </dependency> | ||
| </dependencies> | ||
|
|
||
| <build> | ||
| <plugins> | ||
| <plugin> | ||
| <groupId>org.apache.maven.plugins</groupId> | ||
| <artifactId>maven-compiler-plugin</artifactId> | ||
| <version>2.3.2</version> | ||
| <configuration> | ||
| <encoding>utf-8</encoding> | ||
| <source>${java.version}</source> | ||
| <target>${java.version}</target> | ||
| </configuration> | ||
| </plugin> | ||
| </plugins> | ||
| </build> | ||
|
|
||
| </project> |
72 changes: 72 additions & 0 deletions
72
demo/java/src/main/java/com/pingcap/ticdc/cdc/KafkaMessage.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,72 @@ | ||
| /* | ||
| * Copyright 2020 PingCAP, Inc. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.pingcap.ticdc.cdc; | ||
|
|
||
| public class KafkaMessage { | ||
| private byte[] key; | ||
| private byte[] value; | ||
| private int partition; | ||
| private long offset; | ||
| private long timestamp; | ||
|
|
||
| public KafkaMessage() { | ||
| } | ||
|
|
||
| public KafkaMessage(byte[] key, byte[] value) { | ||
| this.key = key; | ||
| this.value = value; | ||
| } | ||
|
|
||
| public int getPartition() { | ||
| return partition; | ||
| } | ||
|
|
||
| public void setPartition(int partition) { | ||
| this.partition = partition; | ||
| } | ||
|
|
||
| public long getOffset() { | ||
| return offset; | ||
| } | ||
|
|
||
| public void setOffset(long offset) { | ||
| this.offset = offset; | ||
| } | ||
|
|
||
| public long getTimestamp() { | ||
| return timestamp; | ||
| } | ||
|
|
||
| public void setTimestamp(long timestamp) { | ||
| this.timestamp = timestamp; | ||
| } | ||
|
|
||
| public byte[] getKey() { | ||
| return key; | ||
| } | ||
|
|
||
| public void setKey(byte[] key) { | ||
| this.key = key; | ||
| } | ||
|
|
||
| public byte[] getValue() { | ||
| return value; | ||
| } | ||
|
|
||
| public void setValue(byte[] value) { | ||
| this.value = value; | ||
| } | ||
| } |
Oops, something went wrong.