From 835a9704cc46d2c29873b0660ef425deefc299fe Mon Sep 17 00:00:00 2001 From: chiacyu Date: Sun, 15 Sep 2024 22:54:00 +0800 Subject: [PATCH 1/4] Removed blacklist from MM2 --- .../connect/mirror/DefaultConfigPropertyFilter.java | 8 +------- .../apache/kafka/connect/mirror/MirrorSourceConfig.java | 9 +-------- .../kafka/connect/mirror/MirrorMakerConfigTest.java | 4 ---- 3 files changed, 2 insertions(+), 19 deletions(-) diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultConfigPropertyFilter.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultConfigPropertyFilter.java index a464e8f93522..d41334a0249f 100644 --- a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultConfigPropertyFilter.java +++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultConfigPropertyFilter.java @@ -29,7 +29,6 @@ public class DefaultConfigPropertyFilter implements ConfigPropertyFilter { public static final String CONFIG_PROPERTIES_EXCLUDE_CONFIG = "config.properties.exclude"; - public static final String CONFIG_PROPERTIES_EXCLUDE_ALIAS_CONFIG = "config.properties.blacklist"; public static final String USE_DEFAULTS_FROM = "use.defaults.from"; private static final String USE_DEFAULTS_FROM_DOC = "Which cluster's defaults (source or target) to use " + "when syncing topic configurations that have default values."; @@ -75,11 +74,6 @@ static class ConfigPropertyFilterConfig extends AbstractConfig { CONFIG_PROPERTIES_EXCLUDE_DEFAULT, Importance.HIGH, CONFIG_PROPERTIES_EXCLUDE_DOC) - .define(CONFIG_PROPERTIES_EXCLUDE_ALIAS_CONFIG, - Type.LIST, - null, - Importance.HIGH, - "Deprecated. Use " + CONFIG_PROPERTIES_EXCLUDE_CONFIG + " instead.") .define(USE_DEFAULTS_FROM, Type.STRING, USE_DEFAULTS_FROM_DEFAULT, @@ -89,7 +83,7 @@ static class ConfigPropertyFilterConfig extends AbstractConfig { ConfigPropertyFilterConfig(Map props) { super(DEF, ConfigUtils.translateDeprecatedConfigs(props, new String[][]{ - {CONFIG_PROPERTIES_EXCLUDE_CONFIG, CONFIG_PROPERTIES_EXCLUDE_ALIAS_CONFIG}}), false); + {CONFIG_PROPERTIES_EXCLUDE_CONFIG}}), false); } Pattern excludePattern() { diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConfig.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConfig.java index 974db4057b8f..60c051e7e008 100644 --- a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConfig.java +++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConfig.java @@ -46,7 +46,6 @@ public class MirrorSourceConfig extends MirrorConnectorConfig { + " Excludes take precedence over includes."; public static final String CONFIG_PROPERTIES_EXCLUDE = DefaultConfigPropertyFilter.CONFIG_PROPERTIES_EXCLUDE_CONFIG; - public static final String CONFIG_PROPERTIES_EXCLUDE_ALIAS = DefaultConfigPropertyFilter.CONFIG_PROPERTIES_EXCLUDE_ALIAS_CONFIG; public static final String CONFIG_PROPERTIES_EXCLUDE_DEFAULT = DefaultConfigPropertyFilter.CONFIG_PROPERTIES_EXCLUDE_DEFAULT; private static final String CONFIG_PROPERTIES_EXCLUDE_DOC = "Topic config properties that should not be replicated. Supports " + "comma-separated property names and regexes."; @@ -104,7 +103,7 @@ public class MirrorSourceConfig extends MirrorConnectorConfig { public MirrorSourceConfig(Map props) { super(CONNECTOR_CONFIG_DEF, ConfigUtils.translateDeprecatedConfigs(props, new String[][]{ {TOPICS_EXCLUDE, TOPICS_EXCLUDE_ALIAS}, - {CONFIG_PROPERTIES_EXCLUDE, CONFIG_PROPERTIES_EXCLUDE_ALIAS}})); + {CONFIG_PROPERTIES_EXCLUDE}})); } public MirrorSourceConfig(ConfigDef configDef, Map props) { @@ -233,12 +232,6 @@ private static ConfigDef defineSourceConfig(ConfigDef baseConfig) { CONFIG_PROPERTIES_EXCLUDE_DEFAULT, ConfigDef.Importance.HIGH, CONFIG_PROPERTIES_EXCLUDE_DOC) - .define( - CONFIG_PROPERTIES_EXCLUDE_ALIAS, - ConfigDef.Type.LIST, - null, - ConfigDef.Importance.HIGH, - "Deprecated. Use " + CONFIG_PROPERTIES_EXCLUDE + " instead.") .define( TOPIC_FILTER_CLASS, ConfigDef.Type.CLASS, diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java index 03f3bd6aaa88..a6a5511298d1 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java @@ -177,7 +177,6 @@ public void testConfigBackwardsCompatibility() { "clusters", "a, b", "groups.blacklist", "group-7", "topics.blacklist", "topic3", - "config.properties.blacklist", "property-3", "topic.filter.class", DefaultTopicFilter.class.getName())); SourceAndTarget sourceAndTarget = new SourceAndTarget("source", "target"); Map connectorProps = mirrorConfig.connectorBaseConfig(sourceAndTarget, @@ -189,9 +188,6 @@ public void testConfigBackwardsCompatibility() { assertEquals(Collections.singletonList("topic3"), filterConfig.getList("topics.exclude"), "Topics exclude should be backwards compatible."); - assertEquals(Collections.singletonList("property-3"), sourceConfig.getList("config.properties.exclude"), - "Config properties exclude should be backwards compatible."); - MirrorCheckpointConfig checkpointConfig = new MirrorCheckpointConfig(connectorProps); assertEquals(Collections.singletonList("group-7"), checkpointConfig.getList("groups.exclude"), "Groups exclude should be backwards compatible."); From 8da650e700c1ca39793eb99ebdaeec61cebe5e6a Mon Sep 17 00:00:00 2001 From: chiacyu Date: Tue, 17 Sep 2024 16:57:54 +0800 Subject: [PATCH 2/4] fixed unit test and update docs --- .../kafka/connect/mirror/MirrorSourceConfigTest.java | 2 +- docs/upgrade.html | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConfigTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConfigTest.java index e4d6082fa7fc..bda9a1f19ecd 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConfigTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConfigTest.java @@ -69,7 +69,7 @@ public void testConfigPropertyMatching() { @Test public void testConfigBackwardsCompatibility() { MirrorSourceConfig config = new MirrorSourceConfig( - makeProps("config.properties.blacklist", "prop1", + makeProps("config.properties.exclude", "prop1", "topics.blacklist", "topic-1")); assertFalse(config.configPropertyFilter().shouldReplicateConfigProperty("prop1")); assertTrue(config.configPropertyFilter().shouldReplicateConfigProperty("prop2")); diff --git a/docs/upgrade.html b/docs/upgrade.html index d9a1fb393818..a88aad32a372 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -51,6 +51,14 @@
Notable changes in 4 The --whitelist option was removed from the kafka-console-consumer command line tool. Please use --include instead. +
  • + The --whitelist option was removed from the kafka-console-consumer command line tool. + Please use --include instead. +
    • +
  • + The config.properties.blacklist was removed from the kafka.connect.mirror.MirrorSourceConfig + Please use config.properties.exclude instead. +
    • From bc695ab9779169ec712d4fc55c3a73b50f8a90aa Mon Sep 17 00:00:00 2001 From: chiacyu Date: Tue, 17 Sep 2024 18:01:18 +0800 Subject: [PATCH 3/4] Removed topic blacklist config --- .../kafka/connect/mirror/DefaultTopicFilter.java | 10 ++-------- .../kafka/connect/mirror/MirrorSourceConfig.java | 9 +-------- .../kafka/connect/mirror/MirrorSourceConfigTest.java | 2 +- docs/upgrade.html | 4 ++++ 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultTopicFilter.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultTopicFilter.java index 2fa4e193776a..20d4d71df531 100644 --- a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultTopicFilter.java +++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultTopicFilter.java @@ -33,7 +33,6 @@ public class DefaultTopicFilter implements TopicFilter { public static final String TOPICS_INCLUDE_DEFAULT = ".*"; public static final String TOPICS_EXCLUDE_CONFIG = "topics.exclude"; - public static final String TOPICS_EXCLUDE_CONFIG_ALIAS = "topics.blacklist"; private static final String TOPICS_EXCLUDE_DOC = "List of topics and/or regexes that should not be replicated."; public static final String TOPICS_EXCLUDE_DEFAULT = ".*[\\-\\.]internal, .*\\.replica, __.*"; @@ -72,16 +71,11 @@ static class TopicFilterConfig extends AbstractConfig { Type.LIST, TOPICS_EXCLUDE_DEFAULT, Importance.HIGH, - TOPICS_EXCLUDE_DOC) - .define(TOPICS_EXCLUDE_CONFIG_ALIAS, - Type.LIST, - null, - Importance.HIGH, - "Deprecated. Use " + TOPICS_EXCLUDE_CONFIG + " instead."); + TOPICS_EXCLUDE_DOC); TopicFilterConfig(Map props) { super(DEF, ConfigUtils.translateDeprecatedConfigs(props, new String[][]{ - {TOPICS_EXCLUDE_CONFIG, TOPICS_EXCLUDE_CONFIG_ALIAS}}), false); + {TOPICS_EXCLUDE_CONFIG}}), false); } Pattern includePattern() { diff --git a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConfig.java b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConfig.java index 60c051e7e008..e80ae3b14bc7 100644 --- a/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConfig.java +++ b/connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConfig.java @@ -40,7 +40,6 @@ public class MirrorSourceConfig extends MirrorConnectorConfig { public static final String TOPICS_DEFAULT = DefaultTopicFilter.TOPICS_INCLUDE_DEFAULT; private static final String TOPICS_DOC = "Topics to replicate. Supports comma-separated topic names and regexes."; public static final String TOPICS_EXCLUDE = DefaultTopicFilter.TOPICS_EXCLUDE_CONFIG; - public static final String TOPICS_EXCLUDE_ALIAS = DefaultTopicFilter.TOPICS_EXCLUDE_CONFIG_ALIAS; public static final String TOPICS_EXCLUDE_DEFAULT = DefaultTopicFilter.TOPICS_EXCLUDE_DEFAULT; private static final String TOPICS_EXCLUDE_DOC = "Excluded topics. Supports comma-separated topic names and regexes." + " Excludes take precedence over includes."; @@ -102,7 +101,7 @@ public class MirrorSourceConfig extends MirrorConnectorConfig { public MirrorSourceConfig(Map props) { super(CONNECTOR_CONFIG_DEF, ConfigUtils.translateDeprecatedConfigs(props, new String[][]{ - {TOPICS_EXCLUDE, TOPICS_EXCLUDE_ALIAS}, + {TOPICS_EXCLUDE}, {CONFIG_PROPERTIES_EXCLUDE}})); } @@ -220,12 +219,6 @@ private static ConfigDef defineSourceConfig(ConfigDef baseConfig) { TOPICS_EXCLUDE_DEFAULT, ConfigDef.Importance.HIGH, TOPICS_EXCLUDE_DOC) - .define( - TOPICS_EXCLUDE_ALIAS, - ConfigDef.Type.LIST, - null, - ConfigDef.Importance.HIGH, - "Deprecated. Use " + TOPICS_EXCLUDE + " instead.") .define( CONFIG_PROPERTIES_EXCLUDE, ConfigDef.Type.LIST, diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConfigTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConfigTest.java index bda9a1f19ecd..e9024cfd2baa 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConfigTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConfigTest.java @@ -70,7 +70,7 @@ public void testConfigPropertyMatching() { public void testConfigBackwardsCompatibility() { MirrorSourceConfig config = new MirrorSourceConfig( makeProps("config.properties.exclude", "prop1", - "topics.blacklist", "topic-1")); + "topics.exclude", "topic-1")); assertFalse(config.configPropertyFilter().shouldReplicateConfigProperty("prop1")); assertTrue(config.configPropertyFilter().shouldReplicateConfigProperty("prop2")); assertFalse(config.topicFilter().shouldReplicateTopic("topic-1")); diff --git a/docs/upgrade.html b/docs/upgrade.html index a88aad32a372..b600d4a9e61e 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -59,6 +59,10 @@
    Notable changes in 4 The config.properties.blacklist was removed from the kafka.connect.mirror.MirrorSourceConfig Please use config.properties.exclude instead. +
  • + The topics.blacklist was removed from the kafka.connect.mirror.MirrorSourceConfig + Please use topics.exclude instead. +
    • From 7112bae5682eeda4230b7ea64f87ab3f17f7f24f Mon Sep 17 00:00:00 2001 From: chiacyu Date: Wed, 18 Sep 2024 22:59:03 +0800 Subject: [PATCH 4/4] Fixed unit test and restore doc --- .../apache/kafka/connect/mirror/MirrorMakerConfigTest.java | 4 ++-- docs/upgrade.html | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java index a6a5511298d1..c3991af322c2 100644 --- a/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java +++ b/connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java @@ -176,7 +176,7 @@ public void testConfigBackwardsCompatibility() { MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps( "clusters", "a, b", "groups.blacklist", "group-7", - "topics.blacklist", "topic3", + "topics.exclude", "topic3", "topic.filter.class", DefaultTopicFilter.class.getName())); SourceAndTarget sourceAndTarget = new SourceAndTarget("source", "target"); Map connectorProps = mirrorConfig.connectorBaseConfig(sourceAndTarget, @@ -198,7 +198,7 @@ public void testConfigBackwardsCompatibility() { public void testConfigBackwardsCompatibilitySourceTarget() { MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps( "clusters", "a, b", - "source->target.topics.blacklist", "topic3", + "source->target.topics.exclude", "topic3", "source->target.groups.blacklist", "group-7", "topic.filter.class", DefaultTopicFilter.class.getName())); SourceAndTarget sourceAndTarget = new SourceAndTarget("source", "target"); diff --git a/docs/upgrade.html b/docs/upgrade.html index b600d4a9e61e..ded9472c9d7e 100644 --- a/docs/upgrade.html +++ b/docs/upgrade.html @@ -51,10 +51,6 @@
    Notable changes in 4 The --whitelist option was removed from the kafka-console-consumer command line tool. Please use --include instead. -
  • - The --whitelist option was removed from the kafka-console-consumer command line tool. - Please use --include instead. -
  • The config.properties.blacklist was removed from the kafka.connect.mirror.MirrorSourceConfig Please use config.properties.exclude instead.