Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

KAFKA-17529: Remove blacklist from MM2 #17202

Open
wants to merge 4 commits into
base: trunk
Choose a base branch
from
+17 −39
Open

KAFKA-17529: Remove blacklist from MM2 #17202

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
835a970
Removed blacklist from MM2
chiacyu Sep 15, 2024
8da650e
fixed unit test and update docs
chiacyu Sep 17, 2024
bc695ab
Removed topic blacklist config
chiacyu Sep 17, 2024
7112bae
Fixed unit test and restore doc
chiacyu Sep 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions ...ect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultConfigPropertyFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@
public class DefaultConfigPropertyFilter implements ConfigPropertyFilter {

public static final String CONFIG_PROPERTIES_EXCLUDE_CONFIG = "config.properties.exclude";
public static final String CONFIG_PROPERTIES_EXCLUDE_ALIAS_CONFIG = "config.properties.blacklist";
public static final String USE_DEFAULTS_FROM = "use.defaults.from";
private static final String USE_DEFAULTS_FROM_DOC = "Which cluster's defaults (source or target) to use "
+ "when syncing topic configurations that have default values.";
Expand Down Expand Up @@ -75,11 +74,6 @@ static class ConfigPropertyFilterConfig extends AbstractConfig {
CONFIG_PROPERTIES_EXCLUDE_DEFAULT,
Importance.HIGH,
CONFIG_PROPERTIES_EXCLUDE_DOC)
.define(CONFIG_PROPERTIES_EXCLUDE_ALIAS_CONFIG,
Type.LIST,
null,
Importance.HIGH,
"Deprecated. Use " + CONFIG_PROPERTIES_EXCLUDE_CONFIG + " instead.")
.define(USE_DEFAULTS_FROM,
Type.STRING,
USE_DEFAULTS_FROM_DEFAULT,
Expand All @@ -89,7 +83,7 @@ static class ConfigPropertyFilterConfig extends AbstractConfig {

ConfigPropertyFilterConfig(Map<String, ?> props) {
super(DEF, ConfigUtils.translateDeprecatedConfigs(props, new String[][]{
{CONFIG_PROPERTIES_EXCLUDE_CONFIG, CONFIG_PROPERTIES_EXCLUDE_ALIAS_CONFIG}}), false);
{CONFIG_PROPERTIES_EXCLUDE_CONFIG}}), false);
}

Pattern excludePattern() {
Expand Down
10 changes: 2 additions & 8 deletions connect/mirror/src/main/java/org/apache/kafka/connect/mirror/DefaultTopicFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ public class DefaultTopicFilter implements TopicFilter {
public static final String TOPICS_INCLUDE_DEFAULT = ".*";

public static final String TOPICS_EXCLUDE_CONFIG = "topics.exclude";
public static final String TOPICS_EXCLUDE_CONFIG_ALIAS = "topics.blacklist";
private static final String TOPICS_EXCLUDE_DOC = "List of topics and/or regexes that should not be replicated.";
public static final String TOPICS_EXCLUDE_DEFAULT = ".*[\\-\\.]internal, .*\\.replica, __.*";

Expand Down Expand Up @@ -72,16 +71,11 @@ static class TopicFilterConfig extends AbstractConfig {
Type.LIST,
TOPICS_EXCLUDE_DEFAULT,
Importance.HIGH,
TOPICS_EXCLUDE_DOC)
.define(TOPICS_EXCLUDE_CONFIG_ALIAS,
Type.LIST,
null,
Importance.HIGH,
"Deprecated. Use " + TOPICS_EXCLUDE_CONFIG + " instead.");
TOPICS_EXCLUDE_DOC);

TopicFilterConfig(Map<String, ?> props) {
super(DEF, ConfigUtils.translateDeprecatedConfigs(props, new String[][]{
{TOPICS_EXCLUDE_CONFIG, TOPICS_EXCLUDE_CONFIG_ALIAS}}), false);
{TOPICS_EXCLUDE_CONFIG}}), false);
}

Pattern includePattern() {
Expand Down
18 changes: 2 additions & 16 deletions connect/mirror/src/main/java/org/apache/kafka/connect/mirror/MirrorSourceConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,11 @@ public class MirrorSourceConfig extends MirrorConnectorConfig {
public static final String TOPICS_DEFAULT = DefaultTopicFilter.TOPICS_INCLUDE_DEFAULT;
private static final String TOPICS_DOC = "Topics to replicate. Supports comma-separated topic names and regexes.";
public static final String TOPICS_EXCLUDE = DefaultTopicFilter.TOPICS_EXCLUDE_CONFIG;
public static final String TOPICS_EXCLUDE_ALIAS = DefaultTopicFilter.TOPICS_EXCLUDE_CONFIG_ALIAS;
public static final String TOPICS_EXCLUDE_DEFAULT = DefaultTopicFilter.TOPICS_EXCLUDE_DEFAULT;
private static final String TOPICS_EXCLUDE_DOC = "Excluded topics. Supports comma-separated topic names and regexes."
+ " Excludes take precedence over includes.";

public static final String CONFIG_PROPERTIES_EXCLUDE = DefaultConfigPropertyFilter.CONFIG_PROPERTIES_EXCLUDE_CONFIG;
public static final String CONFIG_PROPERTIES_EXCLUDE_ALIAS = DefaultConfigPropertyFilter.CONFIG_PROPERTIES_EXCLUDE_ALIAS_CONFIG;
public static final String CONFIG_PROPERTIES_EXCLUDE_DEFAULT = DefaultConfigPropertyFilter.CONFIG_PROPERTIES_EXCLUDE_DEFAULT;
private static final String CONFIG_PROPERTIES_EXCLUDE_DOC = "Topic config properties that should not be replicated. Supports "
+ "comma-separated property names and regexes.";
Expand Down Expand Up @@ -103,8 +101,8 @@ public class MirrorSourceConfig extends MirrorConnectorConfig {

public MirrorSourceConfig(Map<String, String> props) {
super(CONNECTOR_CONFIG_DEF, ConfigUtils.translateDeprecatedConfigs(props, new String[][]{
{TOPICS_EXCLUDE, TOPICS_EXCLUDE_ALIAS},
{CONFIG_PROPERTIES_EXCLUDE, CONFIG_PROPERTIES_EXCLUDE_ALIAS}}));
{TOPICS_EXCLUDE},
{CONFIG_PROPERTIES_EXCLUDE}}));
}

public MirrorSourceConfig(ConfigDef configDef, Map<String, String> props) {
Expand Down Expand Up @@ -221,24 +219,12 @@ private static ConfigDef defineSourceConfig(ConfigDef baseConfig) {
TOPICS_EXCLUDE_DEFAULT,
ConfigDef.Importance.HIGH,
TOPICS_EXCLUDE_DOC)
.define(
TOPICS_EXCLUDE_ALIAS,
ConfigDef.Type.LIST,
null,
ConfigDef.Importance.HIGH,
"Deprecated. Use " + TOPICS_EXCLUDE + " instead.")
.define(
CONFIG_PROPERTIES_EXCLUDE,
ConfigDef.Type.LIST,
CONFIG_PROPERTIES_EXCLUDE_DEFAULT,
ConfigDef.Importance.HIGH,
CONFIG_PROPERTIES_EXCLUDE_DOC)
.define(
CONFIG_PROPERTIES_EXCLUDE_ALIAS,
ConfigDef.Type.LIST,
null,
ConfigDef.Importance.HIGH,
"Deprecated. Use " + CONFIG_PROPERTIES_EXCLUDE + " instead.")
.define(
TOPIC_FILTER_CLASS,
ConfigDef.Type.CLASS,
Expand Down
8 changes: 2 additions & 6 deletions connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorMakerConfigTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -176,8 +176,7 @@ public void testConfigBackwardsCompatibility() {
MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps(
"clusters", "a, b",
"groups.blacklist", "group-7",
"topics.blacklist", "topic3",
"config.properties.blacklist", "property-3",
"topics.exclude", "topic3",
"topic.filter.class", DefaultTopicFilter.class.getName()));
SourceAndTarget sourceAndTarget = new SourceAndTarget("source", "target");
Map<String, String> connectorProps = mirrorConfig.connectorBaseConfig(sourceAndTarget,
Expand All @@ -189,9 +188,6 @@ public void testConfigBackwardsCompatibility() {
assertEquals(Collections.singletonList("topic3"), filterConfig.getList("topics.exclude"),
"Topics exclude should be backwards compatible.");

assertEquals(Collections.singletonList("property-3"), sourceConfig.getList("config.properties.exclude"),
"Config properties exclude should be backwards compatible.");

MirrorCheckpointConfig checkpointConfig = new MirrorCheckpointConfig(connectorProps);
assertEquals(Collections.singletonList("group-7"), checkpointConfig.getList("groups.exclude"),
"Groups exclude should be backwards compatible.");
Expand All @@ -202,7 +198,7 @@ public void testConfigBackwardsCompatibility() {
public void testConfigBackwardsCompatibilitySourceTarget() {
MirrorMakerConfig mirrorConfig = new MirrorMakerConfig(makeProps(
"clusters", "a, b",
"source->target.topics.blacklist", "topic3",
"source->target.topics.exclude", "topic3",
"source->target.groups.blacklist", "group-7",
"topic.filter.class", DefaultTopicFilter.class.getName()));
SourceAndTarget sourceAndTarget = new SourceAndTarget("source", "target");
Expand Down
4 changes: 2 additions & 2 deletions connect/mirror/src/test/java/org/apache/kafka/connect/mirror/MirrorSourceConfigTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,8 @@ public void testConfigPropertyMatching() {
@Test
public void testConfigBackwardsCompatibility() {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

we don't need this test because the blacklist-related configs are removed, right?

MirrorSourceConfig config = new MirrorSourceConfig(
makeProps("config.properties.blacklist", "prop1",
"topics.blacklist", "topic-1"));
makeProps("config.properties.exclude", "prop1",
"topics.exclude", "topic-1"));
assertFalse(config.configPropertyFilter().shouldReplicateConfigProperty("prop1"));
assertTrue(config.configPropertyFilter().shouldReplicateConfigProperty("prop2"));
assertFalse(config.topicFilter().shouldReplicateTopic("topic-1"));
Expand Down
8 changes: 8 additions & 0 deletions docs/upgrade.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,14 @@ <h5><a id="upgrade_400_notable" href="#upgrade_400_notable">Notable changes in 4
The <code>--whitelist</code> option was removed from the <code>kafka-console-consumer</code> command line tool.
Please use <code>--include</code> instead.
</li>
<li>
The <code>config.properties.blacklist</code> was removed from the <code>kafka.connect.mirror.MirrorSourceConfig</code>
Please use <code>config.properties.exclude</code> instead.
</li>
<li>
The <code>topics.blacklist</code> was removed from the <code>kafka.connect.mirror.MirrorSourceConfig</code>
Please use <code>topics.exclude</code> instead.
</li>
</ul>
</ul>

Expand Down