[MM-62662] add ability to compare tables with matching names only

.github/workflows/ci.yml

@@ -7,7 +7,7 @@ on:
         - ci*
 
 env:
-  go-version: "1.19.5"
+  go-version: "1.22.6"
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
@@ -27,7 +27,7 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@639cd343e1d3b897ff35927a75193d57cfcba299 # 3.6.0
         with:
-          version: v1.53
+          version: v1.63
 
   test:
     name: Test

cmd/dbcmp/main.go

@@ -27,6 +27,7 @@ func main() {
 	rootCmd.PersistentFlags().String("source", "", "source database dsn")
 	rootCmd.PersistentFlags().String("target", "", "target database dsn")
 	rootCmd.Flags().StringSlice("exclude", []string{}, "exclude tables from comparison, takes comma-separated values.")
+	rootCmd.Flags().StringSlice("include", []string{}, "include only matching tables for comparison, takes comma-separated values.")
 	rootCmd.Flags().Int("page-size", 1000, "page size for each checksum comparison.")
 
 	if err := rootCmd.Execute(); err != nil {
@@ -51,6 +52,11 @@ func runRootCmdFn(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	incl, err := cmd.Flags().GetStringSlice("include")
+	if err != nil {
+		return err
+	}
+
 	pageSize, err := cmd.Flags().GetInt("page-size")
 	if err != nil {
 		return err
@@ -62,6 +68,7 @@ func runRootCmdFn(cmd *cobra.Command, args []string) error {
 
 	diffs, err := store.Compare(source, target, store.CompareOptions{
 		ExcludePatterns: excl,
+		IncludePatterns: incl,
 		PageSize:        pageSize,
 	})
 	if err != nil {

go.mod

@@ -1,9 +1,10 @@
 module github.com/mattermost/dbcmp
 
-go 1.20
+go 1.22
 
 require (
 	github.com/Masterminds/squirrel v1.5.4
+	github.com/blang/semver/v4 v4.0.0
 	github.com/brianvoe/gofakeit/v6 v6.23.0
 	github.com/go-sql-driver/mysql v1.7.1
 	github.com/jmoiron/sqlx v1.3.5
@@ -14,7 +15,6 @@ require (
 )
 
 require (
-	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/uuid v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

go.sum

@@ -23,6 +23,7 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
 github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg=
 github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
 github.com/pborman/uuid v1.2.1 h1:+ZZIw58t/ozdjRaXh/3awHfmWRbzYxJoAdNJxe/3pvw=
 github.com/pborman/uuid v1.2.1/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
@@ -36,6 +37,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

internal/store/compare.go

@@ -7,6 +7,7 @@ import (
 
 type CompareOptions struct {
 	ExcludePatterns []string
+	IncludePatterns []string
 	Verbose         bool
 	PageSize        int
 }
@@ -35,16 +36,20 @@ func Compare(srcDSN, dstDSN string, opts CompareOptions) ([]string, error) {
 	}
 
 	excl := sliceToMap(opts.ExcludePatterns)
+	incl := sliceToMap(opts.IncludePatterns)
 
-	// find a more elegant solution fo this
-	// essentially we want to exclude some
-	// patterns from comparing.
-	for k := range srcTables {
-		for e := range excl {
-			if strings.Contains(k, strings.ToLower(e)) {
-				delete(srcTables, k)
-			}
-		}
+	if len(incl) > 0 && len(excl) > 0 {
+		return nil, fmt.Errorf("include and exclude flags cannot be used together")
+	}
+
+	if len(incl) > 0 {
+		// include removes elements from the input map if they are not included.
+		// works with exact match and case-insensitive.
+		srcTables = filterMap(srcTables, opts.IncludePatterns, include)
+	} else if len(excl) > 0 {
+		// exclude removes elements from the input map by the given keys.
+		// filteration is case-insensitive and made with strings.Contains.
+		srcTables = filterMap(srcTables, opts.ExcludePatterns, exclude)
 	}
 
 	var mismatchs []string

internal/store/compare_test.go

@@ -8,48 +8,89 @@ import (
 )
 
 func TestCompare(t *testing.T) {
-	// compare empty databases
-	mismatches, err := Compare(mysqlTestDSN, pgsqlTestDSN, CompareOptions{})
-	require.NoError(t, err)
-	require.Empty(t, mismatches)
+	t.Run("Compare empty database", func(t *testing.T) {
+		mismatches, err := Compare(mysqlTestDSN, pgsqlTestDSN, CompareOptions{})
+		require.NoError(t, err)
+		require.Empty(t, mismatches)
+	})
 
-	// compare empty databases
-	mismatches, err = Compare(mysqlLegacyTestDSN, pgsqlTestDSN, CompareOptions{})
-	require.NoError(t, err)
-	require.Empty(t, mismatches)
+	t.Run("Compare empty database (legacy)", func(t *testing.T) {
+		mismatches, err := Compare(mysqlLegacyTestDSN, pgsqlTestDSN, CompareOptions{})
+		require.NoError(t, err)
+		require.Empty(t, mismatches)
+	})
 
 	ec := rand.Intn(100) + 20 // we add 20 to ensure pagination gets triggered
 	h := newTestHelper(t).SeedTableData(ec)
 	defer h.Teardown()
 
-	mismatches, err = Compare(mysqlTestDSN, pgsqlTestDSN, CompareOptions{
-		PageSize: 20,
+	t.Run("Compare databases with same data", func(t *testing.T) {
+		mismatches, err := Compare(mysqlTestDSN, pgsqlTestDSN, CompareOptions{
+			PageSize: 20,
+		})
+		require.NoError(t, err)
+		require.Empty(t, mismatches)
 	})
-	require.NoError(t, err)
-	require.Empty(t, mismatches)
 
-	mismatches, err = Compare(mysqlLegacyTestDSN, pgsqlTestDSN, CompareOptions{
-		PageSize: 20,
+	t.Run("Compare databases with same data (legacy)", func(t *testing.T) {
+		mismatches, err := Compare(mysqlLegacyTestDSN, pgsqlTestDSN, CompareOptions{
+			PageSize: 20,
+		})
+		require.NoError(t, err)
+		require.Empty(t, mismatches)
 	})
-	require.NoError(t, err)
-	require.Empty(t, mismatches)
 
-	mismatches, err = Compare(pgsqlTestDSN, mysqlTestDSN, CompareOptions{
-		PageSize: 20,
+	t.Run("Compare databases with other way around", func(t *testing.T) {
+		mismatches, err := Compare(pgsqlTestDSN, mysqlTestDSN, CompareOptions{
+			PageSize: 20,
+		})
+		require.NoError(t, err)
+		require.Empty(t, mismatches)
 	})
-	require.NoError(t, err)
-	require.Empty(t, mismatches)
 
-	mysqldb, ok := h.dbInstances["mysql"]
-	require.True(t, ok)
+	t.Run("Compare databases when there is data change", func(t *testing.T) {
+		mysqldb, ok := h.dbInstances["mysql"]
+		require.True(t, ok)
+
+		// delete random entry
+		_, err := mysqldb.sqlDB.Query("DELETE FROM Table1 LIMIT 1")
+		require.NoError(t, err)
+
+		mismatches, err := Compare(pgsqlTestDSN, mysqlTestDSN, CompareOptions{
+			PageSize: 20,
+		})
+		require.NoError(t, err)
+		require.Len(t, mismatches, 1)
+	})
+
+	t.Run("Assert exclude and include flags", func(t *testing.T) {
+		// test with exclude patterns
+		mismatches, err := Compare(mysqlTestDSN, pgsqlTestDSN, CompareOptions{
+			ExcludePatterns: []string{"Table1"},
+		})
+		require.NoError(t, err)
+		require.Empty(t, mismatches)
+
+		// Table2 is the same
+		mismatches, err = Compare(mysqlTestDSN, pgsqlTestDSN, CompareOptions{
+			IncludePatterns: []string{"Table2"},
+		})
+		require.NoError(t, err)
+		require.Empty(t, mismatches)
 
-	// delete random entry
-	_, err = mysqldb.sqlDB.Query("DELETE FROM Table1 LIMIT 1")
-	require.NoError(t, err)
+		// test with include patterns
+		mismatches, err = Compare(mysqlTestDSN, pgsqlTestDSN, CompareOptions{
+			IncludePatterns: []string{"Table1"},
+		})
+		require.NoError(t, err)
+		require.Len(t, mismatches, 1)
 
-	mismatches, err = Compare(pgsqlTestDSN, mysqlTestDSN, CompareOptions{
-		PageSize: 20,
+		// test with both include and exclude patterns
+		_, err = Compare(mysqlTestDSN, pgsqlTestDSN, CompareOptions{
+			IncludePatterns: []string{"Table1"},
+			ExcludePatterns: []string{"Table2"},
+		})
+		require.Error(t, err)
+		require.Contains(t, err.Error(), "include and exclude flags cannot be used together")
 	})
-	require.NoError(t, err)
-	require.Len(t, mismatches, 1)
 }

internal/store/util.go

@@ -1,11 +1,19 @@
 package store
 
 import (
+	"maps"
 	"strings"
 
 	"github.com/go-sql-driver/mysql"
 )
 
+type filterMode int
+
+const (
+	include filterMode = iota
+	exclude
+)
+
 func normalizeDSN(dataSource string) (string, error) {
 	if strings.HasPrefix(dataSource, "postgres") {
 		return dataSource, nil
@@ -32,3 +40,32 @@ func sliceToMap(inc []string) map[string]struct{} {
 
 	return m
 }
+
+func filterMap[V any](m map[string]V, keys []string, mode filterMode) map[string]V {
+	result := maps.Clone(m)
+
+	keySet := make(map[string]struct{}, len(keys))
+	for _, k := range keys {
+		keySet[strings.ToLower(k)] = struct{}{}
+	}
+
+	switch mode {
+	case include:
+		maps.DeleteFunc(result, func(k string, _ V) bool {
+			_, exists := keySet[strings.ToLower(k)]
+			return !exists
+		})
+	case exclude:
+		for k := range result {
+			for e := range keySet {
+				if strings.Contains(k, strings.ToLower(e)) {
+					delete(result, k)
+				}
+			}
+		}
+	default:
+		panic("unknown filter mode")
+	}
+
+	return result
+}

internal/store/util_test.go

@@ -0,0 +1,90 @@
+package store
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestFilterMap(t *testing.T) {
+	tests := []struct {
+		name     string
+		inputMap map[string]int
+		keys     []string
+		mode     filterMode
+		expected map[string]int
+	}{
+		{
+			name: "include keys",
+			inputMap: map[string]int{
+				"key1": 1,
+				"key2": 2,
+				"key3": 3,
+			},
+			keys: []string{"key1", "key3"},
+			mode: include,
+			expected: map[string]int{
+				"key1": 1,
+				"key3": 3,
+			},
+		},
+		{
+			name: "exclude keys",
+			inputMap: map[string]int{
+				"key1": 1,
+				"key2": 2,
+				"key3": 3,
+			},
+			keys: []string{"key1", "key3"},
+			mode: exclude,
+			expected: map[string]int{
+				"key2": 2,
+			},
+		},
+		{
+			name: "include no keys",
+			inputMap: map[string]int{
+				"key1": 1,
+				"key2": 2,
+				"key3": 3,
+			},
+			keys:     []string{},
+			mode:     include,
+			expected: map[string]int{},
+		},
+		{
+			name: "exclude no keys",
+			inputMap: map[string]int{
+				"key1": 1,
+				"key2": 2,
+				"key3": 3,
+			},
+			keys: []string{},
+			mode: exclude,
+			expected: map[string]int{
+				"key1": 1,
+				"key2": 2,
+				"key3": 3,
+			},
+		},
+		{
+			name: "exclude keys match",
+			inputMap: map[string]int{
+				"key1": 1,
+				"key2": 2,
+				"key3": 3,
+			},
+			keys:     []string{"key"},
+			mode:     exclude,
+			expected: map[string]int{},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result := filterMap(tt.inputMap, tt.keys, tt.mode)
+			if !reflect.DeepEqual(result, tt.expected) {
+				t.Errorf("filterMap() = %v, want %v", result, tt.expected)
+			}
+		})
+	}
+}