feat: EXPOSED-552 Include DROP statements for unmapped columns for migration (#2249)

Author: joc-a

exposed-core/api/exposed-core.api

@@ -664,6 +664,7 @@ public final class org/jetbrains/exposed/sql/Database {
 	public final fun getDialect ()Lorg/jetbrains/exposed/sql/vendors/DatabaseDialect;
 	public final fun getIdentifierManager ()Lorg/jetbrains/exposed/sql/statements/api/IdentifierManagerApi;
 	public final fun getSupportsAlterTableWithAddColumn ()Z
+	public final fun getSupportsAlterTableWithDropColumn ()Z
 	public final fun getSupportsMultipleResultSets ()Z
 	public final fun getUrl ()Ljava/lang/String;
 	public final fun getUseNestedTransactions ()Z
@@ -3508,6 +3509,7 @@ public abstract class org/jetbrains/exposed/sql/statements/api/ExposedDatabaseMe
 	public abstract fun getIdentifierManager ()Lorg/jetbrains/exposed/sql/statements/api/IdentifierManagerApi;
 	public abstract fun getSchemaNames ()Ljava/util/List;
 	public abstract fun getSupportsAlterTableWithAddColumn ()Z
+	public abstract fun getSupportsAlterTableWithDropColumn ()Z
 	public abstract fun getSupportsMultipleResultSets ()Z
 	public abstract fun getSupportsSelectForUpdate ()Z
 	public abstract fun getTableNames ()Ljava/util/Map;

exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Database.kt

@@ -69,6 +69,11 @@ class Database private constructor(
         LazyThreadSafetyMode.NONE
     ) { metadata { supportsAlterTableWithAddColumn } }
 
+    /** Whether the database supports ALTER TABLE with a drop column clause. */
+    val supportsAlterTableWithDropColumn by lazy(
+        LazyThreadSafetyMode.NONE
+    ) { metadata { supportsAlterTableWithDropColumn } }
+
     /** Whether the database supports getting multiple result sets from a single execute. */
     val supportsMultipleResultSets by lazy(LazyThreadSafetyMode.NONE) { metadata { supportsMultipleResultSets } }
 

exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/statements/api/ExposedDatabaseMetadata.kt

@@ -30,6 +30,9 @@ abstract class ExposedDatabaseMetadata(val database: String) {
     /** Whether the database supports `ALTER TABLE` with an add column clause. */
     abstract val supportsAlterTableWithAddColumn: Boolean
 
+    /** Whether the database supports `ALTER TABLE` with a drop column clause. */
+    abstract val supportsAlterTableWithDropColumn: Boolean
+
     /** Whether the database supports getting multiple result sets from a single execute. */
     abstract val supportsMultipleResultSets: Boolean
 

exposed-jdbc/api/exposed-jdbc.api

@@ -45,6 +45,7 @@ public final class org/jetbrains/exposed/sql/statements/jdbc/JdbcDatabaseMetadat
 	public final fun getMetadata ()Ljava/sql/DatabaseMetaData;
 	public fun getSchemaNames ()Ljava/util/List;
 	public fun getSupportsAlterTableWithAddColumn ()Z
+	public fun getSupportsAlterTableWithDropColumn ()Z
 	public fun getSupportsMultipleResultSets ()Z
 	public fun getSupportsSelectForUpdate ()Z
 	public fun getTableNames ()Ljava/util/Map;

exposed-jdbc/src/main/kotlin/org/jetbrains/exposed/sql/statements/jdbc/JdbcDatabaseMetadataImpl.kt

@@ -50,6 +50,7 @@ class JdbcDatabaseMetadataImpl(database: String, val metadata: DatabaseMetaData)
     override val defaultIsolationLevel: Int by lazyMetadata { defaultTransactionIsolation }
 
     override val supportsAlterTableWithAddColumn by lazyMetadata { supportsAlterTableWithAddColumn() }
+    override val supportsAlterTableWithDropColumn by lazyMetadata { supportsAlterTableWithDropColumn() }
     override val supportsMultipleResultSets by lazyMetadata { supportsMultipleResultSets() }
     override val supportsSelectForUpdate: Boolean by lazyMetadata { supportsSelectForUpdate() }
 

exposed-migration/api/exposed-migration.api

@@ -1,5 +1,7 @@
 public final class MigrationUtils {
 	public static final field INSTANCE LMigrationUtils;
+	public final fun dropUnmappedColumnsStatements ([Lorg/jetbrains/exposed/sql/Table;Z)Ljava/util/List;
+	public static synthetic fun dropUnmappedColumnsStatements$default (LMigrationUtils;[Lorg/jetbrains/exposed/sql/Table;ZILjava/lang/Object;)Ljava/util/List;
 	public final fun generateMigrationScript ([Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;Z)Ljava/io/File;
 	public static synthetic fun generateMigrationScript$default (LMigrationUtils;[Lorg/jetbrains/exposed/sql/Table;Ljava/lang/String;Ljava/lang/String;ZILjava/lang/Object;)Ljava/io/File;
 	public final fun statementsRequiredForDatabaseMigration ([Lorg/jetbrains/exposed/sql/Table;Z)Ljava/util/List;

exposed-migration/src/main/kotlin/MigrationUtils.kt

@@ -76,7 +76,8 @@ object MigrationUtils {
             checkMissingSequences(tables = tables, withLogs).flatMap { it.createStatement() }
         }
         val alterStatements = logTimeSpent("Preparing alter table statements", withLogs) {
-            addMissingColumnsStatements(tables = tablesToAlter.toTypedArray(), withLogs)
+            addMissingColumnsStatements(tables = tablesToAlter.toTypedArray(), withLogs) +
+                dropUnmappedColumnsStatements(tables = tablesToAlter.toTypedArray(), withLogs)
         }
 
         val modifyTablesStatements = logTimeSpent("Checking mapping consistence", withLogs) {
@@ -90,6 +91,48 @@ object MigrationUtils {
         return allStatements
     }
 
+    /**
+     * Returns the SQL statements that drop any columns that exist in the database but are not defined in [tables].
+     *
+     * By default, a description for each intermediate step, as well as its execution time, is logged at the INFO level.
+     * This can be disabled by setting [withLogs] to `false`.
+     *
+     * **Note:** Some dialects, like SQLite, do not support `ALTER TABLE DROP COLUMN` syntax completely.
+     * Please check the documentation.
+     */
+    fun dropUnmappedColumnsStatements(vararg tables: Table, withLogs: Boolean = true): List<String> {
+        if (tables.isEmpty()) return emptyList()
+
+        val statements = mutableListOf<String>()
+
+        val dbSupportsAlterTableWithDropColumn = TransactionManager.current().db.supportsAlterTableWithDropColumn
+
+        if (dbSupportsAlterTableWithDropColumn) {
+            val existingTablesColumns = logTimeSpent("Extracting table columns", withLogs) {
+                currentDialect.tableColumns(*tables)
+            }
+
+            val tr = TransactionManager.current()
+
+            tables.forEach { table ->
+                val existingColumns = existingTablesColumns[table].orEmpty().toSet()
+                val tableColumns = table.columns.toSet()
+                val mappedColumns = existingColumns.mapNotNull { columnMetadata ->
+                    val mappedCol = tableColumns.find { column -> columnMetadata.name.equals(column.nameUnquoted(), ignoreCase = true) }
+                    if (mappedCol != null) columnMetadata else null
+                }.toSet()
+                val unmappedColumns = existingColumns.subtract(mappedColumns)
+                unmappedColumns.forEach {
+                    statements.add(
+                        "ALTER TABLE ${tr.identity(table)} DROP COLUMN ${tr.db.identifierManager.quoteIdentifierWhenWrongCaseOrNecessary(it.name)}"
+                    )
+                }
+            }
+        }
+
+        return statements
+    }
+
     /**
      * Log Exposed table mappings <-> real database mapping problems and returns DDL Statements to fix them, including
      * DROP/DELETE statements (unlike [checkMappingConsistence])

exposed-tests/src/test/kotlin/org/jetbrains/exposed/sql/tests/shared/ddl/DatabaseMigrationTests.kt

@@ -12,6 +12,7 @@ import org.jetbrains.exposed.sql.tests.DatabaseTestsBase
 import org.jetbrains.exposed.sql.tests.TestDB
 import org.jetbrains.exposed.sql.tests.currentDialectTest
 import org.jetbrains.exposed.sql.tests.inProperCase
+import org.jetbrains.exposed.sql.tests.shared.assertEqualCollections
 import org.jetbrains.exposed.sql.tests.shared.assertEqualLists
 import org.jetbrains.exposed.sql.tests.shared.assertEquals
 import org.jetbrains.exposed.sql.tests.shared.assertTrue
@@ -132,6 +133,45 @@ class DatabaseMigrationTests : DatabaseTestsBase() {
         }
     }
 
+    @Test
+    fun testDropUnmappedColumnsStatementsIdentical() {
+        val t1 = object : Table("foo") {
+            val col1 = integer("col1")
+            val col2 = integer("CoL2")
+            val col3 = integer("\"CoL3\"")
+        }
+
+        val t2 = object : Table("foo") {
+            val col1 = integer("col1")
+            val col2 = integer("CoL2")
+            val col3 = integer("\"CoL3\"")
+        }
+
+        withTables(t1) {
+            val statements = MigrationUtils.dropUnmappedColumnsStatements(t2, withLogs = false)
+            assertEqualCollections(statements, emptyList())
+        }
+    }
+
+    @Test
+    fun testDropUnmappedColumns() {
+        val t1 = object : Table("foo") {
+            val id = integer("id")
+            val name = text("name")
+        }
+
+        val t2 = object : Table("foo") {
+            val id = integer("id")
+        }
+
+        withTables(excludeSettings = listOf(TestDB.SQLITE, TestDB.ORACLE), t1) {
+            assertEqualCollections(MigrationUtils.statementsRequiredForDatabaseMigration(t1, withLogs = false), emptyList())
+
+            val statements = MigrationUtils.statementsRequiredForDatabaseMigration(t2, withLogs = false)
+            assertEquals(1, statements.size)
+        }
+    }
+
     @Test
     fun testAddNewPrimaryKeyOnExistingColumn() {
         val tableName = "tester"