Skip to content

Commit 576e1d5

Browse files
bog-walkbog-walk
authored
fix!: EXPOSED-691 [PostgreSQL] Restrict dropping unmapped sequences to related tables only (#2357)
* fix: [PostgreSQL] Restrict dropping unmapped sequences to related tables only - Add existingSequences() method that checks for sequences that are directly related to tables provided as arguments. This is currently only feasible with PostgreSQL. - Switch MigrationUtils method to use this instead of sequences() when generating SQL to drop sequences. - Adjust tests accordingly. * fix!: EXPOSED-691 [PostgreSQL] Restrict dropping unmapped sequences to related tables only - Update KDocs & Breaking changes doc * fix!: EXPOSED-691 [PostgreSQL] Restrict dropping unmapped sequences to related tables only - Remove investigative tests - Update tests with found issue link
1 parent 4c52764 commit 576e1d5

File tree

11 files changed

+213
-10
lines changed

11 files changed

+213
-10
lines changed

‎documentation-website/Writerside/topics/Breaking-Changes.md

+6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,11 @@
11
# Breaking Changes
22

3+
## 0.59.0
4+
* [PostgreSQL] `MigrationUtils.statementsRequiredForDatabaseMigration(*tables)` used to potentially return `DROP` statements for any database sequence not
5+
mapped to an Exposed table object. Now it only checks against database sequences that have a relational dependency on any of the specified tables
6+
(for example, any sequence automatically associated with a `SERIAL` column registered to `IdTable`). An unbound sequence created manually
7+
via the `CREATE SEQUENCE` command will no longer be checked and will not generate a `DROP` statement.
8+
39
## 0.57.0
410
* Insert, Upsert, and Replace statements will no longer implicitly send all default values (except for client-side default values) in every SQL request.
511
This change will reduce the amount of data Exposed sends to the database and make Exposed rely more on the database's default values.

‎exposed-core/api/exposed-core.api

+5
Original file line numberDiff line numberDiff line change
@@ -2226,6 +2226,7 @@ public final class org/jetbrains/exposed/sql/Sequence {
22262226
public final fun getMinValue ()Ljava/lang/Long;
22272227
public final fun getName ()Ljava/lang/String;
22282228
public final fun getStartWith ()Ljava/lang/Long;
2229+
public fun toString ()Ljava/lang/String;
22292230
}
22302231

22312232
public abstract class org/jetbrains/exposed/sql/SetOperation : org/jetbrains/exposed/sql/AbstractQuery {
@@ -3564,6 +3565,7 @@ public abstract class org/jetbrains/exposed/sql/statements/api/ExposedDatabaseMe
35643565
public abstract fun columns ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
35653566
public abstract fun existingIndices ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
35663567
public abstract fun existingPrimaryKeys ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
3568+
public abstract fun existingSequences ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
35673569
public final fun getDatabase ()Ljava/lang/String;
35683570
public abstract fun getDatabaseDialectName ()Ljava/lang/String;
35693571
public abstract fun getDatabaseProductVersion ()Ljava/lang/String;
@@ -3837,6 +3839,7 @@ public abstract interface class org/jetbrains/exposed/sql/vendors/DatabaseDialec
38373839
public abstract fun dropSchema (Lorg/jetbrains/exposed/sql/Schema;Z)Ljava/lang/String;
38383840
public abstract fun existingIndices ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
38393841
public abstract fun existingPrimaryKeys ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
3842+
public abstract fun existingSequences ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
38403843
public abstract fun getDataTypeProvider ()Lorg/jetbrains/exposed/sql/vendors/DataTypeProvider;
38413844
public abstract fun getDatabase ()Ljava/lang/String;
38423845
public abstract fun getDefaultReferenceOption ()Lorg/jetbrains/exposed/sql/ReferenceOption;
@@ -3888,6 +3891,7 @@ public final class org/jetbrains/exposed/sql/vendors/DatabaseDialect$DefaultImpl
38883891
public static fun dropSchema (Lorg/jetbrains/exposed/sql/vendors/DatabaseDialect;Lorg/jetbrains/exposed/sql/Schema;Z)Ljava/lang/String;
38893892
public static fun existingIndices (Lorg/jetbrains/exposed/sql/vendors/DatabaseDialect;[Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
38903893
public static fun existingPrimaryKeys (Lorg/jetbrains/exposed/sql/vendors/DatabaseDialect;[Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
3894+
public static fun existingSequences (Lorg/jetbrains/exposed/sql/vendors/DatabaseDialect;[Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
38913895
public static fun getDefaultReferenceOption (Lorg/jetbrains/exposed/sql/vendors/DatabaseDialect;)Lorg/jetbrains/exposed/sql/ReferenceOption;
38923896
public static fun getLikePatternSpecialChars (Lorg/jetbrains/exposed/sql/vendors/DatabaseDialect;)Ljava/util/Map;
38933897
public static fun getNeedsQuotesWhenSymbolsInNames (Lorg/jetbrains/exposed/sql/vendors/DatabaseDialect;)Z
@@ -4335,6 +4339,7 @@ public abstract class org/jetbrains/exposed/sql/vendors/VendorDialect : org/jetb
43354339
public fun dropSchema (Lorg/jetbrains/exposed/sql/Schema;Z)Ljava/lang/String;
43364340
public fun existingIndices ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
43374341
public fun existingPrimaryKeys ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
4342+
public fun existingSequences ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
43384343
protected fun fillConstraintCacheForTables (Ljava/util/List;)V
43394344
public final fun filterCondition (Lorg/jetbrains/exposed/sql/Index;)Ljava/lang/String;
43404345
protected final fun getAllTableNamesCache ()Ljava/util/Map;

‎exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/Sequence.kt

+2
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,8 @@ class Sequence(
2929
val identifier
3030
get() = TransactionManager.current().db.identifierManager.cutIfNecessaryAndQuote(name)
3131

32+
override fun toString(): String = "Sequence(identifier=$identifier)"
33+
3234
/** The SQL statements that create this sequence. */
3335
val ddl: List<String>
3436
get() = createStatement()

‎exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/statements/api/ExposedDatabaseMetadata.kt

+13
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package org.jetbrains.exposed.sql.statements.api
22

33
import org.jetbrains.exposed.sql.ForeignKeyConstraint
44
import org.jetbrains.exposed.sql.Index
5+
import org.jetbrains.exposed.sql.Sequence
56
import org.jetbrains.exposed.sql.Table
67
import org.jetbrains.exposed.sql.vendors.ColumnMetadata
78
import org.jetbrains.exposed.sql.vendors.PrimaryKeyMetadata
@@ -64,6 +65,18 @@ abstract class ExposedDatabaseMetadata(val database: String) {
6465
/** Returns a map with the [PrimaryKeyMetadata] in each of the specified [tables]. */
6566
abstract fun existingPrimaryKeys(vararg tables: Table): Map<Table, PrimaryKeyMetadata?>
6667

68+
/**
69+
* Returns a map with all the defined sequences that hold a relation to the specified [tables] in the database.
70+
*
71+
* **Note** PostgreSQL is currently the only database that maps relational dependencies for sequences created when
72+
* a SERIAL column is registered to an `IdTable`. Using this method with any other database returns an empty map.
73+
*
74+
* Any sequence created using the CREATE SEQUENCE command will be ignored
75+
* as it is not necessarily bound to any particular table. Sequences that are used in a table via triggers will also
76+
* not be returned.
77+
*/
78+
abstract fun existingSequences(vararg tables: Table): Map<Table, List<Sequence>>
79+
6780
/** Returns a list of the names of all sequences in the database. */
6881
abstract fun sequences(): List<String>
6982

‎exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/DatabaseDialect.kt

+12
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,18 @@ interface DatabaseDialect {
116116
/** Returns a map with the primary key metadata in each of the specified [tables]. */
117117
fun existingPrimaryKeys(vararg tables: Table): Map<Table, PrimaryKeyMetadata?> = emptyMap()
118118

119+
/**
120+
* Returns a map with all the defined sequences that hold a relation to the specified [tables] in the database.
121+
*
122+
* **Note** PostgreSQL is currently the only database that maps relational dependencies for sequences created when
123+
* a SERIAL column is registered to an `IdTable`. Using this method with any other database returns an empty map.
124+
*
125+
* Any sequence created using the CREATE SEQUENCE command will be ignored
126+
* as it is not necessarily bound to any particular table. Sequences that are used in a table via triggers will also
127+
* not be returned.
128+
*/
129+
fun existingSequences(vararg tables: Table): Map<Table, List<Sequence>> = emptyMap()
130+
119131
/** Returns a list of the names of all sequences in the database. */
120132
fun sequences(): List<String>
121133

‎exposed-core/src/main/kotlin/org/jetbrains/exposed/sql/vendors/VendorDialect.kt

+3
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,9 @@ abstract class VendorDialect(
129129
override fun existingPrimaryKeys(vararg tables: Table): Map<Table, PrimaryKeyMetadata?> =
130130
TransactionManager.current().db.metadata { existingPrimaryKeys(*tables) }
131131

132+
override fun existingSequences(vararg tables: Table): Map<Table, List<Sequence>> =
133+
TransactionManager.current().db.metadata { existingSequences(*tables) }
134+
132135
override fun sequences(): List<String> =
133136
TransactionManager.current().db.metadata { sequences() }
134137

‎exposed-jdbc/api/exposed-jdbc.api

+1
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ public final class org/jetbrains/exposed/sql/statements/jdbc/JdbcDatabaseMetadat
3838
public fun columns ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
3939
public fun existingIndices ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
4040
public fun existingPrimaryKeys ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
41+
public fun existingSequences ([Lorg/jetbrains/exposed/sql/Table;)Ljava/util/Map;
4142
public fun getDatabaseDialectName ()Ljava/lang/String;
4243
public fun getDatabaseProductVersion ()Ljava/lang/String;
4344
public fun getDefaultIsolationLevel ()I

‎exposed-jdbc/src/main/kotlin/org/jetbrains/exposed/sql/statements/jdbc/JdbcDatabaseMetadataImpl.kt

+57
Original file line numberDiff line numberDiff line change
@@ -327,6 +327,63 @@ class JdbcDatabaseMetadataImpl(database: String, val metadata: DatabaseMetaData)
327327
}
328328
}
329329

330+
override fun existingSequences(vararg tables: Table): Map<Table, List<Sequence>> {
331+
if (currentDialect !is PostgreSQLDialect) return emptyMap()
332+
333+
val transaction = TransactionManager.current()
334+
return tables.associateWith { table ->
335+
val (_, tableSchema) = tableCatalogAndSchema(table)
336+
transaction.exec(
337+
"""
338+
SELECT seq_details.sequence_name,
339+
seq_details.start,
340+
seq_details.increment,
341+
seq_details.max,
342+
seq_details.min,
343+
seq_details.cache,
344+
seq_details.cycle
345+
FROM pg_catalog.pg_namespace tns
346+
INNER JOIN pg_catalog.pg_class t ON tns.oid = t.relnamespace AND t.relkind IN ('p', 'r')
347+
INNER JOIN pg_catalog.pg_depend d ON t.oid = d.refobjid
348+
LEFT OUTER JOIN (
349+
SELECT s.relname AS sequence_name,
350+
seq.seqstart AS start,
351+
seq.seqincrement AS increment,
352+
seq.seqmax AS max,
353+
seq.seqmin AS min,
354+
seq.seqcache AS cache,
355+
seq.seqcycle AS cycle,
356+
s.oid AS seq_id
357+
FROM pg_catalog.pg_sequence seq
358+
JOIN pg_catalog.pg_class s ON s.oid = seq.seqrelid AND s.relkind = 'S'
359+
JOIN pg_catalog.pg_namespace sns ON s.relnamespace = sns.oid
360+
WHERE sns.nspname = '$tableSchema'
361+
) seq_details ON seq_details.seq_id = d.objid
362+
WHERE tns.nspname = '$tableSchema' AND t.relname = '${table.nameInDatabaseCaseUnquoted()}'
363+
""".trimIndent()
364+
) { rs ->
365+
val tmpSequences = mutableListOf<Sequence>()
366+
while (rs.next()) {
367+
rs.getString("sequence_name")?.let {
368+
tmpSequences.add(
369+
Sequence(
370+
it,
371+
rs.getLong("start"),
372+
rs.getLong("increment"),
373+
rs.getLong("min"),
374+
rs.getLong("max"),
375+
rs.getBoolean("cycle"),
376+
rs.getLong("cache")
377+
)
378+
)
379+
}
380+
}
381+
rs.close()
382+
tmpSequences
383+
}.orEmpty()
384+
}
385+
}
386+
330387
@Suppress("MagicNumber")
331388
override fun sequences(): List<String> {
332389
val sequences = mutableListOf<String>()

‎exposed-migration/src/main/kotlin/MigrationUtils.kt

+28-5
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,14 @@ import org.jetbrains.exposed.sql.SchemaUtils.checkExcessiveForeignKeyConstraints
55
import org.jetbrains.exposed.sql.SchemaUtils.checkExcessiveIndices
66
import org.jetbrains.exposed.sql.SchemaUtils.checkMappingConsistence
77
import org.jetbrains.exposed.sql.SchemaUtils.createStatements
8-
import org.jetbrains.exposed.sql.SchemaUtils.statementsRequiredToActualizeScheme
98
import org.jetbrains.exposed.sql.Sequence
109
import org.jetbrains.exposed.sql.Table
1110
import org.jetbrains.exposed.sql.exists
1211
import org.jetbrains.exposed.sql.exposedLogger
1312
import org.jetbrains.exposed.sql.transactions.TransactionManager
1413
import org.jetbrains.exposed.sql.vendors.H2Dialect
1514
import org.jetbrains.exposed.sql.vendors.MysqlDialect
15+
import org.jetbrains.exposed.sql.vendors.PostgreSQLDialect
1616
import org.jetbrains.exposed.sql.vendors.SQLiteDialect
1717
import org.jetbrains.exposed.sql.vendors.currentDialect
1818
import java.io.File
@@ -58,14 +58,22 @@ object MigrationUtils {
5858

5959
/**
6060
* Returns the SQL statements that need to be executed to make the existing database schema compatible with
61-
* the table objects defined using Exposed. Unlike [statementsRequiredToActualizeScheme], DROP/DELETE statements are
62-
* included.
61+
* the table objects defined using Exposed. Unlike `SchemaUtils.statementsRequiredToActualizeScheme()`,
62+
* DROP/DELETE statements are included.
6363
*
6464
* **Note:** Some databases, like **SQLite**, only support `ALTER TABLE ADD COLUMN` syntax in very restricted cases,
6565
* which may cause unexpected behavior when adding some missing columns. For more information,
6666
* refer to the relevant documentation.
6767
* For SQLite, see [ALTER TABLE restrictions](https://www.sqlite.org/lang_altertable.html#alter_table_add_column).
6868
*
69+
* **Note:** If this method is called on a **PostgreSQL** database, it will check for a mapping inconsistency
70+
* between the specified [tables] and existing sequences that have a relational dependency on any of these [tables]
71+
* (for example, any sequence automatically associated with a `SERIAL` column registered to `IdTable`). This means
72+
* that an unbound sequence created manually via the `CREATE SEQUENCE` command will no longer be checked and will
73+
* not generate a DROP statement.
74+
* When called on other databases, such an inconsistency will be checked against all sequences from the database,
75+
* potentially returning DROP statements for any sequence unlinked or unrelated to [tables].
76+
*
6977
* By default, a description for each intermediate step, as well as its execution time, is logged at the INFO level.
7078
* This can be disabled by setting [withLogs] to `false`.
7179
*/
@@ -139,7 +147,12 @@ object MigrationUtils {
139147

140148
/**
141149
* Log Exposed table mappings <-> real database mapping problems and returns DDL Statements to fix them, including
142-
* DROP/DELETE statements (unlike [checkMappingConsistence])
150+
* DROP/DELETE statements (unlike [checkMappingConsistence]).
151+
*
152+
* **Note:** If this method is called on a PostgreSQL database, only sequences with a relational dependency on any
153+
* of the specified [tables] will be checked for a mapping inconsistency. When called on other databases, all sequences
154+
* from the database will be checked, potentially returning SQL statements to drop any sequences that are unlinked
155+
* or unrelated to [tables].
143156
*/
144157
private fun mappingConsistenceRequiredStatements(vararg tables: Table, withLogs: Boolean = true): List<String> {
145158
return checkMissingIndices(tables = tables, withLogs).flatMap { it.createStatement() } +
@@ -287,6 +300,7 @@ object MigrationUtils {
287300
}
288301
}
289302

303+
// all possible sequences checked, as 'mappedSequences' is the limiting factor, not 'existingSequencesNames'
290304
val existingSequencesNames: Set<String> = currentDialect.sequences().toSet()
291305

292306
val missingSequences = mutableSetOf<Sequence>()
@@ -303,6 +317,10 @@ object MigrationUtils {
303317
* Checks all [tables] for any that have sequences that exist in the database but are not mapped in the code. If
304318
* found, this function also logs the SQL statements that can be used to drop these sequences.
305319
*
320+
* **Note:** If this method is called on a PostgreSQL database, only sequences with a relational dependency on any
321+
* of the specified [tables] will be checked for a mapping in Exposed code. When called on other databases, all sequences
322+
* from the database will be checked, potentially returning any [Sequence] unlinked or unrelated to [tables].
323+
*
306324
* @return List of sequences that are unmapped and can be dropped.
307325
*/
308326
private fun checkUnmappedSequences(vararg tables: Table, withLogs: Boolean): List<Sequence> {
@@ -316,7 +334,12 @@ object MigrationUtils {
316334
}
317335
}
318336

319-
val existingSequencesNames: Set<String> = currentDialect.sequences().toSet()
337+
val existingSequencesNames: Set<String> = if (currentDialect is PostgreSQLDialect) {
338+
// only sequences with related links to [tables] are checked, to avoid dropping every unmapped sequence
339+
currentDialect.existingSequences(*tables).values.flatMap { it.map { it.name } }
340+
} else {
341+
currentDialect.sequences()
342+
}.toSet()
320343

321344
val unmappedSequences = mutableSetOf<Sequence>()
322345

0 commit comments

Comments
 (0)