diff --git a/pom.xml b/pom.xml index 357c527..8133d05 100644 --- a/pom.xml +++ b/pom.xml @@ -19,10 +19,10 @@ UTF-8 - 1.9.22 + 2.1.0 1.9.10 11 - 2.16.1 + 2.18.2 3.1.2 @@ -71,10 +71,12 @@ org.apache.maven.pluginsmaven-surefire-plugin${version.surefire} false + ${project.build.directory}/workdir - ${project.build.testSourceDirectory}/../data ${project.build.directory} + ${project.build.testSourceDirectory}/../data + ${project.build.directory}/testOutput true @@ -85,9 +87,10 @@ false + ${project.build.directory} ${testRunCmd} ${project.build.testSourceDirectory}/../data - ${project.build.directory} + ${project.build.directory}/testOutput true @@ -264,10 +267,10 @@ org.jetbrains.kotlinkotlin-stdlib-jdk8${version.kotlin} org.jetbrains.kotlinkotlin-reflect${version.kotlin} - org.hsqldbhsqldb2.7.2 + org.hsqldbhsqldb2.7.4 - org.apache.commonscommons-lang33.12.0 - commons-iocommons-io2.11.0 + org.apache.commonscommons-lang33.17.0 + commons-iocommons-io2.18.0 org.slf4jslf4j-api2.0.9 - ch.qos.logbacklogback-classic1.4.14 - ch.qos.logbacklogback-core1.4.14 + ch.qos.logbacklogback-classic1.5.12 + ch.qos.logbacklogback-core1.5.12 - org.junit.jupiterjunit-jupiter5.10.1test - org.assertjassertj-core3.25.2test - org.apache.commonscommons-csv1.10.0test + org.junit.jupiterjunit-jupiter5.11.3test + org.assertjassertj-core3.26.3test + org.apache.commonscommons-csv1.12.0test org.jetbrains.kotlinkotlin-test${version.kotlin}test diff --git a/src/main/kotlin/cz/dynawest/csvcruncher/HsqlDbHelper.kt b/src/main/kotlin/cz/dynawest/csvcruncher/HsqlDbHelper.kt index c5613ba..9176fd5 100644 --- a/src/main/kotlin/cz/dynawest/csvcruncher/HsqlDbHelper.kt +++ b/src/main/kotlin/cz/dynawest/csvcruncher/HsqlDbHelper.kt @@ -98,7 +98,7 @@ class HsqlDbHelper(val jdbcConn: Connection) { } catch (ex: SQLSyntaxErrorException) { if (ex.message!!.contains("object not found:")) { - throw throwHintForObjectNotFound(ex, this) + throw throwHintForObjectNotFound(ex, this, sql) } throw CsvCruncherException(""" | Seems your SQL contains errors: diff --git a/src/main/kotlin/cz/dynawest/csvcruncher/app/OptionsEnums.kt b/src/main/kotlin/cz/dynawest/csvcruncher/app/OptionsEnums.kt index aa1e871..9248527 100644 --- a/src/main/kotlin/cz/dynawest/csvcruncher/app/OptionsEnums.kt +++ b/src/main/kotlin/cz/dynawest/csvcruncher/app/OptionsEnums.kt @@ -2,7 +2,6 @@ package cz.dynawest.csvcruncher.app import org.apache.commons.lang3.EnumUtils import java.util.* -import java.util.function.Function import java.util.stream.Collectors class OptionsEnums { @@ -56,10 +55,9 @@ class OptionsEnums { companion object { const val PARAM_NAME = "combineInputs" val optionValues: List - get() = EnumUtils.getEnumList(CombineInputFiles::class.java).stream() - .map(Function { it.optionValue }) - .filter { obj: String? -> Objects.nonNull(obj) } - .collect(Collectors.toList()) + get() = EnumUtils.getEnumList(CombineInputFiles::class.java) + .map { it.optionValue } + .filterNotNull() } } diff --git a/src/main/kotlin/cz/dynawest/csvcruncher/converters/json/JsonFileFlattener.kt b/src/main/kotlin/cz/dynawest/csvcruncher/converters/json/JsonFileFlattener.kt index 4f65c83..00fec5b 100644 --- a/src/main/kotlin/cz/dynawest/csvcruncher/converters/json/JsonFileFlattener.kt +++ b/src/main/kotlin/cz/dynawest/csvcruncher/converters/json/JsonFileFlattener.kt @@ -62,7 +62,7 @@ class JsonFileFlattener : FileTabularizer { // Expect an array of objects -> rows // TODO: Or expect map of objects -> then the property name is a first column, and the objects props the further columns - val locationBefore = jsonParser.currentLocation + val locationBefore = jsonParser.currentLocation() val nextToken = jsonParser.nextToken() if (nextToken !== JsonToken.START_ARRAY) { throw ItemsArraySproutNotFound("Items JSON Array not found after traversing over path '$sproutPath', found: $nextToken at $locationBefore") @@ -79,11 +79,11 @@ class JsonFileFlattener : FileTabularizer { for (nextStep in itemsArrayPath) { val nextToken = jsonParser.nextToken() if (nextToken != JsonToken.START_OBJECT) - throw ItemsArraySproutNotFound(itemsArrayPath, jsonParser.currentLocation) + throw ItemsArraySproutNotFound(itemsArrayPath, jsonParser.currentLocation()) do { val nextFieldName = jsonParser.nextFieldName() - ?: throw ItemsArraySproutNotFound(itemsArrayPath, jsonParser.currentLocation) + ?: throw ItemsArraySproutNotFound(itemsArrayPath, jsonParser.currentLocation()) if (nextFieldName != nextStep.name) { jsonParser.skipChildren() diff --git a/src/main/kotlin/cz/dynawest/csvcruncher/util/FilesUtils.kt b/src/main/kotlin/cz/dynawest/csvcruncher/util/FilesUtils.kt index 0884b28..f70203c 100644 --- a/src/main/kotlin/cz/dynawest/csvcruncher/util/FilesUtils.kt +++ b/src/main/kotlin/cz/dynawest/csvcruncher/util/FilesUtils.kt @@ -428,7 +428,7 @@ object FilesUtils { do { if (!lineIterator.hasNext()) throw IllegalStateException("No first line with columns definition (format: [# ] [\"][\"] [, ...]) in: " + file.path) - line = lineIterator.nextLine().trim { it <= ' ' } + line = lineIterator.next().trim { it <= ' ' } } while (line!!.startsWith(CSV_COMMENT_PREFIX)) line = StringUtils.stripStart(line, "#") diff --git a/src/main/kotlin/cz/dynawest/csvcruncher/util/HsqlDbTableCreator.kt b/src/main/kotlin/cz/dynawest/csvcruncher/util/HsqlDbTableCreator.kt index 5eb9990..2bbf483 100644 --- a/src/main/kotlin/cz/dynawest/csvcruncher/util/HsqlDbTableCreator.kt +++ b/src/main/kotlin/cz/dynawest/csvcruncher/util/HsqlDbTableCreator.kt @@ -183,7 +183,7 @@ class HsqlDbTableCreator(private val hsqlDbHelper: HsqlDbHelper) { private fun setUpTableIndexes(tableName: String, indexedColumns: List) { for (col in indexedColumns) { - val rnd = RandomStringUtils.randomAlphanumeric(3) + val rnd = RandomStringUtils.insecure().nextAlphanumeric(3) val sql = "CREATE INDEX ${quote("idx_${tableName}_$rnd")} ON ${quote(tableName)} (${quote(col)})" log.debug("Creating index, SQL: $sql") hsqlDbHelper.executeSql(sql, "Failed to create index: ") diff --git a/src/main/kotlin/cz/dynawest/csvcruncher/util/HsqldbErrorHandling.kt b/src/main/kotlin/cz/dynawest/csvcruncher/util/HsqldbErrorHandling.kt index b0c71dd..a645076 100644 --- a/src/main/kotlin/cz/dynawest/csvcruncher/util/HsqldbErrorHandling.kt +++ b/src/main/kotlin/cz/dynawest/csvcruncher/util/HsqldbErrorHandling.kt @@ -10,7 +10,7 @@ object HsqldbErrorHandling { /** * Analyzes the exception against the given DB connection and rethrows an exception with a message containing the available objects as a hint. */ - fun throwHintForObjectNotFound(ex: SQLSyntaxErrorException, helper: HsqlDbHelper): CsvCruncherException { + fun throwHintForObjectNotFound(ex: SQLSyntaxErrorException, helper: HsqlDbHelper, sql: String): CsvCruncherException { val notFoundIsColumn = analyzeWhatWasNotFound(ex.message!!) val tableNames = helper.formatListOfAvailableTables(notFoundIsColumn) val hintMsg = if (notFoundIsColumn) """ @@ -23,6 +23,8 @@ object HsqldbErrorHandling { | Looks like you are referring to a table that was not created. | This could mean that you have a typo in the input file name, | or maybe you use --combineInputs but try to use the original inputs. + | Or it is this known bug: https://github.com/OndraZizka/csv-cruncher/issues/149 + | SQL: $sql | These tables are actually available: | """.trimMargin() diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/CsvCruncherTestUtils.kt b/src/test/kotlin/cz/dynawest/csvcruncher/CsvCruncherTestUtils.kt index 9ea4629..8ea8497 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/CsvCruncherTestUtils.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/CsvCruncherTestUtils.kt @@ -1,27 +1,30 @@ package cz.dynawest.csvcruncher +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testOutputDir import cz.dynawest.csvcruncher.app.App import org.apache.commons.csv.CSVFormat import org.apache.commons.csv.CSVRecord import org.apache.commons.lang3.StringUtils import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.TestInfo import java.io.* import java.nio.file.Path import java.nio.file.Paths object CsvCruncherTestUtils { - /** - * @return Path to the default test data dir. - */ - val testDataDir: Path - get() = Paths.get(System.getProperty("user.dir")).resolve("src/test/data/") + //val testDataDir: Path get() = Paths.get(System.getProperty("user.dir")).resolve("src/test/data/") - /** - * @return Path to the default test output dir. - */ + /** @return Path to the default test data dir. */ + val testDataDir = Path.of(System.getProperty("test.data.dir")) + //.also { it.toFile().mkdirs() } + .also { it.toFile().isFile && throw Exception("SysProp test.data.dir needs to point to a directory with the tests data. Was: $it") } + + /** @return Path to the default test output dir. */ val testOutputDir: Path - get() = Paths.get(System.getProperty("user.dir")).resolve("target/testResults/") + get() = Paths.get(System.getProperty("test.output.dir")) + //.also { it.toFile().mkdirs() } + .also { it.toFile().isFile && throw Exception("SysProp test.output.dir needs to point to a directory, was: $it") } /** * Runs CSV Cruncher with the guven command, which is | separated arguments. @@ -106,4 +109,9 @@ object CsvCruncherTestUtils { throw RuntimeException(e) } } -} \ No newline at end of file +} + + +fun TestInfo.defaultCsvOutputPath() = "$testOutputDir/${testClass.get().simpleName}_${testMethod.get().name}.csv".let { Path.of(it) } + +fun TestInfo.defaultJsonOutputPath() = "$testOutputDir/${testClass.get().simpleName}_${testMethod.get().name}.json".let { Path.of(it) } diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/TestUtils.kt b/src/test/kotlin/cz/dynawest/csvcruncher/TestUtils.kt deleted file mode 100644 index 7786e30..0000000 --- a/src/test/kotlin/cz/dynawest/csvcruncher/TestUtils.kt +++ /dev/null @@ -1,8 +0,0 @@ -package cz.dynawest.csvcruncher - -import org.junit.jupiter.api.TestInfo -import java.nio.file.Path - -fun TestInfo.defaultCsvOutputPath() = "target/results/${testClass.get().simpleName}_${testMethod.get().name}.csv".let { Path.of(it) } - -fun TestInfo.defaultJsonOutputPath() = "target/results/${testClass.get().simpleName}_${testMethod.get().name}.json".let { Path.of(it) } diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/app/OptionsParsingTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/app/OptionsParsingTest.kt index f988ee1..3693bfc 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/app/OptionsParsingTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/app/OptionsParsingTest.kt @@ -1,19 +1,23 @@ package cz.dynawest.csvcruncher.app +import cz.dynawest.csvcruncher.CsvCruncherTestUtils +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testOutputDir import org.assertj.core.api.Assertions import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import java.nio.file.Path class OptionsParsingTest { - @Test fun testOptionsParsing() { + @Test fun testOptionsParsing(testInfo: TestInfo) { val command = "--json | --combineInputs" + - " | --exclude=.*/LOAD.*\\.csv" + - " | -in | src/test/data/sample-collab/apollo_recording_group/" + - " | -out | target/testResults/apollo_recording_group.csv" + - " | -sql | SELECT * FROM apollo_recording_group" + - " | -initSql | foo/init1.sql | foo/init2.sql" + " | --exclude=.*/LOAD.*\\.csv" + + " | -in | ${CsvCruncherTestUtils.testDataDir}/sample-collab/apollo_recording_group/" + + " | -out | $testOutputDir/apollo_recording_group.csv" + + " | -sql | SELECT * FROM apollo_recording_group" + + " | -initSql | foo/init1.sql | foo/init2.sql" val arguments = command.split("|").map { it.trim() } val options = OptionsParser.parseArgs(arguments.toTypedArray())!! @@ -27,10 +31,10 @@ class OptionsParsingTest { // Per import/export Assertions.assertThat(options.importArguments).size().isEqualTo(1) - Assertions.assertThat(options.importArguments).element(0).extracting{it.path}.isEqualTo(Path.of("src/test/data/sample-collab/apollo_recording_group/")) + Assertions.assertThat(options.importArguments).element(0).extracting{it.path}.isEqualTo(Path.of("$testDataDir/sample-collab/apollo_recording_group/")) Assertions.assertThat(options.exportArguments).size().isEqualTo(1) - Assertions.assertThat(options.exportArguments).element(0).extracting{it.path}.isEqualTo(Path.of("target/testResults/apollo_recording_group.csv")) + Assertions.assertThat(options.exportArguments).element(0).extracting{it.path}.isEqualTo(Path.of("$testOutputDir/apollo_recording_group.csv")) Assertions.assertThat(options.exportArguments).element(0).extracting{it.sqlQuery}.isEqualTo("SELECT * FROM apollo_recording_group") Assertions.assertThat(options.initSqlArguments).size().isEqualTo(2) diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/it/BasicIntegTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/it/BasicIntegTest.kt index 6b2c500..c101928 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/it/BasicIntegTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/it/BasicIntegTest.kt @@ -1,6 +1,7 @@ package cz.dynawest.csvcruncher.it import cz.dynawest.csvcruncher.CsvCruncherTestUtils +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir import cz.dynawest.csvcruncher.defaultCsvOutputPath import cz.dynawest.csvcruncher.defaultJsonOutputPath import org.assertj.core.api.Assertions.assertThat @@ -26,7 +27,7 @@ class BasicIntegTest { @Test fun singleImportSingleExportWithJson(testInfo: TestInfo) { - val inPath = Paths.get("src/test/data/eapBuilds.csv") + val inPath = Paths.get("$testDataDir/eapBuilds.csv") val outputPath = testInfo.defaultCsvOutputPath() val command = @@ -41,7 +42,7 @@ class BasicIntegTest { @Test fun singleImportSingleExport_defaultSql(testInfo: TestInfo) { - val inPath = Paths.get("src/test/data/eapBuilds.csv") + val inPath = Paths.get("$testDataDir/eapBuilds.csv") val outputPath = testInfo.defaultCsvOutputPath() val command = """-in | $inPath | -out | $outputPath""" @@ -52,7 +53,7 @@ class BasicIntegTest { @Test fun singleImportSingleExport_jsonFileExtension(testInfo: TestInfo) { - val inPath = Paths.get("src/test/data/eapBuilds.csv") + val inPath = Paths.get("$testDataDir/eapBuilds.csv") val outputPath = testInfo.defaultJsonOutputPath() val command = """-in | $inPath | -out | $outputPath""" @@ -63,10 +64,10 @@ class BasicIntegTest { @Test fun test_initSqlScript(testInfo: TestInfo) { - val inPath = Paths.get("src/test/data/eapBuilds.csv") + val inPath = Paths.get("$testDataDir/eapBuilds.csv") val outputPath = testInfo.defaultCsvOutputPath() - val initSqlFile = Paths.get("src/test/data/init.sql") + val initSqlFile = Paths.get("$testDataDir/init.sql") val command = """-initSql | $initSqlFile | -in | $inPath | -out | $outputPath | -sql | SELECT jobName FROM eapBuilds LIMIT 1""" CsvCruncherTestUtils.runCruncherWithArguments(command) @@ -76,7 +77,7 @@ class BasicIntegTest { @Test fun realData_json_redditAll(testInfo: TestInfo) { - val inPath = Paths.get("src/test/data/json/redditAll.json") + val inPath = Paths.get("$testDataDir/json/redditAll.json") val outputPath = testInfo.defaultCsvOutputPath() val command = """-in | $inPath | -itemsAt | /data/children | -out | $outputPath | -sql | SELECT * FROM REDDITALL_JSON""" @@ -87,7 +88,7 @@ class BasicIntegTest { @Test fun test_basic_json_types(testInfo: TestInfo) { - val inPath = Paths.get("src/test/data/json/jsonTypes_all.json") + val inPath = Paths.get("$testDataDir/json/jsonTypes_all.json") val outputPath = testInfo.defaultCsvOutputPath() val command = """-in | $inPath | -itemsAt | / | -out | $outputPath | -sql | SELECT * FROM jsonTypes_all_json""" @@ -98,7 +99,7 @@ class BasicIntegTest { @Test fun test_basic_json_noParentDirOfOutput(testInfo: TestInfo) { - val inPath = Paths.get("src/test/data/json/redditAll.json") + val inPath = Paths.get("$testDataDir/json/redditAll.json") val outputPath = testInfo.defaultJsonOutputPath() Path.of(outputPath.pathString + ".csv").deleteIfExists() // Temp file can remain after debugging @@ -108,7 +109,7 @@ class BasicIntegTest { @Test @Disabled("Currently we just overwrite to simplify experimenting. Maybe let's have --noOverwrite?") fun testDoesNotOverwrite(testInfo: TestInfo) { - val inPath = Paths.get("src/test/data/json/redditAll.json") + val inPath = Paths.get("$testDataDir/json/redditAll.json") val outputPath = testInfo.defaultCsvOutputPath() val command = """-in | $inPath | -itemsAt | /data/children | -out | $outputPath | -sql | SELECT * FROM REDDITALL_JSON""" diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/it/BooleanColumnDetectionTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/it/BooleanColumnDetectionTest.kt index e6d1216..85d93ea 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/it/BooleanColumnDetectionTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/it/BooleanColumnDetectionTest.kt @@ -1,9 +1,12 @@ package cz.dynawest.csvcruncher.it import cz.dynawest.csvcruncher.CsvCruncherTestUtils +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testOutputDir import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import java.io.* import java.nio.file.Path import java.nio.file.Paths @@ -17,12 +20,12 @@ import kotlin.io.path.deleteIfExists */ class BooleanColumnDetectionTest { - private var inPath: Path = Paths.get("src/test/data/boolTable.csv") + private var inPath: Path = Paths.get("$testDataDir/boolTable.csv") @Test @Throws(Exception::class) - fun testBooleanColumns() { - val outputDir = Paths.get("target/testResults/testBooleanColumns.csv") + fun testBooleanColumns(testInfo: TestInfo) { + val outputDir = Paths.get("$testOutputDir/testBooleanColumns.csv") outputDir.deleteIfExists() val command = "--json" + @@ -69,8 +72,8 @@ class BooleanColumnDetectionTest { @Test @Disabled @Throws(Exception::class) - fun invalidCombination_noSqlWithoutPerTableQuery() { - val outputDir = Paths.get("target/testResults/testBooleanColumns.csv") + fun invalidCombination_noSqlWithoutPerTableQuery(testInfo: TestInfo) { + val outputDir = Paths.get("$testOutputDir/testBooleanColumns.csv") val command = "--json" + " | -in | " + inPath + // " | --exclude=.*/LOAD.*\\.csv" + " | -out | " + outputDir diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/it/ChangedSchemaTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/it/ChangedSchemaTest.kt index 83491da..af38d59 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/it/ChangedSchemaTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/it/ChangedSchemaTest.kt @@ -5,13 +5,14 @@ import cz.dynawest.csvcruncher.CsvCruncherTestUtils import cz.dynawest.csvcruncher.app.Options2 import cz.dynawest.csvcruncher.app.OptionsEnums import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import java.util.* class ChangedSchemaTest { @Test @Throws(Exception::class) - fun changedSchema() { + fun changedSchema(testInfo: TestInfo) { val testDataDir = CsvCruncherTestUtils.testDataDir.resolve("sample-changedSchema") val testOutputDir = CsvCruncherTestUtils.testOutputDir.resolve("sample-changedSchema") val options = Options2() diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/it/InputAliasTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/it/InputAliasTest.kt index f37db33..787ae1b 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/it/InputAliasTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/it/InputAliasTest.kt @@ -2,17 +2,20 @@ package cz.dynawest.csvcruncher.it import cz.dynawest.csvcruncher.CsvCruncherException import cz.dynawest.csvcruncher.CsvCruncherTestUtils +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testOutputDir import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import org.junit.jupiter.api.assertThrows import java.nio.file.Paths import kotlin.io.path.deleteIfExists class InputAliasTest { - val outputPath = Paths.get("target/results/testInputAliasing.csv") + val outputPath = Paths.get("$testOutputDir/testInputAliasing.csv") @BeforeEach @AfterEach fun cleanup(){ @@ -20,17 +23,17 @@ class InputAliasTest { } @Test - fun testInputAliasing_basic() { - val inPath = Paths.get("src/test/data/eapBuilds.csv") + fun testInputAliasing_basic(testInfo: TestInfo) { + val inPath = Paths.get("$testDataDir/eapBuilds.csv") val command = """-in | $inPath | -as | foo | -out | $outputPath | -sql | SELECT * FROM foo""" CsvCruncherTestUtils.runCruncherWithArguments(command) } @Test - fun testInputAliasing_wrongNameInSql() { - val inPath = Paths.get("src/test/data/eapBuilds.csv") - val outputPath = Paths.get("target/results/testInputAliasing.csv") + fun testInputAliasing_wrongNameInSql(testInfo: TestInfo) { + val inPath = Paths.get("$testDataDir/eapBuilds.csv") + val outputPath = Paths.get("$testOutputDir/testInputAliasing.csv") assertThrows { val command = """-in | $inPath | -as | foo | -out | $outputPath | -sql | SELECT * FROM bar""" @@ -39,16 +42,16 @@ class InputAliasTest { } @Test @Disabled(" File names normalized to table names collide: .../data/eapBuilds.csv, .../data/eapBuilds.csv") - fun testInputAliasing_sameInputUnderTwoAliases() { - val inPath = Paths.get("src/test/data/eapBuilds.csv") + fun testInputAliasing_sameInputUnderTwoAliases(testInfo: TestInfo) { + val inPath = Paths.get("$testDataDir/eapBuilds.csv") val command = "-in | $inPath | -as | foo1 | -in | $inPath | -as | foo2 | -out | $outputPath | -sql | SELECT * FROM foo1 UNION SELECT * FROM foo2" CsvCruncherTestUtils.runCruncherWithArguments(command) } @Test @Disabled(" File names normalized to table names collide: .../data/eapBuilds.csv, .../data/eapBuilds.csv") - fun testInputAliasing_sameInputUnderDuplicateAliases() { - val inPath = Paths.get("src/test/data/eapBuilds.csv") + fun testInputAliasing_sameInputUnderDuplicateAliases(testInfo: TestInfo) { + val inPath = Paths.get("$testDataDir/eapBuilds.csv") val command = "-in | $inPath | -as | foo1 | -in | $inPath | -as | foo1 | -out | $outputPath | -sql | SELECT * FROM foo1" diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/it/MovedFromMavenTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/it/MovedFromMavenTest.kt index 0539fca..9f34096 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/it/MovedFromMavenTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/it/MovedFromMavenTest.kt @@ -1,18 +1,21 @@ package cz.dynawest.csvcruncher.it import cz.dynawest.csvcruncher.CsvCruncherTestUtils +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testOutputDir import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo class MovedFromMavenTest { @Test - fun test_testCrunch_simple() { + fun test_testCrunch_simple(testInfo: TestInfo) { val command = """ | --json=entries | --rowNumbers - | -in | src/test/data/eapBuilds.csv - | -out | target/results/result.csv + | -in | $testDataDir/eapBuilds.csv + | -out | $testOutputDir/result.csv | -sql | SELECT jobName, buildNumber, config, ar, arFile, deployDur, warmupDur, scale, CAST(warmupDur AS DOUBLE) / CAST(deployDur AS DOUBLE) AS warmupSlower FROM eapBuilds ORDER BY deployDur @@ -22,13 +25,13 @@ class MovedFromMavenTest { } @Test - fun test_testCrunch_combineInputFile() { + fun test_testCrunch_combineInputFile(testInfo: TestInfo) { val command = """ | --json=entries | --rowNumbers | --combineInputs - | -in | src/test/data/eapBuilds.csv - | -out | target/results/result.csv + | -in | $testDataDir/eapBuilds.csv + | -out | $testOutputDir/result.csv | -sql | SELECT jobName, buildNumber, config, ar, arFile, deployDur, warmupDur, scale, CAST(warmupDur AS DOUBLE) / CAST(deployDur AS DOUBLE) AS warmupSlower FROM eapBuilds ORDER BY deployDur @@ -39,15 +42,15 @@ class MovedFromMavenTest { @Test @Disabled("No test files in apollo_session??") - fun test_testCrunch_combineInputFiles_perRootSubDir() { + fun test_testCrunch_combineInputFiles_perRootSubDir(testInfo: TestInfo) { val command = """ | --json=entries | --rowNumbers | --combineInputs=concat | --combineDirs=all | --exclude=.*/LOAD.*\.csv - | -in | src/test/data/sampleMultiFilesPerDir/apollo_session/ - | -out | target/results/result.csv + | -in | $testDataDir/sampleMultiFilesPerDir/apollo_session/ + | -out | $testOutputDir/result.csv | -sql | SELECT session_uid, name, session_type, created_time, modified_date FROM concat ORDER BY session_type, created_time DESC """ @@ -56,12 +59,12 @@ class MovedFromMavenTest { } @Test @Suppress("UNUSED_VARIABLE") @Disabled - fun test_testCrunch_combineInputFiles_selectStar_negative() { + fun test_testCrunch_combineInputFiles_selectStar_negative(testInfo: TestInfo) { val command = """ | --json | --combineInputs | --rowNumbers | --exclude=.*/LOAD.*\.csv - | -in | src/test/data/sampleMultiFilesPerDir/session_telephony_pins/ - | -out | target/results/session_telephony_pins.csv + | -in | $testDataDir/sampleMultiFilesPerDir/session_telephony_pins/ + | -out | $testOutputDir/session_telephony_pins.csv | -sql | SELECT * FROM session_telephony_pins """ // Suppress output. This will fail because the input files don't match. @@ -69,12 +72,12 @@ class MovedFromMavenTest { } @Test - fun test_testCrunch_combineInputFiles_selectStar_qualified() { + fun test_testCrunch_combineInputFiles_selectStar_qualified(testInfo: TestInfo) { val command = """ | --json | --combineInputs | --rowNumbers | --exclude=.*/LOAD.*\.csv - | -in | src/test/data/sampleMultiFilesPerDir/session_telephony_pins/ - | -out | target/results/session_telephony_pins.csv + | -in | $testDataDir/sampleMultiFilesPerDir/session_telephony_pins/ + | -out | $testOutputDir/session_telephony_pins.csv | -sql | SELECT session_telephony_pins.* FROM session_telephony_pins """ CsvCruncherTestUtils.runCruncherWithArguments(command) @@ -82,12 +85,12 @@ class MovedFromMavenTest { } @Test - fun test_testCrunch_collab_ARG() { + fun test_testCrunch_collab_ARG(testInfo: TestInfo) { val command = """ | --json | --combineInputs | --exclude=.*/LOAD.*\.csv - | -in | src/test/data/sample-collab/apollo_recording_group/ - | -out | target/results/apollo_recording_group.csv + | -in | $testDataDir/sample-collab/apollo_recording_group/ + | -out | $testOutputDir/apollo_recording_group.csv | -sql | SELECT * FROM apollo_recording_group """ CsvCruncherTestUtils.runCruncherWithArguments(command) @@ -95,12 +98,12 @@ class MovedFromMavenTest { } @Test - fun test_testCrunch_collab_STP() { + fun test_testCrunch_collab_STP(testInfo: TestInfo) { val command = """ | --json | --combineInputs | --exclude=.*/LOAD.*\.csv - | -in | src/test/data/sample-collab/session_telephony_pins/ - | -out | target/results/session_telephony_pins.csv + | -in | $testDataDir/sample-collab/session_telephony_pins/ + | -out | $testOutputDir/session_telephony_pins.csv | -sql | SELECT * FROM session_telephony_pins """ CsvCruncherTestUtils.runCruncherWithArguments(command) @@ -108,14 +111,14 @@ class MovedFromMavenTest { } @Test - fun test_testVersion() { + fun test_testVersion(testInfo: TestInfo) { val command = """-v""" CsvCruncherTestUtils.runCruncherWithArguments(command) // TBD: Validate the result. } @Test - fun test_testHelp() { + fun test_testHelp(testInfo: TestInfo) { val command = """-h""" CsvCruncherTestUtils.runCruncherWithArguments(command) // TBD: Validate the result. diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/it/OptionsCombinationsTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/it/OptionsCombinationsTest.kt index b2e7c32..9c92b11 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/it/OptionsCombinationsTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/it/OptionsCombinationsTest.kt @@ -2,12 +2,16 @@ package cz.dynawest.csvcruncher.it import cz.dynawest.csvcruncher.CrucherConfigException import cz.dynawest.csvcruncher.CsvCruncherTestUtils +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testOutputDir import cz.dynawest.csvcruncher.util.FilesUtils.parseColumnsFromFirstCsvLine import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertIterableEquals import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.BeforeAll import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import org.junit.jupiter.api.assertThrows import java.nio.file.Paths @@ -15,70 +19,63 @@ import java.nio.file.Paths * TODO: Add the verifications. */ class OptionsCombinationsTest { - /** - * ${testRunCmd} - * --json=entries - * --rowNumbers - * -in | src/test/data/eapBuilds.csv - * -out | target/results/result.csv - * -sql | SELECT jobName, buildNumber, config, ar, arFile, deployDur, warmupDur, scale, - * CAST(warmupDur AS DOUBLE) / CAST(deployDur AS DOUBLE) AS warmupSlower - * FROM eapBuilds ORDER BY deployDur' - */ + @Test - fun testSimple() { + fun testSimple(testInfo: TestInfo) { val command = " | --rowNumbers" + - " | -in | | src/test/data/eapBuilds.csv" + - " | -out | target/testResults/testSimple.csv" + + " | -in | $testDataDir/eapBuilds.csv" + + " | -out | $testOutputDir/testSimple.csv" + " | -sql | SELECT jobName, buildNumber, config, ar, arFile, deployDur, warmupDur, scale," + " CAST(warmupDur AS DOUBLE) / CAST(deployDur AS DOUBLE) AS warmupSlower" + " FROM eapBuilds ORDER BY deployDur" CsvCruncherTestUtils.runCruncherWithArguments(command) - val resultCsv = Paths.get("target/testResults/testSimple.csv").toFile() + + val resultCsv = Paths.get("$testOutputDir/testSimple.csv").toFile() assertTrue(resultCsv.exists()) val columnNames = parseColumnsFromFirstCsvLine(resultCsv) - assertEquals(columnNames.size.toLong(), 9, "Column names fit") - assertEquals(columnNames[8].lowercase(), "warmupslower") + assertEquals(9, columnNames.size.toLong(), "Column names fit") + assertEquals("warmupslower", columnNames[8].lowercase()) // TODO: Add content verifications. } @Test - fun testSimpleJson() { + fun testSimpleJson(testInfo: TestInfo) { val command = " | --json=entries" + " | --rowNumbers" + - " | -in | | src/test/data/eapBuilds.csv" + - " | -out | target/testResults/testSimple.csv" + + " | -in | $testDataDir/eapBuilds.csv" + + " | -out | $testOutputDir/testSimple.csv" + " | -sql | SELECT jobName, buildNumber, config, ar, arFile, deployDur, warmupDur, scale," + " CAST(warmupDur AS DOUBLE) / CAST(deployDur AS DOUBLE) AS warmupSlower" + " FROM eapBuilds ORDER BY deployDur" CsvCruncherTestUtils.runCruncherWithArguments(command) - val resultCsv = Paths.get("target/testResults/testSimple.json").toFile() + + val resultCsv = Paths.get("$testOutputDir/testSimple.json").toFile() assertTrue(resultCsv.exists()) // TODO: Add JSON verifications. } @Test - fun testOutputToStdout() { + fun testOutputToStdout(testInfo: TestInfo) { val command = " | --json=entries" + " | --rowNumbers" + - " | -in | | src/test/data/eapBuilds.csv" + + " | -in | $testDataDir/eapBuilds.csv" + " | -out | -" + " | -sql | SELECT CONCAT('This should appear on STDOUT. ', jobName) AS msg" + " FROM eapBuilds ORDER BY deployDur LIMIT 3" CsvCruncherTestUtils.runCruncherWithArguments(command) // If printed to stdout, the output goes to a temp file which is deleted at the end. - val resultCsv = Paths.get("target/testResults/testOutputToStdout.json").toFile() + val resultCsv = Paths.get("$testOutputDir/testOutputToStdout.json").toFile() assertTrue(!resultCsv.exists()) } @Test @Disabled("CHARACTER bug - https://github.com/OndraZizka/csv-cruncher/issues/122") - fun regression_i122_stringReducedToCharacter() { + fun regression_i122_stringReducedToCharacter(testInfo: TestInfo) { val command = " | --json=entries" + " | --rowNumbers" + - " | -in | | src/test/data/eapBuilds.csv" + + " | -in | $testDataDir/eapBuilds.csv" + " | -out | -" + " | -sql | SELECT 'This should appear on STDOUT' AS msg" + " FROM eapBuilds ORDER BY deployDur LIMIT 3" @@ -86,127 +83,144 @@ class OptionsCombinationsTest { } @Test - fun combineInputFile() { + fun combineInputFile(testInfo: TestInfo) { val command = - " | --rowNumbers" + - " | --combineInputs" + - " | -in | src/test/data/eapBuilds.csv" + - " | -out | target/testResults/combineInputFile.csv" + - " | -sql | SELECT jobName, buildNumber, config, ar, arFile, deployDur, warmupDur, scale," + + " | --rowNumbers" + + " | --combineInputs" + + " | -in | $testDataDir/eapBuilds.csv" + + " | -out | $testOutputDir/combineInputFile.csv" + + " | -sql | SELECT jobName, buildNumber, config, ar, arFile, deployDur, warmupDur, scale," + " CAST(warmupDur AS DOUBLE) / CAST(deployDur AS DOUBLE) AS warmupSlower" + " FROM eapBuilds ORDER BY deployDur" CsvCruncherTestUtils.runCruncherWithArguments(command) - val resultCsv = Paths.get("target/testResults/combineInputFile.csv").toFile() + + val resultCsv = Paths.get("$testOutputDir/combineInputFile.csv").toFile() assertTrue(resultCsv.exists()) val columnNames = parseColumnsFromFirstCsvLine(resultCsv) - assertEquals(columnNames.size.toLong(), 9, "Column names fit") - assertEquals(columnNames[8].lowercase(), "warmupslower") + assertEquals(9, columnNames.size.toLong(), "Column names fit") + assertEquals("warmupslower", columnNames[8].lowercase()) } @Test - fun combineInputDir_defaultSql() { + fun combineInputDir_defaultSql(testInfo: TestInfo) { val command = - " | -in | src/test/data/sample-multiFile-all" + - " | --combineInputs" + - " | -out | target/testResults/combineInputDir.csv" + " | -in | $testDataDir/sample-multiFile-all" + + " | --combineInputs" + + " | -out | $testOutputDir/combineInputDir.csv" CsvCruncherTestUtils.runCruncherWithArguments(command) - val resultCsv = Paths.get("target/testResults/combineInputDir.csv").toFile() + + val resultCsv = Paths.get("$testOutputDir/combineInputDir.csv").toFile() assertTrue(resultCsv.exists()) val columnNames = parseColumnsFromFirstCsvLine(resultCsv) - assertEquals(columnNames.size.toLong(), 9, "Column names fit") - assertEquals(columnNames[8].lowercase(), "warmupslower") + assertEquals(13, columnNames.size.toLong()) { "Column names fit: $columnNames" } + assertIterableEquals("Op,recording_group_id,status,storage_base_url,owner_id,room_id,session_id,requested_ts,created_ts,deleted_ts,updated_ts,is_deleted,acc_consumer_id".split(","), columnNames) } + /** + * Fails because of: + * $table placeholder not replaced when dir input with non-canonical JSONs is used. #149 + * https://github.com/OndraZizka/csv-cruncher/issues/149 + * + * Fails in extractColumnsInfoFrom1LineSelect() + * when querying + * SQL: SELECT $table.* FROM $table LIMIT 1 + * + * The solution is not trivial, it needs to determine the output columns, using some merging of common columns, + * probably utilizing SQL's NATURAL JOIN to reuse the same-named columns. + */ @Test - fun combineInputDir_JsonAndCsv_defaultSql_issue149() { + fun combineInputDir_JsonAndCsv_defaultSql_issue149(testInfo: TestInfo) { val command = - " | -in | src/test/data/sample-multiFile-json+csv" + - " | --combineInputs" + - " | -out | target/testResults/combineInputDir_json+csv.csv" + " | -in | $testDataDir/sample-multiFile-json+csv" + + " | --combineInputs" + + " | -out | $testOutputDir/combineInputDir_json+csv.csv" CsvCruncherTestUtils.runCruncherWithArguments(command) - val resultCsv = Paths.get("target/testResults/combineInputDir_json+csv.csv").toFile() + + val resultCsv = Paths.get("$testOutputDir/combineInputDir_json+csv.csv").toFile() assertTrue(resultCsv.exists()) val columnNames = parseColumnsFromFirstCsvLine(resultCsv) - assertEquals(columnNames.size.toLong(), 9, "Column names fit") - assertEquals(columnNames[8].lowercase(), "warmupslower") + assertEquals(9, columnNames.size.toLong(), "Column names fit") + assertEquals("warmupslower", columnNames[8].lowercase()) } @Test - fun combineInputFiles_sort() { + fun combineInputFiles_sort(testInfo: TestInfo) { val command = "--json=entries" + - " | --rowNumbers" + - " | --combineInputs=concat" + - " | --combineDirs=all" + - " | --sortInputFileGroups" + - " | -in | src/test/data/sample-multiFile-all" + - " | -out | target/testResults/combineInputFiles_sort.csv | --overwrite" + - " | -sql | SELECT sample_multifile_all.* FROM sample_multifile_all" + " | --rowNumbers" + + " | --combineInputs=concat" + + " | --combineDirs=all" + + " | --sortInputFileGroups" + + " | -in | $testDataDir/sample-multiFile-all" + + " | -out | $testOutputDir/combineInputFiles_sort.csv | --overwrite" + + " | -sql | SELECT sample_multifile_all.* FROM sample_multifile_all" CsvCruncherTestUtils.runCruncherWithArguments(command) - val csvFile = Paths.get("target/testResults/combineInputFiles_sort.csv").toFile() + val csvFile = Paths.get("$testOutputDir/combineInputFiles_sort.csv").toFile() CsvCruncherTestUtils.checkThatIdsAreIncrementing(listOf(csvFile), 3, true) } @Test @Disabled("Not yet implemented") - fun combine_perRootSubDir() { + fun combine_perRootSubDir(testInfo: TestInfo) { val command = "--json=entries" + - " | --rowNumbers" + - " | --combineInputs=concat" + - " | --combineDirs=perInputSubdir" + - " | --exclude=.*/LOAD.*\\.csv" + - " | -in | src/test/data/sample-collab/" + - " | -out | target/testResults/combine_perRootSubDir.csv" + - " | -sql | SELECT session_uid, name, session_type, created_time, modified_date" + + " | --rowNumbers" + + " | --combineInputs=concat" + + " | --combineDirs=perInputSubdir" + + " | --exclude=.*/LOAD.*\\.csv" + + " | -in | $testDataDir/sample-collab/" + + " | -out | $testOutputDir/combine_perRootSubDir.csv" + + " | -sql | SELECT session_uid, name, session_type, created_time, modified_date" + " FROM concat ORDER BY session_type, created_time DESC" CsvCruncherTestUtils.runCruncherWithArguments(command) - val resultCsv = Paths.get("target/testResults/combine_perRootSubDir.csv").toFile() + + val resultCsv = Paths.get("$testOutputDir/combine_perRootSubDir.csv").toFile() assertTrue(resultCsv.exists()) val columnNames = parseColumnsFromFirstCsvLine(resultCsv) - assertEquals(columnNames.size.toLong(), 5, "Column names fit") - assertEquals(columnNames[4].lowercase(), "modified_date") + assertEquals(5, columnNames.size.toLong(), "Column names fit") + assertEquals("modified_date", columnNames[4].lowercase()) // TODO: Add content verifications. } @Test - fun combine_selectStar_negative() { + fun combine_selectStar_negative(testInfo: TestInfo) { val command = "--json | --combineInputs | --rowNumbers" + - " | --exclude=.*/LOAD.*\\.csv" + - " | -in | src/test/data/sampleMultiFilesPerDir/session_telephony_pins/" + - " | -out | target/results/session_telephony_pins.csv" + - " | -sql | SELECT * FROM session_telephony_pins" + " | --exclude=.*/LOAD.*\\.csv" + + " | -in | $testDataDir/sampleMultiFilesPerDir/session_telephony_pins/" + + " | -out | $testOutputDir/session_telephony_pins.csv" + + " | -sql | SELECT * FROM session_telephony_pins" assertThrows { CsvCruncherTestUtils.runCruncherWithArguments(command) } } @Test - fun combine_selectStar_qualified() { + fun combine_selectStar_qualified(testInfo: TestInfo) { // cruncherCounter, Op,id,uuid,session_id,pin,pin_type,pin_access_type,enrollment_id,created_time,modified_time // 123456..., I,9999,950c2668-794b-4cf9-894a-af6aea5bf5d5,1000,1234567891,0,0,,2018-08-02 07:34:55.303000,2018-08-02 07:34:55.303000 - val inputCsv = "target/testResults/combine_selectStar_qualified.csv" + val inputCsv = "$testOutputDir/combine_selectStar_qualified.csv" val command = "--json | --combineInputs | --rowNumbers" + - " | --exclude=.*/LOAD.*\\.csv" + - " | -in | src/test/data/sampleMultiFilesPerDir/session_telephony_pins/" + - " | -out | $inputCsv" + - " | -sql | SELECT session_telephony_pins.* FROM session_telephony_pins" + " | --exclude=.*/LOAD.*\\.csv" + + " | -in | $testDataDir/sampleMultiFilesPerDir/session_telephony_pins/" + + " | -out | $inputCsv" + + " | -sql | SELECT session_telephony_pins.* FROM session_telephony_pins" CsvCruncherTestUtils.runCruncherWithArguments(command) + val resultCsv = Paths.get(inputCsv).toFile() assertTrue(resultCsv.exists()) val columnNames = parseColumnsFromFirstCsvLine(resultCsv) - assertEquals(columnNames.size.toLong(), 10, "Column names fit") - assertEquals(columnNames[9].lowercase(), "modified_time") + assertEquals(10, columnNames.size.toLong(), "Column names fit") + assertEquals("modified_time", columnNames[9].lowercase()) val shouldBeNull = CsvCruncherTestUtils.getCsvCellValue(resultCsv, 1, 8) assertEquals(null, shouldBeNull, "row 1, col 9 should be null") } @Test - fun collab_ApolloRecGroup() { + fun collab_ApolloRecGroup(testInfo: TestInfo) { val command = "--json | --combineInputs" + - " | --exclude=.*/LOAD.*\\.csv" + - " | -in | src/test/data/sample-collab/apollo_recording_group/" + - " | -out | target/testResults/apollo_recording_group.csv" + - " | -sql | SELECT * FROM apollo_recording_group" + " | --exclude=.*/LOAD.*\\.csv" + + " | -in | $testDataDir/sample-collab/apollo_recording_group/" + + " | -out | $testOutputDir/apollo_recording_group.csv" + + " | -sql | SELECT * FROM apollo_recording_group" CsvCruncherTestUtils.runCruncherWithArguments(command) @@ -214,27 +228,27 @@ class OptionsCombinationsTest { } @Test - fun collab_SessTelPins() { + fun collab_SessTelPins(testInfo: TestInfo) { // Op,id,uuid,session_id,pin,pin_type,pin_access_type,enrollment_id,created_time,modified_time // I,9999,950c2668-794b-4cf9-894a-af6aea5bf5d5,1000,1234567891,0,0,,2018-08-02 07:34:55.303000,2018-08-02 07:34:55.303000 val command = "--json | --combineInputs" + - " | --exclude=.*/LOAD.*\\.csv" + - " | -in | src/test/data/sample-collab/session_telephony_pins/" + - " | -out | target/testResults/session_telephony_pins.csv" + - " | -sql | SELECT * FROM session_telephony_pins" + " | --exclude=.*/LOAD.*\\.csv" + + " | -in | $testDataDir/sample-collab/session_telephony_pins/" + + " | -out | $testOutputDir/session_telephony_pins.csv" + + " | -sql | SELECT * FROM session_telephony_pins" CsvCruncherTestUtils.runCruncherWithArguments(command) // TODO: Add the verifications. } @Test - fun testVersion() { + fun testVersion(testInfo: TestInfo) { val command = "-v" CsvCruncherTestUtils.runCruncherWithArguments(command) } @Test - fun testHelp() { + fun testHelp(testInfo: TestInfo) { val command = "-h" CsvCruncherTestUtils.runCruncherWithArguments(command) } diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/it/QueryPerInputSubpartTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/it/QueryPerInputSubpartTest.kt index e3aa854..9fbe8c0 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/it/QueryPerInputSubpartTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/it/QueryPerInputSubpartTest.kt @@ -2,9 +2,12 @@ package cz.dynawest.csvcruncher.it import cz.dynawest.csvcruncher.CrucherConfigException import cz.dynawest.csvcruncher.CsvCruncherTestUtils +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testOutputDir import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.assertTrue import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import java.nio.file.Path import java.nio.file.Paths import java.util.* @@ -14,11 +17,11 @@ import java.util.* */ class QueryPerInputSubpartTest { - var inPath: Path = Paths.get("src/test/data/sample-queryPerInputSubpart/oauth_consumer") + var inPath: Path = Paths.get("$testDataDir/sample-queryPerInputSubpart/oauth_consumer") @Test - fun queryPerInputSubpart() { - val outputDir = Paths.get("target/testResults/queryPerInputSubpart.csv") + fun queryPerInputSubpart(testInfo: TestInfo) { + val outputDir = Paths.get("$testOutputDir/queryPerInputSubpart.csv") val command = "--json | --combineInputs | --queryPerInputSubpart | --rowNumbers" + " | -in | " + inPath + // " | --exclude=.*/LOAD.*\\.csv" + " | -out | " + outputDir + @@ -30,8 +33,8 @@ class QueryPerInputSubpartTest { @Test @Throws(Exception::class) - fun queryPerInputSubpart_defaultSQL() { - val outputDir = Paths.get("target/testResults/queryPerInputSubpart_defaultSQL.csv") + fun queryPerInputSubpart_defaultSQL(testInfo: TestInfo) { + val outputDir = Paths.get("$testOutputDir/queryPerInputSubpart_defaultSQL.csv") val command = "--json | --combineInputs | --queryPerInputSubpart | --rowNumbers" + " | -in | " + inPath + // " | --exclude=.*/LOAD.*\\.csv" + " | -out | " + outputDir @@ -42,10 +45,10 @@ class QueryPerInputSubpartTest { @Test @Throws(Exception::class) - fun queryPerInputSubpart_negative() { + fun queryPerInputSubpart_negative(testInfo: TestInfo) { val command = "--json | --combineInputs | --queryPerInputSubpart | --rowNumbers" + " | -in | " + inPath + // " | --exclude=.*/LOAD.*\\.csv" + - " | -out | target/testResults/queryPerInputSubpart_negative.csv" + + " | -out | $testOutputDir/queryPerInputSubpart_negative.csv" + " | -sql | SELECT oauth_consumer.* FROM oauth_consumer" try { CsvCruncherTestUtils.runCruncherWithArguments(command) diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonFileTabularizerTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonFileTabularizerTest.kt index b13e3f4..cf8c7f3 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonFileTabularizerTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonFileTabularizerTest.kt @@ -6,11 +6,12 @@ import cz.dynawest.csvcruncher.test.json.JsonTestUtils.prepareEntriesFromFile import cz.dynawest.csvcruncher.util.logger import org.assertj.core.api.Assertions import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo class JsonFileTabularizerTest { @Test - fun testConvertJson_01_arrayAtRoot_sameProperties() { + fun testConvertJson_01_arrayAtRoot_sameProperties(testInfo: TestInfo) { val entries = prepareEntriesFromFile("data/basic/01-arrayAtRoot-sameProperties.json", "/") Assertions.assertThat(entries).size().isEqualTo(3) @@ -21,7 +22,7 @@ class JsonFileTabularizerTest { } @Test - fun `testConvertJson_02-arrayAtRoot-propertiesMissing`() { + fun `testConvertJson_02-arrayAtRoot-propertiesMissing`(testInfo: TestInfo) { val entries = prepareEntriesFromFile("data/basic/02-arrayAtRoot-propertiesMissing.json", "/") Assertions.assertThat(entries).size().isEqualTo(3) @@ -34,7 +35,7 @@ class JsonFileTabularizerTest { } @Test - fun `testConvertJson_03-arrayAtRoot-varyingProperties`() { + fun `testConvertJson_03-arrayAtRoot-varyingProperties`(testInfo: TestInfo) { val entries = prepareEntriesFromFile("data/basic/03-arrayAtRoot-varyingProperties.json", "/") Assertions.assertThat(entries).size().isEqualTo(3) @@ -50,7 +51,7 @@ class JsonFileTabularizerTest { } @Test - fun `testConvertJson_04-arrayAtRoot-differentTypes`() { + fun `testConvertJson_04-arrayAtRoot-differentTypes`(testInfo: TestInfo) { val entries = prepareEntriesFromFile("data/basic/04-arrayAtRoot-differentTypes.json", "/") Assertions.assertThat(entries).size().isEqualTo(5) @@ -63,7 +64,7 @@ class JsonFileTabularizerTest { } @Test - fun `testConvertJson_10-arrayAtRoot-nestedObject`() { + fun `testConvertJson_10-arrayAtRoot-nestedObject`(testInfo: TestInfo) { val entries = prepareEntriesFromFile("data/basic/10-arrayAtRoot-nestedObject.json", "/") Assertions.assertThat(entries).size().isEqualTo(3) @@ -79,7 +80,7 @@ class JsonFileTabularizerTest { @Test - fun `testConvertJson_11-arrayAtRoot-nestedArray`() { + fun `testConvertJson_11-arrayAtRoot-nestedArray`(testInfo: TestInfo) { val entries = prepareEntriesFromFile("data/basic/11-arrayAtRoot-nestedArray.json", "/") Assertions.assertThat(entries).size().isEqualTo(3) @@ -94,7 +95,7 @@ class JsonFileTabularizerTest { } @Test - fun `testConvertJson_12-arrayInObject-sameProperties`() { + fun `testConvertJson_12-arrayInObject-sameProperties`(testInfo: TestInfo) { val entries = prepareEntriesFromFile("data/basic/12-arrayInObject-sameProperties.json", "/items") Assertions.assertThat(entries).size().isEqualTo(3) @@ -105,7 +106,7 @@ class JsonFileTabularizerTest { } @Test - fun `testConvertJson_21_redditAll`() { + fun `testConvertJson_21_redditAll`(testInfo: TestInfo) { val entries = prepareEntriesFromFile("$testDataDir/json/redditAll.json", "/data/children") Assertions.assertThat(entries).size().isEqualTo(25) diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonTestUtils.kt b/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonTestUtils.kt index e0af1e4..ec8079d 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonTestUtils.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonTestUtils.kt @@ -25,16 +25,15 @@ object JsonTestUtils { val entries = mutableListOf>() converter.visitEntries(inputStream, Path.of(itemsArraySprout), object : EntryProcessor { override fun processEntry(entry: FlattenedEntrySequence) { - val flattenedPropsByName = entry.flattenedProperties.associateBy { myProp -> myProp.name } + val flattenedPropsByName: Map = entry.flattenedProperties.associateBy { myProp -> myProp.name } entries.add(flattenedPropsByName) - log.info("Entry (10 of ${flattenedPropsByName.size}:" + flattenedPropsByName.entries.take(10).map {"\n ${it.key} = ${it.value}"}) + + val countHint = if (flattenedPropsByName.size <= 10) "" else " (10 of ${flattenedPropsByName.size})" + log.info("Entry$countHint:" + flattenedPropsByName.entries.take(10).map {"${it.key} = ${it.value.toCsvString()}"}) } }) return entries } - private val log = logger() - - val PROJECT_ROOT_TOKEN = "{project}" } \ No newline at end of file diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonToCsvConversionTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonToCsvConversionTest.kt index 7016054..248bb88 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonToCsvConversionTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/test/json/JsonToCsvConversionTest.kt @@ -5,6 +5,7 @@ import cz.dynawest.csvcruncher.converters.TabularPropertiesMetadataCollector import cz.dynawest.csvcruncher.converters.json.JsonFileFlattener import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import java.io.File import java.io.FileInputStream import java.io.FileOutputStream @@ -14,7 +15,7 @@ import kotlin.io.path.isRegularFile class JsonToCsvConversionTest { @Test - fun testJsonToCsvConversion() { + fun testJsonToCsvConversion(testInfo: TestInfo) { val testInputPath = "target/testData/json/github_data.json" val testOutputPath = "target/testData/json/github_data_${System.currentTimeMillis()}.csv" diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/test/json/TabularPropertiesMetadataCollectorTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/test/json/TabularPropertiesMetadataCollectorTest.kt index f0fc72f..f258486 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/test/json/TabularPropertiesMetadataCollectorTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/test/json/TabularPropertiesMetadataCollectorTest.kt @@ -1,13 +1,14 @@ package cz.dynawest.csvcruncher.test.json -import cz.dynawest.csvcruncher.converters.json.ItemsArraySproutNotFound -import cz.dynawest.csvcruncher.converters.json.JsonFileFlattener import cz.dynawest.csvcruncher.converters.PropertyInfo import cz.dynawest.csvcruncher.converters.TabularPropertiesMetadataCollector +import cz.dynawest.csvcruncher.converters.json.ItemsArraySproutNotFound +import cz.dynawest.csvcruncher.converters.json.JsonFileFlattener import cz.dynawest.csvcruncher.util.logger import cz.dynawest.util.ResourceLoader -import org.assertj.core.api.Assertions.* +import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import org.junit.jupiter.api.assertThrows import java.io.FileInputStream import java.io.InputStream @@ -18,7 +19,7 @@ import kotlin.test.assertEquals class TabularPropertiesMetadataCollectorTest { @Test - fun `testConvertJson_01-arrayAtRoot-nestedArray`() { + fun `testConvertJson_01-arrayAtRoot-nestedArray`(testInfo: TestInfo) { val collectedProperties = collectProperties("data/basic/01-arrayAtRoot-sameProperties.json") assertThat(collectedProperties).size().isEqualTo(2) @@ -28,7 +29,7 @@ class TabularPropertiesMetadataCollectorTest { } @Test - fun `testConvertJson_02-arrayAtRoot-propertiesMissing`() { + fun `testConvertJson_02-arrayAtRoot-propertiesMissing`(testInfo: TestInfo) { val collectedProperties = collectProperties("data/basic/02-arrayAtRoot-propertiesMissing.json") assertThat(collectedProperties).size().isEqualTo(3) @@ -39,7 +40,7 @@ class TabularPropertiesMetadataCollectorTest { } @Test - fun `testConvertJson_03-arrayAtRoot-varyingProperties`() { + fun `testConvertJson_03-arrayAtRoot-varyingProperties`(testInfo: TestInfo) { val collectedProperties = collectProperties("data/basic/03-arrayAtRoot-varyingProperties.json") assertThat(collectedProperties).size().isEqualTo(3) @@ -50,7 +51,7 @@ class TabularPropertiesMetadataCollectorTest { } @Test - fun `testConvertJson_04-arrayAtRoot-differentTypes`() { + fun `testConvertJson_04-arrayAtRoot-differentTypes`(testInfo: TestInfo) { val collectedProperties = collectProperties("data/basic/04-arrayAtRoot-differentTypes.json", "/") assertThat(collectedProperties).size().isEqualTo(2) @@ -63,7 +64,7 @@ class TabularPropertiesMetadataCollectorTest { } @Test - fun `testConvertJson_10-arrayAtRoot-nestedObject`() { + fun `testConvertJson_10-arrayAtRoot-nestedObject`(testInfo: TestInfo) { val collectedProperties = collectProperties("data/basic/10-arrayAtRoot-nestedObject.json", "/") assertThat(collectedProperties).size().isEqualTo(4) @@ -75,7 +76,7 @@ class TabularPropertiesMetadataCollectorTest { } @Test - fun `testConvertJson_12-arrayInObject-sameProperties`() { + fun `testConvertJson_12-arrayInObject-sameProperties`(testInfo: TestInfo) { val collectedProperties = collectProperties("data/basic/12-arrayInObject-sameProperties.json", "/items") assertThat(collectedProperties).size().isEqualTo(2) @@ -86,14 +87,14 @@ class TabularPropertiesMetadataCollectorTest { } @Test - fun `testConvertJson_12-arrayInObject-sameProperties_mainArrayNotFound`() { + fun `testConvertJson_12-arrayInObject-sameProperties_mainArrayNotFound`(testInfo: TestInfo) { assertThrows { collectProperties("data/basic/12-arrayInObject-sameProperties.json", "/") } } @Test //@Disabled("A bug in walkThroughToTheCollectionOfMainItems()") - fun `testConvertJson_real_youtube`() { + fun `testConvertJson_real_youtube`(testInfo: TestInfo) { val collectedProperties = collectProperties("data/real/youtube.json", "/items") val propsList = collectedProperties.map { "\n * $it" }.joinToString() @@ -105,7 +106,7 @@ class TabularPropertiesMetadataCollectorTest { } @Test - fun `testConvertJson_large_github`() { + fun `testConvertJson_large_github`(testInfo: TestInfo) { val testInputPath = "./target/testData/json/github_data.json" if (!Path.of(testInputPath).isRegularFile()) return // It's ok, maybe the test is not run through Maven, so it was not unzipped. diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/test/params/ParamsParsingTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/test/params/ParamsParsingTest.kt index 6b69909..9bdb6df 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/test/params/ParamsParsingTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/test/params/ParamsParsingTest.kt @@ -4,13 +4,14 @@ import cz.dynawest.csvcruncher.app.Options2 import cz.dynawest.csvcruncher.app.OptionsParser import org.assertj.core.api.Assertions import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import java.nio.file.Path class ParamsParsingTest { - @Test fun paramsParsing_singleImport() { + @Test fun paramsParsing_singleImport(testInfo: TestInfo) { - val opt = parseTestOptions("""-in | input/path | -itemsAt | /data/children | -out | /output/path | -sql | SELECT * FROM redditAll """) + val opt = parseTestOptions("""-in | input/path | -itemsAt | /data/children | -out | /output/path | -sql | SELECT * FROM redditAll """) Assertions.assertThat(opt.importArguments).size().isEqualTo(1) Assertions.assertThat(opt.importArguments.first().path).isEqualTo(Path.of("input/path")) @@ -22,9 +23,9 @@ class ParamsParsingTest { Assertions.assertThat(opt.exportArguments.first().alias).isNull() } - @Test fun paramsParsing_twoImports() { + @Test fun paramsParsing_twoImports(testInfo: TestInfo) { - val opt = parseTestOptions("""-in | input/path | -itemsAt | /data/children | -in | input2/path | -out | /output/path | -sql | SELECT * FROM redditAll """) + val opt = parseTestOptions("""-in | input/path | -itemsAt | /data/children | -in | input2/path | -out | /output/path | -sql | SELECT * FROM redditAll """) Assertions.assertThat(opt.importArguments).size().isEqualTo(2) Assertions.assertThat(opt.importArguments.first().path).isEqualTo(Path.of("input/path")) @@ -40,9 +41,9 @@ class ParamsParsingTest { Assertions.assertThat(opt.exportArguments.first().sqlQuery).isEqualTo("SELECT * FROM redditAll") } - @Test fun paramsParsing_twoExports() { + @Test fun paramsParsing_twoExports(testInfo: TestInfo) { - val opt = parseTestOptions("""-in | input/path | -out | /output1/path | -sql | SELECT * FROM redditAll1 | -out | /output2/path | -sql | SELECT * FROM redditAll2 """) + val opt = parseTestOptions("""-in | input/path | -out | /output1/path | -sql | SELECT * FROM redditAll1 | -out | /output2/path | -sql | SELECT * FROM redditAll2 """) Assertions.assertThat(opt.importArguments).size().isEqualTo(1) diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/util/FilesUtilsTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/util/FilesUtilsTest.kt index a393591..dc73405 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/util/FilesUtilsTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/util/FilesUtilsTest.kt @@ -3,6 +3,7 @@ package cz.dynawest.csvcruncher.util import cz.dynawest.csvcruncher.* +import cz.dynawest.csvcruncher.CsvCruncherTestUtils.testDataDir import cz.dynawest.csvcruncher.app.Options2 import cz.dynawest.csvcruncher.app.OptionsEnums import cz.dynawest.csvcruncher.util.FilesUtils.combineInputFiles @@ -16,6 +17,7 @@ import cz.dynawest.csvcruncher.util.FilesUtils.parseColumnsFromFirstCsvLine import org.junit.jupiter.api.Assertions import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo import java.io.IOException import java.nio.file.Path import java.nio.file.Paths @@ -25,7 +27,7 @@ import java.util.regex.Pattern class FilesUtilsTest { @Test - fun filterFilePaths() { + fun filterFilePaths(testInfo: TestInfo) { val paths: MutableList = ArrayList() paths.add(Paths.get("foo.bar")) paths.add(Paths.get("foo.foo")) @@ -58,19 +60,19 @@ class FilesUtilsTest { } @Test - fun concatFiles() { + fun concatFiles(testInfo: TestInfo) { } @Test - fun sortInputPaths() { + fun sortInputPaths(testInfo: TestInfo) { } @Test - fun convertResultToJson() { + fun convertResultToJson(testInfo: TestInfo) { } @Test - fun deriveNameForCombinedFile() { + fun deriveNameForCombinedFile(testInfo: TestInfo) { val fileGroup = HashMap>() fileGroup[Paths.get("foo")] = emptyList() val usedConcatFilePaths = HashSet() @@ -80,7 +82,7 @@ class FilesUtilsTest { } @Test - fun deriveNameForCombinedFile_dir() { + fun deriveNameForCombinedFile_dir(testInfo: TestInfo) { val fileGroup = HashMap>() fileGroup[Paths.get("foo/bar.csv")] = emptyList() val usedConcatFilePaths = HashSet() @@ -122,7 +124,7 @@ class FilesUtilsTest { @Test @Throws(IOException::class) - fun combineInputFiles_changedSchema() { + fun combineInputFiles_changedSchema(testInfo: TestInfo) { val options = Options2() options.newImportArgument().apply { path = testDataDir.resolve("sample-changedSchema") } options.excludePathsRegex = Pattern.compile(".*/LOAD.*\\.csv") @@ -148,7 +150,7 @@ class FilesUtilsTest { } @Test - fun expandDirectories() { + fun expandDirectories(testInfo: TestInfo) { val options = Options2() val inputPaths = Arrays.asList(testDataDir.resolve("sample-changedSchema")) options.newImportArgument().apply { path = inputPaths[0] } @@ -167,9 +169,9 @@ class FilesUtilsTest { @Test @Throws(IOException::class) - fun parseColsFromFirstCsvLine() { - val csvFileWithHeader = testDataDir.resolve("sample-collab/session_telephony_pins/20180918-132721852.csv") - val colNames = parseColumnsFromFirstCsvLine(csvFileWithHeader.toFile()) + fun parseColsFromFirstCsvLine(testInfo: TestInfo) { + val csvFileWithHeader = testDataDir.toFile().resolve("sample-collab/session_telephony_pins/20180918-132721852.csv") + val colNames = parseColumnsFromFirstCsvLine(csvFileWithHeader) // Op,id,uuid,session_id,pin,pin_type,pin_access_type,enrollment_id,created_time,modified_time Assertions.assertEquals(10, colNames.size.toLong(), "Proper column count") @@ -180,7 +182,6 @@ class FilesUtilsTest { } companion object { - var testDataDir = CsvCruncherTestUtils.testDataDir var testOutputDir = CsvCruncherTestUtils.testOutputDir } } \ No newline at end of file diff --git a/src/test/kotlin/cz/dynawest/csvcruncher/util/HsqlDbHelperTest.kt b/src/test/kotlin/cz/dynawest/csvcruncher/util/HsqlDbHelperTest.kt index c5e5662..74f0177 100644 --- a/src/test/kotlin/cz/dynawest/csvcruncher/util/HsqlDbHelperTest.kt +++ b/src/test/kotlin/cz/dynawest/csvcruncher/util/HsqlDbHelperTest.kt @@ -4,11 +4,12 @@ import cz.dynawest.csvcruncher.HsqlDbHelper import cz.dynawest.csvcruncher.HsqlDbHelper.Companion.quoteIdentifiersInQuery import org.assertj.core.api.Assertions.assertThat import org.junit.jupiter.api.Test +import org.junit.jupiter.api.TestInfo class HsqlDbHelperTest { @Test - fun test_quoteColumnNames_1() { + fun test_quoteColumnNames_1(testInfo: TestInfo) { val sql = "SELECT a.bc.d FROM table1 WHERE a.bc.d=1 OR (a.bc.d>2 AND a.bc.d<4)" val sqlReplaced = quoteIdentifiersInQuery(sql, listOf("a.bc.d")) @@ -16,7 +17,7 @@ class HsqlDbHelperTest { } @Test - fun test_quoteColumnNames_2() { + fun test_quoteColumnNames_2(testInfo: TestInfo) { val sql = "SELECT a.bc.d, bc+a.bc.d FROM table1 WHERE a.bc.d=1 OR (a.bc.d>2 AND a.bc.d<4)" val sqlReplaced = quoteIdentifiersInQuery(sql, listOf("a.bc.d", "bc")) @@ -24,14 +25,14 @@ class HsqlDbHelperTest { } @Test - fun test_quoteColumnNames_aaaa() { + fun test_quoteColumnNames_aaaa(testInfo: TestInfo) { val sql = "SELECT a.a, a+\"a\", a+a, aa+aa, aa, aaa, aa.aa, (a