Skip to content

Commit 18665cc

Browse files
maheshrajusIndhumathi27
authored andcommitted
[CARBONDATA-4214] inserting NULL value when timestamp value received from FROM_UNIXTIME(0)
Why is this PR needed? Filling null in case of timestamp value is received from FROM_UNIXTIME(0) as spark original insert rdd value[internalRow] received in this case zero. if the original column value[internalRow] is zero then in insert flow adding NULL and giving NULL to spark. When query happens on the same column received NULL value instead of timestamp value. Problem code: if (internalRow.getLong(index) == 0) { internalRow.setNullAt(index) } What changes were proposed in this PR? Removed the null filling check for zero value case and if internalRow value is non null/empty then only set the internalRow timestamp value. Does this PR introduce any user interface change? No Is any new testcase added? Yes This closes apache#4154
1 parent d5cb011 commit 18665cc

File tree

2 files changed

+31
-3
lines changed

2 files changed

+31
-3
lines changed

integration/spark/src/main/scala/org/apache/spark/sql/execution/command/management/CommonLoadUtils.scala

+2-3
Original file line numberDiff line numberDiff line change
@@ -781,9 +781,8 @@ object CommonLoadUtils {
781781
internalRowOriginal
782782
}
783783
for (index <- timeStampIndex) {
784-
if (internalRow.getLong(index) == 0) {
785-
internalRow.setNullAt(index)
786-
} else {
784+
// timestmap value can be set other than null/empty case
785+
if (!internalRow.isNullAt(index)) {
787786
internalRow.setLong(
788787
index,
789788
internalRow.getLong(index) / TimeStampGranularityTypeValue.MILLIS_SECONDS.getValue)

integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCase.scala

+29
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
package org.apache.carbondata.spark.testsuite.allqueries
1919

20+
import java.sql.Timestamp
21+
2022
import org.apache.spark.sql.{Row, SaveMode}
2123
import org.apache.spark.sql.test.util.QueryTest
2224
import org.scalatest.BeforeAndAfterAll
@@ -73,6 +75,33 @@ class AllDataTypesTestCase extends QueryTest with BeforeAndAfterAll {
7375
}
7476
}
7577

78+
test("insert data with from_unixtime(0) and query") {
79+
sql("drop table if exists time_carbon1")
80+
sql("drop table if exists time_parquet")
81+
sql("create table if not exists time_carbon1(time1 timestamp) stored as carbondata")
82+
sql("create table if not exists time_parquet(time1 timestamp) stored as parquet")
83+
sql("insert into time_carbon1 select from_unixtime(0)")
84+
sql("insert into time_parquet select from_unixtime(0)")
85+
sql("insert into time_carbon1 select from_unixtime(123456)")
86+
sql("insert into time_parquet select from_unixtime(123456)")
87+
checkAnswer(sql("select * from time_carbon1"),
88+
sql("select * from time_parquet"))
89+
sql("drop table if exists time_carbon1")
90+
sql("drop table if exists time_parquet")
91+
}
92+
93+
test("insert data with empty/null and query") {
94+
sql("drop table if exists time_carbon2")
95+
sql("create table if not exists time_carbon2(time1 timestamp) stored as carbondata")
96+
sql("insert into time_carbon2 select null")
97+
checkAnswer(sql("select count(*) from time_carbon2"), Seq(Row(1)))
98+
checkAnswer(sql("select * from time_carbon2"), Seq(Row(null)))
99+
sql("insert into time_carbon2 select ''")
100+
checkAnswer(sql("select count(*) from time_carbon2"), Seq(Row(2)))
101+
checkAnswer(sql("select * from time_carbon2"), Seq(Row(null), Row(null)))
102+
sql("drop table if exists time_carbon2")
103+
}
104+
76105
// Test-24
77106
test("select channelsId, sum(channelsId+ 10) Total from Carbon_automation_test group by channelsId order by Total") {
78107
checkAnswer(

0 commit comments

Comments
 (0)