From 649e2ccfba863442561430c4ad83a147227b9bc5 Mon Sep 17 00:00:00 2001
From: Gera Shegalov <gera@apache.org>
Date: Fri, 20 Oct 2023 13:02:31 -0700
Subject: [PATCH] Add a spark-shell smoke test to premerge and nightly

Contributes to #5704

Signed-off-by: Gera Shegalov <gera@apache.org>
---
 integration_tests/run_pyspark_from_build.sh | 13 ++++++++++++-
 jenkins/spark-premerge-build.sh             |  1 +
 jenkins/spark-tests.sh                      |  2 ++
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/integration_tests/run_pyspark_from_build.sh b/integration_tests/run_pyspark_from_build.sh
index 853fae66316..fa4dffc8073 100755
--- a/integration_tests/run_pyspark_from_build.sh
+++ b/integration_tests/run_pyspark_from_build.sh
@@ -309,7 +309,18 @@ EOF
     fi
     export PYSP_TEST_spark_rapids_memory_gpu_allocSize=${PYSP_TEST_spark_rapids_memory_gpu_allocSize:-'1536m'}
 
-    if ((${#TEST_PARALLEL_OPTS[@]} > 0));
+    SPARK_SHELL_SMOKE_TEST="${SPARK_SHELL_SMOKE_TEST:-0}"
+    if [[ "$SPARK_SHELL_SMOKE_TEST" != "0" ]]; then
+        echo "Running spark-shell smoke test..."
+        <<< 'spark.range(100).agg(Map("id" -> "sum")).collect()' \
+            "$SPARK_HOME"/bin/spark-shell \
+                --master local-cluster[1,1,1024] \
+                --jars "${PYSP_TEST_spark_jars}" \
+                --conf spark.plugins=com.nvidia.spark.SQLPlugin \
+                --conf spark.deploy.maxExecutorRetries=0 2>/dev/null \
+            | grep -F 'res0: Array[org.apache.spark.sql.Row] = Array([4950])'
+        echo "SUCCESS spark-shell smoke test..."
+    elif ((${#TEST_PARALLEL_OPTS[@]} > 0));
     then
         exec python "${RUN_TESTS_COMMAND[@]}" "${TEST_PARALLEL_OPTS[@]}" "${TEST_COMMON_OPTS[@]}"
     else
diff --git a/jenkins/spark-premerge-build.sh b/jenkins/spark-premerge-build.sh
index 9b509208986..56997695f39 100755
--- a/jenkins/spark-premerge-build.sh
+++ b/jenkins/spark-premerge-build.sh
@@ -88,6 +88,7 @@ mvn_verify() {
 
     # Triggering here until we change the jenkins file
     rapids_shuffle_smoke_test
+    SPARK_SHELL_SMOKE_TEST=1 ./integration_tests/run_pyspark_from_build.sh
 }
 
 rapids_shuffle_smoke_test() {
diff --git a/jenkins/spark-tests.sh b/jenkins/spark-tests.sh
index 4a062f63871..c3896d05342 100755
--- a/jenkins/spark-tests.sh
+++ b/jenkins/spark-tests.sh
@@ -270,6 +270,8 @@ TEST_MODE=${TEST_MODE:-'DEFAULT'}
 if [[ $TEST_MODE == "DEFAULT" ]]; then
   ./run_pyspark_from_build.sh
 
+  SPARK_SHELL_SMOKE_TEST=1 ./integration_tests/run_pyspark_from_build.sh
+
   # ParquetCachedBatchSerializer cache_test
   PYSP_TEST_spark_sql_cache_serializer=com.nvidia.spark.ParquetCachedBatchSerializer \
     ./run_pyspark_from_build.sh -k cache_test