Add a spark-shell smoke test to premerge and nightly

Contributes to NVIDIA#5704

Signed-off-by: Gera Shegalov <[email protected]>

integration_tests/run_pyspark_from_build.sh

@@ -309,7 +309,18 @@ EOF
     fi
     export PYSP_TEST_spark_rapids_memory_gpu_allocSize=${PYSP_TEST_spark_rapids_memory_gpu_allocSize:-'1536m'}
 
-    if ((${#TEST_PARALLEL_OPTS[@]} > 0));
+    SPARK_SHELL_SMOKE_TEST="${SPARK_SHELL_SMOKE_TEST:-0}"
+    if [[ "$SPARK_SHELL_SMOKE_TEST" != "0" ]]; then
+        echo "Running spark-shell smoke test..."
+        <<< 'spark.range(100).agg(Map("id" -> "sum")).collect()' \
+            "$SPARK_HOME"/bin/spark-shell \
+                --master local-cluster[1,1,1024] \
+                --jars "${PYSP_TEST_spark_jars}" \
+                --conf spark.plugins=com.nvidia.spark.SQLPlugin \
+                --conf spark.deploy.maxExecutorRetries=0 2>/dev/null \
+            | grep -F 'res0: Array[org.apache.spark.sql.Row] = Array([4950])'
+        echo "SUCCESS spark-shell smoke test..."
+    elif ((${#TEST_PARALLEL_OPTS[@]} > 0));
     then
         exec python "${RUN_TESTS_COMMAND[@]}" "${TEST_PARALLEL_OPTS[@]}" "${TEST_COMMON_OPTS[@]}"
     else

jenkins/spark-premerge-build.sh

@@ -88,6 +88,7 @@ mvn_verify() {
 
     # Triggering here until we change the jenkins file
     rapids_shuffle_smoke_test
+    SPARK_SHELL_SMOKE_TEST=1 ./integration_tests/run_pyspark_from_build.sh
 }
 
 rapids_shuffle_smoke_test() {

jenkins/spark-tests.sh

@@ -270,6 +270,8 @@ TEST_MODE=${TEST_MODE:-'DEFAULT'}
 if [[ $TEST_MODE == "DEFAULT" ]]; then
   ./run_pyspark_from_build.sh
 
+  SPARK_SHELL_SMOKE_TEST=1 ./integration_tests/run_pyspark_from_build.sh
+
   # ParquetCachedBatchSerializer cache_test
   PYSP_TEST_spark_sql_cache_serializer=com.nvidia.spark.ParquetCachedBatchSerializer \
     ./run_pyspark_from_build.sh -k cache_test