Code Review Update 2nd:

* Removed redundant dependency: `bcprov-jdk15on-1.69.jar` * Relocated Tencent COS Constants, and reduced magic values in COS with constants * Isolated cloud storage configuration in `common.properties` -> `resource-center.yaml` * Fixed CodeQL: Uncontrolled data used in path expression
apache · Sep 22, 2024 · 3dfe6fd · 3dfe6fd
1 parent c491779
commit 3dfe6fd
Show file tree

Hide file tree

Showing 8 changed files with 98 additions and 62 deletions.
diff --git a/...cheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java b/...cheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java
@@ -125,9 +125,6 @@ public final class Constants {
     public static final String HUAWEI_CLOUD_OBS_BUCKET_NAME = "resource.huawei.cloud.obs.bucket.name";
     public static final String HUAWEI_CLOUD_OBS_END_POINT = "resource.huawei.cloud.obs.endpoint";
 
-    public static final String TENCENT_CLOUD_COS_BUCKET_NAME = "resource.tencent.cloud.cos.bucket.name";
-    public static final String TENCENT_CLOUD_COS_REGION = "resource.tencent.cloud.cos.region";
-
     /**
      * fetch applicationId way
      */

diff --git a/dolphinscheduler-common/src/main/resources/common.properties b/dolphinscheduler-common/src/main/resources/common.properties
@@ -27,52 +27,9 @@ data.basedir.path=/tmp/dolphinscheduler
 resource.storage.type=LOCAL
 # resource store on HDFS/S3 path, resource file will store to this base path, self configuration, please make sure the directory exists on hdfs and have read write permissions. "/dolphinscheduler" is recommended
 resource.storage.upload.base.path=/tmp/dolphinscheduler
-
-# The Azure client ID (Azure Application (client) ID)
-resource.azure.client.id=minioadmin
-# The Azure client secret in the Azure application
-resource.azure.client.secret=minioadmin
-# The Azure data factory subscription ID
-resource.azure.subId=minioadmin
-# The Azure tenant id in the Azure Active Directory
-resource.azure.tenant.id=minioadmin
 # The query interval
 resource.query.interval=10000
 
-# alibaba cloud access key id, required if you set resource.storage.type=OSS
-resource.alibaba.cloud.access.key.id=<your-access-key-id>
-# alibaba cloud access key secret, required if you set resource.storage.type=OSS
-resource.alibaba.cloud.access.key.secret=<your-access-key-secret>
-# alibaba cloud region, required if you set resource.storage.type=OSS
-resource.alibaba.cloud.region=cn-hangzhou
-# oss bucket name, required if you set resource.storage.type=OSS
-resource.alibaba.cloud.oss.bucket.name=dolphinscheduler
-# oss bucket endpoint, required if you set resource.storage.type=OSS
-resource.alibaba.cloud.oss.endpoint=https://oss-cn-hangzhou.aliyuncs.com
-
-# the location of the google cloud credential, required if you set resource.storage.type=GCS
-resource.google.cloud.storage.credential=/path/to/credential
-# gcs bucket name, required if you set resource.storage.type=GCS
-resource.google.cloud.storage.bucket.name=<your-bucket>
-
-# abs container name, required if you set resource.storage.type=ABS
-resource.azure.blob.storage.container.name=<your-container>
-# abs account name, required if you set resource.storage.type=ABS
-resource.azure.blob.storage.account.name=<your-account-name>
-# abs connection string, required if you set resource.storage.type=ABS
-resource.azure.blob.storage.connection.string=<your-connection-string>
-
-
-# huawei cloud access key id, required if you set resource.storage.type=OBS
-resource.huawei.cloud.access.key.id=<your-access-key-id>
-# huawei cloud access key secret, required if you set resource.storage.type=OBS
-resource.huawei.cloud.access.key.secret=<your-access-key-secret>
-# oss bucket name, required if you set resource.storage.type=OBS
-resource.huawei.cloud.obs.bucket.name=dolphinscheduler
-# oss bucket endpoint, required if you set resource.storage.type=OBS
-resource.huawei.cloud.obs.endpoint=obs.cn-southwest-2.huaweicloud.com
-
-
 # if resource.storage.type=HDFS, the user must have the permission to create directories under the HDFS root path
 resource.hdfs.root.user=hdfs
 # if resource.storage.type=S3, the value like: s3a://dolphinscheduler; if resource.storage.type=HDFS and namenode HA is enabled, you need to copy core-site.xml and hdfs-site.xml to conf dir

diff --git a/dolphinscheduler-common/src/main/resources/resource-center.yaml b/dolphinscheduler-common/src/main/resources/resource-center.yaml
@@ -28,3 +28,53 @@ resource:
         region: ap-nanjing
         bucket:
           name: dolphinscheduler
+
+  # Microsoft Azure Storage (ABS) setup, required if you set resource.storage.type=ABS
+  azure:
+    client:
+      id: minioadmin
+      secret: minioadmin
+    subId: minioadmin
+    tenant:
+      id:  minioadmin
+    blob:
+      storage:
+        connection:
+          string: <your-connection-string>
+        account:
+          name: <your-account-name>
+        container:
+          name: <your-container>
+
+  # Alibaba Cloud Storage (OSS) setup, required if you set resource.storage.type=OSS
+  alibaba:
+    cloud:
+      access:
+        key:
+          id: <your-access-key-id>
+          secret: <your-access-key-secret>
+      region: cn-hangzhou
+      oss:
+        bucket:
+          name: dolphinscheduler
+        endpoint: https://oss-cn-hangzhou.aliyuncs.com
+
+  # Google Cloud Storage (GCS) setup, required if you set resource.storage.type=GCS
+  google:
+    cloud:
+      storage:
+        credential: /path/to/credential
+        bucket:
+          name: <your-bucket>
+
+  # Huawei Cloud Storage (OBS), required if you set resource.storage.type=OBS
+  huawei:
+    cloud:
+      access:
+        key:
+          id: <your-access-key-id>
+          secret: <your-access-key-secret>
+      obs:
+        bucket:
+          name: dolphinscheduler
+        endpoint: obs.cn-southwest-2.huaweicloud.com
diff --git a/...cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageConstants.java b/...cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageConstants.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.plugin.storage.cos;
+
+public class CosStorageConstants {
+
+    // Tencent Cloud Storage (COS) configuration
+    public static final String TENCENT_CLOUD_COS_PROPERTY_PREFIX = "resource.tencent.cloud";
+    public static final String TENCENT_CLOUD_COS_BUCKET_NAME = "resource.tencent.cloud.cos.bucket.name";
+    public static final String TENCENT_CLOUD_COS_REGION = "resource.tencent.cloud.cos.region";
+    public static final String TENCENT_CLOUD_ACCESS_KEY_ID = "resource.tencent.cloud.access.key.id";
+    public static final String TENCENT_CLOUD_ACCESS_KEY_SECRET = "resource.tencent.cloud.access.key.secret";
+
+    public static final String DEFAULT_COS_RESOURCE_UPLOAD_PATH = "/dolphinscheduler";
+}
diff --git a/...-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java b/...-cos/src/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperator.java
@@ -35,6 +35,7 @@
 import java.io.InputStreamReader;
 import java.nio.file.FileAlreadyExistsException;
 import java.nio.file.Files;
+import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.HashSet;
@@ -133,8 +134,14 @@ public void createStorageDir(String directoryAbsolutePath) {
     @Override
     public void download(String srcFilePath, String dstFilePath, boolean overwrite) {
         String cosKey = transformAbsolutePathToCOSKey(srcFilePath);
-
-        File dstFile = Paths.get(dstFilePath).normalize().toFile();
+        Path dsTempFolder = Paths.get(FileUtils.DATA_BASEDIR).normalize().toAbsolutePath();
+        Path fileDownloadPathNormalized = dsTempFolder.resolve(dstFilePath).normalize().toAbsolutePath();
+        if (!fileDownloadPathNormalized.startsWith(dsTempFolder)) {
+            // if the destination file path is NOT in DS temp folder (e.g., '/tmp/dolphinscheduler'),
+            // an IllegalArgumentException should be thrown.
+            throw new IllegalArgumentException("failed to download to " + fileDownloadPathNormalized);
+        }
+        File dstFile = fileDownloadPathNormalized.toFile();
         if (dstFile.isDirectory()) {
             Files.delete(dstFile.toPath());
         } else {

diff --git a/...c/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperatorFactory.java b/...c/main/java/org/apache/dolphinscheduler/plugin/storage/cos/CosStorageOperatorFactory.java
@@ -22,7 +22,6 @@
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperator;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageOperatorFactory;
 import org.apache.dolphinscheduler.plugin.storage.api.StorageType;
-import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
 
 import java.util.Map;
 
@@ -44,14 +43,17 @@ public StorageType getStorageOperate() {
 
     private CosStorageProperties getCosStorageProperties() {
 
-        Map<String, String> cosPropertiesMap = PropertyUtils.getByPrefix("resource.tencent.cloud");
+        Map<String, String> cosPropertiesMap =
+                PropertyUtils.getByPrefix(CosStorageConstants.TENCENT_CLOUD_COS_PROPERTY_PREFIX);
 
         return CosStorageProperties.builder()
-                .region(cosPropertiesMap.get(Constants.TENCENT_CLOUD_COS_REGION))
-                .accessKeyId(cosPropertiesMap.get(TaskConstants.TENCENT_CLOUD_ACCESS_KEY_ID))
-                .accessKeySecret(cosPropertiesMap.get(TaskConstants.TENCENT_CLOUD_ACCESS_KEY_SECRET))
-                .bucketName(cosPropertiesMap.get(Constants.TENCENT_CLOUD_COS_BUCKET_NAME))
-                .resourceUploadPath(cosPropertiesMap.getOrDefault(Constants.RESOURCE_UPLOAD_PATH, "/dolphinscheduler"))
+                .region(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_COS_REGION))
+                .accessKeyId(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_ACCESS_KEY_ID))
+                .accessKeySecret(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_ACCESS_KEY_SECRET))
+                .bucketName(cosPropertiesMap.get(CosStorageConstants.TENCENT_CLOUD_COS_BUCKET_NAME))
+                .resourceUploadPath(
+                        cosPropertiesMap.getOrDefault(Constants.RESOURCE_UPLOAD_PATH,
+                                CosStorageConstants.DEFAULT_COS_RESOURCE_UPLOAD_PATH))
                 .build();
     }
 }
diff --git a/...ler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java b/...ler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/TaskConstants.java
@@ -389,12 +389,6 @@ private TaskConstants() {
     public static final String HUAWEI_CLOUD_ACCESS_KEY_ID = "resource.huawei.cloud.access.key.id";
     public static final String HUAWEI_CLOUD_ACCESS_KEY_SECRET = "resource.huawei.cloud.access.key.secret";
 
-    /**
-     * tencent cloud config
-     */
-    public static final String TENCENT_CLOUD_ACCESS_KEY_ID = "resource.tencent.cloud.access.key.id";
-    public static final String TENCENT_CLOUD_ACCESS_KEY_SECRET = "resource.tencent.cloud.access.key.secret";
-
     /**
      * use for k8s task
      */

diff --git a/tools/dependencies/known-dependencies.txt b/tools/dependencies/known-dependencies.txt
@@ -24,7 +24,6 @@ aws-java-sdk-dms-1.12.300.jar
 aws-json-protocol-2.17.282.jar
 bcpkix-jdk15on-1.69.jar
 bcprov-ext-jdk15on-1.69.jar
-bcprov-jdk15on-1.69.jar
 bcutil-jdk15on-1.69.jar
 bonecp-0.8.0.RELEASE.jar
 bucket4j-core-6.2.0.jar