Revert "HADOOP-15759. AliyunOSS: Update oss-sdk version to 3.0.0. Contributed by Jinhu Wu."

sunilgovind · aajisaka · commit 83ef50fdedfe · 2019-01-05T17:49:18.000+09:00
This reverts commit e4fca6a. Revert "HADOOP-15671. AliyunOSS: Support Assume Roles in AliyunOSS. Contributed by Jinhu Wu." This reverts commit 2b63512. (cherry picked from commit 1f42527) (cherry picked from commit 71bee05)
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
@@ -1261,7 +1261,7 @@
       <dependency>
         <groupId>com.aliyun.oss</groupId>
         <artifactId>aliyun-sdk-oss</artifactId>
-        <version>3.0.0</version>
+        <version>2.8.3</version>
         <exclusions>
           <exclusion>
             <groupId>org.apache.httpcomponents</groupId>
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java
@@ -120,8 +120,7 @@ public synchronized void close() throws IOException {
         if (null == partETags) {
           throw new IOException("Failed to multipart upload to oss, abort it.");
         }
-        store.completeMultipartUpload(key, uploadId,
-            new ArrayList<>(partETags));
+        store.completeMultipartUpload(key, uploadId, partETags);
       }
     } finally {
       removePartFiles();
@@ -130,7 +129,7 @@ public synchronized void close() throws IOException {
   }
 
   @Override
-  public synchronized void write(int b) throws IOException {
+  public void write(int b) throws IOException {
     singleByte[0] = (byte)b;
     write(singleByte, 0, 1);
   }
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -149,7 +149,7 @@ public void initialize(URI uri, Configuration conf,
           "null or empty. Please set proper endpoint with 'fs.oss.endpoint'.");
     }
     CredentialsProvider provider =
-        AliyunOSSUtils.getCredentialsProvider(uri, conf);
+        AliyunOSSUtils.getCredentialsProvider(conf);
     ossClient = new OSSClient(endPoint, provider, clientConf);
     uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(conf,
         MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT);
@@ -168,8 +168,6 @@ public void initialize(URI uri, Configuration conf,
       multipartThreshold = 1024 * 1024 * 1024;
     }
 
-    bucketName = uri.getHost();
-
     String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT);
     if (StringUtils.isNotEmpty(cannedACLName)) {
       CannedAccessControlList cannedACL =
@@ -178,6 +176,7 @@ public void initialize(URI uri, Configuration conf,
     }
 
     maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
+    bucketName = uri.getHost();
   }
 
   /**
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -20,7 +20,6 @@
 
 import java.io.File;
 import java.io.IOException;
-import java.net.URI;
 
 import com.aliyun.oss.common.auth.CredentialsProvider;
 import com.google.common.base.Preconditions;
@@ -96,14 +95,13 @@ public static long calculatePartSize(long contentLength, long minPartSize) {
    * Create credential provider specified by configuration, or create default
    * credential provider if not specified.
    *
-   * @param uri uri passed by caller
    * @param conf configuration
    * @return a credential provider
    * @throws IOException on any problem. Class construction issues may be
    * nested inside the IOE.
    */
-  public static CredentialsProvider getCredentialsProvider(
-      URI uri, Configuration conf) throws IOException {
+  public static CredentialsProvider getCredentialsProvider(Configuration conf)
+      throws IOException {
     CredentialsProvider credentials;
 
     String className = conf.getTrimmed(CREDENTIALS_PROVIDER_KEY);
@@ -119,7 +117,7 @@ public static CredentialsProvider getCredentialsProvider(
         try {
           credentials =
               (CredentialsProvider)credClass.getDeclaredConstructor(
-                  URI.class, Configuration.class).newInstance(uri, conf);
+                  Configuration.class).newInstance(conf);
         } catch (NoSuchMethodException | SecurityException e) {
           credentials =
               (CredentialsProvider)credClass.getDeclaredConstructor()
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AssumedRoleCredentialProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AssumedRoleCredentialProvider.java
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.fs.aliyun.oss;
 
-import com.aliyun.oss.common.utils.AuthUtils;
 import com.aliyun.oss.common.utils.VersionInfoUtils;
 
 /**
@@ -43,27 +42,6 @@ private Constants() {
   public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret";
   public static final String SECURITY_TOKEN = "fs.oss.securityToken";
 
-  // Assume role configurations
-  public static final String ROLE_ARN = "fs.oss.assumed.role.arn";
-  public static final String ASSUMED_ROLE_DURATION =
-      "fs.oss.assumed.role.session.duration";
-  // Default session duration(in seconds)
-  public static final long ASSUMED_ROLE_DURATION_DEFAULT = 30 * 60;
-
-  // Expired factor of sts token
-  // For example, if session duration is 900s and expiredFactor is 0.8
-  // sts token will be refreshed after 900 * 0.8s
-  public static final String ASSUMED_ROLE_STS_EXPIRED_FACTOR =
-      "fs.oss.assumed.role.sts.expiredFactor";
-
-  public static final double ASSUMED_ROLE_STS_EXPIRED_FACTOR_DEFAULT =
-      AuthUtils.DEFAULT_EXPIRED_FACTOR;
-
-  public static final String ASSUMED_ROLE_STS_ENDPOINT =
-      "fs.oss.assumed.role.sts.endpoint";
-  public static final String ASSUMED_ROLE_SESSION_NAME =
-      "fs.oss.assumed.role.session.name";
-
   // Number of simultaneous connections to oss
   public static final String MAXIMUM_CONNECTIONS_KEY =
       "fs.oss.connection.maximum";
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -117,56 +117,6 @@ please raise your issues with them.
        </description>
     </property>
 
-    <property>
-      <name>fs.oss.assumed.role.arn</name>
-      <description>
-        Role ARN for the role to be assumed.
-        Required if the fs.oss.credentials.provider is
-        org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
-      </description>
-    </property>
-
-    <property>
-      <name>fs.oss.assumed.role.sts.endpoint</name>
-      <description>
-        STS Token Service endpoint.
-        Required if the fs.oss.credentials.provider is
-        org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
-      </description>
-    </property>
-
-    <property>
-      <name>fs.oss.assumed.role.session.name</name>
-      <value />
-      <description>
-        Session name for the assumed role, must be valid characters
-        according to Aliyun API. It is optional, will be generated by
-        oss java sdk if it is empty.
-        Only used if the fs.oss.credentials.provider is
-        org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
-      </description>
-    </property>
-
-    <property>
-      <name>fs.oss.assumed.role.session.duration</name>
-      <value />
-      <description>
-        Duration of assumed roles before it is expired. Default is 30 minutes.
-        Only used if the fs.oss.credentials.provider is
-        org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
-      </description>
-    </property>
-
-    <property>
-      <name>fs.oss.assumed.role.sts.expiredFactor</name>
-      <value />
-      <description>
-        Sts token will be refreshed after (expiredFactor * duration) seconds.
-        Only used if the fs.oss.credentials.provider is
-        org.apache.hadoop.fs.aliyun.oss.AssumedRoleCredentialProvider.
-      </description>
-    </property>
-
     <property>
       <name>fs.oss.proxy.host</name>
       <description>Hostname of the (optinal) proxy server for Aliyun OSS connection</description>
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java
@@ -19,7 +19,6 @@
 package org.apache.hadoop.fs.aliyun.oss;
 
 import com.aliyun.oss.common.auth.Credentials;
-import com.aliyun.oss.common.auth.CredentialsProvider;
 import com.aliyun.oss.common.auth.InvalidCredentialsException;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract;
@@ -28,15 +27,9 @@
 import org.junit.Test;
 
 import java.io.IOException;
-import java.lang.reflect.InvocationTargetException;
-import java.net.URI;
 
 import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID;
 import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET;
-import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_SESSION_NAME;
-import static org.apache.hadoop.fs.aliyun.oss.Constants.ASSUMED_ROLE_STS_ENDPOINT;
-import static org.apache.hadoop.fs.aliyun.oss.Constants.CREDENTIALS_PROVIDER_KEY;
-import static org.apache.hadoop.fs.aliyun.oss.Constants.ROLE_ARN;
 import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
 
 /**
@@ -70,54 +63,16 @@ public void testCredentialMissingAccessKeySecret() throws Throwable {
     validateCredential(conf);
   }
 
-  @Test
-  public void testCredentialMissingRoleArn() throws Throwable {
-    Configuration conf = new Configuration();
-    conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
-    conf.set(ROLE_ARN, "");
-    validateCredential(conf);
-  }
-
-  @Test
-  public void testCredentialMissingStsEndpoint() throws Throwable {
-    Configuration conf = new Configuration();
-    conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
-    conf.set(ASSUMED_ROLE_STS_ENDPOINT, "");
-    validateCredential(conf);
-  }
-
-  @Test
-  public void testCredentialInvalidSessionName() throws Throwable {
-    Configuration conf = new Configuration();
-    conf.set(CREDENTIALS_PROVIDER_KEY, AssumedRoleCredentialProvider.NAME);
-    conf.set(ASSUMED_ROLE_SESSION_NAME, "hadoop oss");
-    validateCredential(conf);
-  }
-
-  private void validateCredential(URI uri, Configuration conf) {
+  private void validateCredential(Configuration conf) {
     try {
-      CredentialsProvider provider =
-          AliyunOSSUtils.getCredentialsProvider(uri, conf);
+      AliyunCredentialsProvider provider
+          = new AliyunCredentialsProvider(conf);
       Credentials credentials = provider.getCredentials();
       fail("Expected a CredentialInitializationException, got " + credentials);
     } catch (InvalidCredentialsException expected) {
       // expected
     } catch (IOException e) {
-      Throwable cause = e.getCause();
-      if (cause instanceof InvocationTargetException) {
-        boolean isInstance =
-            ((InvocationTargetException)cause).getTargetException()
-                instanceof InvalidCredentialsException;
-        if (!isInstance) {
-          fail("Unexpected exception.");
-        }
-      } else {
-        fail("Unexpected exception.");
-      }
+      fail("Unexpected exception.");
     }
   }
-
-  private void validateCredential(Configuration conf) {
-    validateCredential(null, conf);
-  }
-}
+}

Original file line number	Diff line number	Diff line change
`@@ -120,8 +120,7 @@ public synchronized void close() throws IOException {`
`120`	`120`	`if (null == partETags) {`
`121`	`121`	`throw new IOException("Failed to multipart upload to oss, abort it.");`
`122`	`122`	`}`
`123`		`- store.completeMultipartUpload(key, uploadId,`
`124`		`- new ArrayList<>(partETags));`
	`123`	`+ store.completeMultipartUpload(key, uploadId, partETags);`
`125`	`124`	`}`
`126`	`125`	`} finally {`
`127`	`126`	`removePartFiles();`
`@@ -130,7 +129,7 @@ public synchronized void close() throws IOException {`
`130`	`129`	`}`
`131`	`130`
`132`	`131`	`@Override`
`133`		`- public synchronized void write(int b) throws IOException {`
	`132`	`+ public void write(int b) throws IOException {`
`134`	`133`	`singleByte[0] = (byte)b;`
`135`	`134`	`write(singleByte, 0, 1);`
`136`	`135`	`}`
Original file line number	Diff line number	Diff line change
`@@ -149,7 +149,7 @@ public void initialize(URI uri, Configuration conf,`
`149`	`149`	`"null or empty. Please set proper endpoint with 'fs.oss.endpoint'.");`
`150`	`150`	`}`
`151`	`151`	`CredentialsProvider provider =`
`152`		`- AliyunOSSUtils.getCredentialsProvider(uri, conf);`
	`152`	`+ AliyunOSSUtils.getCredentialsProvider(conf);`
`153`	`153`	`ossClient = new OSSClient(endPoint, provider, clientConf);`
`154`	`154`	`uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(conf,`
`155`	`155`	`MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT);`
`@@ -168,8 +168,6 @@ public void initialize(URI uri, Configuration conf,`
`168`	`168`	`multipartThreshold = 1024 * 1024 * 1024;`
`169`	`169`	`}`
`170`	`170`
`171`		`- bucketName = uri.getHost();`
`172`		`-`
`173`	`171`	`String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT);`
`174`	`172`	`if (StringUtils.isNotEmpty(cannedACLName)) {`
`175`	`173`	`CannedAccessControlList cannedACL =`
`@@ -178,6 +176,7 @@ public void initialize(URI uri, Configuration conf,`
`178`	`176`	`}`
`179`	`177`
`180`	`178`	`maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);`
	`179`	`+ bucketName = uri.getHost();`
`181`	`180`	`}`
`182`	`181`
`183`	`182`	`/**`