fix: update submodules to latest versions (#59)

Author: kneekey23

Package.swift

@@ -80,7 +80,7 @@ awsCCommonPlatformExcludes.append("source/arch/intel/asm")
 awsCCommonPlatformExcludes.append("source/arch/arm/asm")
 #endif
 
-var awsCIoPlatformExcludes = ["docs", "CODE_OF_CONDUCT.md", "codebuild"] + excludesFromAll
+var awsCIoPlatformExcludes = ["docs", "CODE_OF_CONDUCT.md", "codebuild", "PKCS11.md", "THIRD-PARTY-LICENSES.txt"] + excludesFromAll
 
 #if os(macOS)
 awsCIoPlatformExcludes.append("source/windows")
@@ -116,7 +116,7 @@ var awsCCompressionPlatformExcludes = ["source/huffman_generator/", "CODE_OF_CON
                                        "codebuild"] + excludesFromAll
 
 var awsCHttpPlatformExcludes = ["bin", "integration-testing", "include/aws/http/private",
-                                "CODE_OF_CONDUCT.md", "sanitizer-blacklist.txt"] + excludesFromAll
+                                "CODE_OF_CONDUCT.md", "sanitizer-blacklist.txt", "codebuild/linux-integration-tests.yml"] + excludesFromAll
 let awsCAuthPlatformExcludes = ["CODE_OF_CONDUCT.md"] + excludesFromAll
 let awsCMqttPlatformExcludes = ["bin", "CODE_OF_CONDUCT.md"] + excludesFromAll
 

Source/AwsCommonRuntimeKit/http/HttpClientConnectionManager.swift

@@ -15,15 +15,15 @@ public class HttpClientConnectionManager {
         self.options = options
         self.allocator = allocator
         let shutDownPtr: UnsafeMutablePointer<ShutDownCallbackOptions>? = fromOptionalPointer(ptr: options.shutDownOptions)
-
         var mgrOptions = aws_http_connection_manager_options(bootstrap: options.clientBootstrap.rawValue,
                                                              initial_window_size: options.initialWindowSize,
                                                              socket_options: options.socketOptions.rawValue,
                                                              tls_connection_options: options.tlsOptions?.rawValue,
-                                                             proxy_options: options.proxyOptions?.rawValue,
                                                              monitoring_options: options.monitoringOptions?.rawValue,
                                                              host: options.hostName.awsByteCursor,
                                                              port: options.port,
+                                                             proxy_options: options.proxyOptions?.rawValue,
+                                                             proxy_ev_settings: options.proxyEnvSettings?.rawValue,
                                                              max_connections: options.maxConnections,
                                                              shutdown_complete_user_data: shutDownPtr,
                                                              shutdown_complete_callback: { (userData) in
@@ -36,11 +36,9 @@ public class HttpClientConnectionManager {
                                                                 defer {callbackOptions.deinitializeAndDeallocate()}
                                                                 callbackOptions.pointee.shutDownCallback(
                                                                     callbackOptions.pointee.semaphore)
-                                                             },
-                                                             enable_read_back_pressure:
-                                                                options.enableManualWindowManagement,
-                                                             max_connection_idle_in_milliseconds:
-                                                                options.maxConnectionIdleMs)
+                                                            },
+                                                             enable_read_back_pressure: options.enableManualWindowManagement,
+                                                             max_connection_idle_in_milliseconds: options.maxConnectionIdleMs)
 
         self.manager = aws_http_connection_manager_new(allocator.rawValue, &mgrOptions)
     }

Source/AwsCommonRuntimeKit/http/HttpClientConnectionOptions.swift

@@ -13,6 +13,8 @@ public struct HttpClientConnectionOptions {
     public let port: UInt16
     /// The proxy options for connections in the connection pool
     public let proxyOptions: HttpProxyOptions?
+    /// Configuration for using proxy from environment variable. Only works when proxyOptions is not set.
+    public let proxyEnvSettings: ProxyEnvSettings?
     /// The socket options to use for connections in the connection pool
     public var socketOptions: SocketOptions
     /// The tls options to use for connections in the connection pool
@@ -45,6 +47,7 @@ public struct HttpClientConnectionOptions {
                 initialWindowSize: Int = Int.max,
                 port: UInt16,
                 proxyOptions: HttpProxyOptions?,
+                proxyEnvSettings: ProxyEnvSettings? = nil,
                 socketOptions: SocketOptions,
                 tlsOptions: TlsConnectionOptions?,
                 monitoringOptions: HttpMonitoringOptions?,
@@ -58,6 +61,7 @@ public struct HttpClientConnectionOptions {
         self.initialWindowSize = initialWindowSize
         self.port = port
         self.proxyOptions = proxyOptions
+        self.proxyEnvSettings = proxyEnvSettings
         self.socketOptions = socketOptions
         self.tlsOptions = tlsOptions
         self.monitoringOptions = monitoringOptions

Source/AwsCommonRuntimeKit/http/HttpMonitoringOptions.swift

@@ -19,7 +19,9 @@ public class HttpMonitoringOptions {
 
         let options = aws_http_connection_monitoring_options(
             minimum_throughput_bytes_per_second: UInt64(minThroughputBytesPerSecond),
-            allowable_throughput_failure_interval_seconds: UInt32(allowableThroughputFailureInterval))
+            allowable_throughput_failure_interval_seconds: UInt32(allowableThroughputFailureInterval),
+            statistics_observer_fn: nil,
+            statistics_observer_user_data: nil)
         self.rawValue = fromPointer(ptr: options)
 
     }

Source/AwsCommonRuntimeKit/http/HttpProxyConnectionType.swift

@@ -0,0 +1,31 @@
+//  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//  SPDX-License-Identifier: Apache-2.0.
+import AwsCHttp
+
+public enum HttpProxyConnectionType {
+    /// Deprecated, but 0-valued for backwards compatibility
+    /// If tls options are provided (for the main connection) then treat the proxy as a tunneling proxy
+    /// If tls options are not provided (for the main connection), then treat the proxy as a forwarding proxy
+    case legacy
+    /// Use the proxy to forward http requests.  Attempting to use both this mode and TLS on the tunnel destination
+    /// is a configuration error.
+    case forward
+    /// Use the proxy to establish a connection to a remote endpoint via a CONNECT request through the proxy.
+    /// Works for both plaintext and tls connections.
+    case tunnel
+}
+
+extension HttpProxyConnectionType: RawRepresentable, CaseIterable {
+
+    public init(rawValue: aws_http_proxy_connection_type) {
+        let value = Self.allCases.first(where: {$0.rawValue == rawValue})
+        self = value ?? .forward
+    }
+    public var rawValue: aws_http_proxy_connection_type {
+        switch self {
+        case .legacy:  return AWS_HPCT_HTTP_LEGACY
+        case .forward: return AWS_HPCT_HTTP_FORWARD
+        case .tunnel: return AWS_HPCT_HTTP_TUNNEL
+        }
+    }
+}

Source/AwsCommonRuntimeKit/http/HttpProxyEnvType.swift

@@ -0,0 +1,27 @@
+//  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//  SPDX-License-Identifier: Apache-2.0.
+import AwsCHttp
+
+public enum HttpProxyEnvType {
+    /// Default. Disable reading from environment variable for proxy.
+    case disable
+    /// Enable get proxy URL from environment variable, when the manual proxy options of connection manager is not set.
+    /// env HTTPS_PROXY/https_proxy will be checked when the main connection use tls.
+    /// env HTTP_PROXY/http_proxy will be checked when the main connection NOT use tls.
+    /// The lower case version has precedence.
+    case enable
+}
+
+extension HttpProxyEnvType: RawRepresentable, CaseIterable {
+
+    public init(rawValue: aws_http_proxy_env_var_type) {
+        let value = Self.allCases.first(where: {$0.rawValue == rawValue})
+        self = value ?? .disable
+    }
+    public var rawValue: aws_http_proxy_env_var_type {
+        switch self {
+        case .disable:  return AWS_HPEV_DISABLE
+        case .enable: return AWS_HPEV_ENABLE
+        }
+    }
+}

Source/AwsCommonRuntimeKit/http/ProxyEnvSettings.swift

@@ -0,0 +1,23 @@
+//  Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+//  SPDX-License-Identifier: Apache-2.0.
+import AwsCHttp
+
+public class ProxyEnvSettings {
+    let rawValue: UnsafeMutablePointer<proxy_env_var_settings>
+    public var envVarType: HttpProxyEnvType = .disable
+    public var proxyConnectionType: HttpProxyConnectionType = .forward
+    public var tlsOptions: TlsConnectionOptions?
+
+    public init(envVarType: HttpProxyEnvType = .disable,
+                proxyConnectionType: HttpProxyConnectionType = .forward,
+                tlsOptions: TlsConnectionOptions? = nil) {
+        self.rawValue = allocatePointer()
+        self.envVarType = envVarType
+        self.proxyConnectionType = proxyConnectionType
+        self.tlsOptions = tlsOptions
+    }
+
+    deinit {
+        rawValue.deinitializeAndDeallocate()
+    }
+}

aws-common-runtime/aws-c-auth

@@ -0,0 +1,153 @@
+import argparse
+import os
+import os.path
+import re
+import subprocess
+import sys
+
+
+def run(*args, check=True):
+    return subprocess.run(args, capture_output=True, check=check, universal_newlines=True)
+
+
+def get_submodules():
+    """
+    Return list of submodules for current repo, sorted by name.
+
+    Each item looks like:
+    {
+        'name': 'aws-c-common',
+        'path': 'crt/aws-c-common',
+        'url': 'https://github.com/awslabs/aws-c-common.git',
+    }
+    """
+    if not os.path.exists('.gitmodules'):
+        sys.exit(f'No .gitmodules found in {os.getcwd()}')
+
+    submodules = []
+    start_pattern = re.compile(r'\[submodule')
+    path_pattern = re.compile(r'\s+path = (\S+)')
+    url_pattern = re.compile(r'\s+url = (\S+)')
+
+    current = None
+    with open('.gitmodules', 'r') as f:
+        for line in f.readlines():
+            m = start_pattern.match(line)
+            if m:
+                current = {}
+                submodules.append(current)
+                continue
+
+            m = path_pattern.match(line)
+            if m:
+                current['path'] = m.group(1)
+                current['name'] = os.path.basename(current['path'])
+                continue
+
+            m = url_pattern.match(line)
+            if m:
+                current['url'] = m.group(1)
+                continue
+
+    return sorted(submodules, key=lambda x: x['name'])
+
+
+def get_release_tags():
+    """
+    Return list of release tags for current repo, sorted high to low.
+
+    Each item looks like:
+    {
+        'commit': 'e18f041a0c8d17189f2eae2a32f16e0a7a3f0f1c',
+        'version': 'v0.5.18'
+        'num_tuple': (0,5,18),
+    }
+    """
+    git_output = run('git', 'ls-remote', '--tags').stdout
+    tags = []
+    for line in git_output.splitlines():
+        # line looks like: "e18f041a0c8d17189f2eae2a32f16e0a7a3f0f1c refs/tags/v0.5.18"
+        match = re.match(
+            r'([a-f0-9]+)\s+refs/tags/(v([0-9]+)\.([0-9]+)\.([0-9]+))$', line)
+        if not match:
+            # skip malformed release tags
+            continue
+        tags.append({
+            'commit': match.group(1),
+            'version': match.group(2),
+            'num_tuple': (int(match.group(3)), int(match.group(4)), int(match.group(5))),
+        })
+
+    # sort highest version first
+    return sorted(tags, reverse=True, key=lambda tag: tag['num_tuple'])
+
+
+def get_current_commit():
+    git_output = run('git', 'rev-parse', 'HEAD').stdout
+    return git_output.splitlines()[0]
+
+
+def is_ancestor(ancestor, descendant):
+    """Return whether first commit is an ancestor to the second'"""
+    result = run('git', 'merge-base', '--is-ancestor',
+                 ancestor, descendant, check=False)
+    return result.returncode == 0
+
+
+def get_tag_for_commit(tags, commit):
+    for tag in tags:
+        if tag['commit'] == commit:
+            return tag
+    return None
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Update submodules to latest tags")
+    parser.add_argument('ignore', nargs='*', help="submodules to ignore")
+    parser.add_argument('--dry-run', action='store_true',
+                        help="print without actually updating")
+    args = parser.parse_args()
+
+    root_path = os.getcwd()
+    submodules = get_submodules()
+    name_pad = max([len(x['name']) for x in submodules])
+    for submodule in submodules:
+        name = submodule['name']
+
+        os.chdir(os.path.join(root_path, submodule['path']))
+
+        tags = get_release_tags()
+        current_commit = get_current_commit()
+        current_tag = get_tag_for_commit(tags, current_commit)
+        sync_from = current_tag['version'] if current_tag else current_commit
+
+        if name in args.ignore:
+            print(f"{name:<{name_pad}} {sync_from} (ignored)")
+            continue
+
+        latest_tag = tags[0]
+        sync_to = latest_tag['version']
+
+        # The only time we don't want to sync to the latest release is:
+        # The submodule is at some commit beyond the latest release,
+        # and the CRT team doesn't control this repo so can't just cut a new release
+        if sync_from != sync_to and current_tag is None:
+            if name in ['aws-lc', 's2n', 's2n-tls']:
+                # must fetch tags before we can check their ancestry
+                run('git', 'fetch', '--tags', '--prune', '--prune-tags', '--force')
+                if not is_ancestor(ancestor=current_commit, descendant=sync_to):
+                    sync_to = sync_from
+
+        if sync_from == sync_to:
+            print(f"{name:<{name_pad}} {sync_from} ✓")
+        else:
+            print(f"{name:<{name_pad}} {sync_from} -> {sync_to}")
+            if not args.dry_run:
+                run('git', 'fetch', '--tags', '--prune', '--prune-tags', '--force')
+                run('git', 'checkout', sync_to)
+                run('git', 'submodule', 'update')
+
+
+if __name__ == '__main__':
+    main()