Incremental improvements

* Correct typo in expressvpn_windows config
* Freeze Selenium (to avoid an operadriver regression)
* Pass the $PATH to the test runner on Ubuntu
* Remove vestigial ssh_keys/ directory
* Add an option to write a PID file
* Add an option to punch a hole in the firewall (PF only)
* Only log in color when we detect the terminal can handle it
* Retry DNS lookups if DiG doesn't return any output
* Add support for Network Extension VPNs
* Better cleanups
* Source a .source file on remote devices
* Cleaner SSH implementation
* Add a method to get host IP addresses
* Improved connectivity to remote test devices
* Add a debug tool for ip_responder server
* Minor bugfixes and enhancements

Author: ExpressVPN

configs/expressvpn_macos.py

@@ -1,6 +1,6 @@
 from xv_leak_tools.test_templating.templating import TemplateEvaluator, Replacee, Each
 
-NO_IPV6 = True
+NO_IPV6 = False
 
 STRONGEST_SETTINGS = """Ensure Network Lock, IPv6 leak protection and Only user ExpressVPN DNS
 servers are enabled. Also ensure access to LAN is allowed"""

configs/expressvpn_windows.py

@@ -94,7 +94,7 @@
             'TestWindowsIPResponderDisruptAdapter',
             'TestWindowsIPResponderDisruptEnableNewAdapter',
             'TestWindowsIPResponderDisruptReorderAdapters',
-            'TestWindowsIPResponderDisruptVPNConnection',
+            'TestIPResponderDisruptVPNConnection',
             'TestIPResponderDisruptKillVPNProcess',
             'TestIPResponderDisruptVPNConnection',
 

requirements_macos.txt

@@ -7,6 +7,6 @@ parameterized
 paramiko
 psutil
 pylint
-selenium
+selenium <= 3.6.0
 
 https://pypi.python.org/packages/c8/20/3f767baf52ae14b5fa0c6a531651ecb9dbfdc5f75a3dc8eea740df72270c/pyobjc-framework-SystemConfiguration-3.2.1.tar.gz

run_tests.sh

@@ -17,7 +17,7 @@ done
 unamestr=`uname`
 if [[ "$unamestr" == 'Linux' ]]; then
     echo "Leak tools require root permissions..."
-    sudo -E $RUN_TESTS $@
+    sudo env "PATH=$PATH" $RUN_TESTS $@
 elif [[ "$unamestr" == 'Darwin' ]]; then
     echo "Leak tools require root permissions..."
     sudo $RUN_TESTS $@

servers/dns_server/dns_server.py

@@ -19,7 +19,6 @@
     TXT: QTYPE.TXT,
 }
 
-
 class Record:
     def __init__(self, rdata_type, *args, rtype=None, rname=None, ttl=None, **kwargs):
         if isinstance(rdata_type, RD):
@@ -58,8 +57,7 @@ def as_rr(self, alt_rname):
     def sensible_ttl(self):
         if self._rtype in (QTYPE.NS, QTYPE.SOA):
             return 60 * 60 * 24
-        else:
-            return 300
+        return 300
 
     @property
     def is_soa(self):

servers/ip_responder/watch.sh

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+journalctl --lines 0 --follow _SYSTEMD_UNIT=ip_responder.service
+

ssh_keys/.gitignore

@@ -9,17 +9,26 @@
 import sys
 import tempfile
 import time
+import ipaddress
 
 sys.path.append(os.path.join(os.path.dirname(os.path.realpath(__file__)), '..'))
 
 from xv_leak_tools import tools_user
+from xv_leak_tools.exception import XVEx
 from xv_leak_tools.helpers import exception_to_string, current_os
 from xv_leak_tools.import_helpers import import_by_filename
 from xv_leak_tools.log import L
 from xv_leak_tools.object_parser import object_from_command_line
 from xv_leak_tools.path import windows_safe_path, makedirs_safe
 from xv_leak_tools.test_execution.test_run_context import TestRunContext
 from xv_leak_tools.test_execution.test_runner import TestRunner
+from xv_leak_tools.network.macos.pf_firewall import PFCtl
+
+def punch_hole_in_firewall(pf, ip):
+    if current_os() != 'macos':
+        raise XVEx('Editing the firewall is only supported for PF/macOS')
+    pf.set_rules(["pass in quick from {} no state".format(ip),
+                  "pass out quick to {} no state".format(ip)])
 
 def filter_tests(tests, regex):
     filtered_tests = []
@@ -76,13 +85,26 @@ def main(argv=None):
         '"INFO", "DEBUG", "VERBOSE"].')
     parser.add_argument(
         '-2', '--v2', default=False, action='store_true', help='DEPRECATED: Does nothing')
+    parser.add_argument(
+        '-f', '--firewall_exemption', default=None, metavar="IP", nargs='+',
+        help='Punch a hole in the firewall for the given IP address.')
+    parser.add_argument(
+        '-p', '--pidfile', default=None, metavar='PATH_TO_PIDFILE', nargs=1,
+        help='File to write the PID to.')
     args = parser.parse_args(argv)
 
     args.log_level = args.log_level.upper() if args.log_level else None
     # We'll reconfigure in a second but lets get logging going asap!
     inital_log_level = args.log_level if args.log_level else "INFO"
     TestRunner.configure_logger(level=inital_log_level)
 
+    if args.pidfile:
+        pid = os.getpid()
+        path = os.path.abspath(args.pidfile.pop())
+        L.debug("Writing PID {} to {}".format(pid, path))
+        with open(path, 'w') as pidfile:
+            pidfile.write(str(pid))
+
     if args.output_root is None:
         if current_os() == "windows":
             args.output_root = tempfile.mkdtemp()
@@ -105,6 +127,13 @@ def main(argv=None):
     args.output_directory = os.path.join(args.output_root, args.run_directory)
     makedirs_safe(args.output_directory)
 
+    if args.firewall_exemption:
+        pf = PFCtl()
+        ips = [ipaddress.ip_network(ip).exploded for ip in args.firewall_exemption]
+        ip_list = '{ ' + ', '.join(ips) + ' }'
+        L.info("Punching hole for {}".format(ips))
+        punch_hole_in_firewall(pf, ip_list)
+
     context_dict = import_by_filename(args.context).CONTEXT
     cmd_line_overrides = [
         'run_directory',

ssh_keys/keygen.sh

@@ -28,7 +28,7 @@ def _output_filename(input_file, output_file, output_format):
         if output_file:
             return output_file
 
-        return  "{}.remote.{}".format(
+        return "{}.remote.{}".format(
             os.path.splitext(os.path.basename(input_file))[0], output_format)
 
     @staticmethod

tools/run_tests.py

@@ -76,7 +76,7 @@ def _configure_logger(logger_name, stream_format, file_format, logfile, level, l
             logger.addHandler(file_handler)
 
         if stream_format:
-            formatter = colorlog.ColoredFormatter(fmt=stream_format, log_colors=log_colors)
+            formatter = colorlog.TTYColoredFormatter(fmt=stream_format, log_colors=log_colors)
             stream_handler = logging.StreamHandler(stream=sys.stdout)
             stream_handler.setFormatter(formatter)
             logger.addHandler(stream_handler)

tools/wrap_local_tests.py

@@ -73,6 +73,12 @@ def lookup(self, hostname, timeout, server=None):
         else:
             cmd = ['dig', "+time={}".format(timeout), hostname]
 
-        stdout = self._execute(cmd)[0]
+        # Prevent the output from being empty
+        stdout = None
+        while not stdout:
+            stdout = self._execute(cmd)[0]
+            if not stdout:
+                L.verbose("dig output was empty; doing another lookup.")
+
         L.verbose("dig output: {}".format(stdout))
         return Dig._parse_output(stdout)

xv_leak_tools/log.py

@@ -22,7 +22,7 @@ def block_ip(self, ip):
         # Delay initialize the PFCtl object to prevent VPN application connect from removing our
         # reference to the pf firewall. Some VPN apps take full ownership of the firewall which can
         # mean that the firewall will be disabled unless we initialize here.
-        if self._pfctl == None:
+        if self._pfctl is None:
             self._pfctl = PFCtl()
 
         self._current_rules += MacOSFirewall._block_ip_rules(ip)

xv_leak_tools/test_components/dns_tool/dig.py

@@ -1,6 +1,5 @@
 from xv_leak_tools.factory import Builder
 from xv_leak_tools.test_components.route.macos.macos_route import MacOSRoute
-from xv_leak_tools.test_components.route.windows.windows_route import WindowsRoute
 from xv_leak_tools.test_components.route.linux.linux_route import LinuxRoute
 from xv_leak_tools.test_components.component import ComponentNotSupported
 
@@ -16,5 +15,6 @@ def build(self, device, config):
         elif device.os_name() == 'macos':
             return MacOSRoute(device, config)
         elif device.os_name() == 'windows':
+            from xv_leak_tools.test_components.route.windows.windows_route import WindowsRoute
             return WindowsRoute(device, config)
         raise ComponentNotSupported("Can't build a route component on {}".format(device.os_name()))

xv_leak_tools/test_components/firewall/macos/macos_firewall.py

@@ -1,6 +1,6 @@
 import re
 
-import netaddr
+import netaddr # pylint: disable=import-error
 
 from xv_leak_tools.test_components.route.route import Route, RouteEntry
 from xv_leak_tools.test_device.connector_helper import ConnectorHelper

xv_leak_tools/test_components/route/route_builder.py

@@ -5,6 +5,7 @@
 from abc import ABCMeta, abstractmethod
 
 from xv_leak_tools.exception import XVEx
+from xv_leak_tools.helpers import current_os
 from xv_leak_tools.log import L
 from xv_leak_tools.network.common import RE_IPV4_ADDRESS
 from xv_leak_tools.test_components.vpn_application.vpn_application import VPNApplication
@@ -92,6 +93,40 @@ def detect(self):
         L.debug("Detected the following info about openvpn: {}".format(vpn_info))
         return vpn_info
 
+# TODO: This is very ad-hoc for now. Some VPNs use Network Extensions to implement their VPN, e.g.
+# for IKEv2. Currently haven't found a reliable way to distinguish these VPNs yet so for now they
+# are just classes as "network extension" protocols. This will likely need work.
+class NEDetector(VPNDetector):
+
+    # pylint: disable=too-few-public-methods
+
+    def __init__(self, device):
+        self._device = device
+        self._ne_processes_cache = None
+
+    def _ne_processes(self):
+        if self._ne_processes_cache:
+            return self._ne_processes_cache
+        self._ne_processes_cache = []
+        self._ne_processes_cache += self._device.pgrep('neagent')
+        return self._ne_processes_cache
+
+    def detect(self):
+        if current_os() != "macos":
+            return None
+
+        L.debug("Trying to determine if we're using a macOS network extension")
+        vpn_info = VPNInfo()
+
+        if not self._ne_processes():
+            L.debug('Not using a network extension')
+            return None
+
+        L.info("Detected a VPN network extension (unknown protocol)")
+
+        vpn_info.vpn_processes = self._ne_processes()
+        return vpn_info
+
 class OpenVPNDetector(VPNDetector):
 
     # pylint: disable=too-few-public-methods
@@ -335,7 +370,8 @@ def __init__(self, app_path, device, config):
         self._app_path = app_path
         self._vpn_detectors = [
             OpenVPNDetector(self._device),
-            L2TPDetector(self._device)
+            L2TPDetector(self._device),
+            NEDetector(self._device),
         ]
         self._routes_before_connect = None
 

xv_leak_tools/test_components/route/windows/windows_route.py

@@ -28,12 +28,12 @@ def _check_virtual_env(self):
         '''Check that virtualenv was properly setup on the device. If we don't check and there's
         a problem then this can lead to cryptic errors from the helper function below. Better to
         catch this early on.'''
+        # source doesn't error if the sourced file is missing. This file "shouldn't" ever be
+        # missing but let's check.
         try:
-            # source doesn't error if the sourced fail is missing. This file "shouldn't" ever be
-            # missing but let's check.
             self.check_command(['stat', '.pythonlocation'])
             self.check_command(['source', 'activate'])
-        except XVProcessException as _:
+        except XVProcessException:
             raise XVEx("Virtualenv doesn't appear to be setup properly for device '{}'".format(
                 self._device.device_id()))
 
@@ -56,6 +56,14 @@ def _create_temp_directory(self):
         #     L.warning("Couldn't chown temp directory {} back to user {}".format(
         #         self._device.temp_directory(), self._device.tools_user()))
 
+    def _remove_temp_directory(self):
+        L.debug("Removing temp directory {}".format(self._device.temp_directory()))
+        cmd = ['sudo', '-n', 'rm', '-rf', self._device.temp_directory()]
+        ret, stdout, stderr = self._device.connector().execute(cmd)
+        # TODO: warn instead of raising
+        if ret != 0:
+            raise XVProcessException(cmd, ret, stdout, stderr)
+
     def check_command(self, cmd, root=False):
         cmd = ConnectorHelper._sanitize_cmd(cmd)
         ret, stdout, stderr = self.execute_command(cmd, root)

xv_leak_tools/test_components/vpn_application/desktop_vpn_application.py

@@ -16,7 +16,7 @@
 
 class SimpleSSHConnector(Connector):
 
-    # pylint: disable=too-few-public-methods,too-many-arguments
+    # pylint: disable=too-few-public-methods,too-many-arguments,too-many-instance-attributes
 
     # TODO: Account password shouldn't be needed. Paramiko is either or.
     def __init__(self, ips, username, account_password=None, ssh_key=None, ssh_password=None):
@@ -56,7 +56,6 @@ def _create_route_to_ip(self, ip):
     def _ssh_connect(self):
         connect_errors = ""
         for ip in self._ips:
-            self._create_route_to_ip(ip)
             connect_dict = {
                 'hostname': ip,
                 'username': self._username,
@@ -159,22 +158,19 @@ def execute(self, cmd, root=False):
         if root and self._username.decode() != 'root':
             cmd = ['sudo', '-n'] + cmd
 
+        cmd = ["[", "-f", ".source", "]", "&&", ".", ".source;"] + cmd
+
         L.verbose("SimpleSSHConnector: Executing remote command: '{}'".format(cmd))
 
-        chan = self._ssh_client.get_transport().open_session()
-        chan.settimeout(2)
         # TODO: This will block if stdin on the remote machine blocks. Can't ctrl-c. Consider a
         # custom wrapper with select and timeout
-        chan.exec_command(' '.join(cmd))
-        bufsize = -1
-        # stdin = chan.makefile('wb', bufsize)
-        stdout = chan.makefile('r', bufsize)
-        stderr = chan.makefile_stderr('r', bufsize)
-
-        retcode = chan.recv_exit_status()
-        # paramiko splits output into a list AND keeps the newline. So joining like this puts it
-        # back to how we'd "normally" expect output.
-        return retcode, ''.join(stdout), ''.join(stderr)
+        # get_pty is required for accessing environment variables.
+        output = self._ssh_client.exec_command(' '.join(cmd))
+        return_code = output[1].channel.recv_exit_status()
+        stdout = output[1].read().decode().strip()
+        stderr = output[2].read().decode().strip()
+
+        return return_code, stdout, stderr
 
     def push(self, src, dst):
         self._ensure_connected()

xv_leak_tools/test_device/connector_helper.py

@@ -1,5 +1,6 @@
 import json
 import os
+import ipaddress
 
 from xv_leak_tools.log import L
 from xv_leak_tools.test_framework.test_case import TestCase
@@ -13,6 +14,18 @@ def __init__(self, devices, parameters):
         self.remote_test_config = self.parameters
         self.connector_helper = ConnectorHelper(self.test_device)
 
+    def host_ips(self):
+        guest_ips = self.test_device.ips()
+        guest_subnets = [ipaddress.ip_network(ip).supernet(new_prefix=24) for ip in guest_ips]
+
+        ips = set()
+        for ip in self.test_device.local_ips():
+            for subnet in guest_subnets:
+                if ip in subnet:
+                    ips.add(ip.exploded)
+
+        return ips
+
     def test(self):
         L.info("Running test {} completely on remote device".format(
             self.remote_test_config['name']))
@@ -22,10 +35,11 @@ def test(self):
         self.connector_helper.write_remote_file_from_contents(
             remote_config_path, json.dumps([self.remote_test_config]))
 
-        cmd = ['run_tests.py', self.test_device.output_directory(), '-c', remote_config_path]
-
+        cmd = ['./run_tests.sh', '-o', self.test_device.output_directory(),
+               '-c', remote_config_path, '-x', '../contexts/internal_context.py',
+               '-f', ' '.join(self.host_ips())]
         # TODO: Use .requires_root to figure out if the test needs root.
-        ret, stdout, stderr = self.connector_helper.execute_python(cmd, root=True)
+        ret, stdout, stderr = self.connector_helper.execute_command(cmd, root=True)
 
         # TODO: Reconsider all this. Maybe we shouldn't display the output here as it screws up the
         # logs. Possibly better is to have ERROR log go to stderr then we just dump stderr here
@@ -38,7 +52,7 @@ def test(self):
                 # TODO: Logger not satisfying my needs!
                 print(line)
             L.info('*' * 80)
-            L.info('END stdout from remote machine:')
+            L.info('END stdout from remote machine')
             L.info('*' * 80)
 
         if stderr:
@@ -49,7 +63,7 @@ def test(self):
                 # TODO: Logger not satisfying my needs!
                 print(line)
             L.error('*' * 80)
-            L.error('END stderr from remote machine:')
+            L.error('END stderr from remote machine')
             L.error('*' * 80)
 
         if ret == 0: