fuzzing_ci.sh

#!/usr/bin/env bash
##
# Copyright 2018 Google
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Run fuzz testing using xcodebuild. Given a total fuzzing duration, select
# a fuzzing target to run for half of the fuzzing duration. The remaining
# duration is split equally over the rest of the fuzzing targets. Must be run
# from the project root directory. The script is intended to execute on travis
# cron jobs, but can run locally (on a macOS) from the project root.

# Total time allowed for fuzzing in seconds (25 minutes for now).
readonly ALLOWED_TIME=1500

# An invalid target that is used to retrieve the list of available targets in
# the returned error message.
readonly INVALID_TARGET="INVALID_TARGET"
# The NONE target that does not perform fuzzing. It is valid but useless.
readonly NONE_TARGET="NONE"

# Script exit codes.
readonly EXIT_SUCCESS=0
readonly EXIT_FAILURE=1

# Get day of the year to use it when selecting a main target.
readonly DOY="$(date +%j)"

# How many last lines of the log to output for a failing job. This is to avoid
# exceeding Travis log size limit (4 Mb).
readonly LINES_TO_OUTPUT=500

# Run fuzz testing only if executing on xcode >= 9 because fuzzing requires
# Clang that supports -fsanitize-coverage=trace-pc-guard.
xcode_version="$(xcodebuild -version | head -n 1)"
xcode_version="${xcode_version/Xcode /}"
xcode_major="${xcode_version/.*/}"

if [[ "${xcode_major}" -lt 9 ]]; then
  echo "Unsupported version of xcode."
  exit ${EXIT_SUCCESS}
fi

# Helper to run fuzz tests using xcodebuild. Takes the fuzzing target as the
# first argument and the fuzzing duration as the second argument.
function run_xcode_fuzzing() {
  xcodebuild \
    -workspace 'Firestore/Example/Firestore.xcworkspace' \
    -scheme 'Firestore_FuzzTests_iOS' \
    -sdk 'iphonesimulator' \
    -destination 'platform=iOS Simulator,name=iPhone 7' \
    FUZZING_TARGET="$1" \
    FUZZING_DURATION="$2" \
    test
}

# Run fuzz testing with an INVALID_TARGET and grep the response message line
# that contains the string 'Available targets: {'.
fuzzing_targets_line="$(run_xcode_fuzzing ${INVALID_TARGET} "0" \
    | grep "Available targets: {")"

# The fuzzing_targets_line string contains { TARGET1 TARGET2 ... TARGETN }.
# The following command extracts the targets and splits them into an array
# in the variable all_fuzzing_targets.
read -r -a all_fuzzing_targets <<< "$(echo ${fuzzing_targets_line} \
    | cut -d "{" -f2 | cut -d "}" -f1)"

# Remove the NONE target, if it exists.
for i in "${!all_fuzzing_targets[@]}"; do
  if [[ "${all_fuzzing_targets[${i}]}" == ${NONE_TARGET} ]]; then
    unset all_fuzzing_targets[${i}]
  fi
done

# Count all targets.
readonly fuzzing_targets_count="${#all_fuzzing_targets[@]}"

# Select a primary target to fuzz test for longer. Changes daily. The variable
# todays_primary_target contains the index of the target selected as primary.
todays_primary_target="$(( ${DOY} % ${fuzzing_targets_count} ))"

# Spend half of allowed time on the selected primary target and half on the
# remaining targets.
primary_target_time="$(( ${ALLOWED_TIME} / 2 ))"
other_targets_time="$(( ${ALLOWED_TIME} / 2 / $(( ${fuzzing_targets_count} - 1)) ))"

script_return=${EXIT_SUCCESS}
for i in "${!all_fuzzing_targets[@]}"; do
  fuzzing_duration="${other_targets_time}"
  if [[ "${i}" -eq "${todays_primary_target}" ]]; then
    fuzzing_duration="${primary_target_time}"
  fi
  fuzzing_target="${all_fuzzing_targets[${i}]}"

  echo "Running fuzzing target ${fuzzing_target} for ${fuzzing_duration} seconds..."
  fuzzing_results="$(run_xcode_fuzzing "${fuzzing_target}" "${fuzzing_duration}")"
  # Get the last line of the fuzzing results.
  fuzzing_results_last_line="$(echo "${fuzzing_results}" | tail -n1)"
  # If fuzzing succeeded, the last line will start with "Done"; otherwise,
  # print all fuzzing results.
  if [[ "${fuzzing_results_last_line}" == Done* ]]; then
    echo "Fuzz testing for target ${fuzzing_target} succeeded. Ignore the ** TEST FAILED ** message."
  else
    echo "Error: Fuzz testing for target ${fuzzing_target} failed."
    script_return=${EXIT_FAILURE}
    echo "Fuzzing logs:"
    echo "${fuzzing_results}" | tail -n${LINES_TO_OUTPUT}
    echo "End fuzzing logs"
  fi
done

exit ${script_return}