run_sk_stress_test

#!/usr/bin/env python3
# ===--- run_sk_stress_test -----------------------------------------------===
#
#  This source file is part of the Swift.org open source project
#
#  Copyright (c) 2014 - 2022 Apple Inc. and the Swift project authors
#  Licensed under Apache License v2.0 with Runtime Library Exception
#
#  See https://swift.org/LICENSE.txt for license information
#  See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
#
# ===----------------------------------------------------------------------===

"""A run script to be executed as a Jenkins build step."""

import sys
import os
import json
import argparse
import platform

import common
from typing import List

script_dir = os.path.abspath(os.path.dirname(__file__))


def main() -> int:
    if platform.system() != 'Darwin':
        raise common.UnsupportedPlatform

    common.debug_print('** RUN SOURCEKIT STRESS TESTER **')
    os.chdir(os.path.dirname(__file__))

    args = parse_args()
    common.set_swift_branch(args.swift_branch)
    workspace = common.private_workspace('.')

    if not args.skip_tools_clone:
        common.clone_repos()
        clone_swift_syntax(workspace, args.swift_branch)
        clone_stress_tester(workspace, args.swift_branch)

    if not args.skip_tools_build:
        build_swift_toolchain(workspace, args)

    if not args.skip_runner:
        if not execute_runner(workspace, args):
            return 1

    return 0


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument('swift_branch')
    parser.add_argument('--sandbox', action='store_true')
    parser.add_argument('--projects',
                        metavar='PATH',
                        help='JSON project file',
                        default='projects.json')
    parser.add_argument('--filter-by-tag',
                        metavar='TAG',
                        help='Only run project actions with the given tag',
                        default='sourcekit')
    parser.add_argument('--filter-by-project',
                        metavar='PROJECT',
                        help='Only stress test the given project')
    parser.add_argument('--sourcekit-xfails',
                        metavar='PATH',
                        help='JSON file specifying expected sourcekit failures',
                        default='sourcekit-xfails.json')
    parser.add_argument('--verbose',
                        action='store_true')
    parser.add_argument('--assertions',
                        help='Build Swift with asserts',
                        action='store_true')
    parser.add_argument('--debug',
                        help='Build Swift in debug mode',
                        action='store_true')
    parser.add_argument('--swiftc',
                        metavar='PATH',
                        help='swiftc executable')
    parser.add_argument('--sk-swiftc-wrapper',
                        metavar='PATH',
                        help='sk-swiftc-wrapper executable')
    parser.add_argument('--sk-stress-test',
                        metavar='PATH',
                        help='sk-stress-test executable')
    parser.add_argument('--skip-tools-build',
                        action='store_true')
    parser.add_argument('--skip-ci-steps',
                        action='store_true')
    parser.add_argument('--skip-tools-clone',
                        action='store_true')
    parser.add_argument('--skip-runner',
                        action='store_true')
    parser.add_argument('--add-swift-flags',
                        metavar='FLAGS',
                        help='''
                             add flags to each Swift invocation (note: field
                             names from projects.json enclosed in {} will be
                             replaced with their value)
                             ''',
                        default='')
    parser.add_argument('--add-xcodebuild-flags',
                        metavar='FLAGS',
                        help='''
                             add flags to each xcodebuild invocation (note: field
                             names from projects.json enclosed in {} will be
                             replaced with their value)
                             ''',
                        default='')
    parser.add_argument('--cmake-c-launcher',
                        metavar="PATH",
                        help='the absolute path to set CMAKE_C_COMPILER_LAUNCHER for build-script')
    parser.add_argument('--cmake-cxx-launcher',
                        metavar='PATH',
                        help='the absolute path to set CMAKE_CXX_COMPILER_LAUNCHER for build-script')
    return parser.parse_args()

def get_swiftc_path(workspace: str, args: argparse.Namespace) -> str:
    if args.swiftc:
        return args.swiftc
    else:
        return os.path.join(workspace, 'build/compat_macos/install/toolchain/usr/bin/swiftc')

def get_sk_swiftc_wrapper_path(workspace: str, args: argparse.Namespace) -> str:
    if args.sk_swiftc_wrapper:
        return args.sk_swiftc_wrapper
    else:
        # If not explicitly specified, fall back to finding `sk-swiftc-wrapper` next to `swiftc`
        swiftc_path = get_swiftc_path(workspace, args)
        return os.path.join(os.path.dirname(swiftc_path), 'sk-swiftc-wrapper')

def get_sk_stress_test_path(workspace: str, args: argparse.Namespace) -> str:
    if args.sk_stress_test:
        return args.sk_stress_test
    else:
        # If not explicitly specified, fall back to finding `sk-stress-test` next to `swiftc`
        swiftc_path = get_swiftc_path(workspace, args)
        return os.path.join(os.path.dirname(swiftc_path), 'sk-stress-test')

def get_sandbox_profile_flags() -> List[str]:
    return [
        '--sandbox-profile-xcodebuild',
        '../../../workspace-private/swift-source-compat-suite-sandbox/sandbox_xcodebuild.sb',
        '--sandbox-profile-package',
        '../../../workspace-private/swift-source-compat-suite-sandbox/sandbox_package.sb'
    ]

def clone_stress_tester(workspace: str, swift_branch: str) -> None:
    stress_clone_cmd = [
        'git','clone', '-q', '-b', swift_branch, '--recursive',
        'https://github.com/apple/swift-stress-tester',
        '{}/swift-stress-tester'.format(workspace)
    ]
    common.check_execute(stress_clone_cmd, timeout=-1)

def clone_swift_syntax(workspace: str, swift_branch: str) -> None:
    if swift_branch in ['release/5.7', 'release/5.6',
                        'release/5.5', 'release/5.4']:
        syntax_clone_cmd = [
            'git', 'clone', '-q', '-b', swift_branch, '--recursive',
            'https://github.com/apple/swift-syntax',
            '{}/swift-syntax'.format(workspace)
        ]
        common.check_execute(syntax_clone_cmd, timeout=-1)


def execute_runner(workspace: str, args: argparse.Namespace) -> bool:
    swiftc_path = get_swiftc_path(workspace, args)
    wrapper_path = get_sk_swiftc_wrapper_path(workspace, args)
    stress_tester_path = get_sk_stress_test_path(workspace, args)

    extra_runner_args = []
    if args.sandbox:
        extra_runner_args += get_sandbox_profile_flags()

    if args.add_swift_flags:
        extra_runner_args += ['--add-swift-flags=%s' % args.add_swift_flags]

    if args.add_xcodebuild_flags:
        extra_runner_args += ['--add-xcodebuild-flags=%s' % args.add_xcodebuild_flags]

    if args.filter_by_tag:
        extra_runner_args += ['--include-actions', '"tags" in locals() and "{}" in tags.split()'.format(args.filter_by_tag)]
    
    if args.filter_by_project:
        extra_runner_args += ['--include-repos', 'path == "{}"'.format(args.filter_by_project)]

    runner = StressTesterRunner(wrapper_path, stress_tester_path, swiftc_path, args.projects, args.swift_branch, os.path.abspath(args.sourcekit_xfails))
    passed = runner.run(extra_runner_args)

    return passed


def get_preset_name(args):
    build_type = ''
    if args.debug:
        build_type = build_type + 'D'
    else:
        build_type = build_type + 'R'

    if args.assertions:
        build_type = build_type + 'A'

    return f'sourcekit_stress_test_macos_{build_type}'

def build_swift_toolchain(workspace: str, args: argparse.Namespace) -> None:
    build_command = [
        os.path.join(workspace, 'swift/utils/build-script'),
        f'--preset={get_preset_name(args)}',
        f'install_destdir={workspace}/build/compat_macos/install',
        f'install_prefix=/toolchain/usr',
        f'install_symroot={workspace}/build/compat_macos/symroot',
        f'installable_package={workspace}/build/compat_macos/root.tar.gz',
        f'symbols_package={workspace}/build/compat_macos/root-symbols.tar.gz',
    ]
    common.check_execute(build_command, timeout=9999999)


def processed_files_contain(processed_files: List[str], file_path: str) -> bool:
    """
    Returns `True` if a path in `processed_files` contains `file_path` as a substring, `False` otherwise
    """
    for processed_file in processed_files:
        if file_path in processed_file:
            return True
    return False


class StressTesterRunner(object):
    """sets up the Swift compatibility suite runner to use the stress tester's swiftc-wrapper, executes it, and processes its output for failures."""

    wrapper: str
    stress_tester: str
    swiftc: str
    branch: str

    def __init__(self, wrapper: str, stress_tester: str, swiftc: str, projects_path: str, branch: str, xfails_path: str):
        self.wrapper = wrapper
        self.stress_tester = stress_tester
        self.swiftc = swiftc

        self.xfails_path = xfails_path
        self.projects_path = projects_path
        self.swift_branch = branch

        self.compat_runner_failed = False


    def run(self, extra_runner_args: List[str] = []) -> bool:
        # temporary file paths
        filtered_projects = os.path.join(script_dir, 'stress-tester-projects.json')
        results = os.path.join(script_dir, 'stress-tester-results.json')
        request_durations = os.path.join(script_dir, 'stress-tester-request-durations.json')

        # remove temporary files if they already exist
        self._cleanup([filtered_projects, results, request_durations])

        run_env = {
            'SK_STRESS_TEST': self.stress_tester,
            'SK_STRESS_SWIFTC': self.swiftc,
            'SK_STRESS_SILENT': 'true',
            'SK_STRESS_AST_BUILD_LIMIT': '1000',
            'SK_STRESS_OUTPUT': results,
            'SK_XFAILS_PATH': self.xfails_path,
            'SK_STRESS_ACTIVE_CONFIG': self.swift_branch,
            'SK_STRESS_REWRITE_MODES': 'none concurrent insideout',
            'SK_STRESS_REQUEST_DURATIONS_FILE': request_durations,
            'EnableSwiftBuildSystemIntegration': 'NO'
        }
        run_env.update(os.environ)
        run_cmd = ['./runner.py',
          '--projects', filtered_projects,
          '--verbose',
          '--swiftc', self.swiftc,
          '--override-swift-exec', self.wrapper,
          '--swift-branch', self.swift_branch,
          '--job-type', 'stress-tester',
          '--default-timeout', str(-1),
          '--build-config', 'debug',
          '--only-latest-versions',
          # Don't build projects in parallel because stress testing a single project already utilises the CPU 100% and stress testing multiple causes timeouts.
          '--process-count', '1',
          # archs override is set to arm64 for generic/iOS actions that would otherwise invoke the stress tester for both arm64 and armv7
          '--add-xcodebuild-flags', 'ARCHS={archs_override}']

        if extra_runner_args:
            run_cmd.extend(extra_runner_args)

        self._filter_projects(filtered_projects)
        try:
            common.check_execute(run_cmd, timeout=-1, env=run_env)
        except common.ExecuteCommandFailure:
            self.compat_runner_failed = True

        if os.path.isfile(request_durations):
            print('\n---------- Raw request durations for further analysis ----------\n')
            with open(request_durations, 'r') as request_durations_file:
                print(request_durations_file.read())

        success = self._process_output(results, self.xfails_path)

        return success


    def _process_output(self, results_path: str, xfails_path: str) -> bool:
        if not os.path.isfile(results_path):
            return not self.compat_runner_failed

        with open(results_path, 'r') as results_file:
            results = json.load(results_file)
        with open(xfails_path, 'r') as xfails_file:
            xfails = json.load(xfails_file)

        xfails_not_processed = []
        for xfail in xfails:
            # The XFails start with '*' to match any path prefix. Remove it so we can perform substring matching in processed_files_contain
            xfailed_file = xfail['path']
            if xfailed_file[0] == '*':
                xfailed_file = xfailed_file[1:]
            if not processed_files_contain(results['processedFiles'], xfailed_file):
                xfails_not_processed.append(xfail)

        num_failures = len(results['issues'])
        num_xfails = sum(len(value) for _, value in results['expectedIssues'].items())
        num_xfail_issues = len(results['expectedIssues'])
        unmatched = results['unmatchedExpectedIssues']

        success = self.compat_runner_failed == False and num_failures == 0 and len(unmatched) == 0

        if num_xfails > 0:
            print('Expected stress tester issues:')
            for url, issues in results['expectedIssueMessages'].items():
                for (index, issue) in enumerate(issues):
                    self._print_issue(index, issue, url)
            print('\n========================================')

        if len(xfails_not_processed) > 0:
            print('XFails not processed:')
            print('The files that are covered by these XFails were not tested in the current Stress Tester Run.')
            print('The reason for this might be:')
            print(' * The project that contains the file was not tested in the current run')
            print(' * A different file from the same compiler invocation failed and thus this file was not tested')
            print(' * The file has been removed in the original project')
            print('')
            for xfail in xfails_not_processed:
                print('      {}'.format(json.dumps(xfail)))
            print('\n========================================')

        if num_failures > 0:
            print('Unexpected stress tester issues:')
            for (index, issue) in enumerate(results['issueMessages']):
                self._print_issue(index, issue)
            print('\n========================================')


        print('SourceKit Stress Tester summary:')

        print('  {} underlying source compatibility build'.format('failed' if self.compat_runner_failed else 'passed'))
        
        print('  {} XFails not processed'.format(len(xfails_not_processed)))
        if num_failures > 0:
            print('      > see "XFails not processed" how to handle them. This is an info, not an error')

        print('  {} unexpected stress tester failures'.format(num_failures))
        if num_failures > 0:
            print('      > see "Unexpected stress tester issues" above for individual failures')

        print('  {} expected stress tester failures tracked by {} issues'.format(num_xfails, num_xfail_issues))

        if not self.compat_runner_failed and unmatched:
            print('  {} expected stress tester failures not seen'.format(len(unmatched)))
            print('      > if resolved, remove "{}" from "branches" in the following entries in {} or the entire entry if no branches remain:'.format(self.swift_branch, self.xfails_path))
            for xfail in unmatched:
                print('      {}'.format(json.dumps(xfail)))

        print('========================================')
        print('Result: {result}'.format(result=('PASS' if success else 'FAIL')))
        print('========================================')

        return success

    @staticmethod
    def _print_issue(index: int, issue, url = None):
        if url != None:
            print(u'\n{}. [{}] {}'.format(index + 1, url, issue))
        else:
            print(u'\n{}. {}'.format(index + 1, issue))


    def _filter_projects(self, output: str) -> str:
        with open(self.projects_path) as json_file:
            projects = json.load(json_file)
            for project in projects:
                for action in project['actions']:
                    # This is used in combination with --add-xcodebuild-flags
                    # to prevent running the stress tester over the same files
                    # for difference architectures. generic/iOS actions
                    # normally invoke the wrapper for both arm64 and armv7 and
                    # generic/macOS actions for arm64 and x86_64.
                    if action.get('destination') == 'generic/platform=iOS':
                        action['archs_override'] = 'arm64'
                    elif action.get('destination') == 'generic/platform=macOS':
                        action['archs_override'] = 'x86_64'
                    else:
                        action['archs_override'] = '$ARCHS'

        with open(output, 'w') as outfile:
            json.dump(projects, outfile, indent=4)
        return output

    @staticmethod
    def _cleanup(paths: List[str]) -> None:
        for path in paths:
            try:
                os.remove(path)
            except OSError:
                pass

if __name__ == '__main__':
    sys.exit(main())