From bf925a78576aa830c94a0479845eca037f49b765 Mon Sep 17 00:00:00 2001 From: Phil Davis Date: Tue, 8 Oct 2019 17:35:23 +0545 Subject: [PATCH] Update drone starlark with latest code 20191011 --- .drone.starlark | 520 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 379 insertions(+), 141 deletions(-) diff --git a/.drone.starlark b/.drone.starlark index f508990a..49a1a80b 100644 --- a/.drone.starlark +++ b/.drone.starlark @@ -69,14 +69,15 @@ def beforePipelines(): return codestyle() + jscodestyle() + phpstan() + phan() def stagePipelines(): + buildPipelines = build() jsPipelines = javascript() phpunitPipelines = phptests('phpunit') phpintegrationPipelines = phptests('phpintegration') acceptancePipelines = acceptance() - if (jsPipelines == False) or (phpunitPipelines == False) or (phpintegrationPipelines == False) or (acceptancePipelines == False): + if (buildPipelines == False) or (jsPipelines == False) or (phpunitPipelines == False) or (phpintegrationPipelines == False) or (acceptancePipelines == False): return False - return jsPipelines + phpunitPipelines + phpintegrationPipelines + acceptancePipelines + return buildPipelines + jsPipelines + phpunitPipelines + phpintegrationPipelines + acceptancePipelines def afterPipelines(): return [ @@ -344,6 +345,95 @@ def phan(): return pipelines +def build(): + pipelines = [] + + if 'build' not in config: + return pipelines + + default = { + 'phpVersions': ['7.0'], + 'commands': [ + 'make dist' + ], + 'extraEnvironment': {}, + 'configureTarOnTag': False, + } + + if 'defaults' in config: + if 'build' in config['defaults']: + for item in config['defaults']['build']: + default[item] = config['defaults']['build'][item] + + matrix = config['build'] + + if type(matrix) == "bool": + if matrix: + # the config has 'build' true, so specify an empty dict that will get the defaults + matrix = {} + else: + return pipelines + + params = {} + for item in default: + params[item] = matrix[item] if item in matrix else default[item] + + for phpVersion in params['phpVersions']: + result = { + 'kind': 'pipeline', + 'type': 'docker', + 'name': 'build', + 'workspace' : { + 'base': '/var/www/owncloud', + 'path': 'server/apps/%s' % config['app'] + }, + 'steps': [ + { + 'name': 'build', + 'image': 'owncloudci/php:%s' % phpVersion, + 'pull': 'always', + 'environment': params['extraEnvironment'], + 'commands': params['commands'] + } + ] + ([ + { + 'name': 'github_release', + 'image': 'plugins/github-release', + 'pull': 'always', + 'settings': { + 'checksum': 'sha256', + 'file_exists': 'overwrite', + 'files': 'build/dist/%s.tar.gz' % config['app'], + 'prerelease': True, + }, + 'environment': { + 'GITHUB_TOKEN': { + 'from_secret': 'github_token' + }, + }, + 'when': { + 'event': [ + 'tag' + ] + }, + } + ] if params['configureTarOnTag'] else []), + 'depends_on': [], + 'trigger': { + 'ref': [ + 'refs/pull/**', + 'refs/tags/**' + ] + } + } + + for branch in config['branches']: + result['trigger']['ref'].append('refs/heads/%s' % branch) + + pipelines.append(result) + + return pipelines + def javascript(): pipelines = [] @@ -353,7 +443,7 @@ def javascript(): default = { 'coverage': False, 'logLevel': '2', - 'extraSetup': None, + 'extraSetup': [], 'extraServices': [], 'extraEnvironment': {}, 'extraCommandsBeforeTestRun': [], @@ -389,8 +479,8 @@ def javascript(): installCore('daily-master-qa', 'sqlite', False) + installApp('7.0') + setupServerAndApp('7.0', params['logLevel']) + + params['extraSetup'] + [ - params['extraSetup'], { 'name': 'js-tests', 'image': 'owncloudci/php:7.0', @@ -447,11 +537,13 @@ def phptests(testType): 'coverage': True, 'includeKeyInMatrixName': False, 'logLevel': '2', - 'extraSetup': None, + 'cephS3': False, + 'scalityS3': False, + 'extraSetup': [], 'extraServices': [], 'extraEnvironment': {}, 'extraCommandsBeforeTestRun': [], - 'extraApps': [], + 'extraApps': {}, } if 'defaults' in config: @@ -477,6 +569,15 @@ def phptests(testType): for item in default: params[item] = matrix[item] if item in matrix else default[item] + if ((config['app'] != 'files_primary_s3') and ((params['cephS3'] != False) or (params['scalityS3'] != False))): + # If we are not already 'files_primary_s3' and we need S3 storage, then install the 'files_primary_s3' app + extraAppsDict = { + 'files_primary_s3': 'composer install' + } + for app, command in params['extraApps'].items(): + extraAppsDict[app] = command + params['extraApps'] = extraAppsDict + for phpVersion in params['phpVersions']: if testType == 'phpunit': @@ -512,8 +613,10 @@ def phptests(testType): installApp(phpVersion) + installExtraApps(phpVersion, params['extraApps']) + setupServerAndApp(phpVersion, params['logLevel']) + + setupCeph(params['cephS3']) + + setupScality(params['scalityS3']) + + params['extraSetup'] + [ - params['extraSetup'], { 'name': '%s-tests' % testType, 'image': 'owncloudci/php:%s' % phpVersion, @@ -524,7 +627,11 @@ def phptests(testType): ] } ], - 'services': databaseService(db) + params['extraServices'], + 'services': + databaseService(db) + + cephService(params['cephS3']) + + scalityService(params['scalityS3']) + + params['extraServices'], 'depends_on': [], 'trigger': { 'ref': [ @@ -580,13 +687,19 @@ def acceptance(): 'filterTags': '', 'logLevel': '2', 'emailNeeded': False, - 'extraSetup': None, + 'ldapNeeded': False, + 'cephS3': False, + 'scalityS3': False, + 'extraSetup': [], 'extraServices': [], 'extraEnvironment': {}, 'extraCommandsBeforeTestRun': [], - 'extraApps': [], + 'extraApps': {}, 'useBundledApp': False, 'includeKeyInMatrixName': False, + 'runAllSuites': False, + 'runCoreTests': False, + 'numberOfParts': 1, } if 'defaults' in config: @@ -602,136 +715,176 @@ def acceptance(): else: suites = matrix['suites'] - for suite, shortSuiteName in suites.items(): + for suite, alternateSuiteName in suites.items(): isWebUI = suite.startswith('webUI') isAPI = suite.startswith('api') isCLI = suite.startswith('cli') - if isAPI or isCLI: - default['browsers'] = [''] - params = {} for item in default: params[item] = matrix[item] if item in matrix else default[item] + if isAPI or isCLI: + params['browsers'] = [''] + + needObjectStore = (params['cephS3'] != False) or (params['scalityS3'] != False) + + if ((config['app'] != 'files_primary_s3') and (needObjectStore)): + # If we are not already 'files_primary_s3' and we need S3 object storage, then install the 'files_primary_s3' app + extraAppsDict = { + 'files_primary_s3': 'composer install' + } + for app, command in params['extraApps'].items(): + extraAppsDict[app] = command + params['extraApps'] = extraAppsDict + for server in params['servers']: for browser in params['browsers']: for phpVersion in params['phpVersions']: for db in params['databases']: - name = 'unknown' - - if isWebUI or isAPI or isCLI: - browserString = '' if browser == '' else '-' + browser - keyString = '-' + category if params['includeKeyInMatrixName'] else '' - name = '%s%s-%s%s-%s-php%s' % (shortSuiteName, keyString, server.replace('daily-', '').replace('-qa', ''), browserString, db.replace(':', ''), phpVersion) - maxLength = 50 - nameLength = len(name) - if nameLength > maxLength: - print("Error: generated stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) - errorFound = True - - environment = {} - for env in params['extraEnvironment']: - environment[env] = params['extraEnvironment'][env] - - environment['TEST_SERVER_URL'] = 'http://server' - environment['BEHAT_FILTER_TAGS'] = params['filterTags'] - environment['BEHAT_SUITE'] = suite - - if isWebUI: - environment['SELENIUM_HOST'] = 'selenium' - environment['SELENIUM_PORT'] = '4444' - environment['BROWSER'] = browser - environment['PLATFORM'] = 'Linux' - makeParameter = 'test-acceptance-webui' - - if isAPI: - makeParameter = 'test-acceptance-api' - - if isCLI: - makeParameter = 'test-acceptance-cli' - - if params['emailNeeded']: - environment['MAILHOG_HOST'] = 'email' - - result = { - 'kind': 'pipeline', - 'type': 'docker', - 'name': name, - 'workspace' : { - 'base': '/var/www/owncloud', - 'path': 'testrunner/apps/%s' % config['app'] - }, - 'steps': - installCore(server, db, params['useBundledApp']) + - installTestrunner(phpVersion, params['useBundledApp']) + - ([ - { - 'name': 'install-federation', - 'image': 'owncloudci/core', - 'pull': 'always', - 'settings': { - 'version': server, - 'core_path': '/var/www/owncloud/federated' - } - }, - { - 'name': 'configure-federation', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'commands': [ - 'echo "export TEST_SERVER_FED_URL=http://federated" > /var/www/owncloud/saved-settings.sh', - 'cd /var/www/owncloud/federated', - 'php occ a:l', - 'php occ a:e testing', - 'php occ a:l', - 'php occ config:system:set trusted_domains 1 --value=federated', - 'php occ log:manage --level %s' % params['logLevel'], - 'php occ config:list' - ] + for runPart in range(1, params['numberOfParts'] + 1): + name = 'unknown' + + if isWebUI or isAPI or isCLI: + browserString = '' if browser == '' else '-' + browser + keyString = '-' + category if params['includeKeyInMatrixName'] else '' + partString = '' if params['numberOfParts'] == 1 else '-%d-%d' % (params['numberOfParts'], runPart) + name = '%s%s%s-%s%s-%s-php%s' % (alternateSuiteName, keyString, partString, server.replace('daily-', '').replace('-qa', ''), browserString, db.replace(':', ''), phpVersion) + maxLength = 50 + nameLength = len(name) + if nameLength > maxLength: + print("Error: generated stage name of length", nameLength, "is not supported. The maximum length is " + str(maxLength) + ".", name) + errorFound = True + + environment = {} + for env in params['extraEnvironment']: + environment[env] = params['extraEnvironment'][env] + + environment['TEST_SERVER_URL'] = 'http://server' + environment['BEHAT_FILTER_TAGS'] = params['filterTags'] + + if (params['runAllSuites'] == False): + environment['BEHAT_SUITE'] = suite + else: + environment['DIVIDE_INTO_NUM_PARTS'] = params['numberOfParts'] + environment['RUN_PART'] = runPart + + if isWebUI: + environment['SELENIUM_HOST'] = 'selenium' + environment['SELENIUM_PORT'] = '4444' + environment['BROWSER'] = browser + environment['PLATFORM'] = 'Linux' + if (params['runCoreTests']): + makeParameter = 'test-acceptance-core-webui' + else: + makeParameter = 'test-acceptance-webui' + + if isAPI: + if (params['runCoreTests']): + makeParameter = 'test-acceptance-core-api' + else: + makeParameter = 'test-acceptance-api' + + if isCLI: + if (params['runCoreTests']): + makeParameter = 'test-acceptance-core-cli' + else: + makeParameter = 'test-acceptance-cli' + + if params['emailNeeded']: + environment['MAILHOG_HOST'] = 'email' + + if params['ldapNeeded']: + environment['TEST_EXTERNAL_USER_BACKENDS'] = True + + if (needObjectStore): + environment['OC_TEST_ON_OBJECTSTORE'] = '1' + if (params['cephS3'] != False): + environment['S3_TYPE'] = 'ceph' + if (params['scalityS3'] != False): + environment['S3_TYPE'] = 'scality' + + result = { + 'kind': 'pipeline', + 'type': 'docker', + 'name': name, + 'workspace' : { + 'base': '/var/www/owncloud', + 'path': 'testrunner/apps/%s' % config['app'] }, - owncloudLog('federated') - ] if params['federatedServerNeeded'] else []) + - installApp(phpVersion) + - installExtraApps(phpVersion, params['extraApps']) + - setupServerAndApp(phpVersion, params['logLevel']) + - [ - owncloudLog('server'), - params['extraSetup'], - ] + - fixPermissions(phpVersion, params['federatedServerNeeded']) + - [ - ({ - 'name': 'acceptance-tests', - 'image': 'owncloudci/php:%s' % phpVersion, - 'pull': 'always', - 'environment': environment, - 'commands': params['extraCommandsBeforeTestRun'] + [ - 'touch /var/www/owncloud/saved-settings.sh', - '. /var/www/owncloud/saved-settings.sh', - 'make %s' % makeParameter + 'steps': + installCore(server, db, params['useBundledApp']) + + installTestrunner(phpVersion, params['useBundledApp']) + + ([ + { + 'name': 'install-federation', + 'image': 'owncloudci/core', + 'pull': 'always', + 'settings': { + 'version': server, + 'core_path': '/var/www/owncloud/federated' + } + }, + { + 'name': 'configure-federation', + 'image': 'owncloudci/php:%s' % phpVersion, + 'pull': 'always', + 'commands': [ + 'echo "export TEST_SERVER_FED_URL=http://federated" > /var/www/owncloud/saved-settings.sh', + 'cd /var/www/owncloud/federated', + 'php occ a:l', + 'php occ a:e testing', + 'php occ a:l', + 'php occ config:system:set trusted_domains 1 --value=federated', + 'php occ log:manage --level %s' % params['logLevel'], + 'php occ config:list' + ] + } + ] + owncloudLog('federated') if params['federatedServerNeeded'] else []) + + installApp(phpVersion) + + installExtraApps(phpVersion, params['extraApps']) + + setupServerAndApp(phpVersion, params['logLevel']) + + owncloudLog('server') + + setupCeph(params['cephS3']) + + setupScality(params['scalityS3']) + + params['extraSetup'] + + fixPermissions(phpVersion, params['federatedServerNeeded']) + + [ + ({ + 'name': 'acceptance-tests', + 'image': 'owncloudci/php:%s' % phpVersion, + 'pull': 'always', + 'environment': environment, + 'commands': params['extraCommandsBeforeTestRun'] + [ + 'touch /var/www/owncloud/saved-settings.sh', + '. /var/www/owncloud/saved-settings.sh', + 'make %s' % makeParameter + ] + }), + ], + 'services': + databaseService(db) + + browserService(browser) + + emailService(params['emailNeeded']) + + ldapService(params['ldapNeeded']) + + cephService(params['cephS3']) + + scalityService(params['scalityS3']) + + params['extraServices'] + + owncloudService(server, phpVersion, 'server', '/var/www/owncloud/server', False) + + (owncloudService(server, phpVersion, 'federated', '/var/www/owncloud/federated', False) if params['federatedServerNeeded'] else []), + 'depends_on': [], + 'trigger': { + 'ref': [ + 'refs/pull/**', + 'refs/tags/**' ] - }), - ], - 'services': databaseService(db) - + browserService(browser) - + emailService(params['emailNeeded']) - + params['extraServices'] - + owncloudService(server, phpVersion, 'server', '/var/www/owncloud/server', False) - + (owncloudService(server, phpVersion, 'federated', '/var/www/owncloud/federated', False) if params['federatedServerNeeded'] else []), - 'depends_on': [], - 'trigger': { - 'ref': [ - 'refs/pull/**', - 'refs/tags/**' - ] + } } - } - for branch in config['branches']: - result['trigger']['ref'].append('refs/heads/%s' % branch) + for branch in config['branches']: + result['trigger']['ref'].append('refs/heads/%s' % branch) - pipelines.append(result) + pipelines.append(result) if errorFound: return False @@ -814,8 +967,8 @@ def databaseService(db): return [] -def browserService(name): - if name == 'chrome': +def browserService(browser): + if browser == 'chrome': return [{ 'name': 'selenium', 'image': 'selenium/standalone-chrome-debug:3.141.59-oxygen', @@ -825,7 +978,7 @@ def browserService(name): } }] - if name == 'firefox': + if browser == 'firefox': return [{ 'name': 'selenium', 'image': 'selenium/standalone-firefox-debug:3.8.1', @@ -848,6 +1001,55 @@ def emailService(emailNeeded): return [] +def ldapService(ldapNeeded): + if ldapNeeded: + return [{ + 'name': 'ldap', + 'image': 'osixia/openldap', + 'pull': 'always', + 'environment': { + 'LDAP_DOMAIN': 'owncloud.com', + 'LDAP_ORGANISATION': 'owncloud', + 'LDAP_ADMIN_PASSWORD': 'admin', + 'LDAP_TLS_VERIFY_CLIENT': 'never', + 'HOSTNAME': 'ldap', + } + }] + + return [] + +def scalityService(scalityS3): + if not scalityS3: + return [] + + return [{ + 'name': 'scality', + 'image': 'owncloudci/scality-s3server', + 'pull': 'always', + 'environment': { + 'HOST_NAME': 'scality' + } + }] + + +def cephService(cephS3): + if not cephS3: + return [] + + return [{ + 'name': 'ceph', + 'image': 'owncloudci/ceph:tag-build-master-jewel-ubuntu-16.04', + 'pull': 'always', + 'environment': { + 'NETWORK_AUTO_DETECT': '4', + 'RGW_NAME': 'ceph', + 'CEPH_DEMO_UID': 'owncloud', + 'CEPH_DEMO_ACCESS_KEY': 'owncloud123456', + 'CEPH_DEMO_SECRET_KEY': 'secret123456', + } + }] + + def owncloudService(version, phpVersion, name = 'server', path = '/var/www/owncloud/server', ssl = True): if ssl: environment = { @@ -961,22 +1163,11 @@ def installTestrunner(phpVersion, useBundledApp): }] def installExtraApps(phpVersion, extraApps): - if type(extraApps) == "list": - if extraApps == []: - return [] - apps = {} - for app in extraApps: - apps[app] = None - else: - apps = {} - for app, command in extraApps.items(): - apps[app] = command - commandArray = [] - for app, command in apps.items(): + for app, command in extraApps.items(): commandArray.append('git clone https://github.com/owncloud/%s.git /var/www/owncloud/testrunner/apps/%s' % (app, app)) commandArray.append('cp -r /var/www/owncloud/testrunner/apps/%s /var/www/owncloud/server/apps/' % app) - if ((command != None) and (command != '')): + if (command != ''): commandArray.append('cd /var/www/owncloud/server/apps/%s' % app) commandArray.append(command) commandArray.append('cd /var/www/owncloud/server') @@ -984,6 +1175,9 @@ def installExtraApps(phpVersion, extraApps): commandArray.append('php occ a:e %s' % app) commandArray.append('php occ a:l') + if (commandArray == []): + return [] + return [{ 'name': 'install-extra-apps', 'image': 'owncloudci/php:%s' % phpVersion, @@ -1021,6 +1215,50 @@ def setupServerAndApp(phpVersion, logLevel): ] }] +def setupCeph(cephS3): + if not cephS3: + return [] + + return [{ + 'name': 'setup-ceph', + 'image': 'owncloudci/php:7.0', + 'pull': 'always', + 'commands': [ + 'wait-for-it -t 60 ceph:80', + 'cd /var/www/owncloud/server/apps/files_primary_s3', + 'cp tests/drone/ceph.config.php /var/www/owncloud/server/config', + 'cd /var/www/owncloud/server', + './apps/files_primary_s3/tests/drone/create-bucket.sh', + ] + }] + +def setupScality(scalityS3): + if type(scalityS3) == "bool": + if scalityS3: + # specify an empty dict that will get the defaults + scalityS3 = {} + else: + return [] + + specialConfig = '.' + scalityS3['config'] if 'config' in scalityS3 else '' + configFile = 'scality%s.config.php' % specialConfig + createExtraBuckets = scalityS3['createExtraBuckets'] if 'createExtraBuckets' in scalityS3 else False + + return [{ + 'name': 'setup-scality', + 'image': 'owncloudci/php:7.0', + 'pull': 'always', + 'commands': [ + 'wait-for-it -t 60 scality:8000', + 'cd /var/www/owncloud/server/apps/files_primary_s3', + 'cp tests/drone/%s /var/www/owncloud/server/config' % configFile, + 'cd /var/www/owncloud/server', + 'php occ s3:create-bucket owncloud --accept-warning' + ] + ([ + 'for I in $(seq 1 9); do php ./occ s3:create-bucket owncloud$I --accept-warning; done', + ] if createExtraBuckets else []) + }] + def fixPermissions(phpVersion, federatedServerNeeded): return [{ 'name': 'fix-permissions', @@ -1034,7 +1272,7 @@ def fixPermissions(phpVersion, federatedServerNeeded): }] def owncloudLog(server): - return { + return [{ 'name': 'owncloud-log-%s' % server, 'image': 'owncloud/ubuntu:18.04', 'pull': 'always', @@ -1042,7 +1280,7 @@ def owncloudLog(server): 'commands': [ 'tail -f /var/www/owncloud/%s/data/owncloud.log' % server ] - } + }] def dependsOn(earlierStages, nextStages): for earlierStage in earlierStages: