[DCOS-63346] Update to use SDK v0.57.3 (#460)

* bump sdk v0.57.3

* bump mockito v2.27.0 

* bump gradle v5.1.1

* Bump scheduler jre to v11

frameworks/kafka/build.gradle

@@ -18,8 +18,8 @@ repositories {
 ext {
     junitVer = "4.12"
     systemRulesVer = "1.16.0"
-    mockitoVer = "2.10.0"
-    dcosSDKVer = "0.57.0"
+    mockitoVer = "2.27.0"
+    dcosSDKVer = "0.57.3"
 }
 
 dependencies {

frameworks/kafka/universe/resource.json

@@ -1,7 +1,7 @@
 {
   "assets": {
     "uris": {
-      "jre-tar-gz": "{{jre-url}}",
+      "jre-tar-gz": "{{scheduler-jre-url}}",
       "libmesos-bundle-tar-gz": "{{libmesos-bundle-url}}",
       "bootstrap-zip": "https://downloads.mesosphere.com/dcos-commons/artifacts/{{dcos-sdk-version}}/bootstrap.zip",
       "kafka-tgz": "https://downloads.mesosphere.com/kafka/assets/kafka_{{kafka-version}}.tgz",

frameworks/kafka/versions.sh

@@ -13,4 +13,4 @@
 # instructions at:
 # https://wiki.mesosphere.com/display/ENGINEERING/Uploading+an+asset+to+production
 export TEMPLATE_KAFKA_VERSION="2.12-2.4.0"
-export TEMPLATE_DCOS_SDK_VERSION="0.57.0"
+export TEMPLATE_DCOS_SDK_VERSION="0.57.3"

gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.10-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.1.1-bin.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists

test.sh

@@ -92,6 +92,7 @@ headless="false"
 dind="false"
 interactive="false"
 package_registry="false"
+disable_diag_collect="false"
 docker_options="${DOCKER_OPTIONS:=}"
 docker_command="${DOCKER_COMMAND:=bash ${WORK_DIR}/tools/ci/test_runner.sh ${WORK_DIR}}"
 docker_image="${DOCKER_IMAGE:-mesosphere/dcos-commons:latest}"
@@ -157,6 +158,8 @@ function usage()
   echo "    The AWS profile to use. Only required when using an AWS credentials file with multiple profiles."
   echo "  --dind ${dind}"
   echo "    launch docker daemon inside the container"
+  echo "  --disable-diag-collect ${disable_diag_collect}"
+  echo "    Disable collection of diagnostic logs after a failed test."
   echo
   echo "---"
   echo
@@ -254,6 +257,9 @@ while [[ ${#} -gt 0 ]]; do
     --dind)
       dind="true"
       ;;
+    --disable-diag-collect)
+      disable_diag_collect="true"
+      ;;
     --dcos-files-path)
       if [[ ! -d "${2}" ]]; then echo "Directory not found: ${arg} ${2}"; exit 1; fi
       # Resolve abs path:
@@ -515,6 +521,7 @@ cat >> "${env_file}" <<-EOF
 	S3_BUCKET=${S3_BUCKET}
 	SECURITY=${security}
 	STUB_UNIVERSE_URL=${STUB_UNIVERSE_URL}
+    DISABLE_DIAG=${disable_diag_collect}
 EOF
 
 while read -r line; do
@@ -540,7 +547,7 @@ fi
 ######################### Prepare and run command ##############################
 ################################################################################
 
-docker pull "${docker_image}"
+[ ! -z $(docker images -q "${docker_image}") ] || docker pull "${docker_image}"
 
 set +x
 

testing/sdk_security.py

@@ -195,19 +195,23 @@ def create_service_account(service_account_name: str, service_account_secret: st
     delete_service_account(service_account_name, service_account_secret)
 
     log.info("Create keypair")
-    sdk_cmd.run_cli("security org service-accounts keypair private-key.pem public-key.pem")
+    sdk_cmd.run_cli(
+        "security org service-accounts keypair {account}-private-key.pem {account}-public-key.pem".format(
+            account=service_account_name
+        )
+    )
 
     log.info("Create service account")
     sdk_cmd.run_cli(
-        "security org service-accounts create -p public-key.pem "
+        "security org service-accounts create -p {account}-public-key.pem "
         '-d "Service account for integration tests" "{account}"'.format(
             account=service_account_name
         )
     )
 
     log.info("Create secret")
     sdk_cmd.run_cli(
-        'security secrets create-sa-secret --strict private-key.pem "{account}" "{secret}"'.format(
+        'security secrets create-sa-secret --strict {account}-private-key.pem "{account}" "{secret}"'.format(
             account=service_account_name, secret=service_account_secret
         )
     )
@@ -227,7 +231,10 @@ def delete_service_account(service_account_name: str, service_account_secret: st
     sdk_cmd.run_cli("security org service-accounts delete {name}".format(name=service_account_name))
 
     # Files generated by service-accounts keypair command should get removed
-    for keypair_file in ["private-key.pem", "public-key.pem"]:
+    for keypair_file in [
+        "{account}-private-key.pem".format(account=service_account_name),
+        "{account}-public-key.pem".format(account=service_account_name),
+    ]:
         try:
             os.unlink(keypair_file)
         except OSError:
@@ -282,6 +289,7 @@ def _get_integration_test_foldered_role(service_name: str) -> List[str]:
     role_basename = service_name.strip("/").replace("/", "__")
     return [
         "test__integration__{}-role".format(role_basename),
+        "test",
         "quota",
         "quota__{}-role".format(role_basename),
     ]

tools/publish_http.py

@@ -193,7 +193,7 @@ def add_repo_to_cli(self, repo_url):
                 subprocess.check_call("dcos package repo remove {}".format(repo["name"]).split())
         logger.info("Adding repository: {} {}".format(repo_name, repo_url))
         subprocess.check_call(
-            "dcos package repo add --index=0 {} '{}'".format(repo_name, repo_url).split(" ")
+            "dcos package repo add --index=0 {} {}".format(repo_name, repo_url).split(" ")
         )
         return True
 

tools/universe/package_builder.py

@@ -275,6 +275,8 @@ def _generate_packages_dict(self, package_files):
             package_files[_package_json_filename], object_pairs_hook=collections.OrderedDict
         )
         package_json["releaseVersion"] = 0
+        # `lastUpdated` is supported for all packages v3 and above
+        package_json["lastUpdated"] = round(time.time())
 
         config_json = package_files.get(_config_json_filename)
         if config_json is not None:

tools/universe/s3_uploader.py

@@ -20,6 +20,7 @@ def __init__(self, s3_directory, dry_run=False):
         self._aws_region = os.environ.get("AWS_UPLOAD_REGION", "")
         self._reauth_attempted = False
         self._dry_run = dry_run
+        self._acl = os.environ.get("AWS_UPLOAD_ACL", "public-read")
 
     def get_s3_directory(self):
         return self._s3_directory
@@ -29,7 +30,7 @@ def upload(self, filepath, content_type=None):
         cmdlist = ["aws s3"]
         if self._aws_region:
             cmdlist.append("--region={}".format(self._aws_region))
-        cmdlist.append("cp --acl public-read")
+        cmdlist.append("cp --acl {}".format(self._acl))
         if self._dry_run:
             cmdlist.append("--dryrun")
         if content_type is not None: