diff --git a/hail/src/main/scala/is/hail/backend/BackendServer.scala b/hail/src/main/scala/is/hail/backend/BackendServer.scala index 0217fc1d8d7b..7ce224548c98 100644 --- a/hail/src/main/scala/is/hail/backend/BackendServer.scala +++ b/hail/src/main/scala/is/hail/backend/BackendServer.scala @@ -4,16 +4,17 @@ import is.hail.expr.ir.{IRParser, IRParserEnvironment} import is.hail.utils._ import scala.util.control.NonFatal + +import java.io.Closeable import java.net.InetSocketAddress import java.nio.charset.StandardCharsets import java.util.concurrent._ + import com.sun.net.httpserver.{HttpExchange, HttpHandler, HttpServer} import org.json4s._ import org.json4s.jackson.JsonMethods import org.json4s.jackson.JsonMethods.compact -import java.io.Closeable - case class IRTypePayload(ir: String) case class LoadReferencesFromDatasetPayload(path: String) diff --git a/hail/src/main/scala/is/hail/backend/service/ServiceBackend.scala b/hail/src/main/scala/is/hail/backend/service/ServiceBackend.scala index 668a3ec2286e..c4bb2d40388f 100644 --- a/hail/src/main/scala/is/hail/backend/service/ServiceBackend.scala +++ b/hail/src/main/scala/is/hail/backend/service/ServiceBackend.scala @@ -5,14 +5,17 @@ import is.hail.annotations._ import is.hail.asm4s._ import is.hail.backend._ import is.hail.expr.Validate -import is.hail.expr.ir.{Compile, IR, IRParser, IRParserEnvironment, IRSize, LoweringAnalyses, MakeTuple, SortField, TableIR, TableReader, TypeCheck} +import is.hail.expr.ir.{ + Compile, IR, IRParser, IRParserEnvironment, IRSize, LoweringAnalyses, MakeTuple, SortField, + TableIR, TableReader, TypeCheck, +} import is.hail.expr.ir.analyses.SemanticHash import is.hail.expr.ir.functions.IRFunctionRegistry import is.hail.expr.ir.lowering._ import is.hail.io.fs._ import is.hail.linalg.BlockMatrix -import is.hail.services.JobGroupStates.Failure import is.hail.services.{BatchClient, _} +import is.hail.services.JobGroupStates.Failure import is.hail.types._ import is.hail.types.physical._ import is.hail.types.physical.stypes.PTypeReferenceSingleCodeType @@ -22,17 +25,18 @@ import is.hail.variant.ReferenceGenome import scala.annotation.switch import scala.reflect.ClassTag + import java.io._ import java.nio.charset.StandardCharsets +import java.nio.file.Path import java.util.concurrent._ + import org.apache.log4j.Logger import org.json4s.{DefaultFormats, Formats} import org.json4s.JsonAST._ import org.json4s.jackson.JsonMethods import sourcecode.Enclosing -import java.nio.file.Path - class ServiceBackendContext( val billingProject: String, val remoteTmpDir: String, @@ -181,7 +185,7 @@ class ServiceBackend( val uploadContexts = executor.submit[Unit](() => retryTransientErrors { fs.writePDOS(s"$root/contexts") { os => - var o = 12L * n // 12L = sizeof(Long) + sizeof(Int) + var o = 12L * n // 12L = sizeof(Long) + sizeof(Int) collection.foreach { context => val len = context.length os.writeLong(o) diff --git a/hail/src/main/scala/is/hail/backend/service/Worker.scala b/hail/src/main/scala/is/hail/backend/service/Worker.scala index 5a6ec9f95052..de608b265151 100644 --- a/hail/src/main/scala/is/hail/backend/service/Worker.scala +++ b/hail/src/main/scala/is/hail/backend/service/Worker.scala @@ -1,22 +1,24 @@ package is.hail.backend.service +import is.hail.{HAIL_REVISION, HailContext, HailFeatureFlags} import is.hail.asm4s._ import is.hail.backend.HailTaskContext import is.hail.io.fs._ import is.hail.services._ import is.hail.utils._ -import is.hail.{HAIL_REVISION, HailContext, HailFeatureFlags} -import org.apache.log4j.Logger + +import scala.collection.mutable +import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.duration.Duration +import scala.util.control.NonFatal import java.io._ import java.nio.charset._ import java.nio.file.Path import java.util import java.util.{concurrent => javaConcurrent} -import scala.collection.mutable -import scala.concurrent.duration.Duration -import scala.concurrent.{Await, ExecutionContext, Future} -import scala.util.control.NonFatal + +import org.apache.log4j.Logger class ServiceTaskContext(val partitionId: Int) extends HailTaskContext { override def stageId(): Int = 0 diff --git a/hail/src/main/scala/is/hail/io/fs/AzureStorageFS.scala b/hail/src/main/scala/is/hail/io/fs/AzureStorageFS.scala index a4d99c63fe23..a640fc684195 100644 --- a/hail/src/main/scala/is/hail/io/fs/AzureStorageFS.scala +++ b/hail/src/main/scala/is/hail/io/fs/AzureStorageFS.scala @@ -5,21 +5,21 @@ import is.hail.services.oauth2.AzureCloudCredentials import is.hail.services.retryTransientErrors import is.hail.shadedazure.com.azure.core.credential.AzureSasCredential import is.hail.shadedazure.com.azure.core.util.HttpClientOptions +import is.hail.shadedazure.com.azure.storage.blob.{ + BlobClient, BlobContainerClient, BlobServiceClient, BlobServiceClientBuilder, +} import is.hail.shadedazure.com.azure.storage.blob.models.{ BlobItem, BlobRange, BlobStorageException, ListBlobsOptions, } import is.hail.shadedazure.com.azure.storage.blob.specialized.BlockBlobClient -import is.hail.shadedazure.com.azure.storage.blob.{ - BlobClient, BlobContainerClient, BlobServiceClient, BlobServiceClientBuilder, -} -import java.io.{ByteArrayOutputStream, FileNotFoundException, OutputStream} -import java.nio.file.Paths -import java.time.Duration import scala.collection.JavaConverters._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer -import java.nio.file.Path + +import java.io.{ByteArrayOutputStream, FileNotFoundException, OutputStream} +import java.nio.file.{Path, Paths} +import java.time.Duration class AzureStorageFSURL( val account: String, diff --git a/hail/src/main/scala/is/hail/io/fs/FS.scala b/hail/src/main/scala/is/hail/io/fs/FS.scala index 441bde2cdfdb..f7de8037b74c 100644 --- a/hail/src/main/scala/is/hail/io/fs/FS.scala +++ b/hail/src/main/scala/is/hail/io/fs/FS.scala @@ -1,6 +1,5 @@ package is.hail.io.fs - import is.hail.HailContext import is.hail.backend.BroadcastValue import is.hail.io.compress.{BGzipInputStream, BGzipOutputStream} @@ -10,10 +9,12 @@ import is.hail.utils._ import scala.collection.mutable import scala.io.Source + import java.io._ import java.nio.ByteBuffer import java.nio.file.FileSystems import java.util.zip.GZIPOutputStream + import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream import org.apache.commons.io.IOUtils import org.apache.hadoop diff --git a/hail/src/main/scala/is/hail/io/fs/GoogleStorageFS.scala b/hail/src/main/scala/is/hail/io/fs/GoogleStorageFS.scala index 9ee4c1cad1e6..6a2921f07ded 100644 --- a/hail/src/main/scala/is/hail/io/fs/GoogleStorageFS.scala +++ b/hail/src/main/scala/is/hail/io/fs/GoogleStorageFS.scala @@ -1,21 +1,25 @@ package is.hail.io.fs -import com.google.api.client.googleapis.json.GoogleJsonResponseException -import com.google.cloud.http.HttpTransportOptions -import com.google.cloud.storage.Storage.{BlobGetOption, BlobListOption, BlobSourceOption, BlobWriteOption} -import com.google.cloud.storage.{Option => _, _} -import com.google.cloud.{ReadChannel, WriteChannel} import is.hail.HailFeatureFlags import is.hail.io.fs.FSUtil.dropTrailingSlash import is.hail.io.fs.GoogleStorageFS.RequesterPaysFailure -import is.hail.services.oauth2.GoogleCloudCredentials import is.hail.services.{isTransientError, retryTransientErrors} +import is.hail.services.oauth2.GoogleCloudCredentials import is.hail.utils._ +import scala.jdk.CollectionConverters._ + import java.io.{FileNotFoundException, IOException} import java.nio.ByteBuffer import java.nio.file.{Path, Paths} -import scala.jdk.CollectionConverters._ + +import com.google.api.client.googleapis.json.GoogleJsonResponseException +import com.google.cloud.{ReadChannel, WriteChannel} +import com.google.cloud.http.HttpTransportOptions +import com.google.cloud.storage.{Option => _, _} +import com.google.cloud.storage.Storage.{ + BlobGetOption, BlobListOption, BlobSourceOption, BlobWriteOption, +} case class GoogleStorageFSURL(bucket: String, path: String) extends FSURL { def addPathComponent(c: String): GoogleStorageFSURL = @@ -70,13 +74,13 @@ object GoogleStorageFS { case exc: StorageException => Option(exc.getMessage).exists { message => message == "userProjectMissing" || - (exc.getCode == 400 && message.contains("requester pays")) + (exc.getCode == 400 && message.contains("requester pays")) } case exc: GoogleJsonResponseException => Option(exc.getMessage).exists { message => message == "userProjectMissing" || - (exc.getStatusCode == 400 && message.contains("requester pays")) + (exc.getStatusCode == 400 && message.contains("requester pays")) } case _ => diff --git a/hail/src/main/scala/is/hail/io/fs/RouterFS.scala b/hail/src/main/scala/is/hail/io/fs/RouterFS.scala index b61bad0d74bc..d89985c07800 100644 --- a/hail/src/main/scala/is/hail/io/fs/RouterFS.scala +++ b/hail/src/main/scala/is/hail/io/fs/RouterFS.scala @@ -3,11 +3,12 @@ package is.hail.io.fs import is.hail.HailFeatureFlags import is.hail.services.oauth2.{AzureCloudCredentials, GoogleCloudCredentials} import is.hail.utils.{FastSeq, SerializableHadoopConfiguration} -import org.apache.hadoop.conf.Configuration import java.io.Serializable import java.nio.file.Path +import org.apache.hadoop.conf.Configuration + object RouterFSURL { def apply(fs: FS)(_url: fs.URL): RouterFSURL = RouterFSURL(_url, fs) } diff --git a/hail/src/main/scala/is/hail/io/fs/TerraAzureStorageFS.scala b/hail/src/main/scala/is/hail/io/fs/TerraAzureStorageFS.scala index 98018a8c389c..ad5f97e1abc0 100644 --- a/hail/src/main/scala/is/hail/io/fs/TerraAzureStorageFS.scala +++ b/hail/src/main/scala/is/hail/io/fs/TerraAzureStorageFS.scala @@ -3,14 +3,15 @@ package is.hail.io.fs import is.hail.services.oauth2.AzureCloudCredentials import is.hail.shadedazure.com.azure.storage.blob.BlobServiceClient import is.hail.utils._ + +import scala.collection.mutable + import org.apache.http.client.methods.HttpPost import org.apache.http.client.utils.URIBuilder import org.apache.http.impl.client.HttpClients import org.apache.http.util.EntityUtils -import org.json4s.jackson.JsonMethods import org.json4s.{DefaultFormats, Formats} - -import scala.collection.mutable +import org.json4s.jackson.JsonMethods object TerraAzureStorageFS { private val TEN_MINUTES_IN_MS = 10 * 60 * 1000 diff --git a/hail/src/main/scala/is/hail/services/BatchClient.scala b/hail/src/main/scala/is/hail/services/BatchClient.scala index 5d4589004849..dd17671eeace 100644 --- a/hail/src/main/scala/is/hail/services/BatchClient.scala +++ b/hail/src/main/scala/is/hail/services/BatchClient.scala @@ -3,15 +3,17 @@ package is.hail.services import is.hail.expr.ir.ByteArrayBuilder import is.hail.services.requests.{BatchServiceRequester, Requester} import is.hail.utils._ + +import scala.util.Random + +import java.nio.charset.StandardCharsets +import java.nio.file.Path + import org.apache.http.entity.ByteArrayEntity import org.apache.http.entity.ContentType.APPLICATION_JSON +import org.json4s.{CustomSerializer, DefaultFormats, Extraction, Formats, JInt, JObject, JString} import org.json4s.JsonAST.{JArray, JBool} import org.json4s.jackson.JsonMethods -import org.json4s.{CustomSerializer, DefaultFormats, Extraction, Formats, JInt, JObject, JString} - -import java.nio.charset.StandardCharsets -import java.nio.file.Path -import scala.util.Random case class BatchRequest( billing_project: String, @@ -87,7 +89,7 @@ object JobGroupStates { object BatchClient { def apply(deployConfig: DeployConfig, credentialsFile: Path, env: Map[String, String] = sys.env) - : BatchClient = + : BatchClient = new BatchClient(BatchServiceRequester(deployConfig, credentialsFile, env)) } diff --git a/hail/src/main/scala/is/hail/services/BatchConfig.scala b/hail/src/main/scala/is/hail/services/BatchConfig.scala index 3cd8e3b0c62d..da15265b849c 100644 --- a/hail/src/main/scala/is/hail/services/BatchConfig.scala +++ b/hail/src/main/scala/is/hail/services/BatchConfig.scala @@ -1,11 +1,12 @@ package is.hail.services import is.hail.utils._ -import org.json4s._ -import org.json4s.jackson.JsonMethods import java.nio.file.{Files, Path} +import org.json4s._ +import org.json4s.jackson.JsonMethods + object BatchConfig { def fromConfigFile(file: Path): Option[BatchConfig] = if (!file.toFile.exists()) None diff --git a/hail/src/main/scala/is/hail/services/oauth2.scala b/hail/src/main/scala/is/hail/services/oauth2.scala index 657521600ce3..06c26c653a43 100644 --- a/hail/src/main/scala/is/hail/services/oauth2.scala +++ b/hail/src/main/scala/is/hail/services/oauth2.scala @@ -1,6 +1,5 @@ package is.hail.services -import com.google.auth.oauth2.{GoogleCredentials, ServiceAccountCredentials} import is.hail.services.oauth2.AzureCloudCredentials.EnvVars.AzureApplicationCredentials import is.hail.services.oauth2.GoogleCloudCredentials.EnvVars.GoogleApplicationCredentials import is.hail.shadedazure.com.azure.core.credential.{TokenCredential, TokenRequestContext} @@ -8,12 +7,15 @@ import is.hail.shadedazure.com.azure.identity.{ ClientSecretCredentialBuilder, DefaultAzureCredentialBuilder, } import is.hail.utils.{defaultJSONFormats, using} -import org.json4s.Formats -import org.json4s.jackson.JsonMethods + +import scala.collection.JavaConverters._ import java.io.Serializable import java.nio.file.{Files, Path} -import scala.collection.JavaConverters._ + +import com.google.auth.oauth2.{GoogleCredentials, ServiceAccountCredentials} +import org.json4s.Formats +import org.json4s.jackson.JsonMethods object oauth2 { diff --git a/hail/src/main/scala/is/hail/services/package.scala b/hail/src/main/scala/is/hail/services/package.scala index c0369af14b66..e7887fea509a 100644 --- a/hail/src/main/scala/is/hail/services/package.scala +++ b/hail/src/main/scala/is/hail/services/package.scala @@ -1,16 +1,17 @@ package is.hail +import is.hail.services.requests.ClientResponseException import is.hail.shadedazure.com.azure.storage.common.implementation.Constants import is.hail.utils._ import scala.util.Random + import java.io._ import java.net._ + import com.google.api.client.googleapis.json.GoogleJsonResponseException import com.google.api.client.http.HttpResponseException import com.google.cloud.storage.StorageException -import is.hail.services.requests.ClientResponseException - import javax.net.ssl.SSLException import org.apache.http.{ConnectionClosedException, NoHttpResponseException} import org.apache.http.conn.HttpHostConnectException diff --git a/hail/src/main/scala/is/hail/services/requests.scala b/hail/src/main/scala/is/hail/services/requests.scala index 05a13ed057ca..855c219b6319 100644 --- a/hail/src/main/scala/is/hail/services/requests.scala +++ b/hail/src/main/scala/is/hail/services/requests.scala @@ -2,20 +2,21 @@ package is.hail.services import is.hail.services.oauth2.{CloudCredentials, CloudScopes} import is.hail.utils.{log, _} + +import java.net.URL +import java.nio.file.Path + +import org.apache.http.{HttpEntity, HttpEntityEnclosingRequest} import org.apache.http.client.config.RequestConfig import org.apache.http.client.methods.{HttpGet, HttpPatch, HttpPost, HttpUriRequest} import org.apache.http.entity.ContentType.APPLICATION_JSON import org.apache.http.entity.StringEntity import org.apache.http.impl.client.{CloseableHttpClient, HttpClients} import org.apache.http.util.EntityUtils -import org.apache.http.{HttpEntity, HttpEntityEnclosingRequest} import org.json4s.JValue import org.json4s.JsonAST.JNothing import org.json4s.jackson.JsonMethods -import java.net.URL -import java.nio.file.Path - object requests { class ClientResponseException(val status: Int, message: String) extends Exception(message)