From 9479d43c90ad18049b4338bbdf86dab172db7abd Mon Sep 17 00:00:00 2001 From: Hanyu Cui Date: Mon, 14 Oct 2019 09:25:31 -0700 Subject: [PATCH 01/11] Install 1.15-rc3 for Python using pip --- environment.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 55a0e11..f753a08 100644 --- a/environment.yml +++ b/environment.yml @@ -8,6 +8,7 @@ dependencies: - pandas=0.24.2 - nomkl - protobuf=3.6.1 -- tensorflow=1.14.0 # test - nose=1.3.7 +- pip: + - tensorflow==1.15.0-rc3 From 8743962e71cdba36a4f36f64c62c5ba93301e0f0 Mon Sep 17 00:00:00 2001 From: Hanyu Cui Date: Tue, 15 Oct 2019 12:00:47 -0700 Subject: [PATCH 02/11] Use local maven --- build.sbt | 1 + docker-compose.yml | 1 + project/Build.scala | 2 +- project/Dependencies.scala | 2 +- 4 files changed, 4 insertions(+), 2 deletions(-) create mode 100644 build.sbt diff --git a/build.sbt b/build.sbt new file mode 100644 index 0000000..337c271 --- /dev/null +++ b/build.sbt @@ -0,0 +1 @@ +resolvers += Resolver.mavenLocal diff --git a/docker-compose.yml b/docker-compose.yml index 1b73c79..82751ad 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,3 +9,4 @@ services: volumes: - .:/mnt/tensorframes - ~/.ivy2:/root/.ivy2 + - ~/.m2:/root/.m2 diff --git a/project/Build.scala b/project/Build.scala index 166ff78..4a65f48 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -62,7 +62,7 @@ object Shading extends Build { "com.typesafe.scala-logging" %% "scala-logging-api" % "2.1.2", "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", // TensorFlow dependencies - "org.tensorflow" % "tensorflow" % targetTensorFlowVersion + "org.tensorflow" % "libtensorflow" % targetTensorFlowVersion ) lazy val testDependencies = Seq( diff --git a/project/Dependencies.scala b/project/Dependencies.scala index cdf5ded..d76fe14 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -1,5 +1,5 @@ object Dependencies { // The spark version val targetSparkVersion = sys.props.getOrElse("spark.version", "2.4.4") - val targetTensorFlowVersion = "1.14.0" + val targetTensorFlowVersion = "1.15.0-rc3" } From 5e421a65dacb5735580164ab7e33617a39135704 Mon Sep 17 00:00:00 2001 From: Hanyu Cui Date: Tue, 15 Oct 2019 13:31:56 -0700 Subject: [PATCH 03/11] Copy libtensorflow_jni.so --- Dockerfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2277e99..00cebe2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,14 +41,12 @@ ENV PATH $SPARK_HOME/bin:$PATH ENV PYTHONPATH /opt/spark/python/lib/py4j-0.10.7-src.zip:/opt/spark/python/lib/pyspark.zip:$PYTHONPATH ENV PYSPARK_PYTHON python -# Workaround for https://github.com/tensorflow/tensorflow/issues/30635. -RUN wget https://repo1.maven.org/maven2/org/tensorflow/libtensorflow_jni/1.14.0/libtensorflow_jni-1.14.0.jar && \ - jar xf libtensorflow_jni-1.14.0.jar org/tensorflow/native/linux-x86_64/libtensorflow_framework.so.1 && \ - mv org/tensorflow/native/linux-x86_64/libtensorflow_framework.so.1 /usr/lib && \ - rm libtensorflow_jni-1.14.0.jar +COPY ./libtensorflow_jni.so /usr/lib/ # The tensorframes dir will be mounted here. VOLUME /mnt/tensorframes WORKDIR /mnt/tensorframes + + CMD /bin/bash From 81d75c578bc7adf4ad99143868e945eb74760dc5 Mon Sep 17 00:00:00 2001 From: Hanyu Cui Date: Tue, 15 Oct 2019 16:39:32 -0700 Subject: [PATCH 04/11] Copy libtensorflow_framework.so.1 --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 00cebe2..ec2230c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -42,6 +42,7 @@ ENV PYTHONPATH /opt/spark/python/lib/py4j-0.10.7-src.zip:/opt/spark/python/lib/p ENV PYSPARK_PYTHON python COPY ./libtensorflow_jni.so /usr/lib/ +COPY ./libtensorflow_framework.so.1.15.0 /usr/lib/libtensorflow_framework.so.1 # The tensorframes dir will be mounted here. VOLUME /mnt/tensorframes From ebf9cdc59c54129ccf9e696f7de746b6bc9d0024 Mon Sep 17 00:00:00 2001 From: Hanyu Cui Date: Tue, 15 Oct 2019 16:55:34 -0700 Subject: [PATCH 05/11] Rm build.sbt --- build.sbt | 1 - project/Build.scala | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 build.sbt diff --git a/build.sbt b/build.sbt deleted file mode 100644 index 337c271..0000000 --- a/build.sbt +++ /dev/null @@ -1 +0,0 @@ -resolvers += Resolver.mavenLocal diff --git a/project/Build.scala b/project/Build.scala index 4a65f48..4a0f727 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -13,6 +13,8 @@ object Shading extends Build { import Dependencies._ + resolvers += Resolver.mavenLocal + lazy val commonSettings = Seq( name := "tensorframes", scalaVersion := sys.props.getOrElse("scala.version", "2.11.8"), From 2b20d95b8ba534338f25416e5a91d463ae363eae Mon Sep 17 00:00:00 2001 From: Hanyu Cui Date: Tue, 15 Oct 2019 17:32:44 -0700 Subject: [PATCH 06/11] Remove extra blank lines --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ec2230c..1c18c2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,6 +48,4 @@ COPY ./libtensorflow_framework.so.1.15.0 /usr/lib/libtensorflow_framework.so.1 VOLUME /mnt/tensorframes WORKDIR /mnt/tensorframes - - CMD /bin/bash From 1c8baf44d2d22552c820a4c9424803c3e6791bc3 Mon Sep 17 00:00:00 2001 From: Hanyu Cui Date: Wed, 16 Oct 2019 09:03:51 -0700 Subject: [PATCH 07/11] Ran ./update-tf-proto.sh 1.15.0-rc3 --- .../tensorflow/core/framework/node_def.proto | 11 ++++++++++- .../protobuf/tensorflow/core/framework/op_def.proto | 2 +- .../tensorflow/core/framework/resource_handle.proto | 12 ++++++++++++ .../protobuf/tensorflow/core/framework/types.proto | 2 +- .../tensorflow/core/framework/variable.proto | 2 +- 5 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/main/protobuf/tensorflow/core/framework/node_def.proto b/src/main/protobuf/tensorflow/core/framework/node_def.proto index 73cbc96..3c89f78 100644 --- a/src/main/protobuf/tensorflow/core/framework/node_def.proto +++ b/src/main/protobuf/tensorflow/core/framework/node_def.proto @@ -11,7 +11,7 @@ import "tensorflow/core/framework/attr_value.proto"; message NodeDef { // The name given to this operator. Used for naming inputs, // logging, visualization, etc. Unique within a single GraphDef. - // Must match the regexp "[A-Za-z0-9.][A-Za-z0-9_./]*". + // Must match the regexp "[A-Za-z0-9.][A-Za-z0-9_>./]*". string name = 1; // The operation name. There may be custom parameters in attrs. @@ -70,6 +70,15 @@ message NodeDef { // be {A, B}. This information can be used to map errors originating at the // current node to some top level source code. repeated string original_node_names = 1; + + // This is intended to store the list of names of the functions from the + // original graph that this node was derived. For example if this node, say + // C, was result of a fusion of node A in function FA and node B in function + // FB, then `original_funcs` would be {FA, FB}. If the node is in the top + // level graph, the `original_func` is empty. This information, with the + // `original_node_names` can be used to map errors originating at the + // current ndoe to some top level source code. + repeated string original_func_names = 2; }; // This stores debug information associated with the node. diff --git a/src/main/protobuf/tensorflow/core/framework/op_def.proto b/src/main/protobuf/tensorflow/core/framework/op_def.proto index e44ecc9..9b65e23 100644 --- a/src/main/protobuf/tensorflow/core/framework/op_def.proto +++ b/src/main/protobuf/tensorflow/core/framework/op_def.proto @@ -14,7 +14,7 @@ import "tensorflow/core/framework/types.proto"; // LINT.IfChange message OpDef { // Op names starting with an underscore are reserved for internal use. - // Names should be CamelCase and match the regexp "[A-Z][a-zA-Z0-9_]*". + // Names should be CamelCase and match the regexp "[A-Z][a-zA-Z0-9>_]*". string name = 1; // For describing inputs and outputs. diff --git a/src/main/protobuf/tensorflow/core/framework/resource_handle.proto b/src/main/protobuf/tensorflow/core/framework/resource_handle.proto index a54d3d9..4a03fc7 100644 --- a/src/main/protobuf/tensorflow/core/framework/resource_handle.proto +++ b/src/main/protobuf/tensorflow/core/framework/resource_handle.proto @@ -7,6 +7,9 @@ option java_multiple_files = true; option java_package = "org.tensorflow.framework"; option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; +import "tensorflow/core/framework/tensor_shape.proto"; +import "tensorflow/core/framework/types.proto"; + // Protocol buffer representing a handle to a tensorflow resource. Handles are // not valid across executions, but can be serialized back and forth from within // a single run. @@ -27,4 +30,13 @@ message ResourceHandleProto { // For debug-only, the name of the type pointed to by this handle, if // available. string maybe_type_name = 5; + + // Protocol buffer representing a pair of (data type, tensor shape). + message DtypeAndShape { + DataType dtype = 1; + TensorShapeProto shape = 2; + } + + // Data types and shapes for the underlying resource. + repeated DtypeAndShape dtypes_and_shapes = 6; }; diff --git a/src/main/protobuf/tensorflow/core/framework/types.proto b/src/main/protobuf/tensorflow/core/framework/types.proto index 432fbf5..5356f9f 100644 --- a/src/main/protobuf/tensorflow/core/framework/types.proto +++ b/src/main/protobuf/tensorflow/core/framework/types.proto @@ -67,7 +67,7 @@ enum DataType { DT_UINT64_REF = 123; } // LINT.ThenChange( -// https://www.tensorflow.org/code/tensorflow/c/c_api.h, +// https://www.tensorflow.org/code/tensorflow/c/tf_datatype.h, // https://www.tensorflow.org/code/tensorflow/go/tensor.go, // https://www.tensorflow.org/code/tensorflow/core/framework/tensor.cc, // https://www.tensorflow.org/code/tensorflow/core/framework/types.h, diff --git a/src/main/protobuf/tensorflow/core/framework/variable.proto b/src/main/protobuf/tensorflow/core/framework/variable.proto index 70e5651..b2978c7 100644 --- a/src/main/protobuf/tensorflow/core/framework/variable.proto +++ b/src/main/protobuf/tensorflow/core/framework/variable.proto @@ -7,7 +7,7 @@ option java_outer_classname = "VariableProtos"; option java_multiple_files = true; option java_package = "org.tensorflow.framework"; -// add go_package externally with copybara +option go_package = "github.com/tensorflow/tensorflow/tensorflow/go/core/framework"; // Indicates when a distributed variable will be synced. enum VariableSynchronization { From b1a6a29d8a354c95a1c4ea6ddc2fbb62a53f85f7 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Wed, 16 Oct 2019 11:52:14 -0700 Subject: [PATCH 08/11] use snapshot releases from bintray --- Dockerfile | 3 --- project/Build.scala | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1c18c2c..5551025 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,9 +41,6 @@ ENV PATH $SPARK_HOME/bin:$PATH ENV PYTHONPATH /opt/spark/python/lib/py4j-0.10.7-src.zip:/opt/spark/python/lib/pyspark.zip:$PYTHONPATH ENV PYSPARK_PYTHON python -COPY ./libtensorflow_jni.so /usr/lib/ -COPY ./libtensorflow_framework.so.1.15.0 /usr/lib/libtensorflow_framework.so.1 - # The tensorframes dir will be mounted here. VOLUME /mnt/tensorframes WORKDIR /mnt/tensorframes diff --git a/project/Build.scala b/project/Build.scala index 4a0f727..9a6636d 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -13,7 +13,8 @@ object Shading extends Build { import Dependencies._ - resolvers += Resolver.mavenLocal + resolvers += Resolver.bintrayRepo("meng", "spark-3.0-snapshots") + // resolvers += Resolver.mavenLocal lazy val commonSettings = Seq( name := "tensorframes", From ea92165846b72cbc4769f49018a6b9d82e0f49ab Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Wed, 16 Oct 2019 12:13:16 -0700 Subject: [PATCH 09/11] move resolver setting to common --- project/Build.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/project/Build.scala b/project/Build.scala index 9a6636d..69f7de9 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -13,9 +13,6 @@ object Shading extends Build { import Dependencies._ - resolvers += Resolver.bintrayRepo("meng", "spark-3.0-snapshots") - // resolvers += Resolver.mavenLocal - lazy val commonSettings = Seq( name := "tensorframes", scalaVersion := sys.props.getOrElse("scala.version", "2.11.8"), @@ -40,7 +37,8 @@ object Shading extends Build { tagRelease, setNextVersion, commitNextVersion - ) + ), + resolvers += Resolver.bintrayRepo("meng", "spark-3.0-snapshots") ) lazy val sparkDependencies = Seq( From 77e34bf25584da0b367c3181a4d05d04ab970b0d Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Wed, 16 Oct 2019 15:57:00 -0700 Subject: [PATCH 10/11] explicitly add tf_jni as a dependency --- project/Build.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/project/Build.scala b/project/Build.scala index 69f7de9..1034310 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -63,7 +63,8 @@ object Shading extends Build { "com.typesafe.scala-logging" %% "scala-logging-api" % "2.1.2", "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", // TensorFlow dependencies - "org.tensorflow" % "libtensorflow" % targetTensorFlowVersion + "org.tensorflow" % "libtensorflow" % targetTensorFlowVersion, + "org.tensorflow" % "libtensorflow_jni" % targetTensorFlowVersion ) lazy val testDependencies = Seq( From 754cb967f92ed99d186019c74a79bba61f7a2bf8 Mon Sep 17 00:00:00 2001 From: Xiangrui Meng Date: Wed, 16 Oct 2019 16:47:24 -0700 Subject: [PATCH 11/11] add .so back in docker --- Dockerfile | 5 +++++ project/Build.scala | 3 +-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5551025..54c5d18 100644 --- a/Dockerfile +++ b/Dockerfile @@ -41,6 +41,11 @@ ENV PATH $SPARK_HOME/bin:$PATH ENV PYTHONPATH /opt/spark/python/lib/py4j-0.10.7-src.zip:/opt/spark/python/lib/pyspark.zip:$PYTHONPATH ENV PYSPARK_PYTHON python +# Workaround for https://github.com/tensorflow/tensorflow/issues/30635. +RUN wget -q https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-linux-x86_64-1.15.0-rc3.tar.gz && \ + tar xf libtensorflow_jni-cpu-linux-x86_64-1.15.0-rc3.tar.gz && \ + cp libtensorflow_framework.so.1 /usr/lib + # The tensorframes dir will be mounted here. VOLUME /mnt/tensorframes WORKDIR /mnt/tensorframes diff --git a/project/Build.scala b/project/Build.scala index 1034310..641a88b 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -63,8 +63,7 @@ object Shading extends Build { "com.typesafe.scala-logging" %% "scala-logging-api" % "2.1.2", "com.typesafe.scala-logging" %% "scala-logging-slf4j" % "2.1.2", // TensorFlow dependencies - "org.tensorflow" % "libtensorflow" % targetTensorFlowVersion, - "org.tensorflow" % "libtensorflow_jni" % targetTensorFlowVersion + "org.tensorflow" % "tensorflow" % targetTensorFlowVersion ) lazy val testDependencies = Seq(