Fix 64bit ids

.dockerignore

@@ -0,0 +1,5 @@
+.git
+.gitignore
+.DS_Store
+/target
+/bin

.gitignore

@@ -1,2 +1,3 @@
 target/
 .DS_Store
+/bin

Dockerfile

@@ -0,0 +1,42 @@
+FROM java:8
+MAINTAINER DaWanda <[email protected]>
+
+ARG SBT_VERSION="0.13.8"
+
+ENV SQLTAP_HTTP_PORT="3000" \
+    SQLTAP_THREADS="16" \
+    SQLTAP_SCHEMA="/etc/sqltap-schema.xml" \
+    SCHEMA_URL="" \
+    SQLTAP_OPTS="" \
+    MYSQL_HOST="127.0.0.1" \
+    MYSQL_PORT="3306" \
+    MYSQL_USER="fetch" \
+    MYSQL_DATABASE="test" \
+    MYSQL_NUMCONNS="6" \
+    MYSQL_QUEUELEN="2500" \
+    JMX_PORT="9191" \
+    RMI_BIND="127.0.0.1" \
+    JAVA_XMX="16384M" \
+    CACHE_BACKEND="memcache" \
+    MEMCACHE_HOST="" \
+    MEMCACHE_PORT="11211" \
+    MEMCACHE_QUEUELEN="8192" \
+    MEMCACHE_NUMCONNS="20"
+
+ADD https://dl.bintray.com/sbt/debian/sbt-$SBT_VERSION.deb /tmp/sbt.deb
+RUN dpkg -i /tmp/sbt.deb && rm -f /tmp/sbt.deb
+RUN apt-get update && apt-get install sbt
+
+ADD project /usr/src/project/
+ADD src /usr/src/src/
+ADD build.sbt /usr/src/
+
+RUN cd /usr/src && \
+    sbt assembly && \
+    cp -vpi /usr/src/target/scala-*/sqltap.jar /usr/lib/sqltap.jar && \
+    rm -rf /usr/src/*
+
+EXPOSE $SQLTAP_HTTP_PORT
+
+ADD bootup.sh /bootup.sh
+CMD ["/bootup.sh"]

Makefile

@@ -0,0 +1,22 @@
+# This file is part of the "sqltap" project, http://github.com/dawanda/sqltap>
+#   (c) 2016 Christian Parpart <[email protected]>
+#
+# Licensed under the MIT License (the "License"); you may not use this
+# file except in compliance with the License. You may obtain a copy of
+# the License at: http://opensource.org/licenses/MIT
+
+IMAGE = sqltap
+VERSION = $(shell grep ^version build.sbt | cut -d\" -f2)
+
+image:
+	docker build -t ${IMAGE}:${VERSION} .
+
+push: image
+	docker push ${IMAGE}:${VERSION}
+
+clean:
+	rm -rf target
+
+.PHONY: image clean push
+
+# vim:ts=8:noet

bootup.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+set -ex
+
+SQLTAP_JARFILE="/usr/lib/sqltap.jar"
+
+if [[ "$SCHEMA_URL" != "" ]]; then
+  curl -sL "$SCHEMA_URL" -o "${SQLTAP_SCHEMA}"
+fi
+
+require_arg() {
+  local name="$1"
+  local val="${!name}"
+
+  echo "${name}: ${val}"
+
+  if [[ "${val}" == "" ]]; then
+    echo "Error. Required argument ${name} missing." 1>&2
+    exit 1
+  fi
+}
+
+require_args() {
+  while [[ $# -ne 0 ]]; do
+    require_arg $1
+    shift
+  done
+}
+
+require_args SQLTAP_SCHEMA \
+             MYSQL_PORT \
+             SQLTAP_HTTP_PORT \
+             SQLTAP_THREADS \
+             SCHEMA_URL \
+             MYSQL_HOST \
+             MYSQL_PORT \
+             MYSQL_USER \
+             MYSQL_DATABASE \
+             MYSQL_NUMCONNS \
+             MYSQL_QUEUELEN \
+             JMX_PORT \
+             RMI_BIND \
+             JAVA_XMX \
+             CACHE_BACKEND
+
+opts=""
+opts="$opts --config ${SQLTAP_SCHEMA}"
+opts="$opts --http ${SQLTAP_HTTP_PORT}"
+opts="$opts --disable-keepalive"
+opts="$opts -t ${SQLTAP_THREADS}"
+opts="$opts --mysql-host ${MYSQL_HOST}"
+opts="$opts --mysql-port ${MYSQL_PORT}"
+opts="$opts --mysql-user ${MYSQL_USER}"
+opts="$opts --mysql-database ${MYSQL_DATABASE}"
+opts="$opts --mysql-numconns ${MYSQL_NUMCONNS}"
+opts="$opts --mysql-queuelen ${MYSQL_QUEUELEN}"
+opts="$opts --cache-backend ${CACHE_BACKEND}"
+
+if [[ "${CACHE_BACKEND}" == "memcache" ]]; then
+  require_args MEMCACHE_HOST \
+               MEMCACHE_PORT \
+               MEMCACHE_QUEUELEN \
+               MEMCACHE_NUMCONNS
+
+  opts="$opts --memcache-host ${MEMCACHE_HOST}"
+  opts="$opts --memcache-port ${MEMCACHE_PORT}"
+  opts="$opts --memcache-queuelen ${MEMCACHE_QUEUELEN}"
+  opts="$opts --memcache-numconns ${MEMCACHE_NUMCONNS}"
+fi
+
+opts="$opts ${SQLTAP_OPTS}"
+
+exec java \
+    -Djava.rmi.server.hostname="${RMI_BIND}" \
+    -Dcom.sun.management.jmxremote \
+    -Dcom.sun.management.jmxremote.port="${JMX_PORT}" \
+    -Dcom.sun.management.jmxremote.ssl=false \
+    -Dcom.sun.management.jmxremote.authenticate=false \
+    -Xmx"${JAVA_XMX}" -XX:GCTimeRatio=99 -XX:+UseConcMarkSweepGC \
+    -jar "${SQLTAP_JARFILE}" \
+    $opts

build.sbt

@@ -4,16 +4,18 @@ name := "SQLTap"
 
 organization := "com.paulasmuth"
 
-version := "0.7.21"
+version := "0.8.1"
 
 mainClass in (Compile, run) := Some("com.paulasmuth.sqltap.SQLTap")
 
-scalaSource in Compile <<= baseDirectory(_ / "src")
-
-scalaVersion := "2.9.1"
+scalaVersion := "2.11.7"
 
 assemblySettings
 
-jarName in assembly := "sqltap_0.7.21.jar"
+jarName in assembly := { s"${name.value.toLowerCase}.jar" }
 
 fork in run := true
+
+libraryDependencies += "org.scala-lang.modules" %% "scala-xml" % "1.0.2"
+
+libraryDependencies += "org.scalatest" % "scalatest_2.11" % "2.2.4" % "test"

patches/sqltap-encoding-fix.diff

@@ -0,0 +1,62 @@
+diff --git a/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala b/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala
+index 2145f56..cf17011 100644
+--- a/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala
++++ b/src/com/paulasmuth/sqltap/ExpirationHandlerFactory.scala
+@@ -1,5 +1,5 @@
+ // This file is part of the "SQLTap" project
+-//   (c) 2011-2013 Paul Asmuth <[email protected]>
++//   (c) 2014 Paul Asmuth, Google Inc. <[email protected]>
+ //
+ // Licensed under the MIT License (the "License"); you may not use this
+ // file except in compliance with the License. You may obtain a copy of
+@@ -30,8 +30,10 @@ object ExpirationHandlerFactory {
+       case "noop" =>
+         handler = new NoopExpirationHandler()
+
+-      case "purge" =>
++      case "purge" => {
+         handler = new PurgeExpirationHandler()
++        ReplicationFeed.start()
++      }
+
+       case _ =>
+         throw new ParseException("unknown expiration handler: " + name)
+diff --git a/src/com/paulasmuth/sqltap/HTTPParser.scala b/src/com/paulasmuth/sqltap/HTTPParser.scala
+index 22b27be..33cd26d 100644
+--- a/src/com/paulasmuth/sqltap/HTTPParser.scala
++++ b/src/com/paulasmuth/sqltap/HTTPParser.scala
+@@ -1,5 +1,5 @@
+ // This file is part of the "SQLTap" project
+-//   (c) 2011-2013 Paul Asmuth <[email protected]>
++//   (c) 2014 Paul Asmuth, Google Inc. <[email protected]>
+ //
+ // Licensed under the MIT License (the "License"); you may not use this
+ // file except in compliance with the License. You may obtain a copy of
+@@ -130,7 +130,7 @@ class HTTPParser {
+   }
+
+   def uri_parts() : List[String] = {
+-    val uri = URLDecoder.decode(http_uri, "UTF-8")
++    val uri = URLDecoder.decode(http_uri)
+     var pos = uri.length
+     var cur = pos - 1
+     var ret = new ListBuffer[String]()
+diff --git a/src/com/paulasmuth/sqltap/SQLTap.scala b/src/com/paulasmuth/sqltap/SQLTap.scala
+index 357fc20..05c8921 100644
+--- a/src/com/paulasmuth/sqltap/SQLTap.scala
++++ b/src/com/paulasmuth/sqltap/SQLTap.scala
+@@ -1,5 +1,5 @@
+ // This file is part of the "SQLTap" project
+-//   (c) 2011-2013 Paul Asmuth <[email protected]>
++//   (c) 2014 Paul Asmuth, Google Inc. <[email protected]>
+ //
+ // Licensed under the MIT License (the "License"); you may not use this
+ // file except in compliance with the License. You may obtain a copy of
+@@ -108,7 +108,6 @@ object SQLTap{
+     Manifest.load(new File(Config.get('config_base)))
+     RelationTrace.load(Manifest.resources)
+     ExpirationHandlerFactory.configure(Config.get('expiration_handler))
+-    ReplicationFeed.start()
+
+     val server = new Server(Config.get('threads).toInt)
+     server.run(Config.get('http_port).toInt)

patches/sqltap-log-slow-queries.diff

@@ -0,0 +1,100 @@
+diff -r -u sqltap/src/com/paulasmuth/sqltap/HTTPConnection.scala sqltap-0.7.21/src/com/paulasmuth/sqltap/HTTPConnection.scala
+--- sqltap/src/com/paulasmuth/sqltap/HTTPConnection.scala	2014-02-12 16:13:05.009387526 +0100
++++ sqltap-0.7.21/src/com/paulasmuth/sqltap/HTTPConnection.scala	2014-09-04 14:31:28.000000000 +0200
+@@ -1,10 +1,9 @@
+ // This file is part of the "SQLTap" project
+-//   (c) 2011-2013 Paul Asmuth <[email protected]>
++//   (c) 2014 Paul Asmuth, Google Inc. <[email protected]>
+ //
+ // Licensed under the MIT License (the "License"); you may not use this
+ // file except in compliance with the License. You may obtain a copy of
+ // the License at: http://opensource.org/licenses/MIT
+-
+ package com.paulasmuth.sqltap
+
+ import java.nio.channels.{SocketChannel,SelectionKey}
+@@ -23,6 +22,7 @@
+   private val parser = new HTTPParser()
+   private var state = HTTP_STATE_INIT
+   private var last_event : SelectionKey = null
++  private var last_uri : String = "<unknown>" // for debugging only
+   private var keepalive : Boolean = false
+   private var resp_buf : ByteBuffer = null
+
+@@ -190,6 +190,7 @@
+
+     idle_timer.cancel()
+     stime = System.nanoTime
++    last_uri = parser.http_uri
+     seq += 1
+
+     if (parser.http_version == "1.1")
+@@ -302,8 +303,13 @@
+   }
+
+   def finish() : Unit = {
+-    Statistics.incr('http_request_time_mean,
+-      (System.nanoTime - stime) / 1000000.0)
++    val runtime_millis = (System.nanoTime - stime) / 1000000.0
++    Statistics.incr('http_request_time_mean, runtime_millis)
++
++    if (Config.has_key('log_slow_queries) &&
++        runtime_millis >= Config.get('log_slow_queries).toInt) {
++      Logger.log("[HTTP] [Slow Query] (" + runtime_millis + "ms): " + last_uri)
++    }
+
+     if (!keepalive)
+       return close()
+diff -r -u sqltap/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala sqltap-0.7.21/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala
+--- sqltap/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala	2014-02-12 16:13:05.013389526 +0100
++++ sqltap-0.7.21/src/com/paulasmuth/sqltap/mysql/SQLQuery.scala	2014-09-04 14:31:38.000000000 +0200
+@@ -1,10 +1,9 @@
+ // This file is part of the "SQLTap" project
+-//   (c) 2011-2013 Paul Asmuth <[email protected]>
++//   (c) 2014 Paul Asmuth, Google Inc. <[email protected]>
+ //
+ // Licensed under the MIT License (the "License"); you may not use this
+ // file except in compliance with the License. You may obtain a copy of
+ // the License at: http://opensource.org/licenses/MIT
+-
+ package com.paulasmuth.sqltap.mysql
+
+ import com.paulasmuth.sqltap._
+@@ -44,8 +43,14 @@
+     tok = System.nanoTime
+     qtime = tok - tik
+
+-    Statistics.incr('sql_request_time_mean, qtime / 1000000.0)
+-    Logger.debug("Finished (" + (qtime / 1000000.0) + "ms): " + query)
++    val runtime_millis = qtime / 1000000.0
++    Statistics.incr('sql_request_time_mean, runtime_millis)
++    Logger.debug("Finished (" + runtime_millis + "ms): " + query)
++
++    if (Config.has_key('log_slow_queries) &&
++        runtime_millis >= Config.get('log_slow_queries).toInt) {
++      Logger.log("[SQL] [Slow Query] (" + runtime_millis + "ms): " + query)
++    }
+   }
+
+   def error(err: Throwable) : Unit = {
+diff -r -u sqltap/src/com/paulasmuth/sqltap/SQLTap.scala sqltap-0.7.21/src/com/paulasmuth/sqltap/SQLTap.scala
+--- sqltap/src/com/paulasmuth/sqltap/SQLTap.scala	2014-02-25 17:15:47.049982142 +0100
++++ sqltap-0.7.21/src/com/paulasmuth/sqltap/SQLTap.scala	2014-09-04 14:35:51.000000000 +0200
+@@ -72,6 +71,9 @@
+       else if (args(n) == "--disable-keepalive")
+         { Config.set('http_keepalive, "false"); n += 1 }
+
++      else if (args(n) == "--log-slow-queries")
++        { Config.set('log_slow_queries, args(n+1)); n += 2 }
++
+       else if ((args(n) == "-t") || (args(n) == "--threads"))
+         { Config.set('threads, args(n+1)); n += 2 }
+
+@@ -138,6 +139,7 @@
+     println("  --memcache-queuelen    <num>     max mysql queue size per worker              ")
+     println("  --memcache-numconns    <num>     max number of mysql connections per worker   ")
+     println("  --memcache-mode        <name>    replication mode (copy, shard)               ")
++    println("  --log-slow-queries     <millis>  log all queries with a runtime > val in ms   ")
+     println("  -h, --help                       you're reading it...                         ")
+     println("  -d, --debug                      debug mode                                   ")
+   }

project/plugins.sbt

@@ -0,0 +1,3 @@
+resolvers += Resolver.url("artifactory", url("http://scalasbt.artifactoryonline.com/scalasbt/sbt-plugin-releases"))(Resolver.ivyStylePatterns)
+
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.11.2")