Interfacing lua_dump and lua_tolstring (#38)

* Added Lua#dump API and relevant testing
* Added Lua#toBuffer
* Added Lua#toDirectBuffer
* Updated documentation

build.gradle

@@ -1,5 +1,5 @@
 group 'party.iroiro.luajava'
-version (System.getenv('IS_RELEASE') == 'true' ? '3.1.2' : '3.1.2-SNAPSHOT')
+version(System.getenv('IS_RELEASE') == 'true' ? '3.1.3' : '3.1.3-SNAPSHOT')
 
 buildscript {
     repositories {
@@ -41,14 +41,22 @@ apply from: 'publish.gradle'
 apply from: 'jacoco.gradle'
 
 task allJavadoc(type: Javadoc) {
-    source subprojects.findAll { !['example', 'android', 'android-test'].contains(it.name) }
-            .collect {
-        it.sourceSets.main.allJava
-    }
-    classpath = files(subprojects.findAll { !['example', 'android', 'android-test'].contains(it.name) }
-            .collect {
-        it.sourceSets.main.compileClasspath
-    })
-    exclude '**/party/iroiro/luajava/util/**'
+    Set<String> projects = [
+            'lua51',
+            'lua52',
+            'lua53',
+            'lua54',
+            'luajit',
+            'luajava',
+    ]
+    source subprojects
+            .findAll { projects.contains(it.name) }
+            .collect { it.sourceSets.main.allJava }
+    classpath = files(subprojects.findAll { projects.contains(it.name) }
+            .collect { it.sourceSets.main.compileClasspath })
+    exclude(
+            '**/party/iroiro/luajava/util/**',
+            '**/party/iroiro/luajava/cleaner/**',
+    )
     destinationDir = file("${buildDir}/docs/javadoc")
 }

docs/examples/java.md

@@ -180,3 +180,53 @@ See
 [`unref(int)`](../javadoc/party/iroiro/luajava/Lua.html#unref(int))
 and [`unRef(int, int)`](../javadoc/party/iroiro/luajava/Lua.html#unRef(int,int)) for more info.
 
+### Pre-compiled chunks
+
+According to the Lua reference manual:
+
+> Chunks can also be pre-compiled into binary form;
+> see program luac for details.
+> Programs in source and compiled forms are interchangeable;
+> Lua automatically detects the file type and acts accordingly. 
+
+Replacing Lua source files with pre-compiled chunks speeds up loading.
+However, according to `luac` manual:
+
+> Precompiled chunks are *not* portable across different architectures.
+> Moreover, the internal format of precompiled chunks is likely to change
+> when a new version of Lua is released.  Make sure you save  the  source
+> files of all Lua programs that you precompile.
+
+Binary chunks compiled on one platform may not run on another.
+Since we are using Java / JVM-based languages, this is absolutely not desirable.
+To work around this, you may either:
+1. Provide precompiled chunks for all your target platforms;
+2. Or compile them at runtime for once and just reuse the compiled binaries.
+
+Here is a tiny example for the second approach:
+
+:::: code-group
+::: code-group-item Use lua_dump
+```java
+Lua L = getYourLuaInstance();
+ByteBuffer code = readFromFile("MyScript.lua");
+// L.load(...) pushes on stack a precompiled function
+L.load(code, "MyScript.lua");
+// L.dump() calls lua_dump, dumping the precompiled binary
+ByteBuffer precompiledChunk = L.dump();
+```
+:::
+::: code-group-item Use string.dump
+```java
+Lua L = getYourLuaInstance();
+L.openLibrary("string");
+// string.dump(...) returns the precompiled binary as a Lua string
+L.run("return string.dump(function(a, b) return a + b end)");
+// L.toBuffer(...) stores the precompiled binary into a buffer and returns it
+ByteBuffer precompiledChunk = L.toBuffer(-1);
+```
+:::
+::::
+
+Dumping functions involves some more Lua knowledge such as up-values and environments.
+What these terms mean might differ between versions and is not a topic of this document.

example/suite/src/main/java/party/iroiro/luajava/LuaTestSuite.java

@@ -41,6 +41,7 @@ public LuaTestSuite(T L, LuaTestSupplier<T> constructor) {
     public void test() {
         L.openLibraries();
         LuaScriptSuite.addAssertThrows(L);
+        testDump();
         testException();
         testExternalLoader();
         testGc();
@@ -60,6 +61,97 @@ public void test() {
         testThreads();
     }
 
+    private void testDump() {
+        try (T L = constructor.get();
+             T J = constructor.get()) {
+            // Simple functions
+            assertEquals(OK, L.run("return function(a, b) return a + b end"));
+            ByteBuffer add = L.dump();
+            assertNotNull(add);
+            for (int i = 0; i < 100; i += 17) {
+                for (int j = 0; j < 100; j += 37) {
+                    assertEquals(OK, J.load(add, "addition.lua"));
+                    J.push(i);
+                    J.push(j);
+                    assertEquals(OK, J.pCall(2, 1));
+                    assertEquals(i + j, J.toNumber(-1), 0.000001);
+                    J.pop(1);
+                }
+            }
+            L.pop(1);
+
+            // toBuffer with string.dump
+            L.openLibrary("string");
+            assertEquals(OK, L.run("return string.dump(function(a, b) return a + b end)"));
+            ByteBuffer dumpedAdd = L.toBuffer(-1);
+            assertNotNull(dumpedAdd);
+            for (int i = 0; i < 100; i += 17) {
+                for (int j = 0; j < 100; j += 37) {
+                    assertEquals(OK, J.load(dumpedAdd, "stringDumpedAdd.lua"));
+                    J.push(i);
+                    J.push(j);
+                    assertEquals(OK, J.pCall(2, 1));
+                    assertEquals(i + j, J.toNumber(-1), 0.000001);
+                    J.pop(1);
+                }
+            }
+            L.pop(1);
+
+            // Non-functions
+            L.createTable(0, 0);
+            assertNull(L.dump());
+            L.pop(1);
+            L.pushNil();
+            assertNull(L.dump());
+            L.pop(1);
+
+            // C functions
+            assertEquals(OK, L.run("return java.new"));
+            assertTrue(L.isFunction(-1));
+            assertNull(L.dump());
+            L.pop(1);
+
+            // Functions with up-values
+            assertEquals(OK, L.run("value = 1000"));
+            assertEquals(OK, L.run("return function(b) return b + value end"));
+            ByteBuffer upAdd = L.dump();
+            assertNotNull(upAdd);
+            assertEquals(OK, J.load(upAdd, "upAdd.lua"));
+            J.push(24);
+            assertEquals(RUNTIME, J.pCall(1, 1));
+            assertEquals(OK, J.run("value = 1000"));
+            assertEquals(OK, J.load(upAdd, "upAdd.lua"));
+            J.push(24);
+            assertEquals(OK, J.pCall(1, 1));
+            assertEquals(1024., J.toNumber(-1), 0.000001);
+            L.pop(1);
+
+            // Large string to buffer
+            J.openLibrary("string");
+            // Aiming 1 MB
+            J.run("s = 's'; for i = 1, 11 do s = string.rep(s, 4) end");
+            J.getGlobal("s");
+            int size = 4 * 1024 * 1024;
+            assertEquals(size, J.rawLength(-1));
+            ByteBuffer buffer = J.toBuffer(-1);
+            assertNotNull(buffer);
+            assertEquals(size, buffer.capacity());
+            for (int i = 0; i < size; i++) {
+                assertEquals('s', buffer.get(i));
+            }
+            ByteBuffer direct = J.toDirectBuffer(-1);
+            assertNotNull(direct);
+            assertTrue(direct.isDirect());
+            assertTrue(direct.isReadOnly());
+            assertEquals(size, direct.limit());
+            assertEquals(size, direct.capacity());
+            J.pop(1);
+            J.createTable(0, 0);
+            assertNull(J.toDirectBuffer(-1));
+            J.pop(1);
+        }
+    }
+
     private void testGc() {
         try (T L = constructor.get()) {
             L.createTable(0, 100000);

jni/luajava/jua.cpp

@@ -4,6 +4,8 @@
 #include "jua.h"
 #include "juaapi.h"
 
+#include <stdlib.h>
+
 // For template usage
 const char JAVA_CLASS_META_REGISTRY[] = "__jclass__";
 const char JAVA_OBJECT_META_REGISTRY[] = "__jobject__";
@@ -36,6 +38,7 @@ jmethodID juaapi_import         = NULL;
 jmethodID juaapi_proxy          = NULL;
 jmethodID juaapi_unwrap         = NULL;
 jmethodID juaapi_load           = NULL;
+jmethodID juaapi_allocatedirect = NULL;
 // java.lang.Throwable
 jclass java_lang_throwable_class = NULL;
 jmethodID throwable_getmessage   = NULL;
@@ -162,6 +165,8 @@ int initBindings(JNIEnv * env) {
           "unwrap", "(ILjava/lang/Object;)I");
   juaapi_load = bindJavaStaticMethod(env, juaapi_class,
           "load", "(ILjava/lang/String;)I");
+  juaapi_allocatedirect = bindJavaStaticMethod(env, juaapi_class,
+          "allocateDirect", "(I)Ljava/nio/ByteBuffer;");
   if (java_lang_class_class == NULL
       || java_lang_class_forname == NULL
       || java_lang_throwable_class == NULL
@@ -186,7 +191,8 @@ int initBindings(JNIEnv * env) {
       || juaapi_luaify == NULL
       || juaapi_import == NULL
       || juaapi_proxy == NULL
-      || juaapi_load == NULL) {
+      || juaapi_load == NULL
+      || juaapi_allocatedirect == NULL) {
     return -1;
   } else {
     return 0;
@@ -413,3 +419,91 @@ int luaJ_insertloader(lua_State * L, const char * searchers) {
 void luaJ_gc(lua_State * L) {
   lua_gc(L, LUA_GCCOLLECT, 0);
 }
+
+static jint nextCapacity(jint capacity, jint size) {
+  while (capacity > 0 && capacity < size) {
+    capacity <<= 1;
+  }
+  return capacity;
+}
+
+struct DumpBuffer {
+  unsigned char * buffer = NULL;
+  jint size = 0;
+  jint capacity = 0;
+};
+
+int dumpBufferWriter(lua_State * L, const void * p, size_t sz, void * ud) {
+  DumpBuffer * dump = (DumpBuffer *) ud;
+  jint size = dump->size + sz;
+  if (size < 0) {
+    /* Overflows */
+    return 1;
+  }
+  if (size > dump->capacity) {
+    jint capacity = nextCapacity(dump->capacity, size);
+    if (capacity <= 0) {
+      /* Overflows */
+      return 1;
+    }
+    void * buffer = realloc(dump->buffer, capacity);
+    if (buffer == NULL) {
+      return 1;
+    }
+    dump->capacity = capacity;
+    dump->buffer = (unsigned char *) buffer;
+  }
+  memcpy(dump->buffer + dump->size, p, sz);
+  dump->size = size;
+  return 0;
+}
+
+static jobject toBuffer(JNIEnv * env, lua_State * L, const void * ptr, jint size) {
+  jobject buffer = env->CallStaticObjectMethod(juaapi_class, juaapi_allocatedirect, (jint) size);
+  if (checkIfError(env, L)) {
+    return NULL;
+  }
+  void * addr = env->GetDirectBufferAddress(buffer);
+  memcpy(addr, ptr, size);
+  return buffer;
+}
+
+jobject luaJ_dumptobuffer(lua_State * L) {
+  DumpBuffer dump;
+  dump.size = 0;
+  dump.capacity = 4096;
+  dump.buffer = (unsigned char *) malloc(dump.capacity);
+  if (luaJ_dump(L, dumpBufferWriter, &dump)) {
+    free(dump.buffer);
+    return NULL;
+  }
+  JNIEnv * env = getJNIEnv(L);
+  jobject buffer = toBuffer(env, L, dump.buffer, dump.size);
+  free(dump.buffer);
+  return buffer;
+}
+
+jobject luaJ_tobuffer(lua_State * L, int i) {
+  size_t len;
+  const char * str = lua_tolstring(L, i, &len);
+  if (str == NULL) {
+    return NULL;
+  }
+  JNIEnv * env = getJNIEnv(L);
+  return toBuffer(env, L, str, len);
+}
+
+jobject luaJ_todirectbuffer(lua_State * L, int i) {
+  size_t len;
+  const void * str = lua_tolstring(L, i, &len);
+  if (str == NULL) {
+    return NULL;
+  }
+  JNIEnv * env = getJNIEnv(L);
+  /* Have to const_cast. We are to use it "asReadOnlyBuffer" on the Java side. */
+  jobject buffer = env->NewDirectByteBuffer(const_cast<void *>(str), (jlong) len);
+  if (checkIfError(env, L)) {
+    return NULL;
+  }
+  return buffer;
+}

jni/luajava/jua.h

@@ -55,6 +55,10 @@ void luaJ_pusharray(JNIEnv * env, lua_State * L, jobject array);
 jobject luaJ_toobject(lua_State * L, int index);
 int luaJ_isobject(lua_State * L, int index);
 
+jobject luaJ_dumptobuffer(lua_State * L);
+jobject luaJ_tobuffer(lua_State * L, int i);
+jobject luaJ_todirectbuffer(lua_State * L, int i);
+
 int luaJ_insertloader(lua_State * L, const char * searchers);
 
 int luaJ_invokespecial(JNIEnv * env, lua_State * L,

lua51/jni/mod/luacomp.h

@@ -89,4 +89,8 @@ static int luaJ_initloader(lua_State * L) {
   return luaJ_insertloader(L, "loaders");
 }
 
+static int luaJ_dump (lua_State *L, lua_Writer writer, void *data) {
+  return lua_dump (L, writer, data);
+}
+
 #endif /* !LUACOMP_H */