Convert the encoded DCF and ServerFigmaDocs to Proto

Cargo.toml

@@ -42,6 +42,7 @@ serde-reflection = { version = "0.3" }
 svgtypes = "0.15.1"
 taffy = { version = "0.6", default-features = false, features = ["std", "taffy_tree", "flexbox", "content_size"] }
 testdir = "0.9.1"
+tempfile = "3.11.0"
 thiserror = "1.0"
 unicode-segmentation = "1"
 ureq = "2"

common/build.gradle.kts

@@ -38,6 +38,17 @@ publishing {
     }
 }
 
+sourceSets {
+    test {
+        resources.srcDirs(rootProject.rootDir.resolve("designcompose/src/main/assets"))
+        resources.srcDirs(
+            rootProject.rootDir.resolve("reference-apps/helloworld/helloworld-app/src/main/assets")
+        )
+        // Enable testing of the Validation dcf files once we're able to serialize them again
+        // resources.srcDirs(rootProject.rootDir.resolve("integration-tests/validation/src/main/assets"))
+    }
+}
+
 /**
  * Serde gen task
  *

common/src/main/java/com/android/designcompose/common/FsaasSession.kt → common/src/main/java/com/android/designcompose/common/DCDVersion.kt

@@ -18,5 +18,5 @@ package com.android.designcompose.common
 
 // LINT.IfChange
 // Current serialized doc version
-const val FSAAS_DOC_VERSION = 24
+const val FSAAS_DOC_VERSION = 25
 // LINT.ThenChange(crates/dc_bundle/src/legacy_definition.rs)

common/src/main/java/com/android/designcompose/common/Feedback.kt

@@ -16,6 +16,8 @@
 
 package com.android.designcompose.common
 
+import java.util.logging.Logger
+
 const val TAG = "DesignCompose"
 
 enum class FeedbackLevel {
@@ -212,23 +214,17 @@ abstract class FeedbackImpl {
     }
 }
 
-// This has been commented out since it was only used by the gradle preview plugin which currently
-// has been disabled. If we bring the plugin back we may need to uncomment this code so that the
-// plugin can log feedback messages.
-/*
 // Instance of the Feedback class used in non-Android environments
-
 object Feedback : FeedbackImpl() {
-  private val javaLogger = Logger.getLogger(TAG)
+    private val javaLogger = Logger.getLogger(TAG)
 
-  // Implementation-specific functions
-  override fun logMessage(str: String, level: FeedbackLevel) {
-    when (level) {
-      FeedbackLevel.Debug -> javaLogger.config(str)
-      FeedbackLevel.Info -> javaLogger.info(str)
-      FeedbackLevel.Warn -> javaLogger.warning(str)
-      FeedbackLevel.Error -> javaLogger.severe(str)
+    // Implementation-specific functions
+    override fun logMessage(str: String, level: FeedbackLevel) {
+        when (level) {
+            FeedbackLevel.Debug -> javaLogger.config(str)
+            FeedbackLevel.Info -> javaLogger.info(str)
+            FeedbackLevel.Warn -> javaLogger.warning(str)
+            FeedbackLevel.Error -> javaLogger.severe(str)
+        }
     }
-  }
 }
-*/

common/src/main/java/com/android/designcompose/common/GenericDocContent.kt

@@ -17,13 +17,10 @@
 package com.android.designcompose.common
 
 import com.android.designcompose.definition.DesignComposeDefinition
+import com.android.designcompose.definition.DesignComposeDefinitionHeader
 import com.android.designcompose.definition.view.View
+import com.android.designcompose.live_update.figma.FigmaDocInfo
 import com.android.designcompose.live_update.figma.ServerFigmaDoc
-import com.android.designcompose.serdegen.DesignComposeDefinitionHeader
-import com.android.designcompose.serdegen.FigmaDocInfo
-import com.android.designcompose.serdegen.ViewDataType
-import com.novi.bincode.BincodeDeserializer
-import com.novi.bincode.BincodeSerializer
 import java.io.ByteArrayOutputStream
 import java.io.File
 import java.io.FileOutputStream
@@ -57,10 +54,11 @@ class GenericDocContent(
     @kotlin.ExperimentalUnsignedTypes
     fun toSerializedBytes(feedback: FeedbackImpl): ByteArray? {
         try {
-            val serializer = BincodeSerializer()
-            header.serialize(serializer)
-            document.serialize(serializer)
-            return serializer._bytes.toUByteArray().asByteArray() + imageSessionData
+            val outputStream = ByteArrayOutputStream()
+            header.writeDelimitedTo(outputStream)
+            document.writeDelimitedTo(outputStream)
+            outputStream.write(imageSessionData)
+            return outputStream.toByteArray()
         } catch (error: Throwable) {
             feedback.documentSaveError(error.toString(), docId)
         }
@@ -74,15 +72,15 @@ fun decodeServerBaseDoc(
     docId: DesignDocId,
     feedback: FeedbackImpl,
 ): GenericDocContent? {
-    val deserializer = BincodeDeserializer(docBytes)
-    val header = decodeHeader(deserializer, docId, feedback) ?: return null
+    val docStream = docBytes.inputStream()
+    val header = decodeHeader(docStream, docId, feedback) ?: return null
 
     // Server sends content in the format of ServerFigmaDoc, which has additional data
-    val serverDoc = ServerFigmaDoc.deserialize(deserializer)
-    serverDoc.errors?.forEach { feedback.documentUpdateWarnings(docId, it) }
-    val content = serverDoc.figma_doc.get()
-    val imageSessionData = decodeImageSession(docBytes, deserializer)
-    feedback.documentDecodeSuccess(header.dc_version, header.name, header.last_modified, docId)
+    val serverDoc = ServerFigmaDoc.parseDelimitedFrom(docStream)
+    serverDoc.errorsList?.forEach { feedback.documentUpdateWarnings(docId, it) }
+    val content = serverDoc.figmaDoc
+    val imageSessionData = decodeImageSession(docStream)
+    feedback.documentDecodeSuccess(header.dcVersion, header.name, header.lastModified, docId)
 
     val viewMap = content.views()
     val variantViewMap = createVariantViewMap(viewMap)
@@ -97,31 +95,29 @@ fun decodeServerBaseDoc(
         nodeIdMap,
         imageSessionData.imageSessionData,
         imageSessionData.imageSession,
-        serverDoc.branches,
-        serverDoc.project_files,
+        serverDoc.branchesList,
+        serverDoc.projectFilesList,
     )
 }
 
 /// Read a serialized disk document from the given stream. Deserialize it and deserialize its images
 fun decodeDiskBaseDoc(
-    doc: InputStream,
+    docStream: InputStream,
     docId: DesignDocId,
     feedback: FeedbackImpl,
 ): GenericDocContent? {
-    val docBytes = readDocBytes(doc, docId, feedback)
-    val deserializer = BincodeDeserializer(docBytes)
+    feedback.documentDecodeStart(docId)
 
-    val header = decodeHeader(deserializer, docId, feedback) ?: return null
+    val header = decodeHeader(docStream, docId, feedback) ?: return null
+    val content = DesignComposeDefinition.parseDelimitedFrom(docStream)
 
-    // Disk loads are in the format of DesignComposeDefinition
-    val content = DesignComposeDefinition.deserialize(deserializer)
-    val imageSessionData = decodeImageSession(docBytes, deserializer)
+    val imageSessionData = decodeImageSession(docStream)
     val viewMap = content.views()
     val variantMap = createVariantViewMap(viewMap)
     val variantPropertyMap = createVariantPropertyMap(viewMap)
     val nodeIdMap = createNodeIdMap(viewMap)
 
-    feedback.documentDecodeSuccess(header.dc_version, header.name, header.last_modified, docId)
+    feedback.documentDecodeSuccess(header.dcVersion, header.name, header.lastModified, docId)
 
     return GenericDocContent(
         docId,
@@ -183,56 +179,23 @@ private fun createNodeIdMap(nodes: Map<NodeQuery, View>?): HashMap<String, View>
     val nodeIdMap = HashMap<String, View>()
     fun addViewToMap(view: View) {
         nodeIdMap[view.id] = view
-        (view.data.get().view_data_type.get() as? ViewDataType.Container)?.let { container ->
-            container.value.children.forEach { addViewToMap(it) }
+        if (view.data.hasContainer()) {
+            view.data.container.childrenList.forEach { addViewToMap(it) }
         }
     }
     nodes?.values?.forEach { addViewToMap(it) }
     return nodeIdMap
 }
 
-fun readDocBytes(doc: InputStream, docId: DesignDocId, feedback: FeedbackImpl): ByteArray {
-    // Read the doc from assets or network...
-    feedback.documentDecodeStart(docId)
-    val buffer = ByteArrayOutputStream()
-    var nRead: Int
-    val tmp = ByteArray(128)
-
-    do {
-        nRead = doc.read(tmp, 0, tmp.size)
-        if (nRead != -1) buffer.write(tmp, 0, nRead)
-    } while (nRead != -1)
-    buffer.flush()
-    val docBytes = buffer.toByteArray()
-    feedback.documentDecodeReadBytes(docBytes.size, docId)
-    return docBytes
-}
-
-fun readErrorBytes(errorStream: InputStream?): String {
-    if (errorStream == null) return ""
-
-    val buffer = ByteArrayOutputStream()
-    var nRead: Int
-    val tmp = ByteArray(128)
-
-    do {
-        nRead = errorStream.read(tmp, 0, tmp.size)
-        if (nRead != -1) buffer.write(tmp, 0, nRead)
-    } while (nRead != -1)
-    buffer.flush()
-    val docBytes = buffer.toByteArray()
-    return String(docBytes)
-}
-
 private fun decodeHeader(
-    deserializer: BincodeDeserializer,
+    docStream: InputStream,
     docId: DesignDocId,
     feedback: FeedbackImpl,
 ): DesignComposeDefinitionHeader? {
     // Now attempt to deserialize the doc)
-    val header = DesignComposeDefinitionHeader.deserialize(deserializer)
-    if (header.dc_version != FSAAS_DOC_VERSION) {
-        feedback.documentDecodeVersionMismatch(FSAAS_DOC_VERSION, header.dc_version, docId)
+    val header = DesignComposeDefinitionHeader.parseDelimitedFrom(docStream)
+    if (header.dcVersion != FSAAS_DOC_VERSION) {
+        feedback.documentDecodeVersionMismatch(FSAAS_DOC_VERSION, header.dcVersion, docId)
         return null
     }
     return header
@@ -258,12 +221,9 @@ private data class ImageSession(val imageSessionData: ByteArray, var imageSessio
     }
 }
 
-private fun decodeImageSession(
-    docBytes: ByteArray,
-    deserializer: BincodeDeserializer,
-): ImageSession {
-    // The image session data is a JSON blob attached after the bincode document content.
-    val imageSessionData = docBytes.copyOfRange(deserializer._buffer_offset, docBytes.size)
+private fun decodeImageSession(docStream: InputStream): ImageSession {
+    // The image session data is a JSON blob attached after the proto document content.
+    val imageSessionData = docStream.readBytes()
     val imageSession =
         if (imageSessionData.isNotEmpty()) {
             String(imageSessionData, Charsets.UTF_8)

common/src/test/kotlin/com/android/designcompose/common/DocSerializationTest.kt

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.designcompose.common
+
+import java.io.InputStream
+import kotlin.io.path.createTempFile
+import kotlin.io.path.toPath
+import kotlin.test.assertNotNull
+import org.junit.Test
+
+class DocSerializationTest {
+
+    // Load the DesignSwitcherDoc from the Designcompose module
+    @Test
+    fun decodeDesignSwitcher() {
+        val inputStream: InputStream =
+            DocSerializationTest::class
+                .java
+                .classLoader!!
+                .getResourceAsStream("figma/DesignSwitcherDoc_Ljph4e3sC0lHcynfXpoh9f.dcf")!!
+        assertNotNull(inputStream)
+
+        assertNotNull(decodeDiskBaseDoc(inputStream, DesignDocId("test_doc"), Feedback))
+    }
+
+    // Load the HelloWorldDoc from HelloWorld
+    @Test
+    fun decodeHelloWorld() {
+        val inputStream: InputStream =
+            DocSerializationTest::class
+                .java
+                .classLoader!!
+                .getResourceAsStream("figma/HelloWorldDoc_pxVlixodJqZL95zo2RzTHl.dcf")!!
+        assertNotNull(inputStream)
+
+        assertNotNull(decodeDiskBaseDoc(inputStream, DesignDocId("test_doc"), Feedback))
+    }
+
+    @Test
+    fun loadSaveLoadHelloWorld() {
+        val inputStream: InputStream =
+            DocSerializationTest::class
+                .java
+                .classLoader!!
+                .getResourceAsStream("figma/HelloWorldDoc_pxVlixodJqZL95zo2RzTHl.dcf")!!
+        assertNotNull(inputStream)
+        val doc = decodeDiskBaseDoc(inputStream, DesignDocId("loadSaveLoad"), Feedback)
+        assertNotNull(doc)
+        val savedDoc = createTempFile()
+        doc.save(savedDoc.toFile(), Feedback)
+        val loadedDoc =
+            decodeDiskBaseDoc(
+                savedDoc.toFile().inputStream(),
+                DesignDocId("loadSaveLoad"),
+                Feedback,
+            )
+        assertNotNull(loadedDoc)
+    }
+
+    // Iterate through all dcf files in the figma/ directory and test them
+    @Test
+    fun loadSaveLoadAllDocs() {
+        val resourcesUrl = DocSerializationTest::class.java.classLoader!!.getResource("figma")
+        val resourcesFile = resourcesUrl?.toURI()?.toPath()?.toFile()
+
+        resourcesFile!!
+            .walkTopDown()
+            .filter { it.name.endsWith(".dcf") }
+            .forEach { file ->
+                println("Testing ${file.name}")
+                val doc = decodeDiskBaseDoc(file.inputStream(), DesignDocId(file.name), Feedback)
+                assertNotNull(doc)
+                val savedDoc = createTempFile()
+                doc.save(savedDoc.toFile(), Feedback)
+                val loadedDoc =
+                    decodeDiskBaseDoc(
+                        savedDoc.toFile().inputStream(),
+                        DesignDocId(file.name),
+                        Feedback,
+                    )
+                assertNotNull(loadedDoc)
+            }
+    }
+}

crates/dc_bundle/src/definition.rs

@@ -31,6 +31,42 @@ pub mod view;
 
 include!(concat!(env!("OUT_DIR"), "/designcompose.definition.rs"));
 
+// LINT.IfChange
+pub static CURRENT_VERSION: u32 = 25;
+// Lint.ThenChange(common/src/main/java/com/android/designcompose/common/DCDVersion.kt)
+
+impl DesignComposeDefinitionHeader {
+    pub fn current(
+        last_modified: String,
+        name: String,
+        response_version: String,
+        id: String,
+    ) -> DesignComposeDefinitionHeader {
+        DesignComposeDefinitionHeader {
+            dc_version: CURRENT_VERSION,
+            last_modified,
+            name,
+            response_version,
+            id,
+        }
+    }
+    pub fn current_version() -> u32 {
+        CURRENT_VERSION
+    }
+}
+
+impl fmt::Display for DesignComposeDefinitionHeader {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        // NOTE: Using `write!` here instead of typical `format!`
+        // to keep newlines.
+        write!(
+            f,
+            "DC Version: {}\nDoc ID: {}\nName: {}\nLast Modified: {}\nResponse Version: {}",
+            &self.dc_version, &self.id, &self.name, &self.last_modified, &self.response_version
+        )
+    }
+}
+
 impl DesignComposeDefinition {
     pub fn new(
         views: HashMap<NodeQuery, View>,