From 3fbe3cf1579ca730bb2435a8ea5f745cd94daf82 Mon Sep 17 00:00:00 2001 From: Torsten Grote Date: Wed, 18 Dec 2024 15:24:14 -0300 Subject: [PATCH] Use new Backend.save() method with file backup --- storage/lib/Android.bp | 1 + storage/lib/build.gradle.kts | 1 + .../calyxos/backup/storage/backup/Backup.kt | 33 +++-- .../backup/storage/backup/ChunkWriter.kt | 57 ++++++--- .../calyxos/backup/storage/backup/Chunker.kt | 6 +- .../backup/storage/backup/ZipChunker.kt | 2 +- .../backup/storage/BackupRestoreTest.kt | 61 ++++++--- .../backup/storage/backup/ChunkWriterTest.kt | 118 ++++++++++++------ .../backup/SmallFileBackupIntegrationTest.kt | 7 +- 9 files changed, 202 insertions(+), 84 deletions(-) diff --git a/storage/lib/Android.bp b/storage/lib/Android.bp index 6a49cbe22..6d4c4212f 100644 --- a/storage/lib/Android.bp +++ b/storage/lib/Android.bp @@ -32,6 +32,7 @@ android_library { "com.google.android.material_material", "kotlinx-coroutines-android", "kotlinx-coroutines-core", + "okio-lib", ], plugins: [ "androidx.room_room-compiler-plugin", diff --git a/storage/lib/build.gradle.kts b/storage/lib/build.gradle.kts index 0f714c16b..174e2fd75 100644 --- a/storage/lib/build.gradle.kts +++ b/storage/lib/build.gradle.kts @@ -103,6 +103,7 @@ dependencies { implementation(libs.google.material) implementation(libs.androidx.room.runtime) implementation(libs.google.protobuf.javalite) + implementation(libs.squareup.okio) ksp(group = "androidx.room", name = "room-compiler", version = libs.versions.room.get()) lintChecks(libs.thirdegg.lint.rules) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/Backup.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/Backup.kt index 3b52c419a..e549775f4 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/Backup.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/Backup.kt @@ -18,11 +18,15 @@ import org.calyxos.backup.storage.db.Db import org.calyxos.backup.storage.measure import org.calyxos.backup.storage.scanner.FileScanner import org.calyxos.backup.storage.scanner.FileScannerResult +import org.calyxos.seedvault.core.MemoryLogger +import org.calyxos.seedvault.core.backends.BackendSaver import org.calyxos.seedvault.core.backends.FileBackupFileType import org.calyxos.seedvault.core.backends.IBackendManager import org.calyxos.seedvault.core.backends.TopLevelFolder import org.calyxos.seedvault.core.crypto.KeyManager +import java.io.ByteArrayOutputStream import java.io.IOException +import java.io.OutputStream import java.security.GeneralSecurityException import kotlin.time.Duration @@ -143,15 +147,19 @@ internal class Backup( val smallResult = measure("Backing up $numSmallFiles small files") { smallFileBackup.backupFiles(filesResult.smallFiles, availableChunkIds, backupObserver) } + MemoryLogger.log() val numLargeFiles = filesResult.files.size val largeResult = measure("Backing up $numLargeFiles files") { fileBackup.backupFiles(filesResult.files, availableChunkIds, backupObserver) } + MemoryLogger.log() + chunkWriter.clearBuffer() // TODO does this have effect on memory val result = largeResult + smallResult if (result.isEmpty) return // TODO maybe warn user that nothing could get backed up? val backupSize = result.backupMediaFiles.sumOf { it.size } + result.backupDocumentFiles.sumOf { it.size } val endTime = System.currentTimeMillis() + MemoryLogger.log() val backupSnapshot: BackupSnapshot val snapshotWriteTime = measure { @@ -164,15 +172,26 @@ internal class Backup( .setTimeStart(startTime) .setTimeEnd(endTime) .build() + val bytesOutputStream = ByteArrayOutputStream(backupSnapshot.serializedSize) + bytesOutputStream.write(VERSION.toInt()) + val ad = streamCrypto.getAssociatedDataForSnapshot(startTime) + streamCrypto.newEncryptingStream(streamKey, bytesOutputStream, ad).use { cryptoStream -> + backupSnapshot.writeTo(cryptoStream) + } + MemoryLogger.log() val fileHandle = FileBackupFileType.Snapshot(androidId, startTime) - backend.save(fileHandle).use { outputStream -> - outputStream.write(VERSION.toInt()) - val ad = streamCrypto.getAssociatedDataForSnapshot(startTime) - streamCrypto.newEncryptingStream(streamKey, outputStream, ad) - .use { encryptingStream -> - backupSnapshot.writeTo(encryptingStream) - } + val saver = object : BackendSaver { + override val size: Long = bytesOutputStream.size().toLong() + override val sha256: String? = null + override fun save(outputStream: OutputStream): Long { + val bytes = bytesOutputStream.toByteArray() + outputStream.write(bytes) + return bytes.size.toLong() + } } + backend.save(fileHandle, saver) + bytesOutputStream.reset() // probably won't release memory + MemoryLogger.log() } val snapshotSize = backupSnapshot.serializedSize.toLong() val snapshotSizeStr = Formatter.formatShortFileSize(context, snapshotSize) diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunkWriter.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunkWriter.kt index 0e8b78d68..6c5285066 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunkWriter.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunkWriter.kt @@ -6,9 +6,15 @@ package org.calyxos.backup.storage.backup import android.util.Log +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import okio.Buffer +import okio.buffer +import okio.sink import org.calyxos.backup.storage.backup.Backup.Companion.VERSION import org.calyxos.backup.storage.crypto.StreamCrypto import org.calyxos.backup.storage.db.ChunksCache +import org.calyxos.seedvault.core.backends.BackendSaver import org.calyxos.seedvault.core.backends.FileBackupFileType import org.calyxos.seedvault.core.backends.IBackendManager import java.io.ByteArrayOutputStream @@ -37,7 +43,9 @@ internal class ChunkWriter( ) { private val backend get() = backendManager.backend - private val buffer = ByteArray(bufferSize) + private val semaphore = Semaphore(1) + private val byteBuffer = ByteArray(bufferSize) + private val buffer = Buffer() @Throws(IOException::class, GeneralSecurityException::class) suspend fun writeChunk( @@ -54,12 +62,12 @@ internal class ChunkWriter( val notCached = cachedChunk == null if (isMissing) Log.w(TAG, "Chunk ${chunk.id} is missing (cached: ${!notCached})") if (notCached || isMissing) { // chunk not in storage - writeChunkData(chunk.id) { encryptingStream -> + val size = writeChunkData(chunk.id) { encryptingStream -> copyChunkFromInputStream(inputStream, chunk, encryptingStream) } - if (notCached) chunksCache.insert(chunk.toCachedChunk()) + if (notCached) chunksCache.insert(chunk.toCachedChunk(size)) writtenChunks++ - writtenBytes += chunk.plaintextSize + writtenBytes += size } else { // chunk already uploaded val skipped = inputStream.skip(chunk.plaintextSize) check(chunk.plaintextSize == skipped) { "skipping error" } @@ -67,19 +75,34 @@ internal class ChunkWriter( } val endByte = inputStream.read() check(endByte == -1) { "Stream did continue with $endByte" } - // FIXME the writtenBytes are based on plaintext size, not ciphertext size - // However, they don't seem to be really used for anything at the moment. return ChunkWriterResult(writtenChunks, writtenBytes) } @Throws(IOException::class, GeneralSecurityException::class) - private suspend fun writeChunkData(chunkId: String, writer: (OutputStream) -> Unit) { + private suspend fun writeChunkData(chunkId: String, writer: (OutputStream) -> Unit): Long { val handle = FileBackupFileType.Blob(androidId, chunkId) - backend.save(handle).use { chunkStream -> - chunkStream.write(VERSION.toInt()) + semaphore.withPermit { // only allow one writer using the buffer at a time + buffer.clear() + buffer.writeByte(VERSION.toInt()) val ad = streamCrypto.getAssociatedDataForChunk(chunkId) - streamCrypto.newEncryptingStream(streamKey, chunkStream, ad).use { encryptingStream -> - writer(encryptingStream) + streamCrypto.newEncryptingStream(streamKey, buffer.outputStream(), ad).use { stream -> + writer(stream) + } + val saver = object : BackendSaver { + override val size: Long = buffer.size + override val sha256: String = buffer.sha256().hex() + override fun save(outputStream: OutputStream): Long { + val outputBuffer = outputStream.sink().buffer() + val length = outputBuffer.writeAll(buffer) + // flushing is important here, otherwise data doesn't get fully written! + outputBuffer.flush() + return length + } + } + return try { + backend.save(handle, saver) + } finally { + buffer.clear() } } } @@ -93,9 +116,9 @@ internal class ChunkWriter( var totalBytesRead = 0L do { val sizeLeft = (chunk.plaintextSize - totalBytesRead).toInt() - val bytesRead = inputStream.read(buffer, 0, min(bufferSize, sizeLeft)) + val bytesRead = inputStream.read(byteBuffer, 0, min(bufferSize, sizeLeft)) if (bytesRead == -1) throw IOException("unexpected end of stream for ${chunk.id}") - outputStream.write(buffer, 0, bytesRead) + outputStream.write(byteBuffer, 0, bytesRead) totalBytesRead += bytesRead } while (bytesRead >= 0 && totalBytesRead < chunk.plaintextSize) check(totalBytesRead == chunk.plaintextSize) { @@ -119,10 +142,10 @@ internal class ChunkWriter( if (isMissing) Log.w(TAG, "Chunk ${chunk.id} is missing (cached: ${cachedChunk != null})") if (cachedChunk != null && !isMissing) return false // chunk not yet uploaded - writeChunkData(chunk.id) { encryptingStream -> + val size = writeChunkData(chunk.id) { encryptingStream -> zip.writeTo(encryptingStream) } - if (cachedChunk == null) chunksCache.insert(chunk.toCachedChunk()) + if (cachedChunk == null) chunksCache.insert(chunk.toCachedChunk(size)) return true } @@ -147,4 +170,8 @@ internal class ChunkWriter( lastModifiedTime = FileTime.fromMillis(0) } + fun clearBuffer() { + buffer.clear() + } + } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/Chunker.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/Chunker.kt index dd812eb7c..fc26b9eb4 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/Chunker.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/Chunker.kt @@ -6,7 +6,6 @@ package org.calyxos.backup.storage.backup import org.calyxos.backup.storage.db.CachedChunk -import org.calyxos.seedvault.core.crypto.CoreCrypto import org.calyxos.seedvault.core.toHexString import java.io.IOException import java.io.InputStream @@ -18,11 +17,10 @@ internal data class Chunk( val offset: Long, val plaintextSize: Long, ) { - fun toCachedChunk() = CachedChunk( + fun toCachedChunk(size: Long) = CachedChunk( id = id, refCount = 0, - // FIXME sometimes, the ciphertext size is not as expected - size = 1 + CoreCrypto.expectedCiphertextSize(plaintextSize), + size = size, ) } diff --git a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ZipChunker.kt b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ZipChunker.kt index 295ac82e2..42ff42741 100644 --- a/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ZipChunker.kt +++ b/storage/lib/src/main/java/org/calyxos/backup/storage/backup/ZipChunker.kt @@ -25,7 +25,7 @@ internal data class ZipChunk( val size: Long, var wasUploaded: Boolean = false, ) { - fun toCachedChunk() = CachedChunk(id, 0, size) + fun toCachedChunk(size: Long) = CachedChunk(id, 0, size) } @Suppress("BlockingMethodInNonBlockingContext") diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt index a6f4b27f0..431578a6f 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt @@ -44,6 +44,7 @@ import org.calyxos.backup.storage.restore.Restore import org.calyxos.backup.storage.scanner.FileScanner import org.calyxos.backup.storage.scanner.FileScannerResult import org.calyxos.seedvault.core.backends.Backend +import org.calyxos.seedvault.core.backends.BackendSaver import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob import org.calyxos.seedvault.core.backends.FileBackupFileType.Snapshot import org.calyxos.seedvault.core.backends.IBackendManager @@ -164,15 +165,22 @@ internal class BackupRestoreTest { } returns ByteArrayInputStream(fileDBytes) andThen ByteArrayInputStream(fileDBytes) // output streams and caching - coEvery { backend.save(any()) } returnsMany listOf( - zipChunkOutputStream, mOutputStream, dOutputStream - ) + val saverSlot = slot() + coEvery { backend.save(any(), capture(saverSlot)) } answers { + saverSlot.captured.save(zipChunkOutputStream) + } andThenAnswer { + saverSlot.captured.save(mOutputStream) + } andThenAnswer { + saverSlot.captured.save(dOutputStream) + } every { chunksCache.hasCorruptedChunks(any()) } returns false every { chunksCache.insert(any()) } just Runs every { filesCache.upsert(capture(cachedFiles)) } just Runs // snapshot writing - coEvery { backend.save(capture(snapshotHandle)) } returns snapshotOutputStream + coEvery { backend.save(capture(snapshotHandle), capture(saverSlot)) } answers { + saverSlot.captured.save(snapshotOutputStream) + } every { db.applyInParts(any(), any()) } just Runs backup.runBackup(null) @@ -317,49 +325,64 @@ internal class BackupRestoreTest { every { context.cacheDir } returns tmpDir // output streams for deterministic chunks + val saverSlot = slot() val id040f32 = ByteArrayOutputStream() coEvery { backend.save( Blob( androidId = androidId, name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3", - ) + ), + capture(saverSlot), ) - } returns id040f32 + } answers { + saverSlot.captured.save(id040f32) + } val id901fbc = ByteArrayOutputStream() coEvery { backend.save( Blob( androidId = androidId, name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29", - ) + ), + capture(saverSlot), ) - } returns id901fbc + } answers { + saverSlot.captured.save(id901fbc) + } val id5adea3 = ByteArrayOutputStream() coEvery { backend.save( Blob( androidId = androidId, name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d", - ) + ), + capture(saverSlot), ) - } returns id5adea3 + } answers { + saverSlot.captured.save(id5adea3) + } val id40d00c = ByteArrayOutputStream() coEvery { backend.save( Blob( androidId = androidId, name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67", - ) + ), + capture(saverSlot), ) - } returns id40d00c + } answers { + saverSlot.captured.save(id40d00c) + } every { chunksCache.hasCorruptedChunks(any()) } returns false every { chunksCache.insert(any()) } just Runs every { filesCache.upsert(capture(cachedFiles)) } just Runs // snapshot writing - coEvery { backend.save(capture(snapshotHandle)) } returns snapshotOutputStream + coEvery { backend.save(capture(snapshotHandle), capture(saverSlot)) } answers { + saverSlot.captured.save(snapshotOutputStream) + } every { db.applyInParts(any(), any()) } just Runs backup.runBackup(null) @@ -370,25 +393,29 @@ internal class BackupRestoreTest { Blob( androidId = androidId, name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3", - ) + ), + any(), ) backend.save( Blob( androidId = androidId, name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29", - ) + ), + any(), ) backend.save( Blob( androidId = androidId, name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d", - ) + ), + any(), ) backend.save( Blob( androidId = androidId, name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67", - ) + ), + any(), ) } diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunkWriterTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunkWriterTest.kt index fe7585a81..c635cead6 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunkWriterTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/backup/ChunkWriterTest.kt @@ -11,6 +11,7 @@ import io.mockk.coEvery import io.mockk.every import io.mockk.just import io.mockk.mockk +import io.mockk.slot import kotlinx.coroutines.runBlocking import org.calyxos.backup.storage.backup.Backup.Companion.VERSION import org.calyxos.backup.storage.crypto.StreamCrypto @@ -18,6 +19,7 @@ import org.calyxos.backup.storage.db.ChunksCache import org.calyxos.backup.storage.getRandomString import org.calyxos.backup.storage.mockLog import org.calyxos.seedvault.core.backends.Backend +import org.calyxos.seedvault.core.backends.BackendSaver import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob import org.calyxos.seedvault.core.backends.IBackendManager import org.calyxos.seedvault.core.crypto.CoreCrypto.KEY_SIZE_BYTES @@ -27,6 +29,7 @@ import org.junit.Assert.assertEquals import org.junit.Test import java.io.ByteArrayInputStream import java.io.ByteArrayOutputStream +import java.io.OutputStream import kotlin.random.Random internal class ChunkWriterTest { @@ -77,31 +80,41 @@ internal class ChunkWriterTest { every { chunksCache.get(chunkId2) } returns null every { chunksCache.get(chunkId3) } returns null - // get the output streams for the chunks - coEvery { backend.save(Blob(androidId, chunkId1)) } returns chunk1Output - coEvery { backend.save(Blob(androidId, chunkId2)) } returns chunk2Output - coEvery { backend.save(Blob(androidId, chunkId3)) } returns chunk3Output - // get AD every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1 every { streamCrypto.getAssociatedDataForChunk(chunkId2) } returns ad2 every { streamCrypto.getAssociatedDataForChunk(chunkId3) } returns ad3 // wrap output stream in crypto stream - every { - streamCrypto.newEncryptingStream(streamKey, chunk1Output, ad1) - } returns chunk1Output - every { - streamCrypto.newEncryptingStream(streamKey, chunk2Output, ad2) - } returns chunk2Output - every { - streamCrypto.newEncryptingStream(streamKey, chunk3Output, ad3) - } returns chunk3Output + val streamSlot = slot() + every { streamCrypto.newEncryptingStream(streamKey, capture(streamSlot), ad1) } answers { + streamSlot.captured + } + every { streamCrypto.newEncryptingStream(streamKey, capture(streamSlot), ad2) } answers { + streamSlot.captured + } + every { streamCrypto.newEncryptingStream(streamKey, capture(streamSlot), ad3) } answers { + streamSlot.captured + } + + // save the chunks + val saverSlot = slot() + coEvery { + backend.save(Blob(androidId, chunkId1), capture(saverSlot)) + } answers { + saverSlot.captured.save(chunk1Output) + } + coEvery { backend.save(Blob(androidId, chunkId2), capture(saverSlot)) } answers { + saverSlot.captured.save(chunk2Output) + } + coEvery { backend.save(Blob(androidId, chunkId3), capture(saverSlot)) } answers { + saverSlot.captured.save(chunk3Output) + } // insert chunks into cache after upload - every { chunksCache.insert(chunks[0].toCachedChunk()) } just Runs - every { chunksCache.insert(chunks[1].toCachedChunk()) } just Runs - every { chunksCache.insert(chunks[2].toCachedChunk()) } just Runs + every { chunksCache.insert(chunks[0].toCachedChunk(3)) } just Runs + every { chunksCache.insert(chunks[1].toCachedChunk(3)) } just Runs + every { chunksCache.insert(chunks[2].toCachedChunk(3)) } just Runs chunkWriter.writeChunk(inputStream, chunks, emptyList()) @@ -129,29 +142,34 @@ internal class ChunkWriterTest { val chunk3Output = ByteArrayOutputStream() // only first two chunks are cached (first chunk is missing from storage) - every { chunksCache.get(chunkId1) } returns chunks[0].toCachedChunk() - every { chunksCache.get(chunkId2) } returns chunks[1].toCachedChunk() + every { chunksCache.get(chunkId1) } returns chunks[0].toCachedChunk(1) + every { chunksCache.get(chunkId2) } returns chunks[1].toCachedChunk(2) every { chunksCache.get(chunkId3) } returns null // get and wrap the output stream for chunk that is missing - coEvery { backend.save(Blob(androidId, chunkId1)) } returns chunk1Output + val saverSlot = slot() + coEvery { backend.save(Blob(androidId, chunkId1), capture(saverSlot)) } answers { + saverSlot.captured.save(chunk1Output) + } every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1 + val streamSlot = slot() every { - streamCrypto.newEncryptingStream(streamKey, chunk1Output, bytes(34)) - } returns chunk1Output + streamCrypto.newEncryptingStream(streamKey, capture(streamSlot), bytes(34)) + } answers { + streamSlot.captured + } // insert missing cached chunk into cache after upload - every { chunksCache.insert(chunks[0].toCachedChunk()) } just Runs + every { chunksCache.insert(chunks[0].toCachedChunk(3)) } just Runs // get and wrap the output stream for chunk that isn't cached - coEvery { backend.save(Blob(androidId, chunkId3)) } returns chunk3Output + coEvery { backend.save(Blob(androidId, chunkId3), capture(saverSlot)) } answers { + saverSlot.captured.save(chunk3Output) + } every { streamCrypto.getAssociatedDataForChunk(chunkId3) } returns ad3 - every { - streamCrypto.newEncryptingStream(streamKey, chunk3Output, bytes(34)) - } returns chunk3Output // insert last not cached chunk into cache after upload - every { chunksCache.insert(chunks[2].toCachedChunk()) } just Runs + every { chunksCache.insert(chunks[2].toCachedChunk(3)) } just Runs chunkWriter.writeChunk(inputStream, chunks, listOf(chunkId1)) @@ -183,28 +201,38 @@ internal class ChunkWriterTest { // first and last chunk are not cached every { chunksCache.get(chunkId1) } returns null - every { chunksCache.get(chunkId2) } returns chunks[1].toCachedChunk() + every { chunksCache.get(chunkId2) } returns chunks[1].toCachedChunk(1) every { chunksCache.get(chunkId3) } returns null // get the output streams for the chunks - coEvery { backend.save(Blob(androidId, chunkId1)) } returns chunk1Output - coEvery { backend.save(Blob(androidId, chunkId3)) } returns chunk3Output + val saverSlot = slot() + coEvery { backend.save(Blob(androidId, chunkId1), capture(saverSlot)) } answers { + saverSlot.captured.save(chunk1Output) + } + coEvery { backend.save(Blob(androidId, chunkId3), capture(saverSlot)) } answers { + saverSlot.captured.save(chunk3Output) + } // get AD every { streamCrypto.getAssociatedDataForChunk(chunkId1) } returns ad1 every { streamCrypto.getAssociatedDataForChunk(chunkId3) } returns ad3 // wrap output streams in crypto streams - every { - streamCrypto.newEncryptingStream(streamKey, chunk1Output, ad1) - } returns chunk1Output - every { - streamCrypto.newEncryptingStream(streamKey, chunk3Output, ad3) - } returns chunk3Output + val streamSlot = slot() + every { streamCrypto.newEncryptingStream(streamKey, capture(streamSlot), ad1) } answers { + streamSlot.captured + } + every { streamCrypto.newEncryptingStream(streamKey, capture(streamSlot), ad3) } answers { + streamSlot.captured + } // insert chunks into cache after upload - every { chunksCache.insert(chunks[0].toCachedChunk()) } just Runs - every { chunksCache.insert(chunks[2].toCachedChunk()) } just Runs + every { + chunksCache.insert(chunks[0].toCachedChunk(chunk1Bytes.size.toLong() + 1)) + } just Runs + every { + chunksCache.insert(chunks[2].toCachedChunk(chunk3Bytes.size.toLong() + 1)) + } just Runs chunkWriter.writeChunk(inputStream, chunks, emptyList()) @@ -226,6 +254,18 @@ internal class ChunkWriterTest { ) } + private fun getSaver() = object : BackendSaver { + override val size: Long + get() = TODO("Not yet implemented") + override val sha256: String? + get() = TODO("Not yet implemented") + + override fun save(outputStream: OutputStream): Long { + TODO("Not yet implemented") + } + + } + private fun MockKMatcherScope.bytes(size: Int) = match { it.size == size } diff --git a/storage/lib/src/test/java/org/calyxos/backup/storage/backup/SmallFileBackupIntegrationTest.kt b/storage/lib/src/test/java/org/calyxos/backup/storage/backup/SmallFileBackupIntegrationTest.kt index 9e47cf3ca..f3152b997 100644 --- a/storage/lib/src/test/java/org/calyxos/backup/storage/backup/SmallFileBackupIntegrationTest.kt +++ b/storage/lib/src/test/java/org/calyxos/backup/storage/backup/SmallFileBackupIntegrationTest.kt @@ -12,6 +12,7 @@ import io.mockk.coVerify import io.mockk.every import io.mockk.just import io.mockk.mockk +import io.mockk.slot import kotlinx.coroutines.runBlocking import org.calyxos.backup.storage.api.BackupObserver import org.calyxos.backup.storage.crypto.StreamCrypto @@ -22,6 +23,7 @@ import org.calyxos.backup.storage.getRandomDocFile import org.calyxos.backup.storage.getRandomString import org.calyxos.backup.storage.mockLog import org.calyxos.seedvault.core.backends.Backend +import org.calyxos.seedvault.core.backends.BackendSaver import org.calyxos.seedvault.core.backends.IBackendManager import org.calyxos.seedvault.core.crypto.CoreCrypto.KEY_SIZE_BYTES import org.calyxos.seedvault.core.toHexString @@ -98,7 +100,10 @@ internal class SmallFileBackupIntegrationTest { every { mac.doFinal(any()) } returns chunkId every { chunksCache.get(any()) } returns null every { chunksCache.hasCorruptedChunks(any()) } returns false - coEvery { backend.save(any()) } returns outputStream2 + val saverSlot = slot() + coEvery { backend.save(any(), capture(saverSlot)) } answers { + saverSlot.captured.save(outputStream2) + } every { chunksCache.insert(match { cachedChunk -> cachedChunk.id == chunkId.toHexString() &&