Skip to content

Commit

Permalink
Use new Backend.save() method with file backup
Browse files Browse the repository at this point in the history
  • Loading branch information
@grote
grote committed Dec 18, 2024
1 parent bd3669c commit 3fbe3cf
Show file tree
Hide file tree
Showing 9 changed files with 202 additions and 84 deletions.
1 change: 1 addition & 0 deletions storage/lib/Android.bp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ android_library {
"com.google.android.material_material",
"kotlinx-coroutines-android",
"kotlinx-coroutines-core",
"okio-lib",
],
plugins: [
"androidx.room_room-compiler-plugin",
Expand Down
1 change: 1 addition & 0 deletions storage/lib/build.gradle.kts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,7 @@ dependencies {
implementation(libs.google.material)
implementation(libs.androidx.room.runtime)
implementation(libs.google.protobuf.javalite)
implementation(libs.squareup.okio)

ksp(group = "androidx.room", name = "room-compiler", version = libs.versions.room.get())
lintChecks(libs.thirdegg.lint.rules)
Expand Down
33 changes: 26 additions & 7 deletions storage/lib/src/main/java/org/calyxos/backup/storage/backup/Backup.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,15 @@ import org.calyxos.backup.storage.db.Db
import org.calyxos.backup.storage.measure
import org.calyxos.backup.storage.scanner.FileScanner
import org.calyxos.backup.storage.scanner.FileScannerResult
import org.calyxos.seedvault.core.MemoryLogger
import org.calyxos.seedvault.core.backends.BackendSaver
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.IBackendManager
import org.calyxos.seedvault.core.backends.TopLevelFolder
import org.calyxos.seedvault.core.crypto.KeyManager
import java.io.ByteArrayOutputStream
import java.io.IOException
import java.io.OutputStream
import java.security.GeneralSecurityException
import kotlin.time.Duration

Expand Down Expand Up @@ -143,15 +147,19 @@ internal class Backup(
val smallResult = measure("Backing up $numSmallFiles small files") {
smallFileBackup.backupFiles(filesResult.smallFiles, availableChunkIds, backupObserver)
}
MemoryLogger.log()
val numLargeFiles = filesResult.files.size
val largeResult = measure("Backing up $numLargeFiles files") {
fileBackup.backupFiles(filesResult.files, availableChunkIds, backupObserver)
}
MemoryLogger.log()
chunkWriter.clearBuffer() // TODO does this have effect on memory
val result = largeResult + smallResult
if (result.isEmpty) return // TODO maybe warn user that nothing could get backed up?
val backupSize = result.backupMediaFiles.sumOf { it.size } +
result.backupDocumentFiles.sumOf { it.size }
val endTime = System.currentTimeMillis()
MemoryLogger.log()

val backupSnapshot: BackupSnapshot
val snapshotWriteTime = measure {
Expand All @@ -164,15 +172,26 @@ internal class Backup(
.setTimeStart(startTime)
.setTimeEnd(endTime)
.build()
val bytesOutputStream = ByteArrayOutputStream(backupSnapshot.serializedSize)
bytesOutputStream.write(VERSION.toInt())
val ad = streamCrypto.getAssociatedDataForSnapshot(startTime)
streamCrypto.newEncryptingStream(streamKey, bytesOutputStream, ad).use { cryptoStream ->
backupSnapshot.writeTo(cryptoStream)
}
MemoryLogger.log()
val fileHandle = FileBackupFileType.Snapshot(androidId, startTime)
backend.save(fileHandle).use { outputStream ->
outputStream.write(VERSION.toInt())
val ad = streamCrypto.getAssociatedDataForSnapshot(startTime)
streamCrypto.newEncryptingStream(streamKey, outputStream, ad)
.use { encryptingStream ->
backupSnapshot.writeTo(encryptingStream)
}
val saver = object : BackendSaver {
override val size: Long = bytesOutputStream.size().toLong()
override val sha256: String? = null
override fun save(outputStream: OutputStream): Long {
val bytes = bytesOutputStream.toByteArray()
outputStream.write(bytes)
return bytes.size.toLong()
}
}
backend.save(fileHandle, saver)
bytesOutputStream.reset() // probably won't release memory
MemoryLogger.log()
}
val snapshotSize = backupSnapshot.serializedSize.toLong()
val snapshotSizeStr = Formatter.formatShortFileSize(context, snapshotSize)
Expand Down
57 changes: 42 additions & 15 deletions storage/lib/src/main/java/org/calyxos/backup/storage/backup/ChunkWriter.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,15 @@
package org.calyxos.backup.storage.backup

import android.util.Log
import kotlinx.coroutines.sync.Semaphore
import kotlinx.coroutines.sync.withPermit
import okio.Buffer
import okio.buffer
import okio.sink
import org.calyxos.backup.storage.backup.Backup.Companion.VERSION
import org.calyxos.backup.storage.crypto.StreamCrypto
import org.calyxos.backup.storage.db.ChunksCache
import org.calyxos.seedvault.core.backends.BackendSaver
import org.calyxos.seedvault.core.backends.FileBackupFileType
import org.calyxos.seedvault.core.backends.IBackendManager
import java.io.ByteArrayOutputStream
Expand Down Expand Up @@ -37,7 +43,9 @@ internal class ChunkWriter(
) {

private val backend get() = backendManager.backend
private val buffer = ByteArray(bufferSize)
private val semaphore = Semaphore(1)
private val byteBuffer = ByteArray(bufferSize)
private val buffer = Buffer()

@Throws(IOException::class, GeneralSecurityException::class)
suspend fun writeChunk(
Expand All @@ -54,32 +62,47 @@ internal class ChunkWriter(
val notCached = cachedChunk == null
if (isMissing) Log.w(TAG, "Chunk ${chunk.id} is missing (cached: ${!notCached})")
if (notCached || isMissing) { // chunk not in storage
writeChunkData(chunk.id) { encryptingStream ->
val size = writeChunkData(chunk.id) { encryptingStream ->
copyChunkFromInputStream(inputStream, chunk, encryptingStream)
}
if (notCached) chunksCache.insert(chunk.toCachedChunk())
if (notCached) chunksCache.insert(chunk.toCachedChunk(size))
writtenChunks++
writtenBytes += chunk.plaintextSize
writtenBytes += size
} else { // chunk already uploaded
val skipped = inputStream.skip(chunk.plaintextSize)
check(chunk.plaintextSize == skipped) { "skipping error" }
}
}
val endByte = inputStream.read()
check(endByte == -1) { "Stream did continue with $endByte" }
// FIXME the writtenBytes are based on plaintext size, not ciphertext size
// However, they don't seem to be really used for anything at the moment.
return ChunkWriterResult(writtenChunks, writtenBytes)
}

@Throws(IOException::class, GeneralSecurityException::class)
private suspend fun writeChunkData(chunkId: String, writer: (OutputStream) -> Unit) {
private suspend fun writeChunkData(chunkId: String, writer: (OutputStream) -> Unit): Long {
val handle = FileBackupFileType.Blob(androidId, chunkId)
backend.save(handle).use { chunkStream ->
chunkStream.write(VERSION.toInt())
semaphore.withPermit { // only allow one writer using the buffer at a time
buffer.clear()
buffer.writeByte(VERSION.toInt())
val ad = streamCrypto.getAssociatedDataForChunk(chunkId)
streamCrypto.newEncryptingStream(streamKey, chunkStream, ad).use { encryptingStream ->
writer(encryptingStream)
streamCrypto.newEncryptingStream(streamKey, buffer.outputStream(), ad).use { stream ->
writer(stream)
}
val saver = object : BackendSaver {
override val size: Long = buffer.size
override val sha256: String = buffer.sha256().hex()
override fun save(outputStream: OutputStream): Long {
val outputBuffer = outputStream.sink().buffer()
val length = outputBuffer.writeAll(buffer)
// flushing is important here, otherwise data doesn't get fully written!
outputBuffer.flush()
return length
}
}
return try {
backend.save(handle, saver)
} finally {
buffer.clear()
}
}
}
Expand All @@ -93,9 +116,9 @@ internal class ChunkWriter(
var totalBytesRead = 0L
do {
val sizeLeft = (chunk.plaintextSize - totalBytesRead).toInt()
val bytesRead = inputStream.read(buffer, 0, min(bufferSize, sizeLeft))
val bytesRead = inputStream.read(byteBuffer, 0, min(bufferSize, sizeLeft))
if (bytesRead == -1) throw IOException("unexpected end of stream for ${chunk.id}")
outputStream.write(buffer, 0, bytesRead)
outputStream.write(byteBuffer, 0, bytesRead)
totalBytesRead += bytesRead
} while (bytesRead >= 0 && totalBytesRead < chunk.plaintextSize)
check(totalBytesRead == chunk.plaintextSize) {
Expand All @@ -119,10 +142,10 @@ internal class ChunkWriter(
if (isMissing) Log.w(TAG, "Chunk ${chunk.id} is missing (cached: ${cachedChunk != null})")
if (cachedChunk != null && !isMissing) return false
// chunk not yet uploaded
writeChunkData(chunk.id) { encryptingStream ->
val size = writeChunkData(chunk.id) { encryptingStream ->
zip.writeTo(encryptingStream)
}
if (cachedChunk == null) chunksCache.insert(chunk.toCachedChunk())
if (cachedChunk == null) chunksCache.insert(chunk.toCachedChunk(size))
return true
}

Expand All @@ -147,4 +170,8 @@ internal class ChunkWriter(
lastModifiedTime = FileTime.fromMillis(0)
}

fun clearBuffer() {
buffer.clear()
}

}
6 changes: 2 additions & 4 deletions storage/lib/src/main/java/org/calyxos/backup/storage/backup/Chunker.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
package org.calyxos.backup.storage.backup

import org.calyxos.backup.storage.db.CachedChunk
import org.calyxos.seedvault.core.crypto.CoreCrypto
import org.calyxos.seedvault.core.toHexString
import java.io.IOException
import java.io.InputStream
Expand All @@ -18,11 +17,10 @@ internal data class Chunk(
val offset: Long,
val plaintextSize: Long,
) {
fun toCachedChunk() = CachedChunk(
fun toCachedChunk(size: Long) = CachedChunk(
id = id,
refCount = 0,
// FIXME sometimes, the ciphertext size is not as expected
size = 1 + CoreCrypto.expectedCiphertextSize(plaintextSize),
size = size,
)
}

Expand Down
2 changes: 1 addition & 1 deletion storage/lib/src/main/java/org/calyxos/backup/storage/backup/ZipChunker.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ internal data class ZipChunk(
val size: Long,
var wasUploaded: Boolean = false,
) {
fun toCachedChunk() = CachedChunk(id, 0, size)
fun toCachedChunk(size: Long) = CachedChunk(id, 0, size)
}

@Suppress("BlockingMethodInNonBlockingContext")
Expand Down
61 changes: 44 additions & 17 deletions storage/lib/src/test/java/org/calyxos/backup/storage/BackupRestoreTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ import org.calyxos.backup.storage.restore.Restore
import org.calyxos.backup.storage.scanner.FileScanner
import org.calyxos.backup.storage.scanner.FileScannerResult
import org.calyxos.seedvault.core.backends.Backend
import org.calyxos.seedvault.core.backends.BackendSaver
import org.calyxos.seedvault.core.backends.FileBackupFileType.Blob
import org.calyxos.seedvault.core.backends.FileBackupFileType.Snapshot
import org.calyxos.seedvault.core.backends.IBackendManager
Expand Down Expand Up @@ -164,15 +165,22 @@ internal class BackupRestoreTest {
} returns ByteArrayInputStream(fileDBytes) andThen ByteArrayInputStream(fileDBytes)

// output streams and caching
coEvery { backend.save(any<Blob>()) } returnsMany listOf(
zipChunkOutputStream, mOutputStream, dOutputStream
)
val saverSlot = slot<BackendSaver>()
coEvery { backend.save(any<Blob>(), capture(saverSlot)) } answers {
saverSlot.captured.save(zipChunkOutputStream)
} andThenAnswer {
saverSlot.captured.save(mOutputStream)
} andThenAnswer {
saverSlot.captured.save(dOutputStream)
}
every { chunksCache.hasCorruptedChunks(any()) } returns false
every { chunksCache.insert(any<CachedChunk>()) } just Runs
every { filesCache.upsert(capture(cachedFiles)) } just Runs

// snapshot writing
coEvery { backend.save(capture(snapshotHandle)) } returns snapshotOutputStream
coEvery { backend.save(capture(snapshotHandle), capture(saverSlot)) } answers {
saverSlot.captured.save(snapshotOutputStream)
}
every { db.applyInParts<String>(any(), any()) } just Runs

backup.runBackup(null)
Expand Down Expand Up @@ -317,49 +325,64 @@ internal class BackupRestoreTest {
every { context.cacheDir } returns tmpDir

// output streams for deterministic chunks
val saverSlot = slot<BackendSaver>()
val id040f32 = ByteArrayOutputStream()
coEvery {
backend.save(
Blob(
androidId = androidId,
name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
)
),
capture(saverSlot),
)
} returns id040f32
} answers {
saverSlot.captured.save(id040f32)
}
val id901fbc = ByteArrayOutputStream()
coEvery {
backend.save(
Blob(
androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
),
capture(saverSlot),
)
} returns id901fbc
} answers {
saverSlot.captured.save(id901fbc)
}
val id5adea3 = ByteArrayOutputStream()
coEvery {
backend.save(
Blob(
androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
),
capture(saverSlot),
)
} returns id5adea3
} answers {
saverSlot.captured.save(id5adea3)
}
val id40d00c = ByteArrayOutputStream()
coEvery {
backend.save(
Blob(
androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
),
capture(saverSlot),
)
} returns id40d00c
} answers {
saverSlot.captured.save(id40d00c)
}

every { chunksCache.hasCorruptedChunks(any()) } returns false
every { chunksCache.insert(any<CachedChunk>()) } just Runs
every { filesCache.upsert(capture(cachedFiles)) } just Runs

// snapshot writing
coEvery { backend.save(capture(snapshotHandle)) } returns snapshotOutputStream
coEvery { backend.save(capture(snapshotHandle), capture(saverSlot)) } answers {
saverSlot.captured.save(snapshotOutputStream)
}
every { db.applyInParts<String>(any(), any()) } just Runs

backup.runBackup(null)
Expand All @@ -370,25 +393,29 @@ internal class BackupRestoreTest {
Blob(
androidId = androidId,
name = "040f3204869543c4015d92c04bf875b25ebde55f9645380f4172aa439b2825d3",
)
),
any(),
)
backend.save(
Blob(
androidId = androidId,
name = "901fbcf9a94271fc0455d0052522cab994f9392d0bb85187860282b4beadfb29",
)
),
any(),
)
backend.save(
Blob(
androidId = androidId,
name = "5adea3149fe6cf9c6e3270a52ee2c31bc9dfcef5f2080b583a4dd3b779c9182d",
)
),
any(),
)
backend.save(
Blob(
androidId = androidId,
name = "40d00c1be4b0f89e8b12d47f3658aa42f568a8d02b978260da6d0050e7007e67",
)
),
any(),
)
}

Expand Down
Loading

0 comments on commit 3fbe3cf

Please sign in to comment.