From 11fbe5d34b9950340425f9c7e633a92677bd436c Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 00:09:32 -0700 Subject: [PATCH 01/11] Duplicate html export usecase class for json export usecase --- .../di/module/application/UseCaseModule.kt | 18 + .../ExportDownloadedThreadAsJsonUseCase.kt | 370 ++++++++++++++++++ .../LocalArchiveViewModel.kt | 14 + 3 files changed, 402 insertions(+) create mode 100644 Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/di/module/application/UseCaseModule.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/di/module/application/UseCaseModule.kt index 064d0e368..abbe0fc2d 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/di/module/application/UseCaseModule.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/di/module/application/UseCaseModule.kt @@ -29,6 +29,7 @@ import com.github.k1rakishou.chan.core.usecase.ClearPostingCookies import com.github.k1rakishou.chan.core.usecase.DownloadThemeJsonFilesUseCase import com.github.k1rakishou.chan.core.usecase.ExportBackupFileUseCase import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadAsHtmlUseCase +import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadAsJsonUseCase import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadMediaUseCase import com.github.k1rakishou.chan.core.usecase.ExportFiltersUseCase import com.github.k1rakishou.chan.core.usecase.ExtractPostMapInfoHolderUseCase @@ -291,6 +292,23 @@ class UseCaseModule { ) } + @Provides + @Singleton + fun provideExportDownloadedThreadAsJsonUseCase( + appContext: Context, + appConstants: AppConstants, + fileManager: FileManager, + chanPostRepository: ChanPostRepository + ): ExportDownloadedThreadAsJsonUseCase { + deps("ExportDownloadedThreadAsJsonUseCase") + return ExportDownloadedThreadAsJsonUseCase( + appContext, + appConstants, + fileManager, + chanPostRepository + ) + } + @Provides @Singleton fun provideThreadDownloaderPersistPostsInDatabaseUseCase( diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt new file mode 100644 index 000000000..43c9fcbbe --- /dev/null +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt @@ -0,0 +1,370 @@ +package com.github.k1rakishou.chan.core.usecase + +import android.content.Context +import android.net.Uri +import com.github.k1rakishou.chan.R +import com.github.k1rakishou.chan.features.thread_downloading.ThreadDownloadingDelegate +import com.github.k1rakishou.common.AppConstants +import com.github.k1rakishou.common.ModularResult +import com.github.k1rakishou.common.extractFileName +import com.github.k1rakishou.core_logger.Logger +import com.github.k1rakishou.fsaf.FileManager +import com.github.k1rakishou.fsaf.file.AbstractFile +import com.github.k1rakishou.fsaf.file.FileSegment +import com.github.k1rakishou.fsaf.file.Segment +import com.github.k1rakishou.model.data.descriptor.ChanDescriptor +import com.github.k1rakishou.model.data.post.ChanOriginalPost +import com.github.k1rakishou.model.data.post.ChanPost +import com.github.k1rakishou.model.repository.ChanPostRepository +import com.github.k1rakishou.model.util.ChanPostUtils +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.runInterruptible +import kotlinx.coroutines.withContext +import org.joda.time.DateTimeZone +import org.joda.time.format.DateTimeFormatterBuilder +import org.joda.time.format.ISODateTimeFormat +import java.io.File +import java.util.* +import java.util.regex.Pattern +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream + +class ExportDownloadedThreadAsJsonUseCase( + private val appContext: Context, + private val appConstants: AppConstants, + private val fileManager: FileManager, + private val chanPostRepository: ChanPostRepository +) : ISuspendUseCase> { + + override suspend fun execute(parameter: Params): ModularResult { + return ModularResult.Try { + val outputDirUri = parameter.outputDirUri + val threadDescriptors = parameter.threadDescriptors + val onUpdate = parameter.onUpdate + + withContext(Dispatchers.IO) { + withContext(Dispatchers.Main) { onUpdate(0, threadDescriptors.size) } + + threadDescriptors.forEachIndexed { index, threadDescriptor -> + ensureActive() + + val outputDir = fileManager.fromUri(outputDirUri) + ?: throw ThreadExportException("Failed to get output file for directory: \'$outputDirUri\'") + + val fileName = "${threadDescriptor.siteName()}_${threadDescriptor.boardCode()}_${threadDescriptor.threadNo}.zip" + val outputFile = fileManager.createFile(outputDir, fileName) + ?: throw ThreadExportException("Failed to create output file \'$fileName\' in directory \'${outputDir}\'") + + try { + exportThreadAsJson(outputFile, threadDescriptor) + } catch (error: Throwable) { + fileManager.fromUri(outputDirUri)?.let { file -> + if (fileManager.isFile(file)) { + fileManager.delete(file) + } + } + + throw error + } + + withContext(Dispatchers.Main) { onUpdate(index + 1, threadDescriptors.size) } + } + + withContext(Dispatchers.Main) { onUpdate(threadDescriptors.size, threadDescriptors.size) } + } + } + } + + private suspend fun exportThreadAsJson( + outputFile: AbstractFile, + threadDescriptor: ChanDescriptor.ThreadDescriptor + ) { + val postsLoadResult = chanPostRepository.getThreadPostsFromDatabase(threadDescriptor) + + val chanPosts = if (postsLoadResult is ModularResult.Error) { + throw postsLoadResult.error + } else { + postsLoadResult as ModularResult.Value + postsLoadResult.value.sortedBy { post -> post.postNo() } + } + + if (chanPosts.isEmpty()) { + throw ThreadExportException("Failed to load posts to export") + } + + if (chanPosts.first() !is ChanOriginalPost) { + throw ThreadExportException("First post is not OP") + } + + val outputFileUri = outputFile.getFullPath() + Logger.d(TAG, "exportThreadAsJson exporting ${chanPosts.size} posts into file '$outputFileUri'") + + val outputStream = fileManager.getOutputStream(outputFile) + if (outputStream == null) { + throw ThreadExportException("Failed to open output stream for file '${outputFileUri}'") + } + + runInterruptible { + outputStream.use { os -> + ZipOutputStream(os).use { zos -> + appContext.resources.openRawResource(R.raw.tomorrow).use { cssFileInputStream -> + zos.putNextEntry(ZipEntry("tomorrow.css")) + cssFileInputStream.copyTo(zos) + } + + kotlin.run { + zos.putNextEntry(ZipEntry("thread_data.html")) + HTML_TEMPLATE_START.byteInputStream().use { templateStartStream -> + templateStartStream.copyTo(zos) + } + + chanPosts.forEach { chanPost -> + formatPost(chanPost).byteInputStream().use { formattedPostStream -> + formattedPostStream.copyTo(zos) + } + } + + HTML_TEMPLATE_END.byteInputStream().use { templateEndStream -> + templateEndStream.copyTo(zos) + } + } + + appContext.resources.openRawResource(R.raw.tomorrow).use { cssFileInputStream -> + zos.putNextEntry(ZipEntry("thread_data.json")) + cssFileInputStream.copyTo(zos) + + chanPosts.forEach { chanPost -> + formatPost(chanPost).byteInputStream().use { formattedPostStream -> + formattedPostStream.copyTo(zos) + } + } + } + + val threadMediaDirName = ThreadDownloadingDelegate.formatDirectoryName(threadDescriptor) + val threadMediaDir = File(appConstants.threadDownloaderCacheDir, threadMediaDirName) + + threadMediaDir.listFiles()?.forEach { mediaFile -> + zos.putNextEntry(ZipEntry(mediaFile.name)) + + mediaFile.inputStream().use { mediaFileSteam -> + mediaFileSteam.copyTo(zos) + } + } + } + } + } + + Logger.d(TAG, "exportThreadAsJson done") + } + + private fun formatPost(chanPost: ChanPost): String { + val template = if (chanPost is ChanOriginalPost) { + OP_POST_TEMPLATE + } else { + REGULAR_POST_TEMPLATE + } + + val templateBuilder = StringBuilder(template.length) + val matcher = TEMPLATE_PARAMETER_PATTERN.matcher(template) + + var offset = 0 + + while (matcher.find()) { + val startIndex = matcher.start(0) + val endIndex = matcher.end(0) + + templateBuilder.append(template.substring(offset, startIndex)) + + val templateParam = template.substring(startIndex, endIndex) + .removePrefix("{{") + .removeSuffix("}}") + + val templateValue = when (templateParam) { + "POST_NO" -> chanPost.postDescriptor.postNo.toString() + "ORIGINAL_POST_FILES", + "REGULAR_POST_FILES" -> formatPostFiles(chanPost) + "THREAD_SUBJECT" -> { + chanPost.subject ?: "" + } + "POSTER_NAME" -> { + chanPost.tripcode ?: "" + } + "DATE_TIME_FORMATTED" -> { + DATE_TIME_PRINTER.print(chanPost.timestamp * 1000L) + } + "POST_COMMENT" -> { + chanPost.postComment.originalUnparsedComment ?: "" + } + else -> error("Unknown template parameter: ${templateParam}") + } + + templateBuilder + .append(templateValue) + + offset = endIndex + } + + templateBuilder.append(template.substring(offset, template.length)) + + return templateBuilder.toString() + } + + private fun formatPostFiles(chanPost: ChanPost): String { + if (chanPost.postImages.isEmpty()) { + return "" + } + + val templateBuilder = StringBuilder(128) + templateBuilder + .append("
") + + chanPost.iteratePostImages { chanPostImage -> + val template = if (chanPost is ChanOriginalPost) { + ORIGINAL_POST_FILE_TEMPLATE + } else { + REGULAR_POST_FILE_TEMPLATE + } + + val matcher = TEMPLATE_PARAMETER_PATTERN.matcher(template) + + var offset = 0 + + while (matcher.find()) { + val startIndex = matcher.start(0) + val endIndex = matcher.end(0) + + templateBuilder.append(template.substring(offset, startIndex)) + + val templateParam = template.substring(startIndex, endIndex) + .removePrefix("{{") + .removeSuffix("}}") + + val templateValue = when (templateParam) { + "POST_NO" -> chanPost.postDescriptor.postNo.toString() + "FILE_NAME_WEIGHT_DIMENS" -> { + val fileName = chanPostImage.formatFullOriginalFileName() ?: "" + val weight = ChanPostUtils.getReadableFileSize(chanPostImage.size) + val dimens = "${chanPostImage.imageWidth}x${chanPostImage.imageHeight}" + + "${fileName}, $weight, $dimens" + } + "FULL_IMAGE_NAME" -> chanPostImage.imageUrl?.extractFileName() ?: "" + "THUMBNAIL_NAME" -> chanPostImage.actualThumbnailUrl?.extractFileName() ?: "" + "FILE_WEIGHT" -> ChanPostUtils.getReadableFileSize(chanPostImage.size) + else -> error("Unknown template parameter: ${templateParam}") + } + + templateBuilder + .append(templateValue) + + offset = endIndex + } + + templateBuilder + .append(template.substring(offset, template.length)) + } + + templateBuilder + .append("
") + + return templateBuilder.toString() + } + + class ThreadExportException(message: String) : Exception(message) + + data class Params( + val outputDirUri: Uri, + val threadDescriptors: List, + val onUpdate: (Int, Int) -> Unit + ) + + companion object { + private const val TAG = "ExportDownloadedThreadAsJsonUseCase" + + private val TEMPLATE_PARAMETER_PATTERN = Pattern.compile("\\{\\{\\w+\\}\\}") + + private val DATE_TIME_PRINTER = DateTimeFormatterBuilder() + .append(ISODateTimeFormat.date()) + .appendLiteral(' ') + .append(ISODateTimeFormat.hourMinuteSecond()) + .toFormatter() + .withZone(DateTimeZone.forTimeZone(TimeZone.getDefault())) + + private const val HTML_TEMPLATE_START = """ + + + + + +
+
+
+ """ + + private const val HTML_TEMPLATE_END = """ +
+
+
+
+ + + """ + + private const val OP_POST_TEMPLATE = """ +
+
+ {{ORIGINAL_POST_FILES}} +
+ {{THREAD_SUBJECT}} + + {{POSTER_NAME}} + + {{DATE_TIME_FORMATTED}} No. {{POST_NO}} +
+
{{POST_COMMENT}}
+
+
+ """ + + private const val REGULAR_POST_TEMPLATE = """ +
+
+
+ + {{POSTER_NAME}} + + {{DATE_TIME_FORMATTED}} No. {{POST_NO}} +
+ {{REGULAR_POST_FILES}} +
{{POST_COMMENT}}
+
+
+ """ + + private const val ORIGINAL_POST_FILE_TEMPLATE = """ +
+ +
+ """ + + private const val REGULAR_POST_FILE_TEMPLATE = """ + + """ + + } +} \ No newline at end of file diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveViewModel.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveViewModel.kt index 219b7f7ce..e90267e22 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveViewModel.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveViewModel.kt @@ -16,6 +16,7 @@ import com.github.k1rakishou.chan.core.di.component.viewmodel.ViewModelComponent import com.github.k1rakishou.chan.core.di.module.shared.ViewModelAssistedFactory import com.github.k1rakishou.chan.core.manager.ThreadDownloadManager import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadAsHtmlUseCase +import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadAsJsonUseCase import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadMediaUseCase import com.github.k1rakishou.chan.ui.view.bottom_menu_panel.BottomMenuPanelItem import com.github.k1rakishou.chan.ui.view.bottom_menu_panel.BottomMenuPanelItemId @@ -54,6 +55,7 @@ class LocalArchiveViewModel( private val chanPostRepository: ChanPostRepository, private val threadDownloadProgressNotifier: ThreadDownloadProgressNotifier, private val exportDownloadedThreadAsHtmlUseCase: ExportDownloadedThreadAsHtmlUseCase, + private val exportDownloadedThreadAsJsonUseCase: ExportDownloadedThreadAsJsonUseCase, private val exportDownloadedThreadMediaUseCase: ExportDownloadedThreadMediaUseCase, ) : BaseViewModel() { @@ -483,6 +485,16 @@ class LocalArchiveViewModel( .onError { error -> Logger.e(TAG, "exportThreadsAsHtml() error", error) } } + suspend fun exportThreadsAsJson( + outputDirUri: Uri, + threadDescriptors: List, + onUpdate: (Int, Int) -> Unit + ): ModularResult { + val params = ExportDownloadedThreadAsJsonUseCase.Params(outputDirUri, threadDescriptors, onUpdate) + return exportDownloadedThreadAsJsonUseCase.execute(params) + .onError { error -> Logger.e(TAG, "exportThreadsAsJson() error", error) } + } + suspend fun exportThreadsMedia( outputDirectoryUri: Uri, threadDescriptors: List, @@ -558,6 +570,7 @@ class LocalArchiveViewModel( private val chanPostRepository: ChanPostRepository, private val threadDownloadProgressNotifier: ThreadDownloadProgressNotifier, private val exportDownloadedThreadAsHtmlUseCase: ExportDownloadedThreadAsHtmlUseCase, + private val exportDownloadedThreadAsJsonUseCase: ExportDownloadedThreadAsJsonUseCase, private val exportDownloadedThreadMediaUseCase: ExportDownloadedThreadMediaUseCase, ) : ViewModelAssistedFactory { override fun create(handle: SavedStateHandle): LocalArchiveViewModel { @@ -568,6 +581,7 @@ class LocalArchiveViewModel( chanPostRepository = chanPostRepository, threadDownloadProgressNotifier = threadDownloadProgressNotifier, exportDownloadedThreadAsHtmlUseCase = exportDownloadedThreadAsHtmlUseCase, + exportDownloadedThreadAsJsonUseCase = exportDownloadedThreadAsJsonUseCase, exportDownloadedThreadMediaUseCase = exportDownloadedThreadMediaUseCase ) } From 04091c699456365b205ffdb35bf26cc6203c3913 Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 00:32:27 -0700 Subject: [PATCH 02/11] Add json export to menu --- .../ExportDownloadedThreadAsJsonUseCase.kt | 37 +++---------------- .../LocalArchiveController.kt | 5 +++ Kuroba/app/src/main/res/values/strings.xml | 1 + 3 files changed, 11 insertions(+), 32 deletions(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt index 43c9fcbbe..a229ad023 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt @@ -17,6 +17,7 @@ import com.github.k1rakishou.model.data.post.ChanOriginalPost import com.github.k1rakishou.model.data.post.ChanPost import com.github.k1rakishou.model.repository.ChanPostRepository import com.github.k1rakishou.model.util.ChanPostUtils +import com.google.gson.Gson import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ensureActive import kotlinx.coroutines.runInterruptible @@ -108,38 +109,10 @@ class ExportDownloadedThreadAsJsonUseCase( runInterruptible { outputStream.use { os -> ZipOutputStream(os).use { zos -> - appContext.resources.openRawResource(R.raw.tomorrow).use { cssFileInputStream -> - zos.putNextEntry(ZipEntry("tomorrow.css")) - cssFileInputStream.copyTo(zos) - } - - kotlin.run { - zos.putNextEntry(ZipEntry("thread_data.html")) - HTML_TEMPLATE_START.byteInputStream().use { templateStartStream -> - templateStartStream.copyTo(zos) - } - - chanPosts.forEach { chanPost -> - formatPost(chanPost).byteInputStream().use { formattedPostStream -> - formattedPostStream.copyTo(zos) - } - } - - HTML_TEMPLATE_END.byteInputStream().use { templateEndStream -> - templateEndStream.copyTo(zos) - } - } - - appContext.resources.openRawResource(R.raw.tomorrow).use { cssFileInputStream -> - zos.putNextEntry(ZipEntry("thread_data.json")) - cssFileInputStream.copyTo(zos) - - chanPosts.forEach { chanPost -> - formatPost(chanPost).byteInputStream().use { formattedPostStream -> - formattedPostStream.copyTo(zos) - } - } - } + // threadDescriptor.siteName + threadDescriptor.boardCode + threadNoOrNull + zos.putNextEntry(ZipEntry("thread_data.json")) + Gson gson = new Gson(); + gson.toJson(chanPosts).copyto(zos); val threadMediaDirName = ThreadDownloadingDelegate.formatDirectoryName(threadDescriptor) val threadMediaDir = File(appConstants.threadDownloaderCacheDir, threadMediaDirName) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt index b1397c22e..59f761870 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt @@ -798,6 +798,7 @@ class LocalArchiveController( val items = listOf( FloatingListMenuItem(ACTION_EXPORT_THREADS, getString(R.string.controller_local_archive_export_threads)), FloatingListMenuItem(ACTION_EXPORT_THREAD_MEDIA, getString(R.string.controller_local_archive_export_thread_media)) + FloatingListMenuItem(ACTION_EXPORT_THREAD_JSON, getString(R.string.controller_local_archive_export_thread_json)) ) val floatingListMenuController = FloatingListMenuController( @@ -813,6 +814,9 @@ class LocalArchiveController( ACTION_EXPORT_THREAD_MEDIA -> { exportThreadMedia(selectedItems) } + ACTION_EXPORT_THREAD_JSON -> { + exportThreadAsJson(selectedItems) + } } } } @@ -995,6 +999,7 @@ class LocalArchiveController( private const val ACTION_EXPORT_THREADS = 100 private const val ACTION_EXPORT_THREAD_MEDIA = 101 + private const val ACTION_EXPORT_THREAD_JSON = 102 private val ICON_SIZE = 26.dp private val PROGRESS_SIZE = 20.dp diff --git a/Kuroba/app/src/main/res/values/strings.xml b/Kuroba/app/src/main/res/values/strings.xml index 7f32e58e1..a8998f20b 100644 --- a/Kuroba/app/src/main/res/values/strings.xml +++ b/Kuroba/app/src/main/res/values/strings.xml @@ -1360,6 +1360,7 @@ If you ever log out or if the passcode expires you will have to change reply mod Posts: %1$d, Media: %2$d, Media on disk: %3$s Export threads Export thread media + Export thread json Exported %1$d / %2$d Delete %1$d saved post(s)? From fc8a775e392030d1ea3fce209ba0bad667385437 Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 00:38:19 -0700 Subject: [PATCH 03/11] Formatting --- .../chan/features/thread_downloading/LocalArchiveController.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt index 59f761870..53b30a76c 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt @@ -797,7 +797,7 @@ class LocalArchiveController( LocalArchiveViewModel.MenuItemType.Export -> { val items = listOf( FloatingListMenuItem(ACTION_EXPORT_THREADS, getString(R.string.controller_local_archive_export_threads)), - FloatingListMenuItem(ACTION_EXPORT_THREAD_MEDIA, getString(R.string.controller_local_archive_export_thread_media)) + FloatingListMenuItem(ACTION_EXPORT_THREAD_MEDIA, getString(R.string.controller_local_archive_export_thread_media)), FloatingListMenuItem(ACTION_EXPORT_THREAD_JSON, getString(R.string.controller_local_archive_export_thread_json)) ) From f67d9040ad7b816b5f5cbda1371adf20ee9316b1 Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 19:15:32 -0700 Subject: [PATCH 04/11] Export list of chanPosts to json in zip --- .../ExportDownloadedThreadAsJsonUseCase.kt | 6 ++- .../LocalArchiveController.kt | 43 +++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt index a229ad023..20259e98f 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt @@ -26,6 +26,7 @@ import org.joda.time.DateTimeZone import org.joda.time.format.DateTimeFormatterBuilder import org.joda.time.format.ISODateTimeFormat import java.io.File +import java.io.ByteArrayInputStream import java.util.* import java.util.regex.Pattern import java.util.zip.ZipEntry @@ -111,8 +112,9 @@ class ExportDownloadedThreadAsJsonUseCase( ZipOutputStream(os).use { zos -> // threadDescriptor.siteName + threadDescriptor.boardCode + threadNoOrNull zos.putNextEntry(ZipEntry("thread_data.json")) - Gson gson = new Gson(); - gson.toJson(chanPosts).copyto(zos); + val gson = Gson() + ByteArrayInputStream(gson.toJson(chanPosts).toByteArray()).copyTo(zos) + val threadMediaDirName = ThreadDownloadingDelegate.formatDirectoryName(threadDescriptor) val threadMediaDir = File(appConstants.threadDownloaderCacheDir, threadMediaDirName) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt index 53b30a76c..4faa485cb 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt @@ -870,6 +870,49 @@ class LocalArchiveController( }) } + private fun exportThreadAsJson(threadDescriptors: List) { + if (threadDescriptors.isEmpty()) { + return + } + + fileChooser.openChooseDirectoryDialog(object : DirectoryChooserCallback() { + override fun onCancel(reason: String) { + showToast(R.string.canceled) + } + + override fun onResult(uri: Uri) { + val loadingViewController = LoadingViewController(context, false) + + val job = controllerScope.launch(start = CoroutineStart.LAZY) { + try { + viewModel.exportThreadsAsJson( + outputDirUri = uri, + threadDescriptors = threadDescriptors, + onUpdate = { exported, total -> + val text = context.resources.getString(R.string.controller_local_archive_exported_format, exported, total) + loadingViewController.updateWithText(text) + } + ) + .toastOnError(message = { error -> "Failed to export. Error: ${error.errorMessageOrClassName()}" }) + .toastOnSuccess(message = { "Successfully exported" }) + .ignore() + } finally { + loadingViewController.stopPresenting() + } + } + + loadingViewController.enableCancellation { + if (job.isActive) { + job.cancel() + } + } + + presentController(loadingViewController) + job.start() + } + }) + } + private suspend fun exportThreadMedia(threadDescriptors: List) { fileChooser.openChooseDirectoryDialog(object : DirectoryChooserCallback() { override fun onCancel(reason: String) { From f8a044e2669221efc6ea60085688632a271d581e Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 19:39:33 -0700 Subject: [PATCH 05/11] Append _json to zip filename --- .../chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt index 20259e98f..0949b02bb 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt @@ -54,7 +54,7 @@ class ExportDownloadedThreadAsJsonUseCase( val outputDir = fileManager.fromUri(outputDirUri) ?: throw ThreadExportException("Failed to get output file for directory: \'$outputDirUri\'") - val fileName = "${threadDescriptor.siteName()}_${threadDescriptor.boardCode()}_${threadDescriptor.threadNo}.zip" + val fileName = "${threadDescriptor.siteName()}_${threadDescriptor.boardCode()}_${threadDescriptor.threadNo}_json.zip" val outputFile = fileManager.createFile(outputDir, fileName) ?: throw ThreadExportException("Failed to create output file \'$fileName\' in directory \'${outputDir}\'") From f7cdc82c4f3c998b7b3c05fcc4356fa1d39667f1 Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 19:39:50 -0700 Subject: [PATCH 06/11] cleanup json exporter --- .../ExportDownloadedThreadAsJsonUseCase.kt | 199 ------------------ 1 file changed, 199 deletions(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt index 0949b02bb..807bba58e 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt @@ -110,7 +110,6 @@ class ExportDownloadedThreadAsJsonUseCase( runInterruptible { outputStream.use { os -> ZipOutputStream(os).use { zos -> - // threadDescriptor.siteName + threadDescriptor.boardCode + threadNoOrNull zos.putNextEntry(ZipEntry("thread_data.json")) val gson = Gson() ByteArrayInputStream(gson.toJson(chanPosts).toByteArray()).copyTo(zos) @@ -133,119 +132,6 @@ class ExportDownloadedThreadAsJsonUseCase( Logger.d(TAG, "exportThreadAsJson done") } - private fun formatPost(chanPost: ChanPost): String { - val template = if (chanPost is ChanOriginalPost) { - OP_POST_TEMPLATE - } else { - REGULAR_POST_TEMPLATE - } - - val templateBuilder = StringBuilder(template.length) - val matcher = TEMPLATE_PARAMETER_PATTERN.matcher(template) - - var offset = 0 - - while (matcher.find()) { - val startIndex = matcher.start(0) - val endIndex = matcher.end(0) - - templateBuilder.append(template.substring(offset, startIndex)) - - val templateParam = template.substring(startIndex, endIndex) - .removePrefix("{{") - .removeSuffix("}}") - - val templateValue = when (templateParam) { - "POST_NO" -> chanPost.postDescriptor.postNo.toString() - "ORIGINAL_POST_FILES", - "REGULAR_POST_FILES" -> formatPostFiles(chanPost) - "THREAD_SUBJECT" -> { - chanPost.subject ?: "" - } - "POSTER_NAME" -> { - chanPost.tripcode ?: "" - } - "DATE_TIME_FORMATTED" -> { - DATE_TIME_PRINTER.print(chanPost.timestamp * 1000L) - } - "POST_COMMENT" -> { - chanPost.postComment.originalUnparsedComment ?: "" - } - else -> error("Unknown template parameter: ${templateParam}") - } - - templateBuilder - .append(templateValue) - - offset = endIndex - } - - templateBuilder.append(template.substring(offset, template.length)) - - return templateBuilder.toString() - } - - private fun formatPostFiles(chanPost: ChanPost): String { - if (chanPost.postImages.isEmpty()) { - return "" - } - - val templateBuilder = StringBuilder(128) - templateBuilder - .append("
") - - chanPost.iteratePostImages { chanPostImage -> - val template = if (chanPost is ChanOriginalPost) { - ORIGINAL_POST_FILE_TEMPLATE - } else { - REGULAR_POST_FILE_TEMPLATE - } - - val matcher = TEMPLATE_PARAMETER_PATTERN.matcher(template) - - var offset = 0 - - while (matcher.find()) { - val startIndex = matcher.start(0) - val endIndex = matcher.end(0) - - templateBuilder.append(template.substring(offset, startIndex)) - - val templateParam = template.substring(startIndex, endIndex) - .removePrefix("{{") - .removeSuffix("}}") - - val templateValue = when (templateParam) { - "POST_NO" -> chanPost.postDescriptor.postNo.toString() - "FILE_NAME_WEIGHT_DIMENS" -> { - val fileName = chanPostImage.formatFullOriginalFileName() ?: "" - val weight = ChanPostUtils.getReadableFileSize(chanPostImage.size) - val dimens = "${chanPostImage.imageWidth}x${chanPostImage.imageHeight}" - - "${fileName}, $weight, $dimens" - } - "FULL_IMAGE_NAME" -> chanPostImage.imageUrl?.extractFileName() ?: "" - "THUMBNAIL_NAME" -> chanPostImage.actualThumbnailUrl?.extractFileName() ?: "" - "FILE_WEIGHT" -> ChanPostUtils.getReadableFileSize(chanPostImage.size) - else -> error("Unknown template parameter: ${templateParam}") - } - - templateBuilder - .append(templateValue) - - offset = endIndex - } - - templateBuilder - .append(template.substring(offset, template.length)) - } - - templateBuilder - .append("
") - - return templateBuilder.toString() - } - class ThreadExportException(message: String) : Exception(message) data class Params( @@ -256,90 +142,5 @@ class ExportDownloadedThreadAsJsonUseCase( companion object { private const val TAG = "ExportDownloadedThreadAsJsonUseCase" - - private val TEMPLATE_PARAMETER_PATTERN = Pattern.compile("\\{\\{\\w+\\}\\}") - - private val DATE_TIME_PRINTER = DateTimeFormatterBuilder() - .append(ISODateTimeFormat.date()) - .appendLiteral(' ') - .append(ISODateTimeFormat.hourMinuteSecond()) - .toFormatter() - .withZone(DateTimeZone.forTimeZone(TimeZone.getDefault())) - - private const val HTML_TEMPLATE_START = """ - - - - - -
-
-
- """ - - private const val HTML_TEMPLATE_END = """ -
-
-
-
- - - """ - - private const val OP_POST_TEMPLATE = """ -
-
- {{ORIGINAL_POST_FILES}} -
- {{THREAD_SUBJECT}} - - {{POSTER_NAME}} - - {{DATE_TIME_FORMATTED}} No. {{POST_NO}} -
-
{{POST_COMMENT}}
-
-
- """ - - private const val REGULAR_POST_TEMPLATE = """ -
-
-
- - {{POSTER_NAME}} - - {{DATE_TIME_FORMATTED}} No. {{POST_NO}} -
- {{REGULAR_POST_FILES}} -
{{POST_COMMENT}}
-
-
- """ - - private const val ORIGINAL_POST_FILE_TEMPLATE = """ -
- -
- """ - - private const val REGULAR_POST_FILE_TEMPLATE = """ - - """ - } } \ No newline at end of file From 95a6e2843938333073691d398d52507a399d4368 Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 20:00:36 -0700 Subject: [PATCH 07/11] Cleanup imports --- .../usecase/ExportDownloadedThreadAsJsonUseCase.kt | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt index 807bba58e..a65a38a27 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt @@ -2,32 +2,22 @@ package com.github.k1rakishou.chan.core.usecase import android.content.Context import android.net.Uri -import com.github.k1rakishou.chan.R import com.github.k1rakishou.chan.features.thread_downloading.ThreadDownloadingDelegate import com.github.k1rakishou.common.AppConstants import com.github.k1rakishou.common.ModularResult -import com.github.k1rakishou.common.extractFileName import com.github.k1rakishou.core_logger.Logger import com.github.k1rakishou.fsaf.FileManager import com.github.k1rakishou.fsaf.file.AbstractFile -import com.github.k1rakishou.fsaf.file.FileSegment -import com.github.k1rakishou.fsaf.file.Segment import com.github.k1rakishou.model.data.descriptor.ChanDescriptor import com.github.k1rakishou.model.data.post.ChanOriginalPost -import com.github.k1rakishou.model.data.post.ChanPost import com.github.k1rakishou.model.repository.ChanPostRepository -import com.github.k1rakishou.model.util.ChanPostUtils import com.google.gson.Gson import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.ensureActive import kotlinx.coroutines.runInterruptible import kotlinx.coroutines.withContext -import org.joda.time.DateTimeZone -import org.joda.time.format.DateTimeFormatterBuilder -import org.joda.time.format.ISODateTimeFormat -import java.io.File import java.io.ByteArrayInputStream -import java.util.* +import java.io.File import java.util.regex.Pattern import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream From fd5e16512aee157955989fba303b97ee7ea023dc Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 20:02:42 -0700 Subject: [PATCH 08/11] Filter thumbnails from json media export --- .../usecase/ExportDownloadedThreadAsJsonUseCase.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt index a65a38a27..513bf9546 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt @@ -101,16 +101,16 @@ class ExportDownloadedThreadAsJsonUseCase( outputStream.use { os -> ZipOutputStream(os).use { zos -> zos.putNextEntry(ZipEntry("thread_data.json")) - val gson = Gson() - ByteArrayInputStream(gson.toJson(chanPosts).toByteArray()).copyTo(zos) - + ByteArrayInputStream(Gson().toJson(chanPosts).toByteArray()).copyTo(zos) val threadMediaDirName = ThreadDownloadingDelegate.formatDirectoryName(threadDescriptor) val threadMediaDir = File(appConstants.threadDownloaderCacheDir, threadMediaDirName) - threadMediaDir.listFiles()?.forEach { mediaFile -> - zos.putNextEntry(ZipEntry(mediaFile.name)) + // Use this to skip exporting thumbnails + val matcher = MEDIA_EXCLUDE_PATTERN.matcher(mediaFile.name) + if (matcher.matches()) return@forEach + zos.putNextEntry(ZipEntry(mediaFile.name)) mediaFile.inputStream().use { mediaFileSteam -> mediaFileSteam.copyTo(zos) } @@ -132,5 +132,6 @@ class ExportDownloadedThreadAsJsonUseCase( companion object { private const val TAG = "ExportDownloadedThreadAsJsonUseCase" + private val MEDIA_EXCLUDE_PATTERN = Pattern.compile(".*s\\..+") } } \ No newline at end of file From 1466069c8e0e70e55b4947f79fc304aaaa28e943 Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 20:05:13 -0700 Subject: [PATCH 09/11] Relabel and reorder thread export functions --- .../features/thread_downloading/LocalArchiveController.kt | 2 +- Kuroba/app/src/main/res/values/strings.xml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt index 4faa485cb..e4fea62c1 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt @@ -796,8 +796,8 @@ class LocalArchiveController( } LocalArchiveViewModel.MenuItemType.Export -> { val items = listOf( - FloatingListMenuItem(ACTION_EXPORT_THREADS, getString(R.string.controller_local_archive_export_threads)), FloatingListMenuItem(ACTION_EXPORT_THREAD_MEDIA, getString(R.string.controller_local_archive_export_thread_media)), + FloatingListMenuItem(ACTION_EXPORT_THREADS, getString(R.string.controller_local_archive_export_threads)), FloatingListMenuItem(ACTION_EXPORT_THREAD_JSON, getString(R.string.controller_local_archive_export_thread_json)) ) diff --git a/Kuroba/app/src/main/res/values/strings.xml b/Kuroba/app/src/main/res/values/strings.xml index a8998f20b..a3805293d 100644 --- a/Kuroba/app/src/main/res/values/strings.xml +++ b/Kuroba/app/src/main/res/values/strings.xml @@ -1358,9 +1358,9 @@ If you ever log out or if the passcode expires you will have to change reply mod DOWNLOADING DOWNLOADED Posts: %1$d, Media: %2$d, Media on disk: %3$s - Export threads + Export thread html zip Export thread media - Export thread json + Export thread json zip Exported %1$d / %2$d Delete %1$d saved post(s)? From 43c5e024b9f475bc172e8ca40c1169d7271d7eff Mon Sep 17 00:00:00 2001 From: Yuki Date: Fri, 31 May 2024 20:07:03 -0700 Subject: [PATCH 10/11] Formatting --- .../chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt index 513bf9546..99acf1029 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt @@ -134,4 +134,4 @@ class ExportDownloadedThreadAsJsonUseCase( private const val TAG = "ExportDownloadedThreadAsJsonUseCase" private val MEDIA_EXCLUDE_PATTERN = Pattern.compile(".*s\\..+") } -} \ No newline at end of file +} From fa2d6e5ac834a6b020ade11c10a2b64dc11c4aa1 Mon Sep 17 00:00:00 2001 From: k1rakishou Date: Sat, 1 Jun 2024 14:10:53 +0700 Subject: [PATCH 11/11] Some minor stuff. - Inject Gson into the class instead of creating it every single time. - Close ByteArrayInputStream when copying is done. --- .../chan/core/di/module/application/UseCaseModule.kt | 2 ++ .../chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/di/module/application/UseCaseModule.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/di/module/application/UseCaseModule.kt index abbe0fc2d..9d54c34ed 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/di/module/application/UseCaseModule.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/di/module/application/UseCaseModule.kt @@ -297,6 +297,7 @@ class UseCaseModule { fun provideExportDownloadedThreadAsJsonUseCase( appContext: Context, appConstants: AppConstants, + gson: Gson, fileManager: FileManager, chanPostRepository: ChanPostRepository ): ExportDownloadedThreadAsJsonUseCase { @@ -304,6 +305,7 @@ class UseCaseModule { return ExportDownloadedThreadAsJsonUseCase( appContext, appConstants, + gson, fileManager, chanPostRepository ) diff --git a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt index 99acf1029..4e3ec804a 100644 --- a/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt +++ b/Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt @@ -25,6 +25,7 @@ import java.util.zip.ZipOutputStream class ExportDownloadedThreadAsJsonUseCase( private val appContext: Context, private val appConstants: AppConstants, + private val gson: Gson, private val fileManager: FileManager, private val chanPostRepository: ChanPostRepository ) : ISuspendUseCase> { @@ -101,7 +102,9 @@ class ExportDownloadedThreadAsJsonUseCase( outputStream.use { os -> ZipOutputStream(os).use { zos -> zos.putNextEntry(ZipEntry("thread_data.json")) - ByteArrayInputStream(Gson().toJson(chanPosts).toByteArray()).copyTo(zos) + + ByteArrayInputStream(gson.toJson(chanPosts).toByteArray()) + .use { postsJsonByteArray -> postsJsonByteArray.copyTo(zos) } val threadMediaDirName = ThreadDownloadingDelegate.formatDirectoryName(threadDescriptor) val threadMediaDir = File(appConstants.threadDownloaderCacheDir, threadMediaDirName)