Include thread json in thread export methods

Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/di/module/application/UseCaseModule.kt

@@ -29,6 +29,7 @@ import com.github.k1rakishou.chan.core.usecase.ClearPostingCookies
 import com.github.k1rakishou.chan.core.usecase.DownloadThemeJsonFilesUseCase
 import com.github.k1rakishou.chan.core.usecase.ExportBackupFileUseCase
 import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadAsHtmlUseCase
+import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadAsJsonUseCase
 import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadMediaUseCase
 import com.github.k1rakishou.chan.core.usecase.ExportFiltersUseCase
 import com.github.k1rakishou.chan.core.usecase.ExtractPostMapInfoHolderUseCase
@@ -291,6 +292,25 @@ class UseCaseModule {
     )
   }
 
+  @Provides
+  @Singleton
+  fun provideExportDownloadedThreadAsJsonUseCase(
+    appContext: Context,
+    appConstants: AppConstants,
+    gson: Gson,
+    fileManager: FileManager,
+    chanPostRepository: ChanPostRepository
+  ): ExportDownloadedThreadAsJsonUseCase {
+    deps("ExportDownloadedThreadAsJsonUseCase")
+    return ExportDownloadedThreadAsJsonUseCase(
+      appContext,
+      appConstants,
+      gson,
+      fileManager,
+      chanPostRepository
+    )
+  }
+
   @Provides
   @Singleton
   fun provideThreadDownloaderPersistPostsInDatabaseUseCase(

Kuroba/app/src/main/java/com/github/k1rakishou/chan/core/usecase/ExportDownloadedThreadAsJsonUseCase.kt

@@ -0,0 +1,140 @@
+package com.github.k1rakishou.chan.core.usecase
+
+import android.content.Context
+import android.net.Uri
+import com.github.k1rakishou.chan.features.thread_downloading.ThreadDownloadingDelegate
+import com.github.k1rakishou.common.AppConstants
+import com.github.k1rakishou.common.ModularResult
+import com.github.k1rakishou.core_logger.Logger
+import com.github.k1rakishou.fsaf.FileManager
+import com.github.k1rakishou.fsaf.file.AbstractFile
+import com.github.k1rakishou.model.data.descriptor.ChanDescriptor
+import com.github.k1rakishou.model.data.post.ChanOriginalPost
+import com.github.k1rakishou.model.repository.ChanPostRepository
+import com.google.gson.Gson
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.runInterruptible
+import kotlinx.coroutines.withContext
+import java.io.ByteArrayInputStream
+import java.io.File
+import java.util.regex.Pattern
+import java.util.zip.ZipEntry
+import java.util.zip.ZipOutputStream
+
+class ExportDownloadedThreadAsJsonUseCase(
+  private val appContext: Context,
+  private val appConstants: AppConstants,
+  private val gson: Gson,
+  private val fileManager: FileManager,
+  private val chanPostRepository: ChanPostRepository
+) : ISuspendUseCase<ExportDownloadedThreadAsJsonUseCase.Params, ModularResult<Unit>> {
+
+  override suspend fun execute(parameter: Params): ModularResult<Unit> {
+    return ModularResult.Try {
+      val outputDirUri = parameter.outputDirUri
+      val threadDescriptors = parameter.threadDescriptors
+      val onUpdate = parameter.onUpdate
+
+      withContext(Dispatchers.IO) {
+        withContext(Dispatchers.Main) { onUpdate(0, threadDescriptors.size) }
+
+        threadDescriptors.forEachIndexed { index, threadDescriptor ->
+          ensureActive()
+
+          val outputDir = fileManager.fromUri(outputDirUri)
+            ?: throw ThreadExportException("Failed to get output file for directory: \'$outputDirUri\'")
+
+          val fileName = "${threadDescriptor.siteName()}_${threadDescriptor.boardCode()}_${threadDescriptor.threadNo}_json.zip"
+          val outputFile = fileManager.createFile(outputDir, fileName)
+            ?: throw ThreadExportException("Failed to create output file \'$fileName\' in directory \'${outputDir}\'")
+
+          try {
+            exportThreadAsJson(outputFile, threadDescriptor)
+          } catch (error: Throwable) {
+            fileManager.fromUri(outputDirUri)?.let { file ->
+              if (fileManager.isFile(file)) {
+                fileManager.delete(file)
+              }
+            }
+
+            throw error
+          }
+
+          withContext(Dispatchers.Main) { onUpdate(index + 1, threadDescriptors.size) }
+        }
+
+        withContext(Dispatchers.Main) { onUpdate(threadDescriptors.size, threadDescriptors.size) }
+      }
+    }
+  }
+
+  private suspend fun exportThreadAsJson(
+    outputFile: AbstractFile,
+    threadDescriptor: ChanDescriptor.ThreadDescriptor
+  ) {
+    val postsLoadResult = chanPostRepository.getThreadPostsFromDatabase(threadDescriptor)
+
+    val chanPosts = if (postsLoadResult is ModularResult.Error) {
+      throw postsLoadResult.error
+    } else {
+      postsLoadResult as ModularResult.Value
+      postsLoadResult.value.sortedBy { post -> post.postNo() }
+    }
+
+    if (chanPosts.isEmpty()) {
+      throw ThreadExportException("Failed to load posts to export")
+    }
+
+    if (chanPosts.first() !is ChanOriginalPost) {
+      throw ThreadExportException("First post is not OP")
+    }
+
+    val outputFileUri = outputFile.getFullPath()
+    Logger.d(TAG, "exportThreadAsJson exporting ${chanPosts.size} posts into file '$outputFileUri'")
+
+    val outputStream = fileManager.getOutputStream(outputFile)
+    if (outputStream == null) {
+      throw ThreadExportException("Failed to open output stream for file '${outputFileUri}'")
+    }
+
+    runInterruptible {
+      outputStream.use { os ->
+        ZipOutputStream(os).use { zos ->
+          zos.putNextEntry(ZipEntry("thread_data.json"))
+
+          ByteArrayInputStream(gson.toJson(chanPosts).toByteArray())
+            .use { postsJsonByteArray -> postsJsonByteArray.copyTo(zos) }
+
+          val threadMediaDirName = ThreadDownloadingDelegate.formatDirectoryName(threadDescriptor)
+          val threadMediaDir = File(appConstants.threadDownloaderCacheDir, threadMediaDirName)
+          threadMediaDir.listFiles()?.forEach { mediaFile ->
+            // Use this to skip exporting thumbnails
+            val matcher = MEDIA_EXCLUDE_PATTERN.matcher(mediaFile.name)
+            if (matcher.matches()) return@forEach
+
+            zos.putNextEntry(ZipEntry(mediaFile.name))
+            mediaFile.inputStream().use { mediaFileSteam ->
+              mediaFileSteam.copyTo(zos)
+            }
+          }
+        }
+      }
+    }
+
+    Logger.d(TAG, "exportThreadAsJson done")
+  }
+
+  class ThreadExportException(message: String) : Exception(message)
+
+  data class Params(
+    val outputDirUri: Uri,
+    val threadDescriptors: List<ChanDescriptor.ThreadDescriptor>,
+    val onUpdate: (Int, Int) -> Unit
+  )
+
+  companion object {
+    private const val TAG = "ExportDownloadedThreadAsJsonUseCase"
+    private val MEDIA_EXCLUDE_PATTERN = Pattern.compile(".*s\\..+")
+  }
+}

Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveController.kt

@@ -796,8 +796,9 @@ class LocalArchiveController(
       }
       LocalArchiveViewModel.MenuItemType.Export -> {
         val items = listOf(
+          FloatingListMenuItem(ACTION_EXPORT_THREAD_MEDIA, getString(R.string.controller_local_archive_export_thread_media)),
           FloatingListMenuItem(ACTION_EXPORT_THREADS, getString(R.string.controller_local_archive_export_threads)),
-          FloatingListMenuItem(ACTION_EXPORT_THREAD_MEDIA, getString(R.string.controller_local_archive_export_thread_media))
+          FloatingListMenuItem(ACTION_EXPORT_THREAD_JSON, getString(R.string.controller_local_archive_export_thread_json))
         )
 
         val floatingListMenuController = FloatingListMenuController(
@@ -813,6 +814,9 @@ class LocalArchiveController(
                 ACTION_EXPORT_THREAD_MEDIA -> {
                   exportThreadMedia(selectedItems)
                 }
+                ACTION_EXPORT_THREAD_JSON -> {
+                  exportThreadAsJson(selectedItems)
+                }
               }
             }
           }
@@ -866,6 +870,49 @@ class LocalArchiveController(
     })
   }
 
+  private fun exportThreadAsJson(threadDescriptors: List<ChanDescriptor.ThreadDescriptor>) {
+    if (threadDescriptors.isEmpty()) {
+      return
+    }
+
+    fileChooser.openChooseDirectoryDialog(object : DirectoryChooserCallback() {
+      override fun onCancel(reason: String) {
+        showToast(R.string.canceled)
+      }
+
+      override fun onResult(uri: Uri) {
+        val loadingViewController = LoadingViewController(context, false)
+
+        val job = controllerScope.launch(start = CoroutineStart.LAZY) {
+          try {
+            viewModel.exportThreadsAsJson(
+              outputDirUri = uri,
+              threadDescriptors = threadDescriptors,
+              onUpdate = { exported, total ->
+                val text = context.resources.getString(R.string.controller_local_archive_exported_format, exported, total)
+                loadingViewController.updateWithText(text)
+              }
+            )
+              .toastOnError(message = { error -> "Failed to export. Error: ${error.errorMessageOrClassName()}" })
+              .toastOnSuccess(message = { "Successfully exported" })
+              .ignore()
+          } finally {
+            loadingViewController.stopPresenting()
+          }
+        }
+
+        loadingViewController.enableCancellation {
+          if (job.isActive) {
+            job.cancel()
+          }
+        }
+
+        presentController(loadingViewController)
+        job.start()
+      }
+    })
+  }
+
   private suspend fun exportThreadMedia(threadDescriptors: List<ChanDescriptor.ThreadDescriptor>) {
     fileChooser.openChooseDirectoryDialog(object : DirectoryChooserCallback() {
       override fun onCancel(reason: String) {
@@ -995,6 +1042,7 @@ class LocalArchiveController(
 
     private const val ACTION_EXPORT_THREADS = 100
     private const val ACTION_EXPORT_THREAD_MEDIA = 101
+    private const val ACTION_EXPORT_THREAD_JSON = 102
 
     private val ICON_SIZE = 26.dp
     private val PROGRESS_SIZE = 20.dp

Kuroba/app/src/main/java/com/github/k1rakishou/chan/features/thread_downloading/LocalArchiveViewModel.kt

@@ -16,6 +16,7 @@ import com.github.k1rakishou.chan.core.di.component.viewmodel.ViewModelComponent
 import com.github.k1rakishou.chan.core.di.module.shared.ViewModelAssistedFactory
 import com.github.k1rakishou.chan.core.manager.ThreadDownloadManager
 import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadAsHtmlUseCase
+import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadAsJsonUseCase
 import com.github.k1rakishou.chan.core.usecase.ExportDownloadedThreadMediaUseCase
 import com.github.k1rakishou.chan.ui.view.bottom_menu_panel.BottomMenuPanelItem
 import com.github.k1rakishou.chan.ui.view.bottom_menu_panel.BottomMenuPanelItemId
@@ -54,6 +55,7 @@ class LocalArchiveViewModel(
   private val chanPostRepository: ChanPostRepository,
   private val threadDownloadProgressNotifier: ThreadDownloadProgressNotifier,
   private val exportDownloadedThreadAsHtmlUseCase: ExportDownloadedThreadAsHtmlUseCase,
+  private val exportDownloadedThreadAsJsonUseCase: ExportDownloadedThreadAsJsonUseCase,
   private val exportDownloadedThreadMediaUseCase: ExportDownloadedThreadMediaUseCase,
 ) : BaseViewModel() {
 
@@ -483,6 +485,16 @@ class LocalArchiveViewModel(
       .onError { error -> Logger.e(TAG, "exportThreadsAsHtml() error", error) }
   }
 
+  suspend fun exportThreadsAsJson(
+    outputDirUri: Uri,
+    threadDescriptors: List<ChanDescriptor.ThreadDescriptor>,
+    onUpdate: (Int, Int) -> Unit
+  ): ModularResult<Unit> {
+    val params = ExportDownloadedThreadAsJsonUseCase.Params(outputDirUri, threadDescriptors, onUpdate)
+    return exportDownloadedThreadAsJsonUseCase.execute(params)
+      .onError { error -> Logger.e(TAG, "exportThreadsAsJson() error", error) }
+  }
+
   suspend fun exportThreadsMedia(
     outputDirectoryUri: Uri,
     threadDescriptors: List<ChanDescriptor.ThreadDescriptor>,
@@ -558,6 +570,7 @@ class LocalArchiveViewModel(
     private val chanPostRepository: ChanPostRepository,
     private val threadDownloadProgressNotifier: ThreadDownloadProgressNotifier,
     private val exportDownloadedThreadAsHtmlUseCase: ExportDownloadedThreadAsHtmlUseCase,
+    private val exportDownloadedThreadAsJsonUseCase: ExportDownloadedThreadAsJsonUseCase,
     private val exportDownloadedThreadMediaUseCase: ExportDownloadedThreadMediaUseCase,
   ) : ViewModelAssistedFactory<LocalArchiveViewModel> {
     override fun create(handle: SavedStateHandle): LocalArchiveViewModel {
@@ -568,6 +581,7 @@ class LocalArchiveViewModel(
         chanPostRepository = chanPostRepository,
         threadDownloadProgressNotifier = threadDownloadProgressNotifier,
         exportDownloadedThreadAsHtmlUseCase = exportDownloadedThreadAsHtmlUseCase,
+        exportDownloadedThreadAsJsonUseCase = exportDownloadedThreadAsJsonUseCase,
         exportDownloadedThreadMediaUseCase = exportDownloadedThreadMediaUseCase
       )
     }

Kuroba/app/src/main/res/values/strings.xml

@@ -1358,8 +1358,9 @@ If you ever log out or if the passcode expires you will have to change reply mod
     <string name="controller_local_archive_show_downloading_threads">DOWNLOADING</string>
     <string name="controller_local_archive_show_downloaded_threads">DOWNLOADED</string>
     <string name="controller_local_archive_additional_thread_stats">Posts: %1$d, Media: %2$d, Media on disk: %3$s</string>
-    <string name="controller_local_archive_export_threads">Export threads</string>
+    <string name="controller_local_archive_export_threads">Export thread html zip</string>
     <string name="controller_local_archive_export_thread_media">Export thread media</string>
+    <string name="controller_local_archive_export_thread_json">Export thread json zip</string>
     <string name="controller_local_archive_exported_format">Exported %1$d / %2$d</string>
 
     <string name="controller_saved_posts_delete_many_posts">Delete %1$d saved post(s)?</string>