Repository: Suwayomi/Suwayomi-Server
Pull request #1387: Improve Downloads Handling
Author: login: Syer10
Created: 2025-05-16T17:39:37Z | Merged: 2025-05-16T19:57:53Z

- If a download already exists and it has a page count over the known page count, it is set as complete
- If a download as 1000+ pages, it uses a `%04d` string format, 10000+ pages, it uses a `%05d`, etc
- Page count after a download is updated in the database based on how many image files there are
- Updates page count for downloaded chapters when getting ready to read
- Assume downloaded if first page is found, this means that chapters not marked as read will be assumed downloaded when getting page count. This is because we now store non-downloaded chapters in the system temp folders

Will fix issues with Mihon long page split chapters and 1000+ page chapters


Diff:
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt
index 619a65fc89..3d3463c460 100644
--- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/ChapterDownloadHelper.kt
@@ -22,6 +22,11 @@ object ChapterDownloadHelper {
         index: Int,
     ): Pair<InputStream, String> = provider(mangaId, chapterId).getImage().execute(index)
 
+    fun getImageCount(
+        mangaId: Int,
+        chapterId: Int,
+    ): Int = provider(mangaId, chapterId).getImageCount()
+
     fun delete(
         mangaId: Int,
         chapterId: Int,
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt
index 8c938ce105..19c4f9bc38 100644
--- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/Page.kt
@@ -110,7 +110,7 @@ object Page {
             }
         }
 
-        val fileName = getPageName(index)
+        val fileName = getPageName(index, chapterEntry[ChapterTable.pageCount])
 
         val cacheSaveDir = getChapterCachePath(mangaId, chapterId)
 
@@ -121,5 +121,8 @@ object Page {
     }
 
     /** converts 0 to "001" */
-    fun getPageName(index: Int): String = String.format("%03d", index + 1)
+    fun getPageName(
+        index: Int,
+        pageCount: Int,
+    ): String = String.format("%0${pageCount.toString().length.coerceAtLeast(3)}d", index + 1)
 }
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt
index 5222b1e005..f7c44e7a66 100644
--- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/chapter/ChapterForDownload.kt
@@ -70,7 +70,7 @@ private class ChapterForDownload(
 
         val isMarkedAsDownloaded = chapterEntry[ChapterTable.isDownloaded]
         val doesFirstPageExist = firstPageExists()
-        val isDownloaded = isMarkedAsDownloaded && doesFirstPageExist
+        val isDownloaded = isMarkedAsDownloaded || doesFirstPageExist
 
         log.debug { "isDownloaded= $isDownloaded (isMarkedAsDownloaded= $isMarkedAsDownloaded, doesFirstPageExist= $doesFirstPageExist)" }
 
@@ -80,12 +80,22 @@ private class ChapterForDownload(
             markAsNotDownloaded()
 
             updatePageList()
+        } else {
+            updatePageCount(ChapterDownloadHelper.getImageCount(mangaId, chapterId))
         }
 
         return asDataClass()
     }
 
-    private fun asDataClass() = ChapterTable.toDataClass(chapterEntry)
+    private fun asDataClass() =
+        ChapterTable.toDataClass(
+            transaction {
+                ChapterTable
+                    .selectAll()
+                    .where { ChapterTable.id eq chapterId }
+                    .first()
+            },
+        )
 
     init {
         chapterEntry = freshChapterEntry(optChapterId, optChapterIndex, optMangaId)
@@ -160,19 +170,15 @@ private class ChapterForDownload(
             }
         }
 
-        updatePageCount(pageList, chapterId)
+        updatePageCount(pageList.size)
 
         // chapter was updated
         chapterEntry = freshChapterEntry(chapterId, chapterIndex, mangaId)
     }
 
-    private fun updatePageCount(
-        pageList: List<Page>,
-        chapterId: Int,
-    ) {
+    private fun updatePageCount(pageCount: Int) {
         transaction {
             ChapterTable.update({ ChapterTable.id eq chapterId }) {
-                val pageCount = pageList.size
                 it[ChapterTable.pageCount] = pageCount
                 it[ChapterTable.lastPageRead] = chapterEntry[ChapterTable.lastPageRead].coerceAtMost(pageCount - 1).coerceAtLeast(0)
             }
diff --git a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
index aa891f0d0b..e85c56454e 100644
--- a/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
+++ b/server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
@@ -9,8 +9,10 @@ import kotlinx.coroutines.flow.launchIn
 import kotlinx.coroutines.flow.onEach
 import kotlinx.coroutines.flow.sample
 import org.apache.commons.compress.archivers.zip.ZipArchiveEntry
+import org.jetbrains.exposed.sql.SqlExpressionBuilder.eq
 import org.jetbrains.exposed.sql.selectAll
 import org.jetbrains.exposed.sql.transactions.transaction
+import org.jetbrains.exposed.sql.update
 import suwayomi.tachidesk.manga.impl.Page
 import suwayomi.tachidesk.manga.impl.download.model.DownloadChapter
 import suwayomi.tachidesk.manga.impl.util.createComicInfoFile
@@ -76,6 +78,8 @@ abstract class ChaptersFilesProvider<Type : FileType>(
         return Pair(getImageInputStream(image).buffered(), "image/$imageFileType")
     }
 
+    fun getImageCount(): Int = getImageFiles().filter { it.getName() != COMIC_INFO_FILE }.size
+
     override fun getImage(): RetrieveFile1Args<Int> = RetrieveFile1Args(::getImageImpl)
 
     /**
@@ -100,45 +104,56 @@ abstract class ChaptersFilesProvider<Type : FileType>(
         downloadCacheFolder.mkdirs()
 
         val pageCount = download.chapter.pageCount
-        for (pageNum in 0 until pageCount) {
-            var pageProgressJob: Job? = null
-            val fileName = Page.getPageName(pageNum) // might have to change this to index stored in database
-
-            val pageExistsInFinalDownloadFolder = ImageResponse.findFileNameStartingWith(finalDownloadFolder, fileName) != null
-            val pageExistsInCacheDownloadFolder = ImageResponse.findFileNameStartingWith(cacheChapterDir, fileName) != null
-
-            val doesPageAlreadyExist = pageExistsInFinalDownloadFolder || pageExistsInCacheDownloadFolder
-            if (doesPageAlreadyExist) {
-                continue
-            }
-
-            try {
-                Page
-                    .getPageImage(
-                        mangaId = download.mangaId,
-                        chapterIndex = download.chapterIndex,
-                        index = pageNum,
-                    ) { flow ->
-                        pageProgressJob =
-                            flow
-                                .sample(100)
-                                .distinctUntilChanged()
-                                .onEach {
-                                    download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount
-                                    step(
-                                        null,
-                                        false,
-                                    ) // don't throw on canceled download here since we can't do anything
-                                }.launchIn(scope)
-                    }.first
-                    .close()
-            } finally {
-                // always cancel the page progress job even if it throws an exception to avoid memory leaks
-                pageProgressJob?.cancel()
-            }
-            // TODO: retry on error with 2,4,8 seconds of wait
-            download.progress = ((pageNum + 1).toFloat()) / pageCount
+        if (
+            downloadCacheFolder
+                .listFiles()
+                .orEmpty()
+                .filter { it.name != COMIC_INFO_FILE }
+                .size >= pageCount
+        ) {
+            download.progress = 1f
             step(download, false)
+        } else {
+            for (pageNum in 0 until pageCount) {
+                var pageProgressJob: Job? = null
+                val fileName = Page.getPageName(pageNum, pageCount) // might have to change this to index stored in database
+
+                val pageExistsInFinalDownloadFolder = ImageResponse.findFileNameStartingWith(finalDownloadFolder, fileName) != null
+                val pageExistsInCacheDownloadFolder = ImageResponse.findFileNameStartingWith(cacheChapterDir, fileName) != null
+
+                val doesPageAlreadyExist = pageExistsInFinalDownloadFolder || pageExistsInCacheDownloadFolder
+                if (doesPageAlreadyExist) {
+                    continue
+                }
+
+                try {
+                    Page
+                        .getPageImage(
+                            mangaId = download.mangaId,
+                            chapterIndex = download.chapterIndex,
+                            index = pageNum,
+                        ) { flow ->
+                            pageProgressJob =
+                                flow
+                                    .sample(100)
+                                    .distinctUntilChanged()
+                                    .onEach {
+                                        download.progress = (pageNum.toFloat() + (it.toFloat() * 0.01f)) / pageCount
+                                        step(
+                                            null,
+                                            false,
+                                        ) // don't throw on canceled download here since we can't do anything
+                                    }.launchIn(scope)
+                        }.first
+                        .close()
+                } finally {
+                    // always cancel the page progress job even if it throws an exception to avoid memory leaks
+                    pageProgressJob?.cancel()
+                }
+                // TODO: retry on error with 2,4,8 seconds of wait
+                download.progress = ((pageNum + 1).toFloat()) / pageCount
+                step(download, false)
+            }
         }
 
         createComicInfoFile(
@@ -153,6 +168,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
 
         handleSuccessfulDownload()
 
+        transaction {
+            ChapterTable.update({ ChapterTable.id eq chapterId }) {
+                it[ChapterTable.pageCount] = getImageCount()
+            }
+        }
+
         File(cacheChapterDir).deleteRecursively()
 
         return true
diff --git a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt
index 18a07ca747..3b4d5c5a80 100644
--- a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt
+++ b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/PageTest.kt
@@ -15,16 +15,18 @@ import kotlin.test.assertEquals
 class PageTest : ApplicationTest() {
     @Test
     fun testGetPageName() {
-        val tests = listOf(0, 1, 2, 100)
+        val tests = listOf(0 to 5, 1 to 5, 2 to 5, 100 to 100, 998 to 1000, 1400 to 1500)
 
         val testResults =
-            tests.map {
-                getPageName(it)
+            tests.map { (page, count) ->
+                getPageName(page, count)
             }
 
         assertEquals(testResults[0], "001")
         assertEquals(testResults[1], "002")
         assertEquals(testResults[2], "003")
         assertEquals(testResults[3], "101")
+        assertEquals(testResults[4], "0999")
+        assertEquals(testResults[5], "1401")
     }
 }
diff --git a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt
index 8a8570f5e3..3b319a50e5 100644
--- a/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt
+++ b/server/src/test/kotlin/suwayomi/tachidesk/manga/impl/update/TestUpdater.kt
@@ -45,10 +45,16 @@ class TestUpdater : IUpdater {
 
     override val status: Flow<UpdateStatus>
         get() = TODO("Not yet implemented")
+    override val updates: Flow<UpdateUpdates>
+        get() = TODO("Not yet implemented")
     override val statusDeprecated: StateFlow<UpdateStatus>
         get() = TODO("Not yet implemented")
 
     override fun reset() {
         TODO("Not yet implemented")
     }
+
+    override fun getStatus(): UpdateUpdates {
+        TODO("Not yet implemented")
+    }
 }


Reviews:
user: login: schroda
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-16T19:03:21Z
user: login: Syer10
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-16T19:22:26Z
user: login: Syer10
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-16T19:34:57Z
user: login: schroda
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-16T19:46:24Z
user: login: Syer10
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-16T19:50:03Z
user: login: schroda
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-16T19:56:26Z
user: login: schroda
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-05-16T19:57:08Z

Review comments:
diff_hunk: @@ -153,6 +162,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
 
         handleSuccessfulDownload()
 
+        transaction {
path: server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
user: login: schroda
body: is it possibel to get here without having an up-to-date page count?
created_at: 2025-05-16T19:03:21Z
author_association: COLLABORATOR
diff_hunk: @@ -153,6 +162,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
 
         handleSuccessfulDownload()
 
+        transaction {
path: server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
user: login: Syer10
body: This is mainly for chapters like we get from Mihon that have more pages then expected, so we update our known page-count
created_at: 2025-05-16T19:22:26Z
author_association: MEMBER
diff_hunk: @@ -153,6 +162,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
 
         handleSuccessfulDownload()
 
+        transaction {
path: server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
user: login: Syer10
body: I've also got it so that if the chapter has not been fetched, it will accept a download with any page count. This allows for offline migration of chapter downloads
created_at: 2025-05-16T19:34:57Z
author_association: MEMBER
diff_hunk: @@ -153,6 +162,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
 
         handleSuccessfulDownload()
 
+        transaction {
path: server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
user: login: schroda
body: isn't this already handled in `ChapterForDownoad#asDownloadReady` with the added `updatePageCount(ChapterDownloadHelper.getImageCount(mangaId, chapterId))` in case the chapter is considered to be already downloaded?
created_at: 2025-05-16T19:46:24Z
author_association: COLLABORATOR
diff_hunk: @@ -153,6 +162,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
 
         handleSuccessfulDownload()
 
+        transaction {
path: server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
user: login: Syer10
body: Technically yes, but I think its fine to have it in both places? This allows for more updated data since the user might not read the chapter for a long time
created_at: 2025-05-16T19:50:03Z
author_association: MEMBER
diff_hunk: @@ -153,6 +162,12 @@ abstract class ChaptersFilesProvider<Type : FileType>(
 
         handleSuccessfulDownload()
 
+        transaction {
path: server/src/main/kotlin/suwayomi/tachidesk/manga/impl/download/fileProvider/ChaptersFilesProvider.kt
user: login: schroda
body: atm at least `getChapterDownloadReadyById` is required to be called before a chapter download is possible
created_at: 2025-05-16T19:56:26Z
author_association: COLLABORATOR

PR comments:
user: login: schroda
created_at: 2025-05-16T19:04:32Z
body: > If a download as 1000+ pages, it uses a %04d string format, 10000+ pages, it uses a %05d, etc

does this work with existing downloads that have more than 999 pages?

> Will fix issues with Mihon long page split chapters and 1000+ page chapters

how does it fix the long page split pages?
author_association: COLLABORATOR
user: login: Syer10
created_at: 2025-05-16T19:18:47Z
body: > does this work with existing downloads that have more than 999 pages?

No, currently chapters with 1000+ pages being downloaded end up being named things like `005.jpg` `999.jpg` and `1500.jpg`. This breaks our index sorting of pages that requires the prefixed 0. This is why the fix is needed

> how does it fix the long page split pages?

Long split pages from Mihon take the prefix name and then add a split count after it, like `001_001.jpg` and `001_002.jpg`.

Since we only rely on the index + page count, we can easily map to the correct page as long as we have the correct pagecount. This just means we have to assume the chapter is correct when checking the download and it has a pagecount(which includes the split pages as a seperate page) of equal or higher then we expect
author_association: MEMBER
user: login: schroda
created_at: 2025-05-16T19:44:38Z
body: > No, currently chapters with 100+ pages being downloaded end up being named things like 005.jpg 999.jpg and 1500.jpg. This breaks our index sorting of pages that requires the prefixed 0. This is why the fix is needed

What I meant with this was if the changes can handle reading existing downloads with 1000+ pages, or will it break them, due to the additional leading 0s which the existing page file names won't have?

> Long split pages from Mihon take the prefix name and then add a split count after it, like 001_001.jpg and 001_002.jpg.
>
> Since we only rely on the index + page count, we can easily map to the correct page as long as we have the correct pagecount. > This just means we have to assume the chapter is correct when checking the download and it has a pagecount of equal or > higher then we expect

So the original page still exists with the expected name and it will get selected when the page is requested?

Won't we have to filter out the split pages then to get the correct page count, since we don't actually handle them and instead just ignore them?
author_association: COLLABORATOR
user: login: Syer10
created_at: 2025-05-16T19:48:09Z
body: > What I meant with this was if the changes can handle reading existing downloads with 1000+ pages, or will it break them, due to the additional leading 0s which the existing page file names won't have?

It doesn't change how existing pages are handled, its just a fix for how downloaded pages end up being named when downloaded, based on the total page count. The 1000+ page downloads are broken on thier own currently.



> So the original page still exists with the expected name and it will get selected when the page is requested?
> 
> Won't we have to filter out the split pages then to get the correct page count, since we don't actually handle them and instead just ignore them?

Nope, the original page doesn't exist, just the split pages.

We don't actually use the page name after we create it, the name is just for index sorting. We rely on the page index in our sorted list to decide what page to provide, which is why changing the page count will help fix it
author_association: MEMBER
user: login: schroda
created_at: 2025-05-16T19:52:57Z
body: oh, nevermind, I got confused with the created file name in the retrieve function, but that is only used for the non downloaded case
author_association: COLLABORATOR