优化下载

app/src/main/java/io/legado/app/model/CacheBook.kt

@@ -1,27 +1,33 @@
 package io.legado.app.model
 
+import android.annotation.SuppressLint
 import android.content.Context
-import io.legado.app.R
 import io.legado.app.constant.IntentAction
 import io.legado.app.data.appDb
 import io.legado.app.data.entities.Book
 import io.legado.app.data.entities.BookChapter
 import io.legado.app.data.entities.BookSource
+import io.legado.app.help.BookHelp
 import io.legado.app.model.webBook.WebBook
 import io.legado.app.service.CacheBookService
-import io.legado.app.utils.msg
 import io.legado.app.utils.startService
 import kotlinx.coroutines.CoroutineScope
-import splitties.init.appCtx
+import java.text.SimpleDateFormat
+import java.util.*
 import java.util.concurrent.CopyOnWriteArraySet
+import kotlin.coroutines.CoroutineContext
 
 class CacheBook(val bookSource: BookSource, val book: Book) {
 
     companion object {
 
         val logs = arrayListOf<String>()
-        private val cacheBookMap = hashMapOf<String, CacheBook>()
+        val cacheBookMap = hashMapOf<String, CacheBook>()
 
+        @SuppressLint("ConstantLocale")
+        private val logTimeFormat = SimpleDateFormat("[mm:ss.SSS]", Locale.getDefault())
+
+        @Synchronized
         fun get(bookUrl: String): CacheBook? {
             var cacheBook = cacheBookMap[bookUrl]
             if (cacheBook != null) {
@@ -34,6 +40,7 @@ class CacheBook(val bookSource: BookSource, val book: Book) {
             return cacheBook
         }
 
+        @Synchronized
         fun get(bookSource: BookSource, book: Book): CacheBook {
             var cacheBook = cacheBookMap[book.bookUrl]
             if (cacheBook != null) {
@@ -50,7 +57,7 @@ class CacheBook(val bookSource: BookSource, val book: Book) {
                 if (logs.size > 1000) {
                     logs.removeAt(0)
                 }
-                logs.add(log)
+                logs.add(logTimeFormat.format(Date()) + " " + log)
             }
         }
 
@@ -76,51 +83,118 @@ class CacheBook(val bookSource: BookSource, val book: Book) {
             }
         }
 
-        val downloadCount: Int
+        val waitDownloadCount: Int
+            get() {
+                var count = 0
+                cacheBookMap.forEach {
+                    count += it.value.waitDownloadSet.size
+                }
+                return count
+            }
+
+        val successDownloadCount: Int
             get() {
                 var count = 0
                 cacheBookMap.forEach {
-                    count += it.value.downloadSet.size
+                    count += it.value.successDownloadSet.size
                 }
                 return count
             }
 
+        val onDownloadCount: Int
+            get() {
+                var count = 0
+                cacheBookMap.forEach {
+                    count += it.value.onDownloadSet.size
+                }
+                return count
+            }
+
+    }
+
+    val waitDownloadSet = CopyOnWriteArraySet<Int>()
+    val successDownloadSet = CopyOnWriteArraySet<Int>()
+    val onDownloadSet = CopyOnWriteArraySet<Int>()
+
+    fun addDownload(start: Int, end: Int) {
+        for (i in start..end) {
+            waitDownloadSet.add(i)
+        }
     }
 
-    val downloadSet = CopyOnWriteArraySet<Int>()
+    @Synchronized
+    fun download(scope: CoroutineScope, context: CoroutineContext): Boolean {
+        val chapterIndex = waitDownloadSet.firstOrNull() ?: return false
+        if (onDownloadSet.contains(chapterIndex)) {
+            waitDownloadSet.remove(chapterIndex)
+            return download(scope, context)
+        }
+        val chapter = appDb.bookChapterDao.getChapter(book.bookUrl, chapterIndex) ?: let {
+            waitDownloadSet.remove(chapterIndex)
+            return download(scope, context)
+        }
+        if (BookHelp.hasContent(book, chapter)) {
+            waitDownloadSet.remove(chapterIndex)
+            return download(scope, context)
+        }
+        waitDownloadSet.remove(chapterIndex)
+        onDownloadSet.add(chapterIndex)
+        WebBook.getContent(
+            scope,
+            bookSource,
+            book,
+            chapter,
+            waitConcurrent = true,
+            context = context
+        ).onSuccess { content ->
+            onDownloadSet.remove(chapterIndex)
+            successDownloadSet.add(chapterIndex)
+            addLog("${book.name}-${chapter.title} getContentSuccess")
+            downloadFinish(chapter, content.ifBlank { "No content" })
+        }.onError {
+            onDownloadSet.remove(chapterIndex)
+            waitDownloadSet.add(chapterIndex)
+            addLog("${book.name}-${chapter.title} getContentError${it.localizedMessage}")
+            downloadFinish(chapter, it.localizedMessage ?: "download error")
+        }.onCancel {
+            onDownloadSet.remove(chapterIndex)
+            waitDownloadSet.add(chapterIndex)
+        }
+        return true
+    }
 
+    @Synchronized
     fun download(
         scope: CoroutineScope,
         chapter: BookChapter,
         resetPageOffset: Boolean = false
     ) {
-        if (downloadSet.contains(chapter.index)) {
+        if (onDownloadSet.contains(chapter.index)) {
             return
         }
-        downloadSet.add(chapter.index)
+        onDownloadSet.add(chapter.index)
         WebBook.getContent(scope, bookSource, book, chapter)
             .onSuccess { content ->
-                if (ReadBook.book?.bookUrl == book.bookUrl) {
+                downloadFinish(chapter, content.ifBlank { "No content" }, resetPageOffset)
-                    ReadBook.contentLoadFinish(
-                        book,
-                        chapter,
-                        content.ifBlank { appCtx.getString(R.string.content_empty) },
-                        resetPageOffset = resetPageOffset
-                    )
-                }
             }.onError {
-                if (ReadBook.book?.bookUrl == book.bookUrl) {
+                downloadFinish(chapter, it.localizedMessage ?: "download error", resetPageOffset)
-                    ReadBook.contentLoadFinish(
-                        book,
-                        chapter,
-                        it.msg,
-                        resetPageOffset = resetPageOffset
-                    )
-                }
             }.onFinally {
-                downloadSet.remove(chapter.index)
+                onDownloadSet.remove(chapter.index)
                 ReadBook.removeLoading(chapter.index)
             }
     }
 
+    private fun downloadFinish(
+        chapter: BookChapter,
+        content: String,
+        resetPageOffset: Boolean = false
+    ) {
+        if (ReadBook.book?.bookUrl == book.bookUrl) {
+            ReadBook.contentLoadFinish(
+                book, chapter, content,
+                resetPageOffset = resetPageOffset
+            )
+        }
+    }
+
 }

app/src/main/java/io/legado/app/model/Debug.kt

@@ -22,7 +22,7 @@ object Debug {
     var isChecking: Boolean = false
 
     @SuppressLint("ConstantLocale")
-    private val DEBUG_TIME_FORMAT = SimpleDateFormat("[mm:ss.SSS]", Locale.getDefault())
+    private val debugTimeFormat = SimpleDateFormat("[mm:ss.SSS]", Locale.getDefault())
     private var startTime: Long = System.currentTimeMillis()
 
     @Synchronized
@@ -42,7 +42,7 @@ object Debug {
                 printMsg = HtmlFormatter.format(msg)
             }
             if (showTime) {
-                val time = DEBUG_TIME_FORMAT.format(Date(System.currentTimeMillis() - startTime))
+                val time = debugTimeFormat.format(Date(System.currentTimeMillis() - startTime))
                 printMsg = "$time $printMsg"
             }
             it.printLog(state, printMsg)
@@ -55,7 +55,7 @@ object Debug {
                 }
                 if (showTime && debugTimeMap[sourceUrl] != null) {
                     val time =
-                        DEBUG_TIME_FORMAT.format(Date(System.currentTimeMillis() - debugTimeMap[sourceUrl]!!))
+                        debugTimeFormat.format(Date(System.currentTimeMillis() - debugTimeMap[sourceUrl]!!))
                     printMsg = "$time $printMsg"
                     debugMessageMap[sourceUrl] = printMsg
                 }
@@ -80,7 +80,7 @@ object Debug {
     fun startChecking(source: BookSource) {
         isChecking = true
         debugTimeMap[source.bookSourceUrl] = System.currentTimeMillis()
-        debugMessageMap[source.bookSourceUrl] = "${DEBUG_TIME_FORMAT.format(Date(0))} 开始校验"
+        debugMessageMap[source.bookSourceUrl] = "${debugTimeFormat.format(Date(0))} 开始校验"
     }
 
     fun finishChecking() {
@@ -95,7 +95,7 @@ object Debug {
         if (debugTimeMap[sourceUrl] != null && debugMessageMap[sourceUrl] != null) {
             val spendingTime = System.currentTimeMillis() - debugTimeMap[sourceUrl]!!
             debugTimeMap[sourceUrl] = if(state == "成功") spendingTime else 180000L
-            val printTime = DEBUG_TIME_FORMAT.format(Date(spendingTime))
+            val printTime = debugTimeFormat.format(Date(spendingTime))
             val originalMessage = debugMessageMap[sourceUrl]!!.substringAfter("] ")
             debugMessageMap[sourceUrl] = "$printTime $originalMessage $state"
         }

app/src/main/java/io/legado/app/model/ReadBook.kt

@@ -192,6 +192,7 @@ object ReadBook : CoroutineScope by MainScope() {
         }
         upReadStartTime()
         preDownload()
+        ImageProvider.clearOut(durChapterIndex)
     }
 
     /**
@@ -334,7 +335,6 @@ object ReadBook : CoroutineScope by MainScope() {
         success: (() -> Unit)? = null
     ) {
         Coroutine.async {
-            ImageProvider.clearOut(durChapterIndex)
             if (chapter.index in durChapterIndex - 1..durChapterIndex + 1) {
                 chapter.title = when (AppConfig.chineseConverterType) {
                     1 -> ChineseUtils.t2s(chapter.title)

app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeUrl.kt

@@ -267,7 +267,7 @@ class AnalyzeUrl(
     }
 
     /**
-     * 根据书源并发率等待
+     * 并发判断
      */
     private fun judgmentConcurrent() {
         source ?: return
@@ -321,7 +321,7 @@ class AnalyzeUrl(
      */
     suspend fun getStrResponse(
         jsStr: String? = null,
-        sourceRegex: String? = null,
+        sourceRegex: String? = null
     ): StrResponse {
         if (type != null) {
             return StrResponse(url, StringUtils.byteToHexString(getByteArray()))

app/src/main/java/io/legado/app/model/webBook/WebBook.kt

@@ -278,7 +278,7 @@ object WebBook {
         bookSource: BookSource,
         book: Book,
         bookChapter: BookChapter,
-        nextChapterUrl: String? = null,
+        nextChapterUrl: String? = null
     ): String {
         if (bookSource.getContentRule().content.isNullOrEmpty()) {
             Debug.log(bookSource.bookSourceUrl, "⇒正文规则为空,使用章节链接:${bookChapter.url}")

app/src/main/java/io/legado/app/service/CacheBookService.kt

@@ -7,26 +7,12 @@ import io.legado.app.base.BaseService
 import io.legado.app.constant.AppConst
 import io.legado.app.constant.EventBus
 import io.legado.app.constant.IntentAction
-import io.legado.app.data.appDb
-import io.legado.app.data.entities.Book
-import io.legado.app.data.entities.BookChapter
-import io.legado.app.data.entities.BookSource
 import io.legado.app.help.AppConfig
-import io.legado.app.help.BookHelp
-import io.legado.app.help.coroutine.CompositeCoroutine
-import io.legado.app.help.coroutine.Coroutine
 import io.legado.app.model.CacheBook
-import io.legado.app.model.webBook.WebBook
 import io.legado.app.utils.postEvent
 import io.legado.app.utils.servicePendingIntent
-import io.legado.app.utils.toastOnUi
+import kotlinx.coroutines.*
-import kotlinx.coroutines.asCoroutineDispatcher
-import kotlinx.coroutines.delay
-import kotlinx.coroutines.isActive
-import kotlinx.coroutines.launch
 import splitties.init.appCtx
-import java.util.concurrent.ConcurrentHashMap
-import java.util.concurrent.CopyOnWriteArraySet
 import java.util.concurrent.Executors
 import kotlin.math.min
 
@@ -34,16 +20,8 @@ class CacheBookService : BaseService() {
     private val threadCount = AppConfig.threadCount
     private var cachePool =
         Executors.newFixedThreadPool(min(threadCount, 8)).asCoroutineDispatcher()
-    private var tasks = CompositeCoroutine()
+    private var downloadJob: Job? = null
-    private val bookMap = ConcurrentHashMap<String, Book>()
-    private val bookSourceMap = ConcurrentHashMap<String, BookSource>()
-    private val downloadMap = ConcurrentHashMap<String, CopyOnWriteArraySet<BookChapter>>()
-    private val downloadCount = ConcurrentHashMap<String, DownloadCount>()
-    private val finalMap = ConcurrentHashMap<String, CopyOnWriteArraySet<BookChapter>>()
-    private val downloadingList = CopyOnWriteArraySet<String>()
 
-    @Volatile
-    private var downloadingCount = 0
     private var notificationContent = appCtx.getString(R.string.starting_download)
 
     private val notificationBuilder by lazy {
@@ -65,8 +43,8 @@ class CacheBookService : BaseService() {
         launch {
             while (isActive) {
                 delay(1000)
-                upNotification()
+                upNotificationContent()
-                postEvent(EventBus.UP_DOWNLOAD, downloadMap)
+                postEvent(EventBus.UP_DOWNLOAD, "")
             }
         }
     }
@@ -87,180 +65,48 @@ class CacheBookService : BaseService() {
     }
 
     override fun onDestroy() {
-        tasks.clear()
         cachePool.close()
-        downloadMap.clear()
-        finalMap.clear()
         super.onDestroy()
-        postEvent(EventBus.UP_DOWNLOAD, downloadMap)
+        postEvent(EventBus.UP_DOWNLOAD, "")
-    }
-
-    private fun getBook(bookUrl: String): Book? {
-        var book = bookMap[bookUrl]
-        if (book == null) {
-            synchronized(this) {
-                book = bookMap[bookUrl]
-                if (book == null) {
-                    book = appDb.bookDao.getBook(bookUrl)
-                    if (book == null) {
-                        removeDownload(bookUrl)
-                    }
-                }
-            }
-        }
-        return book
-    }
-
-    private fun getBookSource(bookUrl: String, origin: String): BookSource? {
-        var bookSource = bookSourceMap[origin]
-        if (bookSource == null) {
-            synchronized(this) {
-                bookSource = bookSourceMap[origin]
-                if (bookSource == null) {
-                    bookSource = appDb.bookSourceDao.getBookSource(origin)
-                    if (bookSource == null) {
-                        removeDownload(bookUrl)
-                    }
-                }
-            }
-        }
-        return bookSource
     }
 
     private fun addDownloadData(bookUrl: String?, start: Int, end: Int) {
         bookUrl ?: return
-        if (downloadMap.containsKey(bookUrl)) {
+        val cacheBook = CacheBook.get(bookUrl) ?: return
-            notificationContent = getString(R.string.already_in_download)
+        cacheBook.addDownload(start, end)
-            upNotification()
+        if (downloadJob == null) {
-            toastOnUi(notificationContent)
+            download()
-            return
-        }
-        downloadCount[bookUrl] = DownloadCount()
-        execute(context = cachePool) {
-            appDb.bookChapterDao.getChapterList(bookUrl, start, end).let {
-                if (it.isNotEmpty()) {
-                    val chapters = CopyOnWriteArraySet<BookChapter>()
-                    chapters.addAll(it)
-                    downloadMap[bookUrl] = chapters
-                } else {
-                    CacheBook.addLog("${getBook(bookUrl)?.name} is empty")
-                }
-            }
-            for (i in 0 until threadCount) {
-                if (downloadingCount < threadCount) {
-                    download()
-                }
-            }
         }
     }
 
     private fun removeDownload(bookUrl: String?) {
-        downloadMap.remove(bookUrl)
+        CacheBook.cacheBookMap.remove(bookUrl)
-        finalMap.remove(bookUrl)
     }
 
     private fun download() {
-        downloadingCount += 1
+        downloadJob = launch(cachePool) {
-        val task = Coroutine.async(this, context = cachePool) {
+            while (isActive) {
-            if (!isActive) return@async
+                CacheBook.cacheBookMap.forEach {
-            val bookChapter: BookChapter? = synchronized(this@CacheBookService) {
+                    while (CacheBook.onDownloadCount > threadCount) {
-                downloadMap.forEach {
+                        delay(50)
-                    it.value.forEach { chapter ->
-                        if (!downloadingList.contains(chapter.url)) {
-                            downloadingList.add(chapter.url)
-                            return@synchronized chapter
-                        }
                     }
+                    it.value.download(this, cachePool)
                 }
-                return@synchronized null
-            }
-            if (bookChapter == null) {
-                postDownloading(false)
-            } else {
-                val book = getBook(bookChapter.bookUrl)
-                if (book == null) {
-                    postDownloading(true)
-                    return@async
-                }
-                val bookSource = getBookSource(bookChapter.bookUrl, book.origin)
-                if (bookSource == null) {
-                    postDownloading(true)
-                    return@async
-                }
-                if (!BookHelp.hasImageContent(book, bookChapter)) {
-                    WebBook.getContent(this, bookSource, book, bookChapter, context = cachePool)
-                        .timeout(60000L)
-                        .onError(cachePool) {
-                            synchronized(this) {
-                                downloadingList.remove(bookChapter.url)
-                            }
-                            notificationContent = "getContentError${it.localizedMessage}"
-                            upNotification()
-                        }
-                        .onSuccess(cachePool) {
-                            synchronized(this@CacheBookService) {
-                                downloadCount[book.bookUrl]?.increaseSuccess()
-                                downloadCount[book.bookUrl]?.increaseFinished()
-                                downloadCount[book.bookUrl]?.let {
-                                    upNotification(
-                                        it,
-                                        downloadMap[book.bookUrl]?.size,
-                                        bookChapter.title
-                                    )
-                                }
-                                val chapterMap =
-                                    finalMap[book.bookUrl]
-                                        ?: CopyOnWriteArraySet<BookChapter>().apply {
-                                            finalMap[book.bookUrl] = this
-                                        }
-                                chapterMap.add(bookChapter)
-                                if (chapterMap.size == downloadMap[book.bookUrl]?.size) {
-                                    downloadMap.remove(book.bookUrl)
-                                    finalMap.remove(book.bookUrl)
-                                    downloadCount.remove(book.bookUrl)
-                                }
-                            }
-                        }.onFinally(cachePool) {
-                            postDownloading(true)
-                        }
-                } else {
-                    //无需下载的，设置为增加成功
-                    downloadCount[book.bookUrl]?.increaseSuccess()
-                    downloadCount[book.bookUrl]?.increaseFinished()
-                    postDownloading(true)
-                }
-            }
-        }.onError(cachePool) {
-            notificationContent = "ERROR:${it.localizedMessage}"
-            CacheBook.addLog(notificationContent)
-            upNotification()
-        }
-        tasks.add(task)
-    }
-
-    private fun postDownloading(hasChapter: Boolean) {
-        downloadingCount -= 1
-        if (hasChapter) {
-            download()
-        } else {
-            if (downloadingCount < 1) {
-                stopDownload()
             }
         }
     }
 
     private fun stopDownload() {
-        tasks.clear()
+        CacheBook.cacheBookMap.forEach {
+            it.value.waitDownloadSet.clear()
+        }
         stopSelf()
     }
 
-    private fun upNotification(
+    private fun upNotificationContent() {
-        downloadCount: DownloadCount,
-        totalCount: Int?,
-        content: String
-    ) {
         notificationContent =
-            "进度:" + downloadCount.downloadFinishedCount + "/" + totalCount + ",成功:" + downloadCount.successCount + "," + content
+            "正在下载:${CacheBook.onDownloadCount}/等待中:${CacheBook.waitDownloadCount}/成功:${CacheBook.successDownloadCount}"
+        upNotification()
     }
 
     /**
@@ -272,19 +118,4 @@ class CacheBookService : BaseService() {
         startForeground(AppConst.notificationIdDownload, notification)
     }
 
-    class DownloadCount {
-        @Volatile
-        var downloadFinishedCount = 0 // 下载完成的条目数量
-
-        @Volatile
-        var successCount = 0 //下载成功的条目数量
-
-        fun increaseSuccess() {
-            ++successCount
-        }
-
-        fun increaseFinished() {
-            ++downloadFinishedCount
-        }
-    }
 }

app/src/main/java/io/legado/app/ui/main/MainViewModel.kt

@@ -119,7 +119,7 @@ class MainViewModel(application: Application) : BaseViewModel(application) {
                             var addToCache = false
                             while (!addToCache) {
                                 val cacheBook = CacheBook.get(bookSource, book)
-                                if (CacheBook.downloadCount < 10) {
+                                if (CacheBook.onDownloadCount < 10) {
                                     cacheBook.download(this, chapter)
                                     addToCache = true
                                 } else {