优化下载

pull/1298/head
gedoor 3 years ago
parent a563f51c91
commit f0825b6af2
  1. 124
      app/src/main/java/io/legado/app/model/CacheBook.kt
  2. 10
      app/src/main/java/io/legado/app/model/Debug.kt
  3. 2
      app/src/main/java/io/legado/app/model/ReadBook.kt
  4. 4
      app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeUrl.kt
  5. 2
      app/src/main/java/io/legado/app/model/webBook/WebBook.kt
  6. 211
      app/src/main/java/io/legado/app/service/CacheBookService.kt
  7. 2
      app/src/main/java/io/legado/app/ui/main/MainViewModel.kt

@ -1,27 +1,33 @@
package io.legado.app.model
import android.annotation.SuppressLint
import android.content.Context
import io.legado.app.R
import io.legado.app.constant.IntentAction
import io.legado.app.data.appDb
import io.legado.app.data.entities.Book
import io.legado.app.data.entities.BookChapter
import io.legado.app.data.entities.BookSource
import io.legado.app.help.BookHelp
import io.legado.app.model.webBook.WebBook
import io.legado.app.service.CacheBookService
import io.legado.app.utils.msg
import io.legado.app.utils.startService
import kotlinx.coroutines.CoroutineScope
import splitties.init.appCtx
import java.text.SimpleDateFormat
import java.util.*
import java.util.concurrent.CopyOnWriteArraySet
import kotlin.coroutines.CoroutineContext
class CacheBook(val bookSource: BookSource, val book: Book) {
companion object {
val logs = arrayListOf<String>()
private val cacheBookMap = hashMapOf<String, CacheBook>()
val cacheBookMap = hashMapOf<String, CacheBook>()
@SuppressLint("ConstantLocale")
private val logTimeFormat = SimpleDateFormat("[mm:ss.SSS]", Locale.getDefault())
@Synchronized
fun get(bookUrl: String): CacheBook? {
var cacheBook = cacheBookMap[bookUrl]
if (cacheBook != null) {
@ -34,6 +40,7 @@ class CacheBook(val bookSource: BookSource, val book: Book) {
return cacheBook
}
@Synchronized
fun get(bookSource: BookSource, book: Book): CacheBook {
var cacheBook = cacheBookMap[book.bookUrl]
if (cacheBook != null) {
@ -50,7 +57,7 @@ class CacheBook(val bookSource: BookSource, val book: Book) {
if (logs.size > 1000) {
logs.removeAt(0)
}
logs.add(log)
logs.add(logTimeFormat.format(Date()) + " " + log)
}
}
@ -76,51 +83,118 @@ class CacheBook(val bookSource: BookSource, val book: Book) {
}
}
val downloadCount: Int
val waitDownloadCount: Int
get() {
var count = 0
cacheBookMap.forEach {
count += it.value.waitDownloadSet.size
}
return count
}
val successDownloadCount: Int
get() {
var count = 0
cacheBookMap.forEach {
count += it.value.successDownloadSet.size
}
return count
}
val onDownloadCount: Int
get() {
var count = 0
cacheBookMap.forEach {
count += it.value.downloadSet.size
count += it.value.onDownloadSet.size
}
return count
}
}
val downloadSet = CopyOnWriteArraySet<Int>()
val waitDownloadSet = CopyOnWriteArraySet<Int>()
val successDownloadSet = CopyOnWriteArraySet<Int>()
val onDownloadSet = CopyOnWriteArraySet<Int>()
fun addDownload(start: Int, end: Int) {
for (i in start..end) {
waitDownloadSet.add(i)
}
}
@Synchronized
fun download(scope: CoroutineScope, context: CoroutineContext): Boolean {
val chapterIndex = waitDownloadSet.firstOrNull() ?: return false
if (onDownloadSet.contains(chapterIndex)) {
waitDownloadSet.remove(chapterIndex)
return download(scope, context)
}
val chapter = appDb.bookChapterDao.getChapter(book.bookUrl, chapterIndex) ?: let {
waitDownloadSet.remove(chapterIndex)
return download(scope, context)
}
if (BookHelp.hasContent(book, chapter)) {
waitDownloadSet.remove(chapterIndex)
return download(scope, context)
}
waitDownloadSet.remove(chapterIndex)
onDownloadSet.add(chapterIndex)
WebBook.getContent(
scope,
bookSource,
book,
chapter,
waitConcurrent = true,
context = context
).onSuccess { content ->
onDownloadSet.remove(chapterIndex)
successDownloadSet.add(chapterIndex)
addLog("${book.name}-${chapter.title} getContentSuccess")
downloadFinish(chapter, content.ifBlank { "No content" })
}.onError {
onDownloadSet.remove(chapterIndex)
waitDownloadSet.add(chapterIndex)
addLog("${book.name}-${chapter.title} getContentError${it.localizedMessage}")
downloadFinish(chapter, it.localizedMessage ?: "download error")
}.onCancel {
onDownloadSet.remove(chapterIndex)
waitDownloadSet.add(chapterIndex)
}
return true
}
@Synchronized
fun download(
scope: CoroutineScope,
chapter: BookChapter,
resetPageOffset: Boolean = false
) {
if (downloadSet.contains(chapter.index)) {
if (onDownloadSet.contains(chapter.index)) {
return
}
downloadSet.add(chapter.index)
onDownloadSet.add(chapter.index)
WebBook.getContent(scope, bookSource, book, chapter)
.onSuccess { content ->
if (ReadBook.book?.bookUrl == book.bookUrl) {
ReadBook.contentLoadFinish(
book,
chapter,
content.ifBlank { appCtx.getString(R.string.content_empty) },
resetPageOffset = resetPageOffset
)
}
downloadFinish(chapter, content.ifBlank { "No content" }, resetPageOffset)
}.onError {
downloadFinish(chapter, it.localizedMessage ?: "download error", resetPageOffset)
}.onFinally {
onDownloadSet.remove(chapter.index)
ReadBook.removeLoading(chapter.index)
}
}
private fun downloadFinish(
chapter: BookChapter,
content: String,
resetPageOffset: Boolean = false
) {
if (ReadBook.book?.bookUrl == book.bookUrl) {
ReadBook.contentLoadFinish(
book,
chapter,
it.msg,
book, chapter, content,
resetPageOffset = resetPageOffset
)
}
}.onFinally {
downloadSet.remove(chapter.index)
ReadBook.removeLoading(chapter.index)
}
}
}

@ -22,7 +22,7 @@ object Debug {
var isChecking: Boolean = false
@SuppressLint("ConstantLocale")
private val DEBUG_TIME_FORMAT = SimpleDateFormat("[mm:ss.SSS]", Locale.getDefault())
private val debugTimeFormat = SimpleDateFormat("[mm:ss.SSS]", Locale.getDefault())
private var startTime: Long = System.currentTimeMillis()
@Synchronized
@ -42,7 +42,7 @@ object Debug {
printMsg = HtmlFormatter.format(msg)
}
if (showTime) {
val time = DEBUG_TIME_FORMAT.format(Date(System.currentTimeMillis() - startTime))
val time = debugTimeFormat.format(Date(System.currentTimeMillis() - startTime))
printMsg = "$time $printMsg"
}
it.printLog(state, printMsg)
@ -55,7 +55,7 @@ object Debug {
}
if (showTime && debugTimeMap[sourceUrl] != null) {
val time =
DEBUG_TIME_FORMAT.format(Date(System.currentTimeMillis() - debugTimeMap[sourceUrl]!!))
debugTimeFormat.format(Date(System.currentTimeMillis() - debugTimeMap[sourceUrl]!!))
printMsg = "$time $printMsg"
debugMessageMap[sourceUrl] = printMsg
}
@ -80,7 +80,7 @@ object Debug {
fun startChecking(source: BookSource) {
isChecking = true
debugTimeMap[source.bookSourceUrl] = System.currentTimeMillis()
debugMessageMap[source.bookSourceUrl] = "${DEBUG_TIME_FORMAT.format(Date(0))} 开始校验"
debugMessageMap[source.bookSourceUrl] = "${debugTimeFormat.format(Date(0))} 开始校验"
}
fun finishChecking() {
@ -95,7 +95,7 @@ object Debug {
if (debugTimeMap[sourceUrl] != null && debugMessageMap[sourceUrl] != null) {
val spendingTime = System.currentTimeMillis() - debugTimeMap[sourceUrl]!!
debugTimeMap[sourceUrl] = if(state == "成功") spendingTime else 180000L
val printTime = DEBUG_TIME_FORMAT.format(Date(spendingTime))
val printTime = debugTimeFormat.format(Date(spendingTime))
val originalMessage = debugMessageMap[sourceUrl]!!.substringAfter("] ")
debugMessageMap[sourceUrl] = "$printTime $originalMessage $state"
}

@ -192,6 +192,7 @@ object ReadBook : CoroutineScope by MainScope() {
}
upReadStartTime()
preDownload()
ImageProvider.clearOut(durChapterIndex)
}
/**
@ -334,7 +335,6 @@ object ReadBook : CoroutineScope by MainScope() {
success: (() -> Unit)? = null
) {
Coroutine.async {
ImageProvider.clearOut(durChapterIndex)
if (chapter.index in durChapterIndex - 1..durChapterIndex + 1) {
chapter.title = when (AppConfig.chineseConverterType) {
1 -> ChineseUtils.t2s(chapter.title)

@ -267,7 +267,7 @@ class AnalyzeUrl(
}
/**
* 根据书源并发率等待
* 并发判断
*/
private fun judgmentConcurrent() {
source ?: return
@ -321,7 +321,7 @@ class AnalyzeUrl(
*/
suspend fun getStrResponse(
jsStr: String? = null,
sourceRegex: String? = null,
sourceRegex: String? = null
): StrResponse {
if (type != null) {
return StrResponse(url, StringUtils.byteToHexString(getByteArray()))

@ -278,7 +278,7 @@ object WebBook {
bookSource: BookSource,
book: Book,
bookChapter: BookChapter,
nextChapterUrl: String? = null,
nextChapterUrl: String? = null
): String {
if (bookSource.getContentRule().content.isNullOrEmpty()) {
Debug.log(bookSource.bookSourceUrl, "⇒正文规则为空,使用章节链接:${bookChapter.url}")

@ -7,26 +7,12 @@ import io.legado.app.base.BaseService
import io.legado.app.constant.AppConst
import io.legado.app.constant.EventBus
import io.legado.app.constant.IntentAction
import io.legado.app.data.appDb
import io.legado.app.data.entities.Book
import io.legado.app.data.entities.BookChapter
import io.legado.app.data.entities.BookSource
import io.legado.app.help.AppConfig
import io.legado.app.help.BookHelp
import io.legado.app.help.coroutine.CompositeCoroutine
import io.legado.app.help.coroutine.Coroutine
import io.legado.app.model.CacheBook
import io.legado.app.model.webBook.WebBook
import io.legado.app.utils.postEvent
import io.legado.app.utils.servicePendingIntent
import io.legado.app.utils.toastOnUi
import kotlinx.coroutines.asCoroutineDispatcher
import kotlinx.coroutines.delay
import kotlinx.coroutines.isActive
import kotlinx.coroutines.launch
import kotlinx.coroutines.*
import splitties.init.appCtx
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CopyOnWriteArraySet
import java.util.concurrent.Executors
import kotlin.math.min
@ -34,16 +20,8 @@ class CacheBookService : BaseService() {
private val threadCount = AppConfig.threadCount
private var cachePool =
Executors.newFixedThreadPool(min(threadCount, 8)).asCoroutineDispatcher()
private var tasks = CompositeCoroutine()
private val bookMap = ConcurrentHashMap<String, Book>()
private val bookSourceMap = ConcurrentHashMap<String, BookSource>()
private val downloadMap = ConcurrentHashMap<String, CopyOnWriteArraySet<BookChapter>>()
private val downloadCount = ConcurrentHashMap<String, DownloadCount>()
private val finalMap = ConcurrentHashMap<String, CopyOnWriteArraySet<BookChapter>>()
private val downloadingList = CopyOnWriteArraySet<String>()
private var downloadJob: Job? = null
@Volatile
private var downloadingCount = 0
private var notificationContent = appCtx.getString(R.string.starting_download)
private val notificationBuilder by lazy {
@ -65,8 +43,8 @@ class CacheBookService : BaseService() {
launch {
while (isActive) {
delay(1000)
upNotification()
postEvent(EventBus.UP_DOWNLOAD, downloadMap)
upNotificationContent()
postEvent(EventBus.UP_DOWNLOAD, "")
}
}
}
@ -87,180 +65,48 @@ class CacheBookService : BaseService() {
}
override fun onDestroy() {
tasks.clear()
cachePool.close()
downloadMap.clear()
finalMap.clear()
super.onDestroy()
postEvent(EventBus.UP_DOWNLOAD, downloadMap)
}
private fun getBook(bookUrl: String): Book? {
var book = bookMap[bookUrl]
if (book == null) {
synchronized(this) {
book = bookMap[bookUrl]
if (book == null) {
book = appDb.bookDao.getBook(bookUrl)
if (book == null) {
removeDownload(bookUrl)
}
}
}
}
return book
}
private fun getBookSource(bookUrl: String, origin: String): BookSource? {
var bookSource = bookSourceMap[origin]
if (bookSource == null) {
synchronized(this) {
bookSource = bookSourceMap[origin]
if (bookSource == null) {
bookSource = appDb.bookSourceDao.getBookSource(origin)
if (bookSource == null) {
removeDownload(bookUrl)
}
}
}
}
return bookSource
postEvent(EventBus.UP_DOWNLOAD, "")
}
private fun addDownloadData(bookUrl: String?, start: Int, end: Int) {
bookUrl ?: return
if (downloadMap.containsKey(bookUrl)) {
notificationContent = getString(R.string.already_in_download)
upNotification()
toastOnUi(notificationContent)
return
}
downloadCount[bookUrl] = DownloadCount()
execute(context = cachePool) {
appDb.bookChapterDao.getChapterList(bookUrl, start, end).let {
if (it.isNotEmpty()) {
val chapters = CopyOnWriteArraySet<BookChapter>()
chapters.addAll(it)
downloadMap[bookUrl] = chapters
} else {
CacheBook.addLog("${getBook(bookUrl)?.name} is empty")
}
}
for (i in 0 until threadCount) {
if (downloadingCount < threadCount) {
val cacheBook = CacheBook.get(bookUrl) ?: return
cacheBook.addDownload(start, end)
if (downloadJob == null) {
download()
}
}
}
}
private fun removeDownload(bookUrl: String?) {
downloadMap.remove(bookUrl)
finalMap.remove(bookUrl)
CacheBook.cacheBookMap.remove(bookUrl)
}
private fun download() {
downloadingCount += 1
val task = Coroutine.async(this, context = cachePool) {
if (!isActive) return@async
val bookChapter: BookChapter? = synchronized(this@CacheBookService) {
downloadMap.forEach {
it.value.forEach { chapter ->
if (!downloadingList.contains(chapter.url)) {
downloadingList.add(chapter.url)
return@synchronized chapter
}
}
}
return@synchronized null
}
if (bookChapter == null) {
postDownloading(false)
} else {
val book = getBook(bookChapter.bookUrl)
if (book == null) {
postDownloading(true)
return@async
}
val bookSource = getBookSource(bookChapter.bookUrl, book.origin)
if (bookSource == null) {
postDownloading(true)
return@async
}
if (!BookHelp.hasImageContent(book, bookChapter)) {
WebBook.getContent(this, bookSource, book, bookChapter, context = cachePool)
.timeout(60000L)
.onError(cachePool) {
synchronized(this) {
downloadingList.remove(bookChapter.url)
}
notificationContent = "getContentError${it.localizedMessage}"
upNotification()
}
.onSuccess(cachePool) {
synchronized(this@CacheBookService) {
downloadCount[book.bookUrl]?.increaseSuccess()
downloadCount[book.bookUrl]?.increaseFinished()
downloadCount[book.bookUrl]?.let {
upNotification(
it,
downloadMap[book.bookUrl]?.size,
bookChapter.title
)
}
val chapterMap =
finalMap[book.bookUrl]
?: CopyOnWriteArraySet<BookChapter>().apply {
finalMap[book.bookUrl] = this
}
chapterMap.add(bookChapter)
if (chapterMap.size == downloadMap[book.bookUrl]?.size) {
downloadMap.remove(book.bookUrl)
finalMap.remove(book.bookUrl)
downloadCount.remove(book.bookUrl)
}
}
}.onFinally(cachePool) {
postDownloading(true)
}
} else {
//无需下载的,设置为增加成功
downloadCount[book.bookUrl]?.increaseSuccess()
downloadCount[book.bookUrl]?.increaseFinished()
postDownloading(true)
}
}
}.onError(cachePool) {
notificationContent = "ERROR:${it.localizedMessage}"
CacheBook.addLog(notificationContent)
upNotification()
downloadJob = launch(cachePool) {
while (isActive) {
CacheBook.cacheBookMap.forEach {
while (CacheBook.onDownloadCount > threadCount) {
delay(50)
}
tasks.add(task)
it.value.download(this, cachePool)
}
private fun postDownloading(hasChapter: Boolean) {
downloadingCount -= 1
if (hasChapter) {
download()
} else {
if (downloadingCount < 1) {
stopDownload()
}
}
}
private fun stopDownload() {
tasks.clear()
CacheBook.cacheBookMap.forEach {
it.value.waitDownloadSet.clear()
}
stopSelf()
}
private fun upNotification(
downloadCount: DownloadCount,
totalCount: Int?,
content: String
) {
private fun upNotificationContent() {
notificationContent =
"进度:" + downloadCount.downloadFinishedCount + "/" + totalCount + ",成功:" + downloadCount.successCount + "," + content
"正在下载:${CacheBook.onDownloadCount}/等待中:${CacheBook.waitDownloadCount}/成功:${CacheBook.successDownloadCount}"
upNotification()
}
/**
@ -272,19 +118,4 @@ class CacheBookService : BaseService() {
startForeground(AppConst.notificationIdDownload, notification)
}
class DownloadCount {
@Volatile
var downloadFinishedCount = 0 // 下载完成的条目数量
@Volatile
var successCount = 0 //下载成功的条目数量
fun increaseSuccess() {
++successCount
}
fun increaseFinished() {
++downloadFinishedCount
}
}
}

@ -119,7 +119,7 @@ class MainViewModel(application: Application) : BaseViewModel(application) {
var addToCache = false
while (!addToCache) {
val cacheBook = CacheBook.get(bookSource, book)
if (CacheBook.downloadCount < 10) {
if (CacheBook.onDownloadCount < 10) {
cacheBook.download(this, chapter)
addToCache = true
} else {

Loading…
Cancel
Save