添加多页目录并发访问限制

pull/2481/head
kunfei 2 years ago
parent 20c2daeadb
commit 118be1d2e4
  1. 31
      app/src/main/java/io/legado/app/model/analyzeRule/AnalyzeUrl.kt
  2. 39
      app/src/main/java/io/legado/app/model/webBook/BookChapterList.kt

@ -3,9 +3,9 @@ package io.legado.app.model.analyzeRule
import android.annotation.SuppressLint
import android.util.Base64
import androidx.annotation.Keep
import cn.hutool.core.util.HexUtil
import com.bumptech.glide.load.model.GlideUrl
import com.script.SimpleBindings
import cn.hutool.core.util.HexUtil
import io.legado.app.constant.AppConst.SCRIPT_ENGINE
import io.legado.app.constant.AppConst.UA_NAME
import io.legado.app.constant.AppPattern.JS_PATTERN
@ -279,6 +279,7 @@ class AnalyzeUrl(
/**
* 开始访问,并发判断
*/
@Throws(ConcurrentException::class)
private fun fetchStart(): ConcurrentRecord? {
source ?: return null
val concurrentRate = source.concurrentRate
@ -294,10 +295,13 @@ class AnalyzeUrl(
}
val waitTime: Int = synchronized(fetchRecord) {
try {
if (rateIndex == -1) {
if (!fetchRecord.isConcurrent) {
//并发控制非 次数/毫秒
if (fetchRecord.frequency > 0) {
//已经有访问线程,直接等待
return@synchronized concurrentRate.toInt()
}
//没有线程访问,判断还剩多少时间可以访问
val nextTime = fetchRecord.time + concurrentRate.toInt()
if (System.currentTimeMillis() >= nextTime) {
fetchRecord.time = System.currentTimeMillis()
@ -306,9 +310,11 @@ class AnalyzeUrl(
}
return@synchronized (nextTime - System.currentTimeMillis()).toInt()
} else {
//并发控制为 次数/毫秒
val sj = concurrentRate.substring(rateIndex + 1)
val nextTime = fetchRecord.time + sj.toInt()
if (System.currentTimeMillis() >= nextTime) {
//已经过了限制时间,重置开始时间
fetchRecord.time = System.currentTimeMillis()
fetchRecord.frequency = 1
return@synchronized 0
@ -335,7 +341,7 @@ class AnalyzeUrl(
* 访问结束
*/
private fun fetchEnd(concurrentRecord: ConcurrentRecord?) {
if (concurrentRecord != null && !concurrentRecord.concurrent) {
if (concurrentRecord != null && !concurrentRecord.isConcurrent) {
synchronized(concurrentRecord) {
concurrentRecord.frequency = concurrentRecord.frequency - 1
}
@ -345,6 +351,7 @@ class AnalyzeUrl(
/**
* 访问网站,返回StrResponse
*/
@Throws(ConcurrentException::class)
suspend fun getStrResponseAwait(
jsStr: String? = null,
sourceRegex: String? = null,
@ -414,6 +421,7 @@ class AnalyzeUrl(
}
@JvmOverloads
@Throws(ConcurrentException::class)
fun getStrResponse(
jsStr: String? = null,
sourceRegex: String? = null,
@ -427,6 +435,7 @@ class AnalyzeUrl(
/**
* 访问网站,返回Response
*/
@Throws(ConcurrentException::class)
suspend fun getResponseAwait(): Response {
val concurrentRecord = fetchStart()
try {
@ -457,6 +466,7 @@ class AnalyzeUrl(
}
}
@Throws(ConcurrentException::class)
fun getResponse(): Response {
return runBlocking {
getResponseAwait()
@ -464,6 +474,7 @@ class AnalyzeUrl(
}
@Suppress("UnnecessaryVariable")
@Throws(ConcurrentException::class)
private fun getByteArrayIfDataUri(): ByteArray? {
@Suppress("RegExpRedundantEscape")
val dataUriFindResult = dataUriRegex.find(urlNoQuery)
@ -480,6 +491,7 @@ class AnalyzeUrl(
* 访问网站,返回ByteArray
*/
@Suppress("UnnecessaryVariable", "LiftReturnOrAssignment")
@Throws(ConcurrentException::class)
suspend fun getByteArrayAwait(): ByteArray {
getByteArrayIfDataUri()?.let {
return it
@ -497,6 +509,7 @@ class AnalyzeUrl(
* 访问网站,返回InputStream
*/
@Suppress("LiftReturnOrAssignment")
@Throws(ConcurrentException::class)
suspend fun getInputStreamAwait(): InputStream {
getByteArrayIfDataUri()?.let {
return ByteArrayInputStream(it)
@ -504,6 +517,7 @@ class AnalyzeUrl(
return getResponseAwait().body!!.byteStream()
}
@Throws(ConcurrentException::class)
fun getInputStream(): InputStream {
return runBlocking {
getInputStreamAwait()
@ -679,8 +693,17 @@ class AnalyzeUrl(
}
data class ConcurrentRecord(
val concurrent: Boolean,
/**
* 是否按频率
*/
val isConcurrent: Boolean,
/**
* 开始访问时间
*/
var time: Long,
/**
* 正在访问的个数
*/
var frequency: Int
)

@ -6,15 +6,18 @@ import io.legado.app.data.entities.Book
import io.legado.app.data.entities.BookChapter
import io.legado.app.data.entities.BookSource
import io.legado.app.data.entities.rule.TocRule
import io.legado.app.exception.ConcurrentException
import io.legado.app.exception.NoStackTraceException
import io.legado.app.exception.TocEmptyException
import io.legado.app.help.book.ContentProcessor
import io.legado.app.help.http.StrResponse
import io.legado.app.model.Debug
import io.legado.app.model.analyzeRule.AnalyzeRule
import io.legado.app.model.analyzeRule.AnalyzeUrl
import io.legado.app.utils.isTrue
import kotlinx.coroutines.Dispatchers.IO
import kotlinx.coroutines.async
import kotlinx.coroutines.delay
import kotlinx.coroutines.ensureActive
import kotlinx.coroutines.withContext
import splitties.init.appCtx
@ -61,12 +64,26 @@ object BookChapterList {
var nextUrl = chapterData.second[0]
while (nextUrl.isNotEmpty() && !nextUrlList.contains(nextUrl)) {
nextUrlList.add(nextUrl)
AnalyzeUrl(
val analyzeUrl = AnalyzeUrl(
mUrl = nextUrl,
source = bookSource,
ruleData = book,
headerMapF = bookSource.getHeaderMap()
).getStrResponseAwait().body?.let { nextBody ->
)
var res: StrResponse? = null
var isConcurrent: Boolean
do {
//控制并发访问
isConcurrent = false
try {
res = analyzeUrl.getStrResponseAwait()
} catch (e: ConcurrentException) {
isConcurrent = true
//如果是并发限制等待再次访问
delay(e.waitTime.toLong())
}
} while (!isConcurrent)
res!!.body?.let { nextBody ->
chapterData = analyzeChapterList(
book, nextUrl, nextUrl,
nextBody, tocRule, listRule, bookSource
@ -89,10 +106,22 @@ object BookChapterList {
ruleData = book,
headerMapF = bookSource.getHeaderMap()
)
val res = analyzeUrl.getStrResponseAwait()
var res: StrResponse? = null
var isConcurrent: Boolean
do {
//控制并发访问
isConcurrent = false
try {
res = analyzeUrl.getStrResponseAwait()
} catch (e: ConcurrentException) {
isConcurrent = true
//如果是并发限制等待再次访问
delay(e.waitTime.toLong())
}
} while (!isConcurrent)
analyzeChapterList(
book, urlStr, res.url,
res.body!!, tocRule, listRule, bookSource, false
book, urlStr, res!!.url,
res!!.body!!, tocRule, listRule, bookSource, false
).first
}
}

Loading…
Cancel
Save