|
|
|
@ -18,11 +18,95 @@ |
|
|
|
|
|
|
|
|
|
package xyz.fycz.myreader.webapi.crawler.source.find |
|
|
|
|
|
|
|
|
|
import io.reactivex.Observable |
|
|
|
|
import xyz.fycz.myreader.application.App |
|
|
|
|
import xyz.fycz.myreader.entity.FindKind |
|
|
|
|
import xyz.fycz.myreader.entity.thirdsource.source3.ExploreKind3 |
|
|
|
|
import xyz.fycz.myreader.greendao.entity.rule.BookSource |
|
|
|
|
import xyz.fycz.myreader.util.utils.ACache |
|
|
|
|
import xyz.fycz.myreader.util.utils.GSON |
|
|
|
|
import xyz.fycz.myreader.util.utils.fromJsonArray |
|
|
|
|
import xyz.fycz.myreader.util.utils.isJsonArray |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* @author fengyue |
|
|
|
|
* @date 2022/1/20 15:33 |
|
|
|
|
*/ |
|
|
|
|
class Third3FindCrawler(source: BookSource) : ThirdFindCrawler(source) { |
|
|
|
|
override fun initData(): Observable<Boolean> { |
|
|
|
|
return Observable.create { emitter -> |
|
|
|
|
val exploreUrl = source.findRule.url |
|
|
|
|
if (exploreUrl.isNullOrBlank()) { |
|
|
|
|
emitter.onNext(false) |
|
|
|
|
emitter.onComplete() |
|
|
|
|
return@create |
|
|
|
|
} |
|
|
|
|
val kinds = arrayListOf<ExploreKind3>() |
|
|
|
|
var ruleStr = exploreUrl |
|
|
|
|
kotlin.runCatching { |
|
|
|
|
if (exploreUrl.startsWith("<js>", false) |
|
|
|
|
|| exploreUrl.startsWith("@js:", false) |
|
|
|
|
) { |
|
|
|
|
val aCache = ACache.get(cacheName = "explore") |
|
|
|
|
ruleStr = aCache.getAsString(source.sourceUrl) ?: "" |
|
|
|
|
if (ruleStr.isBlank()) { |
|
|
|
|
val jsStr = if (exploreUrl.startsWith("@")) { |
|
|
|
|
exploreUrl.substring(4) |
|
|
|
|
} else { |
|
|
|
|
exploreUrl.substring(4, exploreUrl.lastIndexOf("<")) |
|
|
|
|
} |
|
|
|
|
ruleStr = source.evalJS(jsStr).toString().trim() |
|
|
|
|
aCache.put(source.sourceUrl, ruleStr) |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if (ruleStr.isJsonArray()) { |
|
|
|
|
GSON.fromJsonArray<ExploreKind3>(ruleStr)?.let { |
|
|
|
|
kinds.addAll(it) |
|
|
|
|
} |
|
|
|
|
} else { |
|
|
|
|
ruleStr.split("(&&|\n)+".toRegex()).forEach { kindStr -> |
|
|
|
|
val kindCfg = kindStr.split("::") |
|
|
|
|
kinds.add(ExploreKind3(kindCfg.getOrNull(0) ?: "", kindCfg.getOrNull(1))) |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
var children = arrayListOf<FindKind>() |
|
|
|
|
var groupName = name |
|
|
|
|
var nameCount = 0 |
|
|
|
|
kinds.forEach { |
|
|
|
|
if (it.title.isBlank()) { |
|
|
|
|
if (children.size > 0) { |
|
|
|
|
nameCount++ |
|
|
|
|
kindsMap[groupName] = children |
|
|
|
|
children = arrayListOf() |
|
|
|
|
} |
|
|
|
|
groupName = "$name[$nameCount]" |
|
|
|
|
} else if (it.url.isNullOrBlank()) { |
|
|
|
|
if (children.size > 0) { |
|
|
|
|
kindsMap[groupName] = children |
|
|
|
|
children = arrayListOf() |
|
|
|
|
} |
|
|
|
|
groupName = it.title.replace("\\s".toRegex(), "") |
|
|
|
|
} else { |
|
|
|
|
val findKindBean = FindKind().apply { |
|
|
|
|
tag = source.sourceUrl |
|
|
|
|
name = it.title.replace("\\s".toRegex(), "") |
|
|
|
|
url = it.url |
|
|
|
|
} |
|
|
|
|
children.add(findKindBean) |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
if (children.size > 0) { |
|
|
|
|
kindsMap[groupName] = children |
|
|
|
|
} |
|
|
|
|
emitter.onNext(true) |
|
|
|
|
}.onFailure { |
|
|
|
|
emitter.onNext(false) |
|
|
|
|
kinds.add(ExploreKind3("ERROR:${it.localizedMessage}", it.stackTraceToString())) |
|
|
|
|
if (App.isDebug()) { |
|
|
|
|
it.printStackTrace() |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
emitter.onComplete() |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
} |