fengyuecanzhu
3 years ago
parent
29cdeec30a
commit
8a777a1f7a
@ -1,182 +0,0 @@ |
||||
package xyz.fycz.myreader.ai; |
||||
|
||||
import android.util.Log; |
||||
|
||||
import java.util.ArrayList; |
||||
import java.util.Arrays; |
||||
import java.util.List; |
||||
import java.util.Random; |
||||
|
||||
import xyz.fycz.myreader.greendao.entity.Book; |
||||
import xyz.fycz.myreader.greendao.entity.Chapter; |
||||
import xyz.fycz.myreader.greendao.service.ChapterService; |
||||
|
||||
/** |
||||
* 预测书籍字数 |
||||
* |
||||
* @author fengyue |
||||
* @date 2021/4/18 16:54 |
||||
*/ |
||||
public class BookWordCountPre { |
||||
private static final String TAG = BookWordCountPre.class.getSimpleName(); |
||||
|
||||
private double lr = 0.001; |
||||
private Book book; |
||||
private List<Chapter> chapters; |
||||
private double[][] trainData; |
||||
private double[][] target; |
||||
private double[][] weights1; |
||||
//private double[][] weights2;
|
||||
private double[][] testData; |
||||
private double median; |
||||
private static Random rand = new Random(); |
||||
|
||||
|
||||
public BookWordCountPre(Book book) { |
||||
this.book = book; |
||||
this.chapters = ChapterService.getInstance().findBookAllChapterByBookId(book.getId()); |
||||
} |
||||
|
||||
//进行训练
|
||||
public boolean train() { |
||||
if (!preData()) { |
||||
Log.i(TAG, String.format("《%s》缓存章节数量过少，无法进行训练", book.getName())); |
||||
return false; |
||||
} |
||||
Log.i(TAG, String.format("《%s》开始进行训练", book.getName())); |
||||
double loss = 0; |
||||
double eps = 0.0000000001; |
||||
double[][] gradient1; |
||||
//double[][] gradient2;
|
||||
double[][] adagrad1 = new double[trainData[0].length][1]; |
||||
//double[][] adagrad2 = new double[trainData[0].length][1];
|
||||
//double[][] dl_dw = new double[trainData[0].length][1];
|
||||
int maxEpoch; |
||||
maxEpoch = 1000 / trainData.length; |
||||
if (maxEpoch < 10) maxEpoch = 10; |
||||
for (int epoch = 0; epoch < maxEpoch; epoch++) { |
||||
shuffle(trainData, target); |
||||
for (int j = 0; j < trainData.length; j++) { |
||||
double[][] oneData = MatrixUtil.to2dMatrix(trainData[j], false); |
||||
double[][] oneTarget = MatrixUtil.to2dMatrix(target[j], true); |
||||
double[][] out = getOut(oneData); |
||||
loss = Math.sqrt(MatrixUtil.sum(MatrixUtil.pow(MatrixUtil.sub(out, oneTarget), 2)) / 2); |
||||
/*dl_dw = MatrixUtil.sub( |
||||
MatrixUtil.add( |
||||
MatrixUtil.dot(MatrixUtil.pow(oneData, 2), weights2), |
||||
MatrixUtil.dot(oneData, weights2)), |
||||
oneTarget |
||||
);*/ |
||||
|
||||
gradient1 = MatrixUtil.dot(MatrixUtil.transpose(oneData), MatrixUtil.sub(out, oneTarget)); |
||||
//gradient1 = MatrixUtil.dot(MatrixUtil.transpose(MatrixUtil.pow(oneData, 2)), dl_dw);
|
||||
//gradient2 = MatrixUtil.dot(MatrixUtil.transpose(oneData), dl_dw);
|
||||
adagrad1 = MatrixUtil.add(adagrad1, MatrixUtil.pow(gradient1, 2)); |
||||
//adagrad2 = MatrixUtil.add(adagrad1, MatrixUtil.pow(gradient2, 2));
|
||||
weights1 = MatrixUtil.sub(weights1, MatrixUtil.divide(MatrixUtil.dot(gradient1, lr), |
||||
MatrixUtil.sqrt(MatrixUtil.add(adagrad1, eps)))); |
||||
/*weights2 = MatrixUtil.sub(weights2, MatrixUtil.divide(MatrixUtil.dot(gradient2, lr), |
||||
MatrixUtil.sqrt(MatrixUtil.add(adagrad2, eps))));*/ |
||||
} |
||||
Log.i(TAG, String.format("《%s》-> epoch=%d，loss=%f", book.getName(), epoch, loss)); |
||||
} |
||||
return true; |
||||
} |
||||
|
||||
//进行预测并获得书籍总字数
|
||||
public int predict() { |
||||
double[][] pre = getOut(testData); |
||||
double[] preVec = MatrixUtil.toVector(pre); |
||||
Arrays.sort(preVec); |
||||
int k = (int) (preVec[preVec.length / 2 + 1] / median); |
||||
//int k = (int) ((MatrixUtil.sum(pre) / pre.length) / median);
|
||||
pre = MatrixUtil.divide(pre, k); |
||||
/*for (int i = 0; i < pre.length; i++) { |
||||
pre[i][0] = median; |
||||
}*/ |
||||
Log.i(TAG, String.format("k=%d->《%s》的预测数据%s", k, book.getName(), |
||||
Arrays.toString(MatrixUtil.toVector(pre)))); |
||||
return (int) (MatrixUtil.sum(pre) + MatrixUtil.sum(target)); |
||||
} |
||||
|
||||
private double[][] getOut(double[][] data) { |
||||
/*return MatrixUtil.add(MatrixUtil.dot(MatrixUtil.pow(data, 2), weights2), |
||||
MatrixUtil.dot(data, weights1));*/ |
||||
return MatrixUtil.dot(data, weights1); |
||||
} |
||||
|
||||
//准备训练数据
|
||||
private boolean preData() { |
||||
rand.setSeed(10); |
||||
List<Chapter> catheChapters = new ArrayList<>(); |
||||
List<Chapter> unCatheChapters = new ArrayList<>(); |
||||
//章节最长标题长度
|
||||
int maxTitleLen = 0; |
||||
//获取已缓存章节
|
||||
for (Chapter chapter : chapters) { |
||||
if (ChapterService.isChapterCached(book.getId(), chapter.getTitle())) { |
||||
catheChapters.add(chapter); |
||||
} else { |
||||
unCatheChapters.add(chapter); |
||||
} |
||||
if (maxTitleLen < chapter.getTitle().length()) { |
||||
maxTitleLen = chapter.getTitle().length(); |
||||
} |
||||
} |
||||
Log.i(TAG, String.format("《%s》已缓存章节数量：%d，最大章节标题长度：%d", |
||||
book.getName(), catheChapters.size(), maxTitleLen)); |
||||
if (catheChapters.size() <= 10) return false; |
||||
//创建训练数据
|
||||
trainData = new double[catheChapters.size()][maxTitleLen + 1]; |
||||
//创建测试数据
|
||||
testData = new double[chapters.size() - catheChapters.size()][maxTitleLen + 1]; |
||||
//创建权重矩阵
|
||||
weights1 = new double[maxTitleLen + 1][1]; |
||||
//weights2 = new double[maxTitleLen + 1][1];
|
||||
//创建目标矩阵
|
||||
target = new double[catheChapters.size()][1]; |
||||
for (int i = 0; i < catheChapters.size(); i++) { |
||||
Chapter chapter = catheChapters.get(i); |
||||
char[] charArr = chapter.getTitle().replaceAll("[(（【{]", "").toCharArray(); |
||||
for (int j = 0; j < charArr.length; j++) { |
||||
trainData[i][j] = charArr[j]; |
||||
} |
||||
trainData[i][maxTitleLen] = 1; |
||||
target[i][0] = ChapterService.countChar(book.getId(), chapter.getTitle()); |
||||
} |
||||
for (int i = 0; i < maxTitleLen + 1; i++) { |
||||
weights1[i][0] = rand.nextDouble(); |
||||
//weights2[i][0] = Math.random();
|
||||
} |
||||
for (int i = 0; i < unCatheChapters.size(); i++) { |
||||
Chapter chapter = unCatheChapters.get(i); |
||||
char[] charArr = chapter.getTitle().toCharArray(); |
||||
for (int j = 0; j < charArr.length; j++) { |
||||
testData[i][j] = charArr[j]; |
||||
} |
||||
testData[i][maxTitleLen] = 1; |
||||
} |
||||
/*double[] tem = MatrixUtil.toVector(target); |
||||
Arrays.sort(tem); |
||||
median = tem[tem.length / 2 + 1];*/ |
||||
median = MatrixUtil.sum(target) / target.length; |
||||
return true; |
||||
} |
||||
|
||||
|
||||
public static <T> void swap(T[] a, int i, int j) { |
||||
T temp = a[i]; |
||||
a[i] = a[j]; |
||||
a[j] = temp; |
||||
} |
||||
|
||||
public static <T> void shuffle(T[]... arr) { |
||||
int length = arr[0].length; |
||||
for (int i = length; i > 0; i--) { |
||||
int randInd = rand.nextInt(i); |
||||
for (T[] ts : arr) { |
||||
swap(ts, randInd, i - 1); |
||||
} |
||||
} |
||||
} |
||||
} |
||||
@ -1,295 +0,0 @@ |
||||
package xyz.fycz.myreader.ai; |
||||
|
||||
/** |
||||
* @author fengyue |
||||
* @date 2021/4/7 16:10 |
||||
*/ |
||||
public class MatrixUtil { |
||||
//矩阵加法 C=A+B
|
||||
public static double[][] add(double[][] m1, double[][] m2) { |
||||
if (m1 == null || m2 == null || |
||||
m1.length != m2.length || |
||||
m1[0].length != m2[0].length) { |
||||
return null; |
||||
} |
||||
|
||||
double[][] m = new double[m1.length][m1[0].length]; |
||||
|
||||
for (int i = 0; i < m.length; ++i) { |
||||
for (int j = 0; j < m[i].length; ++j) { |
||||
m[i][j] = m1[i][j] + m2[i][j]; |
||||
} |
||||
} |
||||
|
||||
return m; |
||||
} |
||||
|
||||
public static double[][] add(double[][] m, double a) { |
||||
if (m == null) { |
||||
return null; |
||||
} |
||||
|
||||
double[][] retM = new double[m.length][m[0].length]; |
||||
|
||||
for (int i = 0; i < retM.length; ++i) { |
||||
for (int j = 0; j < retM[i].length; ++j) { |
||||
retM[i][j] = m[i][j] + a; |
||||
} |
||||
} |
||||
|
||||
return retM; |
||||
} |
||||
|
||||
public static double[][] sub(double[][] m1, double[][] m2) { |
||||
if (m1 == null || m2 == null || |
||||
m1.length != m2.length || |
||||
m1[0].length != m2[0].length) { |
||||
return null; |
||||
} |
||||
|
||||
double[][] m = new double[m1.length][m1[0].length]; |
||||
|
||||
for (int i = 0; i < m.length; ++i) { |
||||
for (int j = 0; j < m[i].length; ++j) { |
||||
m[i][j] = m1[i][j] - m2[i][j]; |
||||
} |
||||
} |
||||
|
||||
return m; |
||||
} |
||||
|
||||
//矩阵转置
|
||||
public static double[][] transpose(double[][] m) { |
||||
if (m == null) return null; |
||||
double[][] mt = new double[m[0].length][m.length]; |
||||
for (int i = 0; i < m.length; ++i) { |
||||
for (int j = 0; j < m[i].length; ++j) { |
||||
mt[j][i] = m[i][j]; |
||||
} |
||||
} |
||||
return mt; |
||||
} |
||||
|
||||
//矩阵相乘 C=A*B
|
||||
public static double[][] dot(double[][] m1, double[][] m2) { |
||||
if (m1 == null || m2 == null || m1[0].length != m2.length) |
||||
return null; |
||||
|
||||
double[][] m = new double[m1.length][m2[0].length]; |
||||
for (int i = 0; i < m1.length; ++i) { |
||||
for (int j = 0; j < m2[0].length; ++j) { |
||||
for (int k = 0; k < m1[i].length; ++k) { |
||||
m[i][j] += m1[i][k] * m2[k][j]; |
||||
} |
||||
} |
||||
} |
||||
|
||||
return m; |
||||
} |
||||
|
||||
//数乘矩阵
|
||||
public static double[][] dot(double[][] m, double k) { |
||||
if (m == null) return null; |
||||
double[][] retM = new double[m.length][m[0].length]; |
||||
for (int i = 0; i < m.length; i++) { |
||||
for (int j = 0; j < m[0].length; j++) { |
||||
retM[i][j] = m[i][j] * k; |
||||
} |
||||
} |
||||
return retM; |
||||
} |
||||
|
||||
//同型矩阵除法
|
||||
public static double[][] divide(double[][] m1, double[][] m2) { |
||||
if (m1 == null || m2 == null || |
||||
m1.length != m2.length || |
||||
m1[0].length != m2[0].length) { |
||||
return null; |
||||
} |
||||
double[][] retM = new double[m1.length][m1[0].length]; |
||||
for (int i = 0; i < retM.length; ++i) { |
||||
for (int j = 0; j < retM[i].length; ++j) { |
||||
retM[i][j] = m1[i][j] / m2[i][j]; |
||||
} |
||||
} |
||||
return retM; |
||||
} |
||||
|
||||
//矩阵除数
|
||||
public static double[][] divide(double[][] m, double k) { |
||||
if (m == null) return null; |
||||
double[][] retM = new double[m.length][m[0].length]; |
||||
for (int i = 0; i < m.length; i++) { |
||||
for (int j = 0; j < m[0].length; j++) { |
||||
retM[i][j] = m[i][j] / k; |
||||
} |
||||
} |
||||
return retM; |
||||
} |
||||
|
||||
//求矩阵行列式（需为方阵）
|
||||
public static double det(double[][] m) { |
||||
if (m == null || m.length != m[0].length) |
||||
return 0; |
||||
|
||||
if (m.length == 1) |
||||
return m[0][0]; |
||||
else if (m.length == 2) |
||||
return det2(m); |
||||
else if (m.length == 3) |
||||
return det3(m); |
||||
else { |
||||
int re = 0; |
||||
for (int i = 0; i < m.length; ++i) { |
||||
re += (((i + 1) % 2) * 2 - 1) * det(companion(m, i, 0)) * m[i][0]; |
||||
} |
||||
return re; |
||||
} |
||||
} |
||||
|
||||
//求二阶行列式
|
||||
public static double det2(double[][] m) { |
||||
if (m == null || m.length != 2 || m[0].length != 2) |
||||
return 0; |
||||
|
||||
return m[0][0] * m[1][1] - m[1][0] * m[0][1]; |
||||
} |
||||
|
||||
//求三阶行列式
|
||||
public static double det3(double[][] m) { |
||||
if (m == null || m.length != 3 || m[0].length != 3) |
||||
return 0; |
||||
|
||||
double re = 0; |
||||
for (int i = 0; i < 3; ++i) { |
||||
int temp1 = 1; |
||||
for (int j = 0, k = i; j < 3; ++j, ++k) { |
||||
temp1 *= m[j][k % 3]; |
||||
} |
||||
re += temp1; |
||||
temp1 = 1; |
||||
for (int j = 0, k = i; j < 3; ++j, --k) { |
||||
if (k < 0) k += 3; |
||||
temp1 *= m[j][k]; |
||||
} |
||||
re -= temp1; |
||||
} |
||||
|
||||
return re; |
||||
} |
||||
|
||||
//求矩阵的逆（需方阵）
|
||||
public static double[][] inv(double[][] m) { |
||||
if (m == null || m.length != m[0].length) |
||||
return null; |
||||
|
||||
double A = det(m); |
||||
double[][] mi = new double[m.length][m[0].length]; |
||||
for (int i = 0; i < m.length; ++i) { |
||||
for (int j = 0; j < m[i].length; ++j) { |
||||
double[][] temp = companion(m, i, j); |
||||
mi[j][i] = (((i + j + 1) % 2) * 2 - 1) * det(temp) / A; |
||||
} |
||||
} |
||||
|
||||
return mi; |
||||
} |
||||
|
||||
//求方阵代数余子式
|
||||
public static double[][] companion(double[][] m, int x, int y) { |
||||
if (m == null || m.length <= x || m[0].length <= y || |
||||
m.length == 1 || m[0].length == 1) |
||||
return null; |
||||
|
||||
double[][] cm = new double[m.length - 1][m[0].length - 1]; |
||||
|
||||
int dx = 0; |
||||
for (int i = 0; i < m.length; ++i) { |
||||
if (i != x) { |
||||
int dy = 0; |
||||
for (int j = 0; j < m[i].length; ++j) { |
||||
if (j != y) { |
||||
cm[dx][dy++] = m[i][j]; |
||||
} |
||||
} |
||||
++dx; |
||||
} |
||||
} |
||||
return cm; |
||||
} |
||||
|
||||
//生成全为0的矩阵
|
||||
public static double[][] zeros(int rows, int cols){ |
||||
return new double[rows][cols]; |
||||
} |
||||
|
||||
//生成全为1的矩阵
|
||||
public static double[][] ones(int rows, int cols){ |
||||
return add(zeros(rows, cols), 1); |
||||
} |
||||
|
||||
public static double sum(double[][] matrix){ |
||||
double sum = 0; |
||||
for (double[] doubles : matrix) { |
||||
for (double aDouble : doubles) { |
||||
sum += aDouble; |
||||
} |
||||
} |
||||
return sum; |
||||
} |
||||
|
||||
public static double[][] pow(double[][] matrix, int exponent){ |
||||
if (matrix == null) return null; |
||||
double[][] retM = new double[matrix.length][matrix[0].length]; |
||||
for (int i = 0; i < matrix.length; i++) { |
||||
for (int j = 0; j < matrix[i].length; j++) { |
||||
retM[i][j] = Math.pow(matrix[i][j], exponent); |
||||
} |
||||
} |
||||
return retM; |
||||
} |
||||
|
||||
public static double[][] sqrt(double[][] matrix){ |
||||
if (matrix == null) return null; |
||||
double[][] retM = new double[matrix.length][matrix[0].length]; |
||||
for (int i = 0; i < matrix.length; i++) { |
||||
for (int j = 0; j < matrix[i].length; j++) { |
||||
retM[i][j] = Math.sqrt(matrix[i][j]); |
||||
} |
||||
} |
||||
return matrix; |
||||
} |
||||
|
||||
public static double[][] to2dMatrix(double[] vector, boolean isCol){ |
||||
if (vector == null) return null; |
||||
double[][] retM; |
||||
if (isCol) { |
||||
retM = new double[vector.length][1]; |
||||
}else { |
||||
retM = new double[1][vector.length]; |
||||
} |
||||
for (int i = 0; i < vector.length; i++) { |
||||
if (isCol) { |
||||
retM[i][0] = vector[i]; |
||||
}else { |
||||
retM[0][i] = vector[i]; |
||||
} |
||||
} |
||||
return retM; |
||||
} |
||||
|
||||
public static double[] toVector(double[][] matrix){ |
||||
double[] retV = null; |
||||
if (matrix.length == 1){ |
||||
retV = new double[matrix[0].length]; |
||||
double[] doubles = matrix[0]; |
||||
System.arraycopy(doubles, 0, retV, 0, doubles.length); |
||||
}else if (matrix[0].length == 1){ |
||||
retV = new double[matrix.length]; |
||||
for (int i = 0; i < matrix.length; i++) { |
||||
retV[i] = matrix[i][0]; |
||||
} |
||||
} |
||||
return retV; |
||||
} |
||||
} |
||||
Loading…
Reference in new issue