轻量级神经网络框架

记录神经网络开发历程

目标

设计一个轻量级易用的C++深度学习框架,首先为了加深对深度学习算法的理解,其次是希望能完成一个深度学习框架,作为公司的技术储备,为后面自然语言处理引擎打下基础。

设计

本框架的设计思想主要参考tensorflow框架,所有的操作都是Tensor,模型都是由许多Tensor通过图的方式连接起来,前向传播以及反向求导都是自动进行,使用的时候可以像普通计算表达式一样通过加减乘除符号来构建基本的模型,用户不需要关注内部实现细节。

依赖库

矩阵计算库 eigen 3.3.4

开发环境

  • 操作系统 Ubuntu 14.4,
  • 编译器 G++ 5.4.0,
  • 调试器 gdb 7.11.1,
  • 构建工具 cmake 3.5.1

编译步骤

1、官网下载eigen的源码,在当前工程的同一级目录解压。比如:项目所在目录为:/home/redtea,那么eigen解压后的目录为:/home/eigen
2、进入CMakeList.txt文件所在目录,执行命令:cmake ./,这一步用于生成Makefile文件
3、执行命令:make,这一步直接生成可执行文件
4、运行测试程序:./test-lsl 1000,是一个基于最小平方差损失的线性回归模型,训练迭代1000次。

源码

源码后期可能以红茶智能技术有限公司公司的名义公开。

Example

test-lsl

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
/*
* test case for LeastSquareLoss
*/
#include <iostream>
#include <stdlib.h>
#include "stdafx.h"
#include "Tensor.h"
#include "TensorOps.h"
#include "Loss.h"
#include "Optimizer.h"
using namespace std;
using namespace Eigen;
using namespace redtea::core;
int main(int argc, char* argv[]) {
if(argc < 2) return 1;
int epoch = atoi(argv[1]);
cout<<"epoch="<<epoch<<endl;
Matrix<type, 5, 2> sample;
sample << 1, 1,
2, 1,
3, 2,
5, 3,
6, 0;
Matrix<type, 5, 2> target;
target << 6, 7.95,
8, 9.4,
13, 15.95,
20, 23.95,
13, 10.1;
/*
auto x = Constant::create(sample);
auto y = Constant::create(target);
auto w = Variable::create(MatrixX::Random(2, 2));
auto b = Variable::create(MatrixX::Random(1, 2));
auto mul = Mul::create(x, w);
auto add = Add::create(mul, b);
LeastSquareLoss loss(add, y);
cout<<"w: "<<w->getOutput()<<endl;
cout<<"b: "<<b->getOutput()<<endl;
Optimizer opti(1e-2);
for(int i=0;i<epoch;i++) {
loss.forward();
loss.backward(opti);
cout<<"o: "<<add->getOutput();
cout<<", w: "<<w->getOutput();
cout<<", b: "<<b->getOutput();
cout<<", l: "<<loss.getTotalLoss()<<endl;
}
*/
Constant x(sample);
Constant y(target);
Variable w(MatrixX::Random(2, 2));
Variable b(MatrixX::Random(1, 2));
// Mul mul = x * w;
Add add = x * w + b;
LeastSquareLoss loss(add, y);
cout<<"w: "<<w.getOutput()<<endl;
cout<<"b: "<<b.getOutput()<<endl;
//AdamOptimizer opti;
//MomentumOptimizer opti(0.8, 1e-3);
//AdadeltaOptimizer opti;
SGDOptimizer opti(1e-3);
opti.minimize(loss);
for(int i=0;i<epoch;i++) {
opti.run();
cout<<"o: "<<add.getOutput();
cout<<", w: "<<w.getOutput();
cout<<", b: "<<b.getOutput();
cout<<", l: "<<loss.getOutput().sum()<<endl;
}
return 0;
}

test-tanh

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
/*
* test case for LeastSquareLoss
*/
#include <iostream>
#include <stdlib.h>
#include "Tensor.h"
#include "TensorOps.h"
#include "Loss.h"
#include "Optimizer.h"
#include "Activation.h"
using namespace std;
using namespace Eigen;
using namespace redtea::core;
int mainabc(int argc, char* argv[]) {
if(argc < 2) return 1;
int epoch = atoi(argv[1]);
cout<<"epoch="<<epoch<<endl;
//prepare test samples
Matrix<type, 100, 2> sample;
sample << -0.017612,14.053064,-1.395634,4.662541,-0.752157,6.538620,-1.322371,7.152853,0.423363,11.054677,0.406704,7.067335,0.667394,12.741452,-2.460150,6.866805,0.569411,9.548755,-0.026632,10.427743,0.850433,6.920334,1.347183,13.175500,1.176813,3.167020,-1.781871,9.097953,-0.566606,5.749003,0.931635,1.589505,-0.024205,6.151823,-0.036453,2.690988,-0.196949,0.444165,1.014459,5.754399,1.985298,3.230619,-1.693453,-0.557540,-0.576525,11.778922,-0.346811,-1.678730,-2.124484,2.672471,1.217916,9.597015,-0.733928,9.098687,-3.642001,-1.618087,0.315985,3.523953,1.416614,9.619232,-0.386323,3.989286,0.556921,8.294984,1.224863,11.587360,-1.347803,-2.406051,1.196604,4.951851,0.275221,9.543647,0.470575,9.332488,-1.889567,9.542662,-1.527893,12.150579,-1.185247,11.309318,-0.445678,3.297303,1.042222,6.105155,-0.618787,10.320986,1.152083,0.548467,0.828534,2.676045,-1.237728,10.549033,-0.683565,-2.166125,0.229456,5.921938,-0.959885,11.555336,0.492911,10.993324,0.184992,8.721488,-0.355715,10.325976,-0.397822,8.058397,0.824839,13.730343,1.507278,5.027866,0.099671,6.835839,-0.344008,10.717485,1.785928,7.718645,-0.918801,11.560217,-0.364009,4.747300,-0.841722,4.119083,0.490426,1.960539,-0.007194,9.075792,0.356107,12.447863,0.342578,12.281162,-0.810823,-1.466018,2.530777,6.476801,1.296683,11.607559,0.475487,12.040035,-0.783277,11.009725,0.074798,11.023650,-1.337472,0.468339,-0.102781,13.763651,-0.147324,2.874846,0.518389,9.887035,1.015399,7.571882,-1.658086,-0.027255,1.319944,2.171228,2.056216,5.019981,-0.851633,4.375691,-1.510047,6.061992,-1.076637,-3.181888,1.821096,10.283990,3.010150,8.401766,-1.099458,1.688274,-0.834872,-1.733869,-0.846637,3.849075,1.400102,12.628781,1.752842,5.468166,0.078557,0.059736,0.089392,-0.715300,1.825662,12.693808,0.197445,9.744638,0.126117,0.922311,-0.679797,1.220530,0.677983,2.556666,0.761349,10.693862,-2.168791,0.143632,1.388610,9.341997,0.317029,14.739025;
Matrix<type, 100, 1> target;
target << -1,1,-1,-1,-1,1,-1,1,-1,-1,1,-1,1,-1,1,1,1,1,1,1,1,1,-1,1,1,-1,-1,1,1,-1,1,1,-1,1,1,-1,-1,-1,-1,-1,1,1,-1,1,1,-1,1,1,-1,-1,-1,-1,-1,-1,1,1,-1,1,-1,1,1,1,-1,-1,-1,1,1,-1,-1,-1,-1,1,-1,1,-1,-1,1,1,1,1,-1,1,-1,1,1,1,1,-1,1,1,1,-1,-1,1,1,1,-1,1,-1,-1;
cout<<"direct mode"<<endl;
Constant x(sample);
Constant y(target);
Variable w(MatrixX::Random(2, 1));
Variable b(MatrixX::Random(1, 1));
Add add = x * w + b;
Tanh act(add);
LeastSquareLoss loss(act, y);
SGDOptimizer opti(1e-3);
opti.minimize(loss);
for(int i=0;i<epoch;i++) {
opti.run();
cout<<"loss: "<<loss.getOutput().sum()<<endl;
}
cout<<"o: "<<act.getOutput()<<endl;
return 0;
}

test-softmax

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
/*
* test case for LeastSquareLoss
*/
#include <iostream>
#include <stdlib.h>
#include "Tensor.h"
#include "TensorOps.h"
#include "Loss.h"
#include "Optimizer.h"
#include "Activation.h"
using namespace std;
using namespace Eigen;
using namespace redtea::core;
int main(int argc, char* argv[]) {
if(argc < 2) return 1;
int epoch = atoi(argv[1]);
cout<<"epoch="<<epoch<<endl;
//prepare test samples
Matrix<type, 100, 2> sample;
sample << -0.017612,14.053064,-1.395634,4.662541,-0.752157,6.538620,-1.322371,7.152853,0.423363,11.054677,0.406704,7.067335,0.667394,12.741452,-2.460150,6.866805,0.569411,9.548755,-0.026632,10.427743,0.850433,6.920334,1.347183,13.175500,1.176813,3.167020,-1.781871,9.097953,-0.566606,5.749003,0.931635,1.589505,-0.024205,6.151823,-0.036453,2.690988,-0.196949,0.444165,1.014459,5.754399,1.985298,3.230619,-1.693453,-0.557540,-0.576525,11.778922,-0.346811,-1.678730,-2.124484,2.672471,1.217916,9.597015,-0.733928,9.098687,-3.642001,-1.618087,0.315985,3.523953,1.416614,9.619232,-0.386323,3.989286,0.556921,8.294984,1.224863,11.587360,-1.347803,-2.406051,1.196604,4.951851,0.275221,9.543647,0.470575,9.332488,-1.889567,9.542662,-1.527893,12.150579,-1.185247,11.309318,-0.445678,3.297303,1.042222,6.105155,-0.618787,10.320986,1.152083,0.548467,0.828534,2.676045,-1.237728,10.549033,-0.683565,-2.166125,0.229456,5.921938,-0.959885,11.555336,0.492911,10.993324,0.184992,8.721488,-0.355715,10.325976,-0.397822,8.058397,0.824839,13.730343,1.507278,5.027866,0.099671,6.835839,-0.344008,10.717485,1.785928,7.718645,-0.918801,11.560217,-0.364009,4.747300,-0.841722,4.119083,0.490426,1.960539,-0.007194,9.075792,0.356107,12.447863,0.342578,12.281162,-0.810823,-1.466018,2.530777,6.476801,1.296683,11.607559,0.475487,12.040035,-0.783277,11.009725,0.074798,11.023650,-1.337472,0.468339,-0.102781,13.763651,-0.147324,2.874846,0.518389,9.887035,1.015399,7.571882,-1.658086,-0.027255,1.319944,2.171228,2.056216,5.019981,-0.851633,4.375691,-1.510047,6.061992,-1.076637,-3.181888,1.821096,10.283990,3.010150,8.401766,-1.099458,1.688274,-0.834872,-1.733869,-0.846637,3.849075,1.400102,12.628781,1.752842,5.468166,0.078557,0.059736,0.089392,-0.715300,1.825662,12.693808,0.197445,9.744638,0.126117,0.922311,-0.679797,1.220530,0.677983,2.556666,0.761349,10.693862,-2.168791,0.143632,1.388610,9.341997,0.317029,14.739025;
Matrix<type, 100, 1> target;
target << 0,1,0,0,0,1,0,1,0,0,1,0,1,0,1,1,1,1,1,1,1,1,0,1,1,0,0,1,1,0,1,1,0,1,1,0,0,0,0,0,1,1,0,1,1,0,1,1,0,0,0,0,0,0,1,1,0,1,0,1,1,1,0,0,0,1,1,0,0,0,0,1,0,1,0,0,1,1,1,1,0,1,0,1,1,1,1,0,1,1,1,0,0,1,1,1,0,1,0,0;
MatrixX targetSoftmax = Matrix<type, 100, 2>::Zero(100, 2);
for(int i=0;i<100;i++) {
if(target(i, 0) > 0.5) targetSoftmax(i, 1) = 1;
else targetSoftmax(i, 0) = 1;
}
/*
auto x = Constant::create(sample);
auto y = Constant::create(targetSoftmax);
//create the network
auto w = Variable::create(MatrixX::Random(2, 2));
auto b = Variable::create(MatrixX::Random(1, 2));
auto mul = Mul::create(x, w);
auto add = Add::create(mul, b);
auto act = Softmax::create(add);
//train
LogisticLoss loss(act, y);
Optimizer opti(1e-2);
for(int i=0;i<epoch;i++) {
loss.reset();
loss.forward();
loss.backward(opti);
cout<<", l: "<<loss.getTotalLoss()<<endl;
}
cout<<"o: "<<act->getOutput()<<endl;
*/
cout<<"direct mode."<<endl;
Constant x(sample);
Constant y(targetSoftmax);
//create the network
Variable w(MatrixX::Random(2, 2));
Variable b(MatrixX::Random(1, 2));
Mul mul(x, w);
Add add(mul, b);
Softmax act(add);
//train
LogisticLoss loss(act, y);
//AdamOptimizer opti;
//AdadeltaOptimizer opti;
//MomentumOptimizer opti;
SGDOptimizer opti(1e-3);
opti.minimize(loss);
for(int i=0;i<epoch;i++) {
opti.run();
cout<<"loss: "<<loss.getOutput().sum()<<endl;
}
cout<<"o: "<<act.getOutput()<<endl;
return 0;
}

test-logistic

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
/*
* test case for LeastSquareLoss
*/
#include <iostream>
#include <stdlib.h>
#include "Tensor.h"
#include "TensorOps.h"
#include "Loss.h"
#include "Optimizer.h"
#include "Activation.h"
#include "data.h"
using namespace std;
using namespace Eigen;
using namespace redtea::core;
int main(int argc, char* argv[]) {
if(argc < 2) return 1;
int epoch = atoi(argv[1]);
cout<<"epoch="<<epoch<<endl;
//prepare test samples
//Matrix<type, 100, 2> sample;
//sample << -0.017612,14.053064,-1.395634,4.662541,-0.752157,6.538620,-1.322371,7.152853,0.423363,11.054677,0.406704,7.067335,0.667394,12.741452,-2.460150,6.866805,0.569411,9.548755,-0.026632,10.427743,0.850433,6.920334,1.347183,13.175500,1.176813,3.167020,-1.781871,9.097953,-0.566606,5.749003,0.931635,1.589505,-0.024205,6.151823,-0.036453,2.690988,-0.196949,0.444165,1.014459,5.754399,1.985298,3.230619,-1.693453,-0.557540,-0.576525,11.778922,-0.346811,-1.678730,-2.124484,2.672471,1.217916,9.597015,-0.733928,9.098687,-3.642001,-1.618087,0.315985,3.523953,1.416614,9.619232,-0.386323,3.989286,0.556921,8.294984,1.224863,11.587360,-1.347803,-2.406051,1.196604,4.951851,0.275221,9.543647,0.470575,9.332488,-1.889567,9.542662,-1.527893,12.150579,-1.185247,11.309318,-0.445678,3.297303,1.042222,6.105155,-0.618787,10.320986,1.152083,0.548467,0.828534,2.676045,-1.237728,10.549033,-0.683565,-2.166125,0.229456,5.921938,-0.959885,11.555336,0.492911,10.993324,0.184992,8.721488,-0.355715,10.325976,-0.397822,8.058397,0.824839,13.730343,1.507278,5.027866,0.099671,6.835839,-0.344008,10.717485,1.785928,7.718645,-0.918801,11.560217,-0.364009,4.747300,-0.841722,4.119083,0.490426,1.960539,-0.007194,9.075792,0.356107,12.447863,0.342578,12.281162,-0.810823,-1.466018,2.530777,6.476801,1.296683,11.607559,0.475487,12.040035,-0.783277,11.009725,0.074798,11.023650,-1.337472,0.468339,-0.102781,13.763651,-0.147324,2.874846,0.518389,9.887035,1.015399,7.571882,-1.658086,-0.027255,1.319944,2.171228,2.056216,5.019981,-0.851633,4.375691,-1.510047,6.061992,-1.076637,-3.181888,1.821096,10.283990,3.010150,8.401766,-1.099458,1.688274,-0.834872,-1.733869,-0.846637,3.849075,1.400102,12.628781,1.752842,5.468166,0.078557,0.059736,0.089392,-0.715300,1.825662,12.693808,0.197445,9.744638,0.126117,0.922311,-0.679797,1.220530,0.677983,2.556666,0.761349,10.693862,-2.168791,0.143632,1.388610,9.341997,0.317029,14.739025;
//Matrix<type, 100, 1> target;
//target << 0,1,0,0,0,1,0,1,0,0,1,0,1,0,1,1,1,1,1,1,1,1,0,1,1,0,0,1,1,0,1,1,0,1,1,0,0,0,0,0,1,1,0,1,1,0,1,1,0,0,0,0,0,0,1,1,0,1,0,1,1,1,0,0,0,1,1,0,0,0,0,1,0,1,0,0,1,1,1,1,0,1,0,1,1,1,1,0,1,1,1,0,0,1,1,1,0,1,0,0;
/*
cout<<"factory mode."<<endl;
auto x = Constant::create(sample);
auto y = Constant::create(target);
//create the network
auto w = Variable::create(MatrixX::Random(2, 1));
auto b = Variable::create(MatrixX::Random(1, 1));
auto mul = Mul::create(x, w);
auto add = Add::create(mul, b);
auto act = Sigmoid::create(add);
//train
LogisticLoss loss(act, y);
Optimizer opti(1e-2);
for(int i=0;i<epoch;i++) {
loss.reset();
loss.forward();
loss.backward(opti);
cout<<", l: "<<loss.getTotalLoss()<<endl;
}
cout<<"o: "<<act->getOutput()<<endl;
*/
MatrixX sample;
MatrixX target;
loadCsv("./test/logistic.txt", sample, target);
sample /= 200;
cout<<"direct mode"<<endl;
Constant x(sample);
Constant y(target);
Variable w(MatrixX::Random(sample.cols(), 1));
Variable b(MatrixX::Random(1, 1));
Mul mul(x, w);
Add add(mul, b);
Sigmoid act(add);
LogisticLoss loss(act, y);
//AdamOptimizer opti;
SGDOptimizer opti(1e-3);
//AdadeltaOptimizer opti;
//MomentumOptimizer opti(0.8, 1e-3);
opti.minimize(loss);
for(int i=0;i<epoch;i++) {
opti.run();
if(i % 100 == 0) {
cout<<"loss: "<<loss.getOutput().mean()<<endl;
}
}
cout<<"o: "<<act.getOutput()<<endl;
cout<<"w: "<<w.getOutput()<<endl;
cout<<"b: "<<b.getOutput()<<endl;
return 0;
}
Data.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
#ifndef __DATA_H
#define __DATA_H
#include<iostream>
#include<fstream>
#include <sstream>
#include<string>
#include "def.h"
using namespace std;
bool loadCsv(const char* csvFile, MatrixX& data, MatrixX& label) {
ifstream in(csvFile);
if(!in) {
cerr<<"Error: file '"<<csvFile<<"' not Found!"<<endl;
}
string line;
vector<vector<string>> strArray;
while (getline(in, line)) {
stringstream ss(line);
string field;
vector<string> lineArray;
while (getline(ss, field, ',')) {
lineArray.push_back(field);
}
strArray.push_back(lineArray);
}
if(strArray.size() == 0 || strArray[0].size() == 0) return false;
data.resize(strArray.size(), strArray[0].size()-1);
label.resize(strArray.size(), 1);
for(int i=0;i<strArray.size();i++) {
for(int j=0;j<data.cols();j++) {
data(i, j) = atof(strArray[i][j].c_str());
}
label(i, 0) = atof(strArray[i][data.cols()].c_str());
}
return true;
}
#endif
/*0.99,0.99,1.00,1.00,1.00,1.00,1.00,1.00,1.00,0.99,0.99*/

test-layer

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
/*
* test case for LeastSquareLoss
*/
#include <iostream>
#include <stdlib.h>
#include "Tensor.h"
#include "Loss.h"
#include "Optimizer.h"
#include "Layer.h"
#include "Activation.h"
using namespace std;
using namespace Eigen;
using namespace redtea::core;
int main(int argc, char* argv[]) {
if(argc < 2) return 1;
int epoch = atoi(argv[1]);
cout<<"epoch="<<epoch<<endl;
Matrix<type, 5, 2> sample;
sample << 1, 1,
2, 1,
3, 2,
5, 3,
6, 0;
Matrix<type, 5, 2> target;
target << 6, 7.95,
8, 9.4,
13, 15.95,
20, 23.95,
13, 10.1;
/*
auto x = Constant::create(sample);
auto y = Constant::create(target);
auto w = Variable::create(MatrixX::Random(2, 2));
auto b = Variable::create(MatrixX::Random(1, 2));
auto mul = Mul::create(x, w);
auto add = Add::create(mul, b);
LeastSquareLoss loss(add, y);
cout<<"w: "<<w->getOutput()<<endl;
cout<<"b: "<<b->getOutput()<<endl;
Optimizer opti(1e-2);
for(int i=0;i<epoch;i++) {
loss.forward();
loss.backward(opti);
cout<<"o: "<<add->getOutput();
cout<<", w: "<<w->getOutput();
cout<<", b: "<<b->getOutput();
cout<<", l: "<<loss.getTotalLoss()<<endl;
}
*/
Constant x(sample);
Constant y(target);
DenseLayer dense(x, 2);
LeastSquareLoss loss(dense, y);
//AdamOptimizer opti;
//MomentumOptimizer opti(0.8, 1e-3);
//AdadeltaOptimizer opti;
SGDOptimizer opti(1e-3);
opti.minimize(loss);
for(int i=0;i<epoch;i++) {
opti.run();
cout<<"output: "<<dense.getOutput();
cout<<", l: "<<loss.getOutput().sum()<<endl;
}
return 0;
}
-------------本文结束 感谢您的阅读-------------
作者GonewithGt
有问题请 留言 或者私信我的 微博
满分是10分的话,这篇文章你给几分