1. 1. : NAIST Computational Linguistic Lab D1 Masayoshi Kondo 4 -
2. 2. 00: Deep Learning : 165 ()
3. 3. XX: () . . .
4. 4. 00: (CNN) (RNN)
5. 5. 00: () - (Deep Learning) / ()
6. 6. 00: (3) - (overtting) .
7. 7. () () () 00: !!
8. 8. 4 4.1 4.2 4.3 4.4 4.5 ()!! ()
9. 9. 4 4.1 4.2 4.3 4.4 4.5
10. 10. () ( or ) -: (13) !! Q : Ans: () ( or ) ()
11. 11. DNN() E E w = E w1 !! E wM " # $ % & ' t w(t+1) = w(t) E W E w (RNNBPTT) E w~ [] []t h[]t+ [] W -: DNN W E
12. 12. - xnL . ( ) En wji (l) = En y y wji (l) = y(xn ) dn( ) t y wji (l) En = 1 2 y(xn ) dn 2 lj i wji (l) 01: E E w = E w1 !! E wM " # $ % & ' t w(t+1) = w(t) E
13. 13. l wji (l) y(x) f f f wji (l) 02: En wji (l) = En y y wji (l) = y(xn ) dn( ) t y wji (l) y(x) = f (u(L) ) = f (W(L) z(L1) + b(l) ) = f (W(L) f (W(L1) z(L2) + b(L1) )+ b(L) ) = f (W(L) f (W(L1) f (! f (W(l) z(l1) + b(l) )!))+ b(L) ) f - ()
14. 14. 4 4.1 4.2 4.3 4.4 4.5
15. 15. -: . .
16. 16. zj (2) = f (uj (2) ) = f ( wji (2) zi (1) i ) zi (1) = xi yj (x) = zj (3) = uj (3) = wji (3) zi (2) i 03: f ( ) z = f (u) = u
17. 17. En wji (3) = (y(x) d)t y wji (3) . y wji (3) =[ 0 ! 0 zi (2) 0 ! 0 ]t y wji (3) En wji (3) = (yj (x) dj )zi (2) ( ) E E w = E w1 !! E wM " # $ % & ' t w(t+1) = w(t) E 04:
18. 18. En wji (2) = En uj (2) uj (2) wji (2) uj (2) wji (2) En uj (2) = En uk (3) uk (3) uj (2) k uj (2) wji (2) = zi (1) uj (2) = wji (2) zi (1) i " # $ % & ' En uk (3) = uk (3) dk uk (3) uj (2) = wkj (3) "f (uj (2) ) En uk (3) uk (3) uj (2) 05:
19. 19. 06: En uj (2) = En uk (3) uk (3) uj (2) k uj (2) wji (2) = zi (1) uj (2) = wji (2) zi (1) i " # $ % & ' En uk (3) = uk (3) dk uk (3) uj (2) = wkj (3) "f (uj (2) ) En wji (2) = En uj (2) uj (2) wji (2) En wji (2) = !f (uj (2) ) wkj (3) (uk (3) dk ) k " # $ % & 'zi (1) ( ) E E w = E w1 !! E wM " # $ % & ' t w(t+1) = w(t) E
20. 20. 4 4.1 4.2 4.3 4.4 4.5
21. 21. j (l) En uj (l) 07: En wji (2) = En uj (2) uj (2) wji (2) l En wji (l) = En uj (l) uj (l) wji (l)
22. 22. En uj (l) = En uk (l+1) uk (l+1) uj (l) k uj (l) wji (l) = zi (l1) uj (l) = wji (l) zi (l1) i # $ % & ' ( l En wji (l) = En uj (l) uj (l) wji (l) j (l) = k (l+1) uk (l+1) uj (l) k uj (l) wji (l) = zi (l1) uj (l) = wji (l) zi (l1) i # $ % & ' ( uk (l+1) uj (l) uk (l+1) = wkj (l+1) zj (l) j = wkj (l+1) f (uj (l) ) j uj (l) uk (l+1) uk (l+1) uj (l) = wkj (l+1) !f (uj (l) ) j (l) En uj (l) 08: j (l) = k (l+1) wkj (l+1) !f (uj (l) )( ) k (l+1) l
23. 23. l En wji (l) = En uj (l) uj (l) wji (l) 09: En wji (l) =j (l) zi (l1) uj (l) wji (l) = zi (l1) uj (l) = wji (l) zi (l1) i # $ % & ' ( j (l) = k (l+1) wkj (l+1) !f (uj (l) )( ) k j (l) En uj (l) uj (l) wji (l) j (l) (l1)ilj j i wji (l)
24. 24. 10: 1.z(1)=xnlu(l)z(l). 2. j (L). 3.l(=L-1, L-2, L-3,,4 ,3, 2)j (L) . 4. l(=2, 3, 4,,L-2 ,L-1 ,L)wji (l) . j (l) = k (l+1) wkj (l+1) !f (uj (l) )( ) k En wji (l) =j (l) zi (l1)
25. 25. 11: En E = En n . E wji (l) = En wij (l) n
26. 26. 4 4.1 4.2 4.3 4.4 4.5 ()
27. 27.