反向传播
#实现正向传播，反向传播和梯度检查#code基于计算图计算前向传播
def forward_propagation(x, y, w):
'''In this function, we will compute the forward propagation '''
# X: input data point, note that in this assignment you are having 5-d data points
# y: output varible
# W: weight array, its of length 9, W[0] corresponds to w1 in graph, W[1] corresponds to w2 in graph,..., W[8] corresponds to w9 in graph.
# you have to return the following variables
# exp= part1 (compute the forward propagation until exp and then store the values in exp)
# tanh =part2(compute the forward propagation until tanh and then store the values in tanh)
# sig = part3(compute the forward propagation until sigmoid and then store the values in sig)
# we are computing one of the values for better understanding
val_1= (w[0]*x[0]+w[1]*x[1]) * (w[0]*x[0]+w[1]*x[1]) + w[5]
exp = np.exp(val_1)
tanh = np.tanh(exp + w[6])
val_2= ((np.sin(w[2]*x[2]))*((w[3]*x[3])+(w[4]*x[4])))+w[7]
sig = sigmoid(val_2)
y_pred = tanh+sig*w[8]
# after computing part1,part2 and part3 compute the value of y' from the main Computational graph using required equations
# write code to compute the value of L=(y-y')^2 and store it in variable loss
loss =pow(y-y_pred,2)
# compute derivative of L w.r.to y' and store it in dy_pred
dy_pred = -2 * (y-y_pred)
# Create a dictionary to store all the intermediate values i.e. dy_pred ,loss,exp,tanh,sigmoid
# we will be using the dictionary to find values in backpropagation, you can add other keys in dictionary as well
forward_dict={}
forward_dict['exp']= exp
forward_dict['sigmoid'] = sig
forward_dict['tanh'] =tanh
forward_dict['loss'] = loss
forward_dict['dy_pred'] = dy_pred
return forward_dict
反向传播
def backward_propagation(x,y,w,forward_dict):
'''In this function, we will compute the backward propagation '''
# forward_dict: the outputs of the forward_propagation() function
# write code to compute the gradients of each weight [w1,w2,w3,...,w9]
# Hint: you can use dict type to store the required variables
# dw1 = # in dw1 compute derivative of L w.r.to w1
dw1 = forward_dict['dy_pred']*(1-(math.pow(forward_dict['tanh'],2)))*forward_dict["exp"]*2*((w[0]*x[0])+(w[1]*x[1]))*x[0]
# dw2 = # in dw2 compute derivative of L w.r.to w2
dw2=forward_dict['dy_pred']*(1-(math.pow(forward_dict['tanh'],2)))*forward_dict["exp"]*2*((w[0]*x[0])+(w[1]*x[1]))*x[1]
# dw3 = # in dw3 compute derivative of L w.r.to w3
dw3 =forward_dict['dy_pred']*(forward_dict['sigmoid']*(1-forward_dict['sigmoid']))*w[8]*((w[3]*x[3])+(w[4]*x[4]))*math.cos(x[2]*w[2])*x[2]
# dw4 = # in dw4 compute derivative of L w.r.to w4
dw4 =forward_dict['dy_pred']*(forward_dict['sigmoid']*(1-forward_dict['sigmoid']))*w[8]*math.sin(x[2]*w[2])*x[3]
# dw5 = # in dw5 compute derivative of L w.r.to w5
dw5 =forward_dict['dy_pred']*(forward_dict['sigmoid']*(1-forward_dict['sigmoid']))*w[8]*math.sin(x[2]*w[2])*x[4]
# dw6 = # in dw6 compute derivative of L w.r.to w6
dw6 = forward_dict['dy_pred']*(1-(math.pow(forward_dict['tanh'],2)))*forward_dict["exp"]
# dw7 = # in dw7 compute derivative of L w.r.to w7
dw7 =forward_dict['dy_pred']*(1-(math.pow(forward_dict['tanh'],2)))
# dw8 = # in dw8 compute derivative of L w.r.to w8
dw8 =forward_dict['dy_pred']*(forward_dict['sigmoid']*(1-forward_dict['sigmoid']))*w[8]
# dw9 = # in dw9 compute derivative of L w.r.to w9
dw9 =forward_dict['dy_pred']*forward_dict['sigmoid']
backward_dict={}
#store the variables dw1,dw2 etc. in a dict as backward_dict['dw1']= dw1,backward_dict['dw2']= dw2...
backward_dict['dw1']= dw1
backward_dict['dw2']= dw2
backward_dict['dw3']= dw3
backward_dict['dw4']= dw4
backward_dict['dw5']= dw5
backward_dict['dw6']= dw6
backward_dict['dw7']= dw7
backward_dict['dw8']= dw8
backward_dict['dw9']= dw9
return backward_dict
def gradient_checking(x,y,w,eps):
# compute the dict value using forward_propagation()
# compute the actual gradients of W using backword_propagation()
forward_dict=forward_propagation(x,y,w)
backward_dict=backward_propagation(x,y,w,forward_dict)
#we are storing the original gradients for the given datapoints in a list
original_gradients_list=list(backward_dict.values())
# make sure that the order is correct i.e. first element in the list corresponds to dw1 ,second element is dw2 etc.
# you can use reverse function if the values are in reverse order
approx_gradients_list=[]
eps=0.0001
w = np.ones(9)*0.1
#now we have to write code for approx gradients, here you have to make sure that you update only one weight at a time
#write your code here and append the approximate gradient value for each weight in approx_gradients_list
for i in range(len(w)):
w_plus =w.copy()
w_plus[i]=w_plus[i]+eps
Loss1=forward_propagation(x,y,w_plus)['loss']
w_sub = w.copy()
w_sub[i]=w_sub[i]-eps
Loss2=forward_propagation(x,y,w_sub)['loss']
approx =(Loss1-Loss2)/(2*eps)
approx_gradients_list.append(approx)
gradient_check_value =[]
for i in range(len(w)):
num = np.linalg.norm(original_gradients_list[i] - approx_gradients_list[i])
den = np.linalg.norm(original_gradients_list[i]) + np.linalg.norm(approx_gradients_list[i])
diff = num / den
gradient_check_value.append(diff)
return gradient_check_value
我正试图从头开始实现以下图的反向传播。当我尝试向前和向后运行grader函数时，传播返回true。然而，当在gradient_checking函数上运行grader函数时，我得到了下面的错误:
def grader_grad_check(value):
print(value)
assert(np.all(value <= 10**-3))
return True
w=[ 0.00271756, 0.01260512, 0.00167639, -0.00207756, 0.00720768,
0.00114524, 0.00684168, 0.02242521, 0.01296444]
eps=10**-7
value= gradient_checking(X[0],y[0],w,eps)
grader_grad_check(value)
([0.9033700837499321, 0.9033700856470759, 1.0, 0.9950783883268165, 0.9950783883310051, 0.1755656033519971, 0.23240434925625725, 0.7442376971131373, 0.03845869617360365], 0.03845869617360365)
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-59-a4c2d0b40187> in <module>()
9 eps=10**-7
10 value= gradient_checking(X[0],y[0],w,eps)
---> 11 grader_grad_check(value)
<ipython-input-59-a4c2d0b40187> in grader_grad_check(value)
1 def grader_grad_check(value):
2 print(value)
----> 3 assert(np.all(value <= 10**-3))
4 return True
5
TypeError: '<=' not supported between instances of 'tuple' and 'float'
图
无需使用";w = np.ones(9)*0.1 ";因为权重已经初始化并传递给了gradient_checking"函数。
在for循环之后使用这段代码，
original_gradients_list=np.array(original_gradients_list)
approx_gradients_list=np.array(approx_gradients_list)
numerator = np.linalg.norm(approx_gradients_list -
original_gradients_list)
denominator = np.linalg.norm(approx_gradients_list ) + np.linalg.norm(
original_gradients_list)
gradient_check_value = numerator / denominator
return gradient_check_value