在TensorFlow中执行梯度下降时，我如何将偏差项与其他权重包含在一起?

Question

我是ML的初学者，一直在遵循Coursera入门教学大纲。我正在尝试使用TensorFlow而不是Octave来实现练习。

我有三个版本-前两个工作良好，第三个不行。我想知道为什么。

注意-我使用TensorFlow。但是绑定是API的1:1映射，所以Python开发人员应该非常熟悉它。

1)。完全手动，工作良好

let gradientDescent (x : Tensor) (y : Tensor) (alpha : Tensor) iters = 
let update (theta : Tensor) =
let delta = 
let h = tf.matmul(x,theta)
let errors = h - y
let s =  tf.matmul((tf.transpose x), errors) 
s / m
theta - alpha * delta
let rec search (theta : Tensor) i =
if i = 0 then
theta
else
search (update theta) (i - 1)
let initTheta = tf.constant([| 0.; 0. |], shape = Shape [| 2; 1 |])      
search initTheta iters 
let ones = tf.ones(Shape [| m; 1 |], dtype = TF_DataType.TF_DOUBLE)
let X = tf.concat([ ones; x ], axis = 1)
let theta = gradientDescent X y alpha iterations

2)。使用梯度带进行带有单独偏置项的自动微分-效果也很好

let gradientDescent (x : Tensor) (y : Tensor) (alpha : float32) iters = 
let W = tf.Variable(0.f, name = "weight")
let b = tf.Variable(0.f, name = "bias")
let optimizer = keras.optimizers.SGD alpha
for _ in 0 .. iters do 
use g = tf.GradientTape()
let h = W * x + b
let loss = tf.reduce_sum(tf.pow(h - y,2)) / (2 * m)
let gradients = g.gradient(loss, struct (b,W))
optimizer.apply_gradients(zip(gradients, struct (b,W)))

tf
.constant(value = [| b.value(); W.value() |], shape = Shape [| 2; 1 |])
.numpy()
.astype(TF_DataType.TF_DOUBLE)
let theta = gradientDescent x y alpha iterations

3)。像以前一样使用梯度磁带，这次包括偏置项作为权重的一部分-这在调用apply_gradients时抛出堆栈溢出异常。

let gradientDescent (x : Tensor) (y : Tensor) (alpha : float32) iters = 
let W = tf.Variable(tf.constant([| 0.; 0. |], shape = Shape [| 2; 1 |]))
let optimizer = keras.optimizers.SGD alpha
for _ in 0 .. iters do 
use g = tf.GradientTape()
let h = tf.matmul(x, W)
let loss = tf.reduce_sum(tf.pow(h - y,2)) / (2 * m)
let gradients = g.gradient(loss, W) // correct gradient tensor returned here
optimizer.apply_gradients(struct(gradients, W)) // boom!

tf
.constant(value = W.value().ToArray<double>(), shape = Shape [| 2; 1 |])
.numpy()
let ones = tf.ones(Shape [| m; 1 |], dtype = TF_DataType.TF_DOUBLE)
let X = tf.concat([ ones; x ], axis = 1)
let theta = gradientDescent X y alpha iterations

Answer 1

我算出来了——optimizer.apply_gradients需要一个iterable

我所要做的就是改变

optimizer.apply_gradients(struct(gradients, W))

optimizer.apply_gradients(zip([|gradients|], [|W|]))

加一点float32/64类型转换