如何在keras中实现maclaurin系列

Question

我正试图通过使用maclaurin系列来实现可扩展的CNN。其基本思想是第一个输入节点可以分解为具有不同阶数和系数的多个节点。将单个节点分解为多个节点可以生成不同于maclaurin级数的非线性线路连接。有人能给我一个如何用maclaurin级数非线性展开来展开CNN的可能想法吗？有什么想法吗？

我不太明白如何将输入节点分解为多个具有不同非线性线连接的节点，这是由maclaurin级数生成的。据我所知，maclaurin级数是一个近似函数，但分解节点在实现方面对我来说不是很直观。如何在python中实现将输入节点分解为多个输入节点？如何轻松实现这一点？知道吗？

我的尝试：

import tensorflow as tf
import numpy as np
import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Conv2D, MaxPooling2D, Dropout, Flatten
from keras.datasets import cifar10
from keras.utils import to_categorical
(train_imgs, train_label), (test_imgs, test_label)= cifar10.load_data()
output_class = np.unique(train_label)
n_class = len(output_class)
nrows_tr, ncols_tr, ndims_tr = train_imgs.shape[1:]
nrows_ts, ncols_ts, ndims_ts = test_imgs.shape[1:]
train_data = train_imgs.reshape(train_imgs.shape[0], nrows_tr, ncols_tr, ndims_tr)
test_data = test_imgs.reshape(test_imgs.shape[0], nrows_ts, ncols_ts, ndims_ts)
input_shape = (nrows_tr, ncols_tr, ndims_tr)
train_data = train_data.astype('float32')
trast_data = test_data.astype('float32')
train_data //= 255
test_data //= 255
train_label_one_hot = to_categorical(train_label)
test_label_one_hot = to_categorical(test_label)
def pown(x,n):
return(x**n)
def expandable_cnn(input_shape, output_shape, approx_order):
inputs=Input(shape=(input_shape))
x= Dense(input_shape)(inputs)
y= Dense(output_shape)(x)
model = Sequential()
model.add(Conv2D(filters=32, kernel_size=(3,3), padding='same', activation="relu", input_shape=input_shape))
model.add(Conv2D(filters=32, kernel_size=(3,3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2,2)))
model.add(Dropout(0.25))
model.add(Flatten())
model.add(Dense(512, activation='relu'))
model.add(Dropout(0.5))
for i in range(2, approx_order+1):
y=add([y, Dense(output_shape)(Activation(lambda x: pown(x, n=i))(x))])
model.add(Dense(n_class, activation='softmax')(y))
return model

但当我运行上面的模型时，我遇到了一堆编译错误和维度错误。我认为CNN模型的Tylor非线性展开方式可能不正确。此外，我不知道如何表示体重。如何做到这一点？关于如何纠正我的尝试，有什么可能的想法吗？

所需输出：

我希望用maclaurin系列非线性展开来扩展CNN，如何使上述实现正确有效？有什么可能的想法或方法吗？

Answer 1

有趣的问题。我已经实现了一个Keras模型，它计算泰勒展开，正如你所描述的：

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Dense, Input, Lambda

def taylor_expansion_network(input_dim, max_pow):
x = Input((input_dim,))
# 1. Raise input x_i to power p_i for each i in [0, max_pow].
def raise_power(x, max_pow):
x_ = x[..., None]  # Shape=(batch_size, input_dim, 1)
x_ = tf.tile(x_, multiples=[1, 1, max_pow + 1])  # Shape=(batch_size, input_dim, max_pow+1)
pows = tf.range(0, max_pow + 1, dtype=tf.float32)  # Shape=(max_pow+1,)
x_p = tf.pow(x_, pows)  # Shape=(batch_size, input_dim, max_pow+1)
x_p_ = x_p[..., None]  # Shape=(batch_size, input_dim, max_pow+1, 1)
return x_p_
x_p_ = Lambda(lambda x: raise_power(x, max_pow))(x)
# 2. Multiply by alpha coefficients
h = LocallyConnected2D(filters=1,
kernel_size=1,  # This layer is computing a_i * x^{p_i} for each i in [0, max_pow]
use_bias=False)(x_p_)  # Shape=(batch_size, input_dim, max_pow+1, 1)
# 3. Compute s_i for each i in [0, max_pow]
def cumulative_sum(h):
h = tf.squeeze(h, axis=-1)  # Shape=(batch_size, input_dim, max_pow+1)
s = tf.cumsum(h, axis=-1)  # s_i = sum_{j=0}^i h_j. Shape=(batch_size, input_dim, max_pow+1)
s_ = s[..., None]  # Shape=(batch_size, input_dim, max_pow+1, 1)
return s_
s_ = Lambda(cumulative_sum)(h)
# 4. Compute sum w_i * s_i each i in [0, max_pow]
s_ = LocallyConnected2D(filters=1,  # This layer is computing w_i * s_i for each i in [0, max_pow]
kernel_size=1,
use_bias=False)(s_)  # Shape=(batch_size, input_dim, max_pow+1)
y = Lambda(lambda s_: tf.reduce_sum(tf.squeeze(s_, axis=-1), axis=-1))(s_)  # Shape=(batch_size, input_dim)
# Return Taylor expansion model
model = Model(inputs=x, outputs=y)
model.summary()
return model

该实现将相同的泰勒展开应用于来自卷积网络的具有形状(batch_size, input_dim=512)的平坦张量的每个元素。

---

UPDATE：正如我们在评论部分所讨论的，这里有一些代码来展示如何修改函数expandable_cnn以集成上面定义的模型：

def expandable_cnn(input_shape, nclass, approx_order):
inputs = Input(shape=(input_shape))
h = inputs
h = Conv2D(filters=32, kernel_size=(3, 3), padding='same', activation='relu', input_shape=input_shape)(h)
h = Conv2D(filters=32, kernel_size=(3, 3), activation='relu')(h)
h = MaxPooling2D(pool_size=(2, 2))(h)
h = Dropout(0.25)(h)
h = Flatten()(h)
h = Dense(512, activation='relu')(h)
h = Dropout(0.5)(h)
taylor_model = taylor_expansion_network(input_dim=512, max_pow=approx_order)
h = taylor_model(h)
h = Activation('relu')(h)
print(h.shape)
h = Dense(nclass, activation='softmax')(h)
model = Model(inputs=inputs, outputs=h)
return model

请注意，我不保证你的模型会起作用(例如，你会得到良好的性能(。我只是根据我对你想要的东西的理解提供了一个解决方案。