Network Layers Reference


                    
                    
                    LayerType

Undocumented

Declaration

Swift

public protocol LayerType


                    
                    
                    LayerType


                    
                    
                    Sequential

A sequential layer that concatenates the computations of two other layers.

With function builders, a sequential layer can be used to express sequential models in a type safe way.

Example:

let model = Sequential {
    Dense<Float, CPU>(inputSize: 32, outputSize: 64)
    Relu<Float, CPU>()
    Dense<Float, CPU>(inputSize: 64, outputSize: 10)
    Softmax<Float, CPU>()
}

model will have a type Sequential<Sequential<Dense<Float, CPU>, Relu<Float, CPU>>, Sequential<Dense<Float, CPU>, Softmax<Float, CPU>>>.

Declaration

Swift

public struct Sequential<First, Second> : LayerType where First : LayerType, Second : LayerType, First.Device == Second.Device, First.Outputs == Second.Inputs, First.Parameter == Second.Parameter

extension Sequential: Codable where First: Codable, Second: Codable


                    
                    
                    Dense

Dense (Linear, Fully connected) layer with no activation function.

Declaration

Swift

public struct Dense<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    Convolution2D

A 2D convolutional layer

Declaration

Swift

public struct Convolution2D<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    TransposedConvolution2D

A 2D transposed (fractionally strided) convolutional layer

Declaration

Swift

public struct TransposedConvolution2D<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    MaxPool2D

A 2D max pooling layer

Declaration

Swift

public struct MaxPool2D<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    AvgPool2D

A 2D average pooling layer

Declaration

Swift

public struct AvgPool2D<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    AdaptiveMaxPool2D

A 2D adaptive max pooling layer that pools its inputs with an automatically computed stride and window size to reach the desired output size

Declaration

Swift

public struct AdaptiveMaxPool2D<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    AdaptiveAvgPool2D

A 2D adaptive average pooling layer that pools its inputs with an automatically computed stride and window size to reach the desired output size

Declaration

Swift

public struct AdaptiveAvgPool2D<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType

RNN

Base protocol for recurrent neural networks.

Declaration

Swift

public protocol RNN : LayerType


                    
                    
                    LSTM

Undocumented

Declaration

Swift

public struct LSTM<Element, Device> : RNN, Codable where Element : RandomizableType, Device : DeviceType

GRU

Undocumented

Declaration

Swift

public struct GRU<Element, Device> : RNN, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    BasicRNN

A ‘vanilla’ RNN. In each step, the RNN performs the transformation matMul(x_t, W) + matMul(h_t-1, U) + b

Declaration

Swift

public struct BasicRNN<Element, Device> : RNN, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    Bidirectional

A bidirectional RNN

Declaration

Swift

public struct Bidirectional<RNNLayer> : LayerType where RNNLayer : RNN

extension Bidirectional: Codable where RNNLayer: Codable


                    
                    
                    Embedding

Transforms discrete values, such as word indices, into a lower dimensional embedding.

Declaration

Swift

public struct Embedding<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    PositionalEncoding

Positional Encoding layer using the encoding method proposed in Attention Is All You Need.

The layer takes an array of Ints as an input, which indicate the number of elements in each sequence of the minibatch. It returns a tensor with the shape [max(inputs), hiddenSize]. It does not mask out positional encodings for padding elements.

Declaration

Swift

public struct PositionalEncoding<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    ScaledDotProductAttention

Computes Scaled Multi-Head Dot Product Attention as introduced by Attention Is All You Need.

Declaration

Swift

public struct ScaledDotProductAttention<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    MultiHeadAttention

Multi-Head Attention Layer following Attention Is All You Need.

Declaration

Swift

public struct MultiHeadAttention<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    TransformerEncoderBlock

Transformer encoder layer consisting of a self-attention and a pointwise feed forward layer as introduced by Attention Is All You Need.

Declaration

Swift

public struct TransformerEncoderBlock<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    TransformerDecoderBlock

Transformer decoder layer consisting of a self attention, encoder attention and a pointwise feed forward layer as introduced by Attention Is All You Need.

Declaration

Swift

public struct TransformerDecoderBlock<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    TransformerEncoder

Transformer encoder sequencing positional encoding and token embedding and multiple transformer encoder layers, as introduced by Attention Is All You Need.

Declaration

Swift

public struct TransformerEncoder<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    TransformerDecoder

Transformer encoder sequencing positional encoding and token embedding and multiple transformer encoder layers, as introduced by Attention Is All You Need.

Declaration

Swift

public struct TransformerDecoder<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    PointwiseFeedForward

Pointwise feed forward layer as introduced in Attention Is All You Need.

The layer sequences a dense layer, GeLU activation, another dense layer and a dropout layer. Furthermore, it has a residual connection and the output is layer normalized.

Declaration

Swift

public struct PointwiseFeedForward<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    LayerNorm

A layer that normalizes its inputs along all dimensions except the batch dimension

Declaration

Swift

public struct LayerNorm<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    BatchNorm

A layer that normalizes its inputs along the batch dimension

Declaration

Swift

public struct BatchNorm<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    Dropout

A dropout layer

Declaration

Swift

public struct Dropout<Element, Device> : LayerType, Codable where Element : RandomizableType, Device : DeviceType


                    
                    
                    Tanh

Element-wise hyperbolic tangent activation layer.

Declaration

Swift

public struct Tanh<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    Sigmoid

Element-wise sigmoid activation layer.

Declaration

Swift

public struct Sigmoid<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    Relu

Element-wise rectified linear unit activation layer.

Declaration

Swift

public struct Relu<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    LeakyRelu

Element-wise leaky linear rectified unit activation layer.

Declaration

Swift

public struct LeakyRelu<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    Softmax

Softmax activation layer

Declaration

Swift

public struct Softmax<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    LogSoftmax

Log Softmax activation layer

Declaration

Swift

public struct LogSoftmax<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    Lambda

Layer wrapping an arbitrary transform provided by a closure.

Declaration

Swift

public struct Lambda<Inputs, Outputs, Element, Device> : LayerType where Element : NumericType, Device : DeviceType


                    
                    
                    Flatten

Layer that flattens its inputs into a tensor of shape [batchSize, -1]

Declaration

Swift

public struct Flatten<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    Reshape

Layer that reshapes its inputs to a given target size, except the batch size

Declaration

Swift

public struct Reshape<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    Concat

Layer that concatenates a list of input tensors along their second dimension

Declaration

Swift

public struct Concat<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    AnyLayer

Type erased layer

Declaration

Swift

public struct AnyLayer<Inputs, Outputs, Parameter, Device> : LayerType where Parameter : NumericType, Device : DeviceType


                    
                    
                    Gelu

Element-wise gaussian error linear unit activation layer.

Declaration

Swift

public struct Gelu<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    Swish

Element-wise Swish activation layer.

Declaration

Swift

public struct Swish<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    Mish

Element-wise Mish activation layer.

Declaration

Swift

public struct Mish<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType


                    
                    
                    LiSHT

Element-wise LiSHT activation layer.

Declaration

Swift

public struct LiSHT<Element, Device> : LayerType, Codable where Element : NumericType, Device : DeviceType