Initial commit: Added files for implementation of SDKN and corresponding examples.

.gitignore

+# Added Tiz
+tb_logs/
+
+
+# Created by .ignore support plugin (hsz.mobi)
+### Python template
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+### VirtualEnv template
+# Virtualenv
+# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
+.Python
+[Bb]in
+[Ii]nclude
+[Ll]ib
+[Ll]ib64
+[Ll]ocal
+[Ss]cripts
+pyvenv.cfg
+.venv
+pip-selfcheck.json
+### JetBrains template
+# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm
+# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
+
+# User-specific stuff:
+.idea/workspace.xml
+.idea/tasks.xml
+.idea/dictionaries
+.idea/vcs.xml
+.idea/jsLibraryMappings.xml
+
+# Sensitive or high-churn files:
+.idea/dataSources.ids
+.idea/dataSources.xml
+.idea/dataSources.local.xml
+.idea/sqlDataSources.xml
+.idea/dynamic.xml
+.idea/uiDesigner.xml
+
+# Gradle:
+.idea/gradle.xml
+.idea/libraries
+
+# Mongo Explorer plugin:
+.idea/mongoSettings.xml
+
+.idea/
+
+## File-based project format:
+*.iws
+
+## Plugin-specific files:
+
+# IntelliJ
+/out/
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# JIRA plugin
+atlassian-ide-plugin.xml
+
+# Crashlytics plugin (for Android Studio and IntelliJ)
+com_crashlytics_export_strings.xml
+crashlytics.properties
+crashlytics-build.properties
+fabric.properties

README.md

+# structured-deep-kernel-networks
+
+
+
+
+In order to view tb_logs:
+    tensorboard --logdir=path_to_dir                        # view one folder
+    tensorboard --logdir=path_to_dir                        # view several folder

main_basic.py

+# Simple comparison of SDKN approach vs NN on a 1D toy problem.
+
+# Some imports and random seed
+import math
+import time
+
+import numpy as np
+import torch
+import torch.optim as optim
+from matplotlib import pyplot as plt
+
+from utils.models import NN, SDKN
+
+torch.random.manual_seed(2)
+
+# Create SDKN model with some arbitrary centers
+centers = torch.tensor([[-1, -.5, 0, .5, 1]]).t()
+model_SDKN = SDKN(centers=centers, dim_input=1, dim_output=1)
+model_NN = NN(dim_input=1, dim_output=1)
+
+# Print total number of parameters
+total_params1 = sum(p.numel() for p in model_SDKN.parameters() if p.requires_grad)
+total_params2 = sum(p.numel() for p in model_NN.parameters() if p.requires_grad)
+print('nr params model1 = {}.'.format(total_params1))
+print('nr params model2 = {}.'.format(total_params2))
+
+# Create training and test data
+X_train = torch.linspace(-1, 1, 1000).view(-1, 1)
+X_test = torch.linspace(-2, 2, 1000).view(-1, 1)
+
+
+# Define some target function for approximation
+def f_func(x):
+    return torch.sin(math.pi * x ** 2)
+
+
+y_train = f_func(X_train)
+
+# Set up training details
+optimizer1 = optim.Adam(model_SDKN.parameters(), lr=.05)
+optimizer2 = optim.Adam(model_NN.parameters(), lr=.05)
+loss = torch.nn.MSELoss(reduction='mean')
+
+batch_size = 200
+n_epoch = int(1e3)
+N = X_train.shape[0]
+n_batches = int(np.floor(N / batch_size))  # some samples might be lost
+
+lambda_lr_decay = lambda epoch: 1 / (1 + .02 * epoch ** 1.2)
+scheduler1 = torch.optim.lr_scheduler.LambdaLR(optimizer1, lr_lambda=lambda_lr_decay)
+scheduler2 = torch.optim.lr_scheduler.LambdaLR(optimizer2, lr_lambda=lambda_lr_decay)
+
+# Start the training and keep track of the (training) losses
+list_loss1 = []
+list_loss2 = []
+t1 = time.time()
+for idx_epoch in range(n_epoch):  # loop over the dataset multiple times
+
+    shuffle = np.random.permutation(X_train.shape[0])  # shuffle the data set
+
+    for idx_batch in range(n_batches):
+        # Set gradients to zero
+        optimizer1.zero_grad()
+        optimizer2.zero_grad()
+
+        # Select a mini-batch of data
+        ind = shuffle[idx_batch * batch_size: min((idx_batch + 1) * batch_size, N)]
+        X_batch = X_train[ind, :]
+
+        # forward + backward + optimize
+        outputs1 = model_SDKN(X_batch)
+        outputs2 = model_NN(X_batch)
+        loss1 = loss(outputs1, y_train[ind, :])
+        loss2 = loss(outputs2, y_train[ind, :])
+        loss1.backward()
+        loss2.backward()
+        optimizer1.step()
+        optimizer2.step()
+
+    t2 = time.time()
+
+    # modify learning rate
+    scheduler1.step()
+    scheduler2.step()
+
+    timediff = t2 - t1
+
+    # Calculate and store loss on whole training set
+    loss_all1 = loss(model_SDKN(X_train), y_train)
+    loss_all2 = loss(model_NN(X_train), y_train)
+    list_loss1.append(loss_all1.item())
+    list_loss2.append(loss_all2.item())
+
+    # Print some training information from time to time
+    if idx_epoch % 10 == 9:
+        print('{:d}, time: {:.1f}: loss1: {:.3e}, loss2: {:.3e}.'.format(
+            idx_epoch + 1, timediff, loss_all1, loss_all2))
+
+# Calculate overall loss after training
+loss_overall_model1 = loss(model_SDKN(X_train), y_train)
+loss_overall_model2 = loss(model_NN(X_train), y_train)
+
+# Several plots, e.g of prediction, residual, loss decay
+plt.figure(1)
+plt.clf()
+plt.plot(X_test, f_func(X_test), 'k')
+plt.plot(X_test, model_SDKN(X_test).detach())
+plt.plot(X_test, model_NN(X_test).detach())
+plt.legend(['function', 'prediction1', 'prediction2'])
+plt.draw()
+
+plt.figure(2)
+plt.clf()
+plt.plot(X_test, model_SDKN(X_test).detach() - f_func(X_test))
+plt.plot(X_test, model_NN(X_test).detach() - f_func(X_test))
+plt.legend(['residual1', 'residual2'])
+plt.draw()
+
+plt.figure(3)
+plt.clf()
+plt.plot(list_loss1)
+plt.plot(list_loss2)
+plt.legend(['loss1', 'loss2'])
+plt.xscale('log')
+plt.yscale('log')
+plt.draw()
+
+plt.show()

main_lightning_NN.py

+# Example file of using pytorch-lightning together with a NN
+
+# Some import statements
+import pytorch_lightning as pl
+from pytorch_lightning.loggers import TensorBoardLogger
+
+from utils import utilities
+from utils import lightning_models
+from utils import settings
+from datetime import date
+
+
+# Load data
+train_loader, val_loader, test_loader = utilities.get_DataLoader()
+
+# Torch model
+model_NN = lightning_models.NN()
+
+# Training
+logger = TensorBoardLogger('tb_logs_' + date.today().strftime('%d.%m.%y') + '/',
+                           name='model_NN')
+trainer = pl.Trainer(max_epochs=settings.num_epochs_nn,
+                     progress_bar_refresh_rate=settings.progress_bar_refresh_rate,
+                     logger=logger,
+                     log_every_n_steps=settings.log_every_n_steps,
+                     gpus=1 if settings.ianscluster else 0
+                     )
+
+trainer.fit(model_NN, train_dataloader=train_loader, val_dataloaders=val_loader)
+
+# Test
+trainer.test(model_NN, dataloaders=test_loader)
\ No newline at end of file

main_lightning_SDKN.py

+# Example file of using pytorch-lightning together with a NN
+
+# Some import statements
+import pytorch_lightning as pl
+from pytorch_lightning.loggers import TensorBoardLogger
+
+from utils import utilities
+from utils import lightning_models
+from utils import settings
+from datetime import date
+
+
+# Load data
+train_loader, val_loader, test_loader = utilities.get_DataLoader()
+
+# Torch model
+for sample, label in train_loader:
+    break
+model_SDKN = lightning_models.SDKN(centers=sample[:5, :])
+
+# Training
+logger = TensorBoardLogger('tb_logs_' + date.today().strftime('%d.%m.%y') + '/',
+                           name='model_SDKN')
+trainer = pl.Trainer(max_epochs=settings.num_epochs_sdkn,
+                     progress_bar_refresh_rate=settings.progress_bar_refresh_rate,
+                     logger=logger,
+                     log_every_n_steps=settings.log_every_n_steps,
+                     gpus=1 if settings.ianscluster else 0
+                     )
+
+trainer.fit(model_SDKN, train_dataloader=train_loader, val_dataloaders=val_loader)
+
+# Test
+trainer.test(model_SDKN, dataloaders=test_loader)
\ No newline at end of file

setup.sh

+set -e
+export BASEDIR="$(cd "$(dirname ${BASH_SOURCE[0]})" ;  pwd -P)"
+cd "${BASEDIR}"
+
+# Create and source virtualenv
+if [ -e "${BASEDIR}/venv/bin/activate" ]; then
+	echo "using existing virtualenv"
+else	
+	echo "creating virtualenv ..."
+	virtualenv --python=python3 venv
+fi
+
+source venv/bin/activate
+
+# Upgrade pip and install libraries (dependencies are also installed)
+pip install --upgrade pip
+pip3 install --ignore-installed torch==1.9.1+cpu torchvision==0.10.1+cpu torchaudio==0.9.1 -f https://download.pytorch.org/whl/torch_stable.html
+pip install pytorch-lightning
+
+
+# Install further stuff
+pip install matplotlib
+
+
+# pip freeze > requirements.txt				# not required

utils/kernels.py

+import torch
+
+
+
+# Implementation of a few kernels
+from abc import ABC, abstractmethod
+
+
+class Kernel(ABC):
+    @abstractmethod
+    def __init__(self):
+        super().__init__()
+
+    @abstractmethod
+    def eval(self):
+        pass
+
+    @abstractmethod
+    def diagonal(self, X):
+        pass
+
+    @abstractmethod
+    def __str__(self):
+        pass
+
+    @abstractmethod
+    def set_params(self, params):
+        pass
+
+
+class RBF(Kernel):
+    @abstractmethod
+    def __init__(self):
+        super(RBF, self).__init__()
+
+    def eval(self, x, y):
+        return self.rbf(self.ep, torch.cdist(x, y))
+
+    def diagonal(self, X):
+        return torch.ones(X.shape[0], 1) * self.rbf(self.ep, torch.tensor(0.0))
+
+    def __str__(self):
+        return self.name + ' [gamma = %2.2e]' % self.ep
+
+    def set_params(self, par):
+        self.ep = par
+
+
+class Gaussian(RBF):
+    def __init__(self, ep=1):
+        self.ep = ep
+        self.name = 'gauss'
+        self.rbf = lambda ep, r: torch.exp(-(ep * r) ** 2)
+
+class Matern(RBF):
+    def __init__(self, ep=1):
+        self.ep = ep
+        self.name = 'matern'
+        self.rbf = lambda ep, r: torch.exp(-ep * r)
+
+class Wendland_order_0(RBF):
+    def __init__(self, ep=1):
+        self.ep = ep
+        # self.rbf = lambda ep, r: torch.clamp(1 - ep*r, min=0)
+        self.rbf = lambda ep, r: torch.nn.functional.relu(1- ep * r)
+        self.name = 'Wendland order 0'
+

utils/lightning_models.py

+# Implementation of both SDKN and NN in the pytorch-lightning framework.
+
+import pytorch_lightning as pl
+import torch
+from torch import nn
+from utils.utilities import ActivFunc
+from utils import kernels
+from torch.nn import functional as F
+from utils import settings
+
+
+class Network(pl.LightningModule):
+    '''
+    General class with training_step, validation_step and test_step method.
+    '''
+
+    def __init__(self):
+        super().__init__()
+
+
+    def training_step(self, batch, batch_idx):
+        x, y = batch
+        y_pred = self(x)
+        loss = F.mse_loss(y_pred, y)
+
+        # Some logs
+        self.log('train/loss', loss, prog_bar=True)
+
+        return loss
+
+    def validation_step(self, batch, batch_idx):
+        x, y = batch
+        y_pred = self(x)
+        loss = F.mse_loss(y_pred, y)
+
+        # Some logs
+        self.log('val/loss', loss, prog_bar=True)
+
+        return loss
+
+    def test_step(self, batch, batch_idx):
+        x, y = batch
+        y_pred = self(x)
+        loss = F.mse_loss(y_pred, y)
+
+        # Some logs
+        self.log('test/loss', loss, prog_bar=True)
+
+        return loss
+
+
+
+class SDKN(Network):
+    '''
+    Implementation of SDKN with initialization, forward and configure_optimizers method.
+    '''
+
+    def __init__(self, centers, learning_rate=1e-3):
+        super().__init__()
+
+        # Some settings
+        self.register_buffer("centers", centers)
+        self.M = self.centers.shape[0]
+        self.learning_rate = learning_rate
+
+        # Define linear maps - hardcoded
+        self.fc1 = nn.Linear(5, 32, bias=False)
+        self.fc2 = nn.Linear(32, 64, bias=False)
+        self.fc3 = nn.Linear(64, 48, bias=False)
+        self.fc4 = nn.Linear(48, 24, bias=False)
+        self.fc5 = nn.Linear(24, 2, bias=False)
+
+        torch.nn.init.kaiming_normal_(self.fc1.weight)
+        torch.nn.init.kaiming_normal_(self.fc2.weight)
+        torch.nn.init.kaiming_normal_(self.fc3.weight)
+        torch.nn.init.kaiming_normal_(self.fc4.weight)
+        torch.nn.init.kaiming_normal_(self.fc5.weight)
+
+        # Define activation function mappings
+        self.activ1 = ActivFunc(32, self.M, kernel=kernels.Gaussian())
+        self.activ2 = ActivFunc(64, self.M, kernel=kernels.Gaussian())
+        self.activ3 = ActivFunc(48, self.M, kernel=kernels.Gaussian())
+        self.activ4 = ActivFunc(24, self.M, kernel=kernels.Gaussian())
+
+
+
+    def forward(self, x):
+        centers = self.centers
+
+        # First fully connect + activation function
+        x = self.fc1(x)
+        centers = self.fc1(centers)
+        x, centers = self.activ1(x, centers)
+
+        # Second fully connect + activation function
+        x = self.fc2(x)
+        centers = self.fc2(centers)
+        x, centers = self.activ2(x, centers)
+
+        # Third fully connect + activation function
+        x = self.fc3(x)
+        centers = self.fc3(centers)
+        x, centers = self.activ3(x, centers)
+
+        # Fourth fully connect + activation function
+        x = self.fc4(x)
+        centers = self.fc4(centers)
+        x, centers = self.activ4(x, centers)
+
+        # Fifth (final) fully connect
+        x = self.fc5(x)
+
+        return x
+
+
+    def configure_optimizers(self):         # Adam + LR scheduler
+        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
+        scheduler = torch.optim.lr_scheduler.StepLR(
+            optimizer, step_size=settings.decayEpochs_sdkn, gamma=settings.decayRate)
+
+        return [optimizer], [scheduler]
+
+
+
+class NN(Network):
+    '''
+    Implementation of NN with initialization, forward and configure_optimizers method.
+    '''
+
+    def __init__(self, learning_rate=1e-3):
+        super().__init__()
+
+        # Some settings
+        self.learning_rate = learning_rate
+
+        # Define linear maps - hardcoded
+        self.fc1 = nn.Linear(5, 32, bias=False)
+        self.fc2 = nn.Linear(32, 64, bias=False)
+        self.fc3 = nn.Linear(64, 48, bias=False)
+        self.fc4 = nn.Linear(48, 24, bias=False)
+        self.fc5 = nn.Linear(24, 2, bias=False)
+
+        torch.nn.init.kaiming_normal_(self.fc1.weight)
+        torch.nn.init.kaiming_normal_(self.fc2.weight)
+        torch.nn.init.kaiming_normal_(self.fc3.weight)
+        torch.nn.init.kaiming_normal_(self.fc4.weight)
+        torch.nn.init.kaiming_normal_(self.fc5.weight)
+
+    def forward(self, x):
+        x = F.relu(self.fc1(x))
+        x = F.relu(self.fc2(x))
+        x = F.relu(self.fc3(x))
+        x = F.relu(self.fc4(x))
+        x = self.fc5(x)
+
+        return x
+
+
+    def configure_optimizers(self):         # Adam + LR scheduler
+        optimizer = torch.optim.Adam(self.parameters(), lr=self.learning_rate)
+        scheduler = torch.optim.lr_scheduler.StepLR(
+            optimizer, step_size=settings.decayEpochs_nn, gamma=settings.decayRate)
+
+        return [optimizer], [scheduler]
\ No newline at end of file

utils/models.py

+# Basic implementation of both SDKN and NN without using the pytorch-lightning framework.
+
+import math
+import torch
+import torch.nn.functional as F
+import torch.nn as nn
+from utils import kernels
+from utils.utilities import ActivFunc
+
+
+# Implementation of an exemplary SDKN model
+class SDKN(torch.nn.Module):  # inherit from the StandardDK class to have the same init
+    def __init__(self, centers, dim_input, dim_output):
+        super(SDKN, self).__init__()
+
+        # General stuff
+        self.centers = centers
+        self.M = self.centers.shape[0]
+
+        # Define linear maps
+        self.width = 10
+        self.fc1 = nn.Linear(dim_input, self.width, bias=False)
+        self.fc2 = nn.Linear(self.width, self.width, bias=False)
+        self.fc3 = nn.Linear(self.width, dim_output, bias=False)
+
+        # Define activation maps
+        self.activ1 = ActivFunc(self.width, self.M, kernel=kernels.Gaussian(ep=1))
+        self.activ2 = ActivFunc(self.width, self.M, kernel=kernels.Wendland_order_0(ep=1))
+
+    def forward(self, x):
+        centers = self.centers
+
+        # First fully connect + activation function
+        x = self.fc1(x)
+        centers = self.fc1(centers)
+        x, centers = self.activ1(x, centers)
+
+        # Second fully connect + activation function
+        x = self.fc2(x)
+        centers = self.fc2(centers)
+        x, centers = self.activ2(x, centers)
+
+        # Third fully connect
+        x = self.fc3(x)
+
+        return x
+
+# Implementation of an exemplary NN model
+class NN(nn.Module):
+
+    def __init__(self, dim_input, dim_output):
+        super(NN, self).__init__()
+
+        self.width = 15
+
+        self.fc1 = nn.Linear(dim_input, self.width)
+        self.fc2 = nn.Linear(self.width, self.width)
+        self.fc3 = nn.Linear(self.width, dim_output)
+
+    def forward(self, x):
+        x = F.relu(self.fc1(x))
+        x = F.relu(self.fc2(x))
+        x = self.fc3(x)
+
+        return x
\ No newline at end of file

utils/settings.py

+import glob
+import os
+
+## Settings to choose
+idx_GRU = 1                             # 1, 2, or 3
+data_type = 'DG_data/'             # 'DG_data', 'FV_data', 'Fourier_data'
+ianscluster = False
+flag_load_only_few_files = True
+
+# Settings for data
+if ianscluster:
+    base_path = '/usr/local/storage/wenzeltn/A_final_data_marius/data/' + data_type   # ianscluster path
+else:
+    # base_path = '/usr/local/storage/wenzeltn/A_final_data_marius/data/' + data_type  # ianscluster path
+    base_path = '/usr/local/home/wenzeltn/A_final_data_marius/data/' + data_type        # anm03 path
+
+
+# Model parameters
+depth = 0  # amount of residual blocks
+kernel = 1  # kernel size of conv layers
+n_hidden_1 = 32  # number of hidden neurons 1
+n_hidden_2 = 64  # number of hidden neurons 2
+
+# Learning parameters
+batch_size = 256  # batch_size used during training
+num_epochs_nn = 50  # number of training epochs
+num_epochs_sdkn = 25  # number of training epochs
+val_split = 0.01  # percentage of training data used as validation data, between (0,1)
+
+# Learning rate
+initialLearningRate = 0.001  # initial learning rate
+decayRate = 0.5  # exponential decay rate
+decayEpochs_nn = 10  # decay steps
+decayEpochs_sdkn = 5  # decay steps
+doStairCase = True  # Use stepwise instead of continuous exp. decay
+
+
+# Tiz settings
+num_workers=4
+log_every_n_steps=50
+progress_bar_refresh_rate=20
\ No newline at end of file

utils/utilities.py

+# Define a dataset class
+import math
+
+import torch
+from torch.utils.data import DataLoader
+from torch.utils.data import Dataset
+
+from utils import kernels
+from utils import settings
+
+
+class TorchDataset(Dataset):
+    '''
+    Implementation of a custom dataset class.
+    '''
+
+    def __init__(self, data_input, data_output):
+        self.data_input = data_input
+        self.data_output = data_output
+
+    def __len__(self):
+        return len(self.data_input)
+
+    def __getitem__(self, idx):
+        return (self.data_input[idx], self.data_output[idx])
+
+
+
+def get_DataLoader(some_unused_imput=0):
+    '''
+    Implementation of a function that returns dataloaders, which can be used via pytorch-lightning.
+    :param some_unused_imput: Any meaningful input can implemented here
+    :return: train-, validation- and test-loader
+    '''
+
+    # Just use some random points and learn the projection onto the first two dimensions squared
+    train_dataset = torch.rand(10000, 5)
+    train_labels = train_dataset[:, :2]**2
+
+    test_dataset = torch.rand(2000, 5)
+    test_labels = test_dataset[:, :2]**2
+
+
+    # Create torchdatasets
+    dataset_train0 = TorchDataset(data_input=train_dataset, data_output=train_labels)
+    dataset_test = TorchDataset(data_input=test_dataset, data_output=test_labels)
+
+    # Split of dataset
+    train_size = int((1 - settings.val_split) * len(dataset_train0))
+    val_size = len(dataset_train0) - train_size
+    dataset_train, dataset_val = torch.utils.data.random_split(dataset_train0, [train_size, val_size])
+
+    # Define dataloaders
+    train_loader = DataLoader(dataset_train, batch_size=settings.batch_size,
+                              num_workers=settings.num_workers)
+    val_loader = DataLoader(dataset_val, batch_size=settings.batch_size,
+                            num_workers=settings.num_workers)
+    test_loader = DataLoader(dataset_test, batch_size=settings.batch_size,
+                             num_workers=settings.num_workers)
+
+    return train_loader, val_loader, test_loader
+
+
+
+
+class ActivFunc(torch.nn.Module):
+    '''
+    Implementation of the single-dimensional kernel layers of the SDKN, which can be viewed
+    as optimizable activation function layers.
+    '''
+
+    def __init__(self, in_features, nr_centers, kernel=None):
+        super().__init__()
+
+        # General stuff
+        self.in_features = in_features
+        self.nr_centers = nr_centers
+        self.nr_centers_id = nr_centers  # number of centers + maybe additional dimension for identity
+
+        # Define kernel if not given
+        if kernel is None:
+            self.kernel = kernels.Wendland_order_0(ep=1)
+        else:
+            self.kernel = kernel
+
+        # Weight parameters
+        self.weight = torch.nn.Parameter(torch.Tensor(self.in_features, self.nr_centers_id))
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        torch.nn.init.kaiming_uniform_(self.weight, a=math.sqrt(5))
+        self.weight.data += .2 * torch.ones(self.weight.data.shape)  # provide some positive mean
+
+    def forward(self, x, centers):
+        cx = torch.cat((centers, x), 0)
+
+        dist_matrix = torch.abs(torch.unsqueeze(cx, 2) - centers.t().view(1, centers.shape[1], self.nr_centers))
+        kernel_matrix = self.kernel.rbf(self.kernel.ep, dist_matrix)
+        cx = torch.sum(kernel_matrix * self.weight, dim=2)
+
+        centers = cx[:self.nr_centers, :]
+        x = cx[self.nr_centers:, :]
+
+        return x, centers
+