# Module `net`

Multifidelity Surrogate Modeling via Directed Networks

Author: Alex Gorodetsky, goroda@umich.edu

Copyright (c) 2020, Alex Gorodetsky

Expand source code
``````"""  Multifidelity Surrogate Modeling via Directed Networks

Author: Alex Gorodetsky, goroda@umich.edu

Copyright (c) 2020, Alex Gorodetsky

"""

import copy
try:
import queue.SimpleQueue as SimpleQueue
except ImportError:
from queue import Queue as SimpleQueue
from functools import partial

import numpy as np

import networkx as nx
# print("NetworkX Version = ", nx.__version__)

import scipy.optimize as sciopt

from functools import partial

pyapprox_is_installed = True
try:
from pyapprox.l1_minimization import nonlinear_basis_pursuit, lasso
except ModuleNotFoundError:
pyapprox_is_installed = False

def least_squares(target, predicted, std=1e0):
""" Evaluate the least squares objective function

Parameters
----------
target : np.ndarray (nobs)
The observations

predicted : np.ndarray (nobs)
The model predictions of the observations

std : float
The standard deviation of the I.I.D noise

Returns
-------
obj : float
The value of the least squares objective function

grad : np.ndarray (nobs)
The gradient of ``obj``
"""
resid = target - predicted
obj = np.dot(resid, resid) * 0.5 * std**-2
grad = - std**-2 * resid

def lin(param, xinput):
"""A linear parametric model

Parameters
----------
param : np.ndarray (nparams)
The parameters of the model

xinput : np.ndarray (nsamples,nparams)
The independent variables of the model

Returns
-------
vals : np.ndarray (nsamples)
Evaluation of the linear model

grad : np.ndarray (nsamples,nparams)
gradient of the linear model with respect to the model parameters
"""
one = np.ones((xinput.shape, 1))
grad = np.concatenate((one, xinput), axis=1)
return param + np.dot(param[1:], xinput.T), grad

def monomial_1d_lin(param, xinput):
"""Linear Model with Monomial basis functions

p+sum(x**p[1:])

Parameters
----------
param : np.ndarray (nparams)
The parameters of the model

xinput : np.ndarray (nsamples,nparams)
The independent variables of the model

Returns
-------
vals : np.ndarray (nsamples)
Evaluation of the linear model

grad : np.ndarray (nsamples,nparams)
gradient of the linear model with respect to the model parameters
"""
basis = xinput**np.arange(param.shape)[np.newaxis, :]
vals = basis.dot(param)

class MFSurrogate():
"""Multifidelity surrogate

A surrogate consists of a graph where the edges and nodes are functions
Each node represents a particular information sources and the edges
describe the relationships between the information sources

"""
def __init__(self, graph, roots, copy_data=True):
"""Initialize a multifidelity surrogate by providing a graph and the
roots of the graph

Parameters
----------
graph : networkx.graph
The graphical representation of the MF network

roots : list
The ids of every root node in the network

copy_data : boolean
True - perform a deep copy of graph and roots
False - just use a shallow copy (this is dangerous as many functions
change the internal shape)
"""
if copy_data:
self.roots = copy.deepcopy(roots)
self.graph = copy.deepcopy(graph)
else:
self.roots = roots
self.graph = graph
print('warning MFSurrogate not copying data. proceed with caution')
self.nparam = graph_to_vec(self.graph).shape

def get_nparam(self):
"""Number of parameters parameterizing the graph

Returns
-------
nparam : integer
The number of all the unknown parameters in the MF surrogate"""
return self.nparam

def forward(self, xinput, target_node):
"""Evaluate the surrogate output at target_node by considering the
subgraph of all ancestors of this node

Parameters
----------
xinput : np.ndarray (nsamples,nparams)
The independent variables of the model

target_node : integer
The id of the node under consideration

Returns
-------
This function adds the following attributes to the underlying graph

eval :
stores the evaluation of the function represented by the
particular node / edge the evaluations at the nodes are
cumulative (summing up all ancestors) whereas the edges are local

internal attribute needed for accounting

parents-left : set
internal attribute needed for accounting

"""
anc = nx.ancestors(self.graph, target_node)
anc_and_target = anc.union(set([target_node]))
relevant_root_nodes = anc.intersection(self.roots)

# Evaluate the target nodes and all ancestral nodes and put the root
# nodes in a queue
queue = SimpleQueue()
for node in anc_and_target:
pval, pgrad = self.graph.nodes[node]['func'](
self.graph.nodes[node]['param'], xinput)
self.graph.nodes[node]['eval'] = pval
self.graph.nodes[node]['parents_left'] = set(
self.graph.predecessors(node))

if node in relevant_root_nodes:
queue.put(node)

while not queue.empty():

node = queue.get()
feval = self.graph.nodes[node]['eval']
for child in self.graph.successors(node):
if child in anc_and_target:
pval, pgrad = self.graph.edges[node, child]['func'](
self.graph.edges[node, child]['param'], xinput)

self.graph.nodes[child]['eval'] += feval * pval

ftile = np.tile(feval.reshape(
(feval.shape, 1)), (1, pgrad.shape))
self.graph.edges[node, child]['eval'] = pval

self.graph.nodes[child]['parents_left'].remove(node)

if self.graph.nodes[child]['parents_left'] == set():
queue.put(child)

return self.graph.nodes[node]['eval'], anc

def backward(self, target_node, deriv_pass, ancestors=None):
"""Perform a backward computation to compute the derivatives for all
parameters that affect the target node

Parameters
----------
target_node : integer
The id of the node under consideration

deriv_pass : np.ndarray (nparams)

Returns
-------
derivative : np.ndarray(nparams)
A vector containing the derivative of all parameters
"""

if ancestors is None:
ancestors = nx.ancestors(self.graph, target_node)

anc_and_target = ancestors.union(set([target_node]))

# Evaluate the node
self.graph.nodes[target_node]['pass_down'] = deriv_pass

# Gradient with respect to beta
self.graph.nodes[target_node]['derivative'] = \
np.dot(self.graph.nodes[target_node]['pass_down'],

queue = SimpleQueue()
queue.put(target_node)

for node in ancestors:
self.graph.nodes[node]['children_left'] = set(
self.graph.successors(node)).intersection(anc_and_target)
self.graph.nodes[node]['pass_down'] = 0.0
self.graph.nodes[node]['derivative'] = 0.0

while not queue.empty():
node = queue.get()

pass_down = self.graph.nodes[node]['pass_down']
for parent in self.graph.predecessors(node):
self.graph.nodes[parent]['pass_down'] += \
pass_down * self.graph.edges[parent, node]['eval']
self.graph.edges[parent, node]['derivative'] = \
self.graph.nodes[parent]['derivative'] += \
np.dot(pass_down * self.graph.edges[parent, node]['eval'],

self.graph.nodes[parent]['children_left'].remove(node)
if self.graph.nodes[parent]['children_left'] == set():
queue.put(parent)

return self.get_derivative()

def set_param(self, param):
"""Set the parameters for the graph

Parameters
----------
param : np.ndarray (nparams)
A flattened array containing all parameters of the MF surrogate
"""
self.graph = vec_to_graph(param, self.graph, attribute='param')

def get_param(self):
"""Get the parameters of the graph """
return graph_to_vec(self.graph, attribute='param')

def get_derivative(self):
"""Get a vector of derivatives of each parameter """
return graph_to_vec(self.graph, attribute='derivative')

def get_evals(self):
"""Get the evaluations at each node """
return [self.graph.nodes[node]['eval'] for node in self.graph.nodes]

def zero_derivatives(self):
"""Set all the derivative attributes to zero

Used prior to computing a new derivative to clear out previous sweep
"""
self.graph = vec_to_graph(
np.zeros(self.nparam), self.graph, attribute='derivative')

def zero_attributes(self):
"""Zero all attributes except 'func' and 'param' """

atts = ['eval', 'pass_down', 'pre_grad', 'derivative',
'children_left', 'parents_left']
for att in atts:
for node in self.graph.nodes:
try:
self.graph.nodes[node][att] = 0.0
except: # what exception is it?
continue
for edge in self.graph.edges:
try:
self.graph.edges[edge][att] = 0.0
except: # what exception is it?
continue

def train(self, param0in, nodes, xtrain, ytrain, stdtrain, niters=200,
func=least_squares,
verbose=False, warmup=True, opts=dict()):
"""Train the multifidelity surrogate.

This is the main entrance point for data-driven training.

Parameters
----------
param0in : np.ndarray (nparams)
The initial guess for the parameters

nodes : list
A list of nodes for which data is available

xtrain : list
A list of input features for each node in *nodes*

ytrain : list
A list of output values for each node in *nodes*

stdtrain : float
The standard devaition for data for each node in *nodes*

niters : integer
The number of optimization iterations

func : callable
A scalar valued objective function with the signature

``func(target, predicted) ->  val (float), grad (np.ndarray)``

where ``target`` is a np.ndarray of shape (nobs)
containing the observations and ``predicted`` is a np.ndarray of
shape (nobs) containing the model predictions of the observations

verbose : integer
The verbosity level

warmup : boolean
Specify whether or not to progressively find a good guess before
optimizing

opts : dictionary
Specify the type of loss function: 'lstsq' for squared error, anything else for L1 regularization, 'lambda' for regularization value

Returns
-------
Upon completion of this function, the parameters of the graph are set
to the values that best fit the data, as defined by *func*
"""
bounds = list(zip([-np.inf]*self.nparam, [np.inf]*self.nparam))
param0 = copy.deepcopy(param0in)

# options = {'maxiter':20, 'disp':False, 'gtol':1e-10, 'ftol':1e-18}
options = {'maxiter':20, 'disp':False, 'gtol':1e-10, 'ftol':1e-18}

# Warming up
if warmup is True:
for node in nodes:

node_list = nodes[node-1:node]
x_list = xtrain[node-1:node]
y_list = ytrain[node-1:node]
std_list = stdtrain[node-1:node]

res = sciopt.minimize(
optimize_obj, param0,
args=(func, self, node_list, x_list, y_list, std_list),
method='L-BFGS-B', jac=True, bounds=bounds,
options=options)

param0 = res.x
for ii in range(self.nparam):
if np.abs(param0[ii]) > 1e-10:
bounds[ii] = (param0[ii]-1e-10, param0[ii]+1e-10)
# print("bounds", bounds)

# Final Training
lossfunc = opts.get('lossfunc','lstsq')
if lossfunc == 'lstsq':
options = {'maxiter':niters, 'disp':verbose, 'gtol':1e-10}
res = sciopt.minimize(
optimize_obj, param0,
args=(func, self, nodes, xtrain, ytrain, stdtrain),
method='L-BFGS-B', jac=True,
options=options)
elif pyapprox_is_installed is True:

obj = partial(
optimize_obj,optf=least_squares,graph=self,nodes=nodes,
xin_l=xtrain, yin_l=ytrain, std_l=stdtrain)
lamda = opts['lambda']
options = {'ftol':1e-12,'disp':False,
'maxiter':1e3, 'method':'slsqp'};
l1_coef, res = lasso(obj,True,None,param0,lamda,options)
#res.x includes slack variables so remove these
res.x=l1_coef
else:
raise Exception("Specified loss is not accepted")

self.set_param(res.x)
return self

def graph_to_vec(graph, attribute='param'):
"""Extract the multifidelity surrogate parameters from the graph

Parameters
----------
graph : networkx.graph
The graphical representation of the MF network

Returns
-------
vec : np.ndarray (nparams)
A flattened array containing all the parameters of the MF network
"""
nodes = graph.nodes
node_params_dict = nx.get_node_attributes(graph, attribute)
node_params = np.concatenate([node_params_dict[n] for n in nodes])

edges = graph.edges
edge_params_dict = nx.get_edge_attributes(graph, attribute)
edge_params = np.concatenate([edge_params_dict[e] for e in edges])

return np.concatenate((node_params, edge_params))

def vec_to_graph(vec, graph, attribute='param'):
"""
Update the parameters of a multifidelity surrogate

Parameters
----------
vec : np.ndarray (nparams)
A flattened array containing all the parameters of the MF network

graph : networkx.graph
The graphical representation of the MF network

Returns
-------
graph : networkx.graph
The updated graphical representation of the MF network with the
parameter values given by ``vec``.
"""
nodes = graph.nodes
ind = 0
for node in nodes:
try:
offset = graph.nodes[node][attribute].shape
except: # What is the exception?
offset = graph.nodes[node]['param'].shape
graph.nodes[node][attribute] = vec[ind:ind + offset]
ind = ind + offset

edges = graph.edges
for edge in edges:
try:
offset = graph.edges[edge][attribute].shape
except: # What is the exception?
offset = graph.edges[edge]['param'].shape

graph.edges[edge][attribute] = vec[ind:ind + offset]
ind = ind + offset

return graph

def optimize_obj(param, optf, graph, nodes, xin_l, yin_l, std_l):
"""Composite optimization objective for a set of nodes

Parameters
----------
param : np.ndarray (nparams)
The parameter values at which to compute the objective value and

optf : callable
A scalar valued objective function with the signature

``optf(target, predicted) ->  float``

where ``target`` is a np.ndarray of shape (nobs)
containing the observations and ``predicted`` is a np.ndarray of
shape (nobs) containing the model predictions of the observations

graph : networkx.graph
The graphical representation of the MF network

nodes : list
A list of nodes for which data is available

xin_l : list
A list of input features for each node in *nodes*

yin_l : list
A list of output values for each node in *nodes*

std_l : float
The standard devaition for data for each node in *nodes*

Returns
-------
final_val : float
The value of the least squares objective function

final_derivative : np.ndarray (nobs)
The gradient of ``obj``
"""
final_derivative = np.zeros((param.shape))
final_val = 0.0

# print("nodes =", nodes)
for node, xin, yout, std in zip(nodes, xin_l, yin_l, std_l):
graph.zero_attributes()
graph.zero_derivatives()
graph.set_param(param)
val, anc = graph.forward(xin, node)

## optimization function takes
new_val, obj_grad = optf(yout, val, std=std)
derivative = graph.backward(node, obj_grad, ancestors=anc)

final_val += new_val
final_derivative += derivative

return final_val, final_derivative

def learn_obj(param, graph, node, x, y, std):
"""
Return the least squares learning objective function

Parameters
----------
param : np.ndarray (nparams)
The parameter values at which to compute the objective value and

graph : networkx.graph
The graphical representation of the MF network

nodes : list
A list of nodes for which data is available

x : list
A list of input features for each node in *nodes*

y : list
A list of output values for each node in *nodes*

std : float
The standard devaition for data for each node in *nodes*

Returns
-------
final_val : float
The value of the least squares objective function
"""
graph.set_param(param)
predict, _ = graph.forward(x, node)
val, _ = least_squares(y, predict, std=std)

return val

def learn_obj_grad(param, graph, node, x, y, std):
"""
Return the gradient of the least squares learning objective function

Parameters
----------
param : np.ndarray (nparams)
The parameter values at which to compute the objective value and

graph : networkx.graph
The graphical representation of the MF network

nodes : list
A list of nodes for which data is available

x : list
A list of input features for each node in *nodes*

y : list
A list of output values for each node in *nodes*

std : float
The standard devaition for data for each node in *nodes*

Returns
-------
final_derivative : np.ndarray (nobs)
The gradient of the objective
"""
graph.zero_derivatives()
graph.set_param(param)
predict, anc = graph.forward(x, node)
_, grad = least_squares(y, predict, std=std)
return graph.get_derivative()

def learn_obj_grad_both(param, graph, node, x, y, std):
"""
Return the value and gradient of the least squares learning objective
function

Parameters
----------
param : np.ndarray (nparams)
The parameter values at which to compute the objective value and

graph : networkx.graph
The graphical representation of the MF network

nodes : list
A list of nodes for which data is available

x : list
A list of input features for each node in *nodes*

y : list
A list of output values for each node in *nodes*

std : float
The standard devaition for data for each node in *nodes*

Returns
-------
val : float
The value of the least squares objective function

final_derivative : np.ndarray (nobs)
The gradient of the objective
"""

graph.zero_derivatives()
graph.set_param(param)
predict, A = graph.forward(x, node)
# print("predict = ", predict.shape)
# print("y = ", y.shape)
val, grad = least_squares(y, predict, std=std)

return val, graph.get_derivative()

#--------------------------------#
# Functions useful for debugging #

def identity(ynotused, predict, std=None):
""" Identity output function """
# f(predict) = predict
return predict, np.ones(predict.shape)

def identity_obj(param, graph, node, x):

graph.set_param(param)
predict, _ = graph.forward(x, node)
return predict

def identity_obj_grad(param, graph, node, x):

graph.zero_derivatives()
graph.set_param(param)
predict, A = graph.forward(x, node)
# print("predict = ", predict)
_, pass_back = identity(None, predict, std=None)
graph.backward(node, pass_back, ancestors=A)

return graph.get_derivative()``````

## Functions

``` def graph_to_vec(graph, attribute='param') ```

Extract the multifidelity surrogate parameters from the graph

## Parameters

`graph` : `networkx.graph`
The graphical representation of the MF network

## Returns

``````vec : np.ndarray (nparams)
A flattened array containing all the parameters of the MF network
``````
Expand source code
``````def graph_to_vec(graph, attribute='param'):
"""Extract the multifidelity surrogate parameters from the graph

Parameters
----------
graph : networkx.graph
The graphical representation of the MF network

Returns
-------
vec : np.ndarray (nparams)
A flattened array containing all the parameters of the MF network
"""
nodes = graph.nodes
node_params_dict = nx.get_node_attributes(graph, attribute)
node_params = np.concatenate([node_params_dict[n] for n in nodes])

edges = graph.edges
edge_params_dict = nx.get_edge_attributes(graph, attribute)
edge_params = np.concatenate([edge_params_dict[e] for e in edges])

return np.concatenate((node_params, edge_params))``````
``` def identity(ynotused, predict, std=None) ```

Identity output function

Expand source code
``````def identity(ynotused, predict, std=None):
""" Identity output function """
# f(predict) = predict
return predict, np.ones(predict.shape)``````
``` def identity_obj(param, graph, node, x) ```
Expand source code
``````def identity_obj(param, graph, node, x):

graph.set_param(param)
predict, _ = graph.forward(x, node)
return predict``````
``` def identity_obj_grad(param, graph, node, x) ```
Expand source code
``````def identity_obj_grad(param, graph, node, x):

graph.zero_derivatives()
graph.set_param(param)
predict, A = graph.forward(x, node)
# print("predict = ", predict)
_, pass_back = identity(None, predict, std=None)
graph.backward(node, pass_back, ancestors=A)

return graph.get_derivative()``````
``` def learn_obj(param, graph, node, x, y, std) ```

Return the least squares learning objective function

## Parameters

`param` : `np.ndarray (nparams)`
The parameter values at which to compute the objective value and gradient
`graph` : `networkx.graph`
The graphical representation of the MF network
`nodes` : `list `
A list of nodes for which data is available
`x` : `list`
A list of input features for each node in nodes
`y` : `list `
A list of output values for each node in nodes
`std` : `float`
The standard devaition for data for each node in nodes

## Returns

`final_val` : `float`
The value of the least squares objective function
Expand source code
``````def learn_obj(param, graph, node, x, y, std):
"""
Return the least squares learning objective function

Parameters
----------
param : np.ndarray (nparams)
The parameter values at which to compute the objective value and

graph : networkx.graph
The graphical representation of the MF network

nodes : list
A list of nodes for which data is available

x : list
A list of input features for each node in *nodes*

y : list
A list of output values for each node in *nodes*

std : float
The standard devaition for data for each node in *nodes*

Returns
-------
final_val : float
The value of the least squares objective function
"""
graph.set_param(param)
predict, _ = graph.forward(x, node)
val, _ = least_squares(y, predict, std=std)

return val``````
``` def learn_obj_grad(param, graph, node, x, y, std) ```

Return the gradient of the least squares learning objective function

## Parameters

`param` : `np.ndarray (nparams)`
The parameter values at which to compute the objective value and gradient
`graph` : `networkx.graph`
The graphical representation of the MF network
`nodes` : `list`
A list of nodes for which data is available
`x` : `list`
A list of input features for each node in nodes
`y` : `list`
A list of output values for each node in nodes
`std` : `float`
The standard devaition for data for each node in nodes

## Returns

`final_derivative` : `np.ndarray (nobs)`
The gradient of the objective
Expand source code
``````def learn_obj_grad(param, graph, node, x, y, std):
"""
Return the gradient of the least squares learning objective function

Parameters
----------
param : np.ndarray (nparams)
The parameter values at which to compute the objective value and

graph : networkx.graph
The graphical representation of the MF network

nodes : list
A list of nodes for which data is available

x : list
A list of input features for each node in *nodes*

y : list
A list of output values for each node in *nodes*

std : float
The standard devaition for data for each node in *nodes*

Returns
-------
final_derivative : np.ndarray (nobs)
The gradient of the objective
"""
graph.zero_derivatives()
graph.set_param(param)
predict, anc = graph.forward(x, node)
_, grad = least_squares(y, predict, std=std)
return graph.get_derivative()``````
``` def learn_obj_grad_both(param, graph, node, x, y, std) ```

Return the value and gradient of the least squares learning objective function

## Parameters

`param` : `np.ndarray (nparams)`
The parameter values at which to compute the objective value and gradient
`graph` : `networkx.graph`
The graphical representation of the MF network
`nodes` : `list `
A list of nodes for which data is available
`x` : `list`
A list of input features for each node in nodes
`y` : `list `
A list of output values for each node in nodes
`std` : `float`
The standard devaition for data for each node in nodes

## Returns

`val` : `float`
The value of the least squares objective function
`final_derivative` : `np.ndarray (nobs)`
The gradient of the objective
Expand source code
``````def learn_obj_grad_both(param, graph, node, x, y, std):
"""
Return the value and gradient of the least squares learning objective
function

Parameters
----------
param : np.ndarray (nparams)
The parameter values at which to compute the objective value and

graph : networkx.graph
The graphical representation of the MF network

nodes : list
A list of nodes for which data is available

x : list
A list of input features for each node in *nodes*

y : list
A list of output values for each node in *nodes*

std : float
The standard devaition for data for each node in *nodes*

Returns
-------
val : float
The value of the least squares objective function

final_derivative : np.ndarray (nobs)
The gradient of the objective
"""

graph.zero_derivatives()
graph.set_param(param)
predict, A = graph.forward(x, node)
# print("predict = ", predict.shape)
# print("y = ", y.shape)
val, grad = least_squares(y, predict, std=std)

return val, graph.get_derivative()``````
``` def least_squares(target, predicted, std=1.0) ```

Evaluate the least squares objective function

## Parameters

`target` : `np.ndarray (nobs)`
The observations
`predicted` : `np.ndarray (nobs)`
The model predictions of the observations
`std` : `float`
The standard deviation of the I.I.D noise

## Returns

`obj` : `float`
The value of the least squares objective function
`grad` : `np.ndarray (nobs)`
The gradient of `obj`
Expand source code
``````def least_squares(target, predicted, std=1e0):
""" Evaluate the least squares objective function

Parameters
----------
target : np.ndarray (nobs)
The observations

predicted : np.ndarray (nobs)
The model predictions of the observations

std : float
The standard deviation of the I.I.D noise

Returns
-------
obj : float
The value of the least squares objective function

grad : np.ndarray (nobs)
The gradient of ``obj``
"""
resid = target - predicted
obj = np.dot(resid, resid) * 0.5 * std**-2
grad = - std**-2 * resid
``` def lin(param, xinput) ```

A linear parametric model

## Parameters

`param` : `np.ndarray (nparams)`

The parameters of the model

`xinput` : `np.ndarray (nsamples,nparams)`

The independent variables of the model

## Returns

`vals` : `np.ndarray (nsamples)`

Evaluation of the linear model

`grad` : `np.ndarray (nsamples,nparams)`

gradient of the linear model with respect to the model parameters

Expand source code
``````def lin(param, xinput):
"""A linear parametric model

Parameters
----------
param : np.ndarray (nparams)
The parameters of the model

xinput : np.ndarray (nsamples,nparams)
The independent variables of the model

Returns
-------
vals : np.ndarray (nsamples)
Evaluation of the linear model

grad : np.ndarray (nsamples,nparams)
gradient of the linear model with respect to the model parameters
"""
one = np.ones((xinput.shape, 1))
grad = np.concatenate((one, xinput), axis=1)
return param + np.dot(param[1:], xinput.T), grad``````
``` def monomial_1d_lin(param, xinput) ```

Linear Model with Monomial basis functions

p+sum(x**p[1:])

## Parameters

`param` : `np.ndarray (nparams)`

The parameters of the model

`xinput` : `np.ndarray (nsamples,nparams)`

The independent variables of the model

## Returns

`vals` : `np.ndarray (nsamples)`
Evaluation of the linear model
`grad` : `np.ndarray (nsamples,nparams)`

gradient of the linear model with respect to the model parameters

Expand source code
``````def monomial_1d_lin(param, xinput):
"""Linear Model with Monomial basis functions

p+sum(x**p[1:])

Parameters
----------
param : np.ndarray (nparams)
The parameters of the model

xinput : np.ndarray (nsamples,nparams)
The independent variables of the model

Returns
-------
vals : np.ndarray (nsamples)
Evaluation of the linear model

grad : np.ndarray (nsamples,nparams)
gradient of the linear model with respect to the model parameters
"""
basis = xinput**np.arange(param.shape)[np.newaxis, :]
vals = basis.dot(param)
``` def optimize_obj(param, optf, graph, nodes, xin_l, yin_l, std_l) ```

Composite optimization objective for a set of nodes

## Parameters

`param` : `np.ndarray (nparams)`
The parameter values at which to compute the objective value and gradient
`optf` : `callable`

A scalar valued objective function with the signature

```optf(target, predicted) -> float```

where `target` is a np.ndarray of shape (nobs) containing the observations and `predicted` is a np.ndarray of shape (nobs) containing the model predictions of the observations

`graph` : `networkx.graph`
The graphical representation of the MF network
`nodes` : `list`
A list of nodes for which data is available
`xin_l` : `list`
A list of input features for each node in nodes
`yin_l` : `list`
A list of output values for each node in nodes
`std_l` : `float`
The standard devaition for data for each node in nodes

## Returns

`final_val` : `float`
The value of the least squares objective function
`final_derivative` : `np.ndarray (nobs)`
The gradient of `obj`
Expand source code
``````def optimize_obj(param, optf, graph, nodes, xin_l, yin_l, std_l):
"""Composite optimization objective for a set of nodes

Parameters
----------
param : np.ndarray (nparams)
The parameter values at which to compute the objective value and

optf : callable
A scalar valued objective function with the signature

``optf(target, predicted) ->  float``

where ``target`` is a np.ndarray of shape (nobs)
containing the observations and ``predicted`` is a np.ndarray of
shape (nobs) containing the model predictions of the observations

graph : networkx.graph
The graphical representation of the MF network

nodes : list
A list of nodes for which data is available

xin_l : list
A list of input features for each node in *nodes*

yin_l : list
A list of output values for each node in *nodes*

std_l : float
The standard devaition for data for each node in *nodes*

Returns
-------
final_val : float
The value of the least squares objective function

final_derivative : np.ndarray (nobs)
The gradient of ``obj``
"""
final_derivative = np.zeros((param.shape))
final_val = 0.0

# print("nodes =", nodes)
for node, xin, yout, std in zip(nodes, xin_l, yin_l, std_l):
graph.zero_attributes()
graph.zero_derivatives()
graph.set_param(param)
val, anc = graph.forward(xin, node)

## optimization function takes
new_val, obj_grad = optf(yout, val, std=std)
derivative = graph.backward(node, obj_grad, ancestors=anc)

final_val += new_val
final_derivative += derivative

return final_val, final_derivative``````
``` def vec_to_graph(vec, graph, attribute='param') ```

Update the parameters of a multifidelity surrogate

## Parameters

`vec` : `np.ndarray (nparams)`
A flattened array containing all the parameters of the MF network
`graph` : `networkx.graph`
The graphical representation of the MF network

## Returns

`graph` : `networkx.graph`
The updated graphical representation of the MF network with the parameter values given by `vec`.
Expand source code
``````def vec_to_graph(vec, graph, attribute='param'):
"""
Update the parameters of a multifidelity surrogate

Parameters
----------
vec : np.ndarray (nparams)
A flattened array containing all the parameters of the MF network

graph : networkx.graph
The graphical representation of the MF network

Returns
-------
graph : networkx.graph
The updated graphical representation of the MF network with the
parameter values given by ``vec``.
"""
nodes = graph.nodes
ind = 0
for node in nodes:
try:
offset = graph.nodes[node][attribute].shape
except: # What is the exception?
offset = graph.nodes[node]['param'].shape
graph.nodes[node][attribute] = vec[ind:ind + offset]
ind = ind + offset

edges = graph.edges
for edge in edges:
try:
offset = graph.edges[edge][attribute].shape
except: # What is the exception?
offset = graph.edges[edge]['param'].shape

graph.edges[edge][attribute] = vec[ind:ind + offset]
ind = ind + offset

return graph``````

## Classes

``` class MFSurrogate (graph, roots, copy_data=True) ```

Multifidelity surrogate

A surrogate consists of a graph where the edges and nodes are functions Each node represents a particular information sources and the edges describe the relationships between the information sources

Initialize a multifidelity surrogate by providing a graph and the roots of the graph

## Parameters

`graph` : `networkx.graph`
The graphical representation of the MF network
`roots` : `list`
The ids of every root node in the network
`copy_data` : `boolean`

True - perform a deep copy of graph and roots False - just use a shallow copy (this is dangerous as many functions change the internal shape)

Expand source code
``````class MFSurrogate():
"""Multifidelity surrogate

A surrogate consists of a graph where the edges and nodes are functions
Each node represents a particular information sources and the edges
describe the relationships between the information sources

"""
def __init__(self, graph, roots, copy_data=True):
"""Initialize a multifidelity surrogate by providing a graph and the
roots of the graph

Parameters
----------
graph : networkx.graph
The graphical representation of the MF network

roots : list
The ids of every root node in the network

copy_data : boolean
True - perform a deep copy of graph and roots
False - just use a shallow copy (this is dangerous as many functions
change the internal shape)
"""
if copy_data:
self.roots = copy.deepcopy(roots)
self.graph = copy.deepcopy(graph)
else:
self.roots = roots
self.graph = graph
print('warning MFSurrogate not copying data. proceed with caution')
self.nparam = graph_to_vec(self.graph).shape

def get_nparam(self):
"""Number of parameters parameterizing the graph

Returns
-------
nparam : integer
The number of all the unknown parameters in the MF surrogate"""
return self.nparam

def forward(self, xinput, target_node):
"""Evaluate the surrogate output at target_node by considering the
subgraph of all ancestors of this node

Parameters
----------
xinput : np.ndarray (nsamples,nparams)
The independent variables of the model

target_node : integer
The id of the node under consideration

Returns
-------
This function adds the following attributes to the underlying graph

eval :
stores the evaluation of the function represented by the
particular node / edge the evaluations at the nodes are
cumulative (summing up all ancestors) whereas the edges are local

internal attribute needed for accounting

parents-left : set
internal attribute needed for accounting

"""
anc = nx.ancestors(self.graph, target_node)
anc_and_target = anc.union(set([target_node]))
relevant_root_nodes = anc.intersection(self.roots)

# Evaluate the target nodes and all ancestral nodes and put the root
# nodes in a queue
queue = SimpleQueue()
for node in anc_and_target:
pval, pgrad = self.graph.nodes[node]['func'](
self.graph.nodes[node]['param'], xinput)
self.graph.nodes[node]['eval'] = pval
self.graph.nodes[node]['parents_left'] = set(
self.graph.predecessors(node))

if node in relevant_root_nodes:
queue.put(node)

while not queue.empty():

node = queue.get()
feval = self.graph.nodes[node]['eval']
for child in self.graph.successors(node):
if child in anc_and_target:
pval, pgrad = self.graph.edges[node, child]['func'](
self.graph.edges[node, child]['param'], xinput)

self.graph.nodes[child]['eval'] += feval * pval

ftile = np.tile(feval.reshape(
(feval.shape, 1)), (1, pgrad.shape))
self.graph.edges[node, child]['eval'] = pval

self.graph.nodes[child]['parents_left'].remove(node)

if self.graph.nodes[child]['parents_left'] == set():
queue.put(child)

return self.graph.nodes[node]['eval'], anc

def backward(self, target_node, deriv_pass, ancestors=None):
"""Perform a backward computation to compute the derivatives for all
parameters that affect the target node

Parameters
----------
target_node : integer
The id of the node under consideration

deriv_pass : np.ndarray (nparams)

Returns
-------
derivative : np.ndarray(nparams)
A vector containing the derivative of all parameters
"""

if ancestors is None:
ancestors = nx.ancestors(self.graph, target_node)

anc_and_target = ancestors.union(set([target_node]))

# Evaluate the node
self.graph.nodes[target_node]['pass_down'] = deriv_pass

# Gradient with respect to beta
self.graph.nodes[target_node]['derivative'] = \
np.dot(self.graph.nodes[target_node]['pass_down'],

queue = SimpleQueue()
queue.put(target_node)

for node in ancestors:
self.graph.nodes[node]['children_left'] = set(
self.graph.successors(node)).intersection(anc_and_target)
self.graph.nodes[node]['pass_down'] = 0.0
self.graph.nodes[node]['derivative'] = 0.0

while not queue.empty():
node = queue.get()

pass_down = self.graph.nodes[node]['pass_down']
for parent in self.graph.predecessors(node):
self.graph.nodes[parent]['pass_down'] += \
pass_down * self.graph.edges[parent, node]['eval']
self.graph.edges[parent, node]['derivative'] = \
self.graph.nodes[parent]['derivative'] += \
np.dot(pass_down * self.graph.edges[parent, node]['eval'],

self.graph.nodes[parent]['children_left'].remove(node)
if self.graph.nodes[parent]['children_left'] == set():
queue.put(parent)

return self.get_derivative()

def set_param(self, param):
"""Set the parameters for the graph

Parameters
----------
param : np.ndarray (nparams)
A flattened array containing all parameters of the MF surrogate
"""
self.graph = vec_to_graph(param, self.graph, attribute='param')

def get_param(self):
"""Get the parameters of the graph """
return graph_to_vec(self.graph, attribute='param')

def get_derivative(self):
"""Get a vector of derivatives of each parameter """
return graph_to_vec(self.graph, attribute='derivative')

def get_evals(self):
"""Get the evaluations at each node """
return [self.graph.nodes[node]['eval'] for node in self.graph.nodes]

def zero_derivatives(self):
"""Set all the derivative attributes to zero

Used prior to computing a new derivative to clear out previous sweep
"""
self.graph = vec_to_graph(
np.zeros(self.nparam), self.graph, attribute='derivative')

def zero_attributes(self):
"""Zero all attributes except 'func' and 'param' """

atts = ['eval', 'pass_down', 'pre_grad', 'derivative',
'children_left', 'parents_left']
for att in atts:
for node in self.graph.nodes:
try:
self.graph.nodes[node][att] = 0.0
except: # what exception is it?
continue
for edge in self.graph.edges:
try:
self.graph.edges[edge][att] = 0.0
except: # what exception is it?
continue

def train(self, param0in, nodes, xtrain, ytrain, stdtrain, niters=200,
func=least_squares,
verbose=False, warmup=True, opts=dict()):
"""Train the multifidelity surrogate.

This is the main entrance point for data-driven training.

Parameters
----------
param0in : np.ndarray (nparams)
The initial guess for the parameters

nodes : list
A list of nodes for which data is available

xtrain : list
A list of input features for each node in *nodes*

ytrain : list
A list of output values for each node in *nodes*

stdtrain : float
The standard devaition for data for each node in *nodes*

niters : integer
The number of optimization iterations

func : callable
A scalar valued objective function with the signature

``func(target, predicted) ->  val (float), grad (np.ndarray)``

where ``target`` is a np.ndarray of shape (nobs)
containing the observations and ``predicted`` is a np.ndarray of
shape (nobs) containing the model predictions of the observations

verbose : integer
The verbosity level

warmup : boolean
Specify whether or not to progressively find a good guess before
optimizing

opts : dictionary
Specify the type of loss function: 'lstsq' for squared error, anything else for L1 regularization, 'lambda' for regularization value

Returns
-------
Upon completion of this function, the parameters of the graph are set
to the values that best fit the data, as defined by *func*
"""
bounds = list(zip([-np.inf]*self.nparam, [np.inf]*self.nparam))
param0 = copy.deepcopy(param0in)

# options = {'maxiter':20, 'disp':False, 'gtol':1e-10, 'ftol':1e-18}
options = {'maxiter':20, 'disp':False, 'gtol':1e-10, 'ftol':1e-18}

# Warming up
if warmup is True:
for node in nodes:

node_list = nodes[node-1:node]
x_list = xtrain[node-1:node]
y_list = ytrain[node-1:node]
std_list = stdtrain[node-1:node]

res = sciopt.minimize(
optimize_obj, param0,
args=(func, self, node_list, x_list, y_list, std_list),
method='L-BFGS-B', jac=True, bounds=bounds,
options=options)

param0 = res.x
for ii in range(self.nparam):
if np.abs(param0[ii]) > 1e-10:
bounds[ii] = (param0[ii]-1e-10, param0[ii]+1e-10)
# print("bounds", bounds)

# Final Training
lossfunc = opts.get('lossfunc','lstsq')
if lossfunc == 'lstsq':
options = {'maxiter':niters, 'disp':verbose, 'gtol':1e-10}
res = sciopt.minimize(
optimize_obj, param0,
args=(func, self, nodes, xtrain, ytrain, stdtrain),
method='L-BFGS-B', jac=True,
options=options)
elif pyapprox_is_installed is True:

obj = partial(
optimize_obj,optf=least_squares,graph=self,nodes=nodes,
xin_l=xtrain, yin_l=ytrain, std_l=stdtrain)
lamda = opts['lambda']
options = {'ftol':1e-12,'disp':False,
'maxiter':1e3, 'method':'slsqp'};
l1_coef, res = lasso(obj,True,None,param0,lamda,options)
#res.x includes slack variables so remove these
res.x=l1_coef
else:
raise Exception("Specified loss is not accepted")

self.set_param(res.x)
return self``````

### Methods

``` def backward(self, target_node, deriv_pass, ancestors=None) ```

Perform a backward computation to compute the derivatives for all parameters that affect the target node

## Parameters

`target_node` : `integer`
The id of the node under consideration
`deriv_pass` : `np.ndarray (nparams)`

## Returns

`derivative` : `np.ndarray(nparams)`
A vector containing the derivative of all parameters
Expand source code
``````def backward(self, target_node, deriv_pass, ancestors=None):
"""Perform a backward computation to compute the derivatives for all
parameters that affect the target node

Parameters
----------
target_node : integer
The id of the node under consideration

deriv_pass : np.ndarray (nparams)

Returns
-------
derivative : np.ndarray(nparams)
A vector containing the derivative of all parameters
"""

if ancestors is None:
ancestors = nx.ancestors(self.graph, target_node)

anc_and_target = ancestors.union(set([target_node]))

# Evaluate the node
self.graph.nodes[target_node]['pass_down'] = deriv_pass

# Gradient with respect to beta
self.graph.nodes[target_node]['derivative'] = \
np.dot(self.graph.nodes[target_node]['pass_down'],

queue = SimpleQueue()
queue.put(target_node)

for node in ancestors:
self.graph.nodes[node]['children_left'] = set(
self.graph.successors(node)).intersection(anc_and_target)
self.graph.nodes[node]['pass_down'] = 0.0
self.graph.nodes[node]['derivative'] = 0.0

while not queue.empty():
node = queue.get()

pass_down = self.graph.nodes[node]['pass_down']
for parent in self.graph.predecessors(node):
self.graph.nodes[parent]['pass_down'] += \
pass_down * self.graph.edges[parent, node]['eval']
self.graph.edges[parent, node]['derivative'] = \
self.graph.nodes[parent]['derivative'] += \
np.dot(pass_down * self.graph.edges[parent, node]['eval'],

self.graph.nodes[parent]['children_left'].remove(node)
if self.graph.nodes[parent]['children_left'] == set():
queue.put(parent)

return self.get_derivative()``````
``` def forward(self, xinput, target_node) ```

Evaluate the surrogate output at target_node by considering the subgraph of all ancestors of this node

## Parameters

`xinput` : `np.ndarray (nsamples,nparams)`
The independent variables of the model
`target_node` : `integer`
The id of the node under consideration

## Returns

`This function adds the following attributes to the underlying graph`

`eval :`
stores the evaluation of the function represented by the particular node / edge the evaluations at the nodes are cumulative (summing up all ancestors) whereas the edges are local
`pre-grad : np.ndarray`
internal attribute needed for accounting
`parents-left : set`
internal attribute needed for accounting
Expand source code
``````def forward(self, xinput, target_node):
"""Evaluate the surrogate output at target_node by considering the
subgraph of all ancestors of this node

Parameters
----------
xinput : np.ndarray (nsamples,nparams)
The independent variables of the model

target_node : integer
The id of the node under consideration

Returns
-------
This function adds the following attributes to the underlying graph

eval :
stores the evaluation of the function represented by the
particular node / edge the evaluations at the nodes are
cumulative (summing up all ancestors) whereas the edges are local

internal attribute needed for accounting

parents-left : set
internal attribute needed for accounting

"""
anc = nx.ancestors(self.graph, target_node)
anc_and_target = anc.union(set([target_node]))
relevant_root_nodes = anc.intersection(self.roots)

# Evaluate the target nodes and all ancestral nodes and put the root
# nodes in a queue
queue = SimpleQueue()
for node in anc_and_target:
pval, pgrad = self.graph.nodes[node]['func'](
self.graph.nodes[node]['param'], xinput)
self.graph.nodes[node]['eval'] = pval
self.graph.nodes[node]['parents_left'] = set(
self.graph.predecessors(node))

if node in relevant_root_nodes:
queue.put(node)

while not queue.empty():

node = queue.get()
feval = self.graph.nodes[node]['eval']
for child in self.graph.successors(node):
if child in anc_and_target:
pval, pgrad = self.graph.edges[node, child]['func'](
self.graph.edges[node, child]['param'], xinput)

self.graph.nodes[child]['eval'] += feval * pval

ftile = np.tile(feval.reshape(
(feval.shape, 1)), (1, pgrad.shape))
self.graph.edges[node, child]['eval'] = pval

self.graph.nodes[child]['parents_left'].remove(node)

if self.graph.nodes[child]['parents_left'] == set():
queue.put(child)

return self.graph.nodes[node]['eval'], anc``````
``` def get_derivative(self) ```

Get a vector of derivatives of each parameter

Expand source code
``````def get_derivative(self):
"""Get a vector of derivatives of each parameter """
return graph_to_vec(self.graph, attribute='derivative')``````
``` def get_evals(self) ```

Get the evaluations at each node

Expand source code
``````def get_evals(self):
"""Get the evaluations at each node """
return [self.graph.nodes[node]['eval'] for node in self.graph.nodes]``````
``` def get_nparam(self) ```

Number of parameters parameterizing the graph

## Returns

`nparam` : `integer`
The number of all the unknown parameters in the MF surrogate
Expand source code
``````def get_nparam(self):
"""Number of parameters parameterizing the graph

Returns
-------
nparam : integer
The number of all the unknown parameters in the MF surrogate"""
return self.nparam``````
``` def get_param(self) ```

Get the parameters of the graph

Expand source code
``````def get_param(self):
"""Get the parameters of the graph """
return graph_to_vec(self.graph, attribute='param')``````
``` def set_param(self, param) ```

Set the parameters for the graph

## Parameters

`param` : `np.ndarray (nparams)`
A flattened array containing all parameters of the MF surrogate
Expand source code
``````def set_param(self, param):
"""Set the parameters for the graph

Parameters
----------
param : np.ndarray (nparams)
A flattened array containing all parameters of the MF surrogate
"""
self.graph = vec_to_graph(param, self.graph, attribute='param')``````
``` def train(self, param0in, nodes, xtrain, ytrain, stdtrain, niters=200, func=<function least_squares>, verbose=False, warmup=True, opts={}) ```

Train the multifidelity surrogate.

This is the main entrance point for data-driven training.

## Parameters

`param0in` : `np.ndarray (nparams)`
The initial guess for the parameters
`nodes` : `list`
A list of nodes for which data is available
`xtrain` : `list`
A list of input features for each node in nodes
`ytrain` : `list`
A list of output values for each node in nodes
`stdtrain` : `float`
The standard devaition for data for each node in nodes
`niters` : `integer`
The number of optimization iterations
`func` : `callable`

A scalar valued objective function with the signature

```func(target, predicted) -> val (float), grad (np.ndarray)```

where `target` is a np.ndarray of shape (nobs) containing the observations and `predicted` is a np.ndarray of shape (nobs) containing the model predictions of the observations

`verbose` : `integer`
The verbosity level
`warmup` : `boolean`
Specify whether or not to progressively find a good guess before optimizing
`opts` : `dictionary`
Specify the type of loss function: 'lstsq' for squared error, anything else for L1 regularization, 'lambda' for regularization value

## Returns

`Upon completion` of `this function, the parameters` of `the graph are set`

to the values that best fit the data, as defined by func

Expand source code
``````def train(self, param0in, nodes, xtrain, ytrain, stdtrain, niters=200,
func=least_squares,
verbose=False, warmup=True, opts=dict()):
"""Train the multifidelity surrogate.

This is the main entrance point for data-driven training.

Parameters
----------
param0in : np.ndarray (nparams)
The initial guess for the parameters

nodes : list
A list of nodes for which data is available

xtrain : list
A list of input features for each node in *nodes*

ytrain : list
A list of output values for each node in *nodes*

stdtrain : float
The standard devaition for data for each node in *nodes*

niters : integer
The number of optimization iterations

func : callable
A scalar valued objective function with the signature

``func(target, predicted) ->  val (float), grad (np.ndarray)``

where ``target`` is a np.ndarray of shape (nobs)
containing the observations and ``predicted`` is a np.ndarray of
shape (nobs) containing the model predictions of the observations

verbose : integer
The verbosity level

warmup : boolean
Specify whether or not to progressively find a good guess before
optimizing

opts : dictionary
Specify the type of loss function: 'lstsq' for squared error, anything else for L1 regularization, 'lambda' for regularization value

Returns
-------
Upon completion of this function, the parameters of the graph are set
to the values that best fit the data, as defined by *func*
"""
bounds = list(zip([-np.inf]*self.nparam, [np.inf]*self.nparam))
param0 = copy.deepcopy(param0in)

# options = {'maxiter':20, 'disp':False, 'gtol':1e-10, 'ftol':1e-18}
options = {'maxiter':20, 'disp':False, 'gtol':1e-10, 'ftol':1e-18}

# Warming up
if warmup is True:
for node in nodes:

node_list = nodes[node-1:node]
x_list = xtrain[node-1:node]
y_list = ytrain[node-1:node]
std_list = stdtrain[node-1:node]

res = sciopt.minimize(
optimize_obj, param0,
args=(func, self, node_list, x_list, y_list, std_list),
method='L-BFGS-B', jac=True, bounds=bounds,
options=options)

param0 = res.x
for ii in range(self.nparam):
if np.abs(param0[ii]) > 1e-10:
bounds[ii] = (param0[ii]-1e-10, param0[ii]+1e-10)
# print("bounds", bounds)

# Final Training
lossfunc = opts.get('lossfunc','lstsq')
if lossfunc == 'lstsq':
options = {'maxiter':niters, 'disp':verbose, 'gtol':1e-10}
res = sciopt.minimize(
optimize_obj, param0,
args=(func, self, nodes, xtrain, ytrain, stdtrain),
method='L-BFGS-B', jac=True,
options=options)
elif pyapprox_is_installed is True:

obj = partial(
optimize_obj,optf=least_squares,graph=self,nodes=nodes,
xin_l=xtrain, yin_l=ytrain, std_l=stdtrain)
lamda = opts['lambda']
options = {'ftol':1e-12,'disp':False,
'maxiter':1e3, 'method':'slsqp'};
l1_coef, res = lasso(obj,True,None,param0,lamda,options)
#res.x includes slack variables so remove these
res.x=l1_coef
else:
raise Exception("Specified loss is not accepted")

self.set_param(res.x)
return self``````
``` def zero_attributes(self) ```

Zero all attributes except 'func' and 'param'

Expand source code
``````def zero_attributes(self):
"""Zero all attributes except 'func' and 'param' """

atts = ['eval', 'pass_down', 'pre_grad', 'derivative',
'children_left', 'parents_left']
for att in atts:
for node in self.graph.nodes:
try:
self.graph.nodes[node][att] = 0.0
except: # what exception is it?
continue
for edge in self.graph.edges:
try:
self.graph.edges[edge][att] = 0.0
except: # what exception is it?
continue``````
``` def zero_derivatives(self) ```

Set all the derivative attributes to zero

Used prior to computing a new derivative to clear out previous sweep

Expand source code
``````def zero_derivatives(self):
"""Set all the derivative attributes to zero

Used prior to computing a new derivative to clear out previous sweep
"""
self.graph = vec_to_graph(
np.zeros(self.nparam), self.graph, attribute='derivative')``````