import math import numpy as np import matplotlib.pyplot as plt import random from typing import Any def f(x): return 3*x**2 + 2*x + 4 def sigmoid(x): return 1/(1+math.exp(-x)) def modsigmoid(x): return 2/(1+math.exp(abs(x))) class Value: def __init__(self, data, _children = (), _op='', label = ''): self.data = data self.grad = 0.0 # represents derivative of the parent node with respec to current node self._prev = set(_children) self._backward = lambda: None self._op = _op self.label = label def __repr__(self): return f'Value(data={self.data})' def __add__(self, other): other = other if isinstance(other, Value) else Value(other) out = Value(self.data+other.data, (self, other), '+') def _backward(): self.grad += 1.0*out.grad # out and self are addresses here, so if it gets executed in outer node then out == currentnode and self and other == children, so even if we are assigning a different address to out in current node, since out was used in this node, out will be current node when executing the function other.grad += 1.0*out.grad out._backward = _backward return out def __mul__(self, other): other = other if isinstance(other, Value) else Value(other) out = Value(self.data*other.data, (self, other), '*') def _backward(): self.grad += other.data*out.grad other.grad += self.data*out.grad out._backward = _backward return out def __pow__(self, other): assert isinstance(other,(int, float)) out = Value(self.data**other, (self,), f'**{other}') def _backward(): self.grad += other*(self.data**(other-1))*out.grad out._backward = _backward return out def __rmul__(self, other): # other*self return self*other def __truediv__(self, other): return self*other**-1 def __neg__(self): return self*-1 def __sub__(self, other): return self + (-other) def __radd__(self, other): return self + other def tanh(self): x = self.data t = (math.exp(2*x) - 1)/(math.exp(2*x) + 1) out = Value(t, (self, ), 'tanh') def _backward(): self.grad += (1 - t**2)*out.grad out._backward = _backward return out def sin(self): x = self.data out = Value(math.sin(x), (self, ), 'sin') def _backward(): self.grad += math.cos(x)*out.grad out._backward = _backward return out def cos(self): x = self.data out = Value(math.cos(x), (self, ), 'cos') def _backward(): self.grad += -math.sin(x)*out.grad out._backward = _backward return out def tan(self): x = self.data out = Value(math.tan(x), (self, ), 'tan') def _backward(): self.grad += (1/math.cos(x)**2)*out.grad out._backward = _backward return out def cot(self): x = self.data out = Value(math.cot(x), (self, ), 'cot') def _backward(): self.grad += -(1/math.sin(x)**2)*out.grad out._backward = _backward return out def sinh(self): x = self.data out = Value(math.sinh(x), (self, ), 'sinh') def _backward(): self.grad += math.cosh(x)*out.grad out._backward = _backward return out def cosh(self): x = self.data out = Value(math.cosh(x), (self, ), 'sinh') def _backward(): self.grad += math.sinh(x)*out.grad out._backward = _backward return out def exp(self): x = self.data out = Value(math.exp(x), (self,), 'exp') def _backward(): self.grad += out.data*out.grad out._backward = _backward return out def reLu(self): x = self.data out = Value(max(0, x), (self, ), 'reLu') def _backward(): if x > 0: self.grad += out.grad else: self.grad += 0 out._backward = _backward return out def sigmoid(self): x = self.data s = sigmoid(x) out = Value(s, (self,), 'sigmoid') def _backward(): self.grad += s*(1 - s)*out.grad out._backward = _backward return out def log(self): x = self.data # print(x) out = Value(math.log(x), (self,), 'log') def _backward(): self.grad += (1/x)*out.grad out._backward = _backward return out def modsigmoid(self): x = self.data s = modsigmoid(x) out = Value(s, (self,), 'modsigmoid') def _backward(): if x >= 0: self.grad += -((2*x)/(x*(1+x)**2))*out.grad else: self.grad += -((2*x)/(-x*(1-x)**2))*out.grad out._backward = _backward return out def sinc(self): if x == 0: print('error 0 not valdid input') return x = self.data out = Value(math.sinx(x)/x, (self, ), 'sinc') def _backward(): self.grad += ((2*x*math.sin(x) - (x**2)*math.cos(x))/(x**4))*out.grad out._backward = _backward return out def backward(self): topo = [] visited = set() def build_topo(v): if v not in visited: visited.add(v) for child in v._prev: build_topo(child) topo.append(v) build_topo(self) self.grad = 1.0 for node in reversed(topo): node._backward() class Neuron: def __init__(self, nin, activation='sigmoid'): self.w = [Value(random.uniform(-2, 2)) for _ in range(nin)] self.b = Value(random.uniform(-2, 2)) self.activation = activation def parameters(self): return self.w + [self.b] def __call__(self, x): # Neuron()(x) act = sum((xi*wi for xi, wi in zip(x, self.w)), self.b) if self.activation == 'sigmoid': out = act.sigmoid() if self.activation == 'reLu': out = act.reLu() if self.activation == 'modsigmoid': out = act.modsigmoid() if self.activation == '': return act return out class Layer: def __init__(self, nin, nout, activation='sigmoid'): self.neurons = [Neuron(nin, activation=activation) for _ in range(nout)] def parameters(self): return [p for neuron in self.neurons for p in neuron.parameters()] def __call__(self, x): outs = [n(x) for n in self.neurons] return outs[0] if len(outs) == 1 else outs class MLP: def __init__(self, nin, nouts): sz = [nin] + nouts self.layers = [Layer(sz[i], sz[i+1]) for i in range(len(nouts))] def __call__(self, x): for layer in self.layers: x = layer(x) return x def parameters(self): return [p for layer in self.layers for p in layer.parameters()] def fit(n, X, Y, epochs, learning_rate): for k in range(epochs): ypred = [n(x) for x in X] loss = sum([(yout - ygt)**2 for ygt, yout in zip(Y, ypred)]) # resets all the grad to zero for our new weight and biases for node in n.parameters(): node.grad = 0 # deposits all the gradients in the nodes loss.backward() # subtracts all the values by a fraction of gradient decent for node in n.parameters(): node.data -= learning_rate*node.grad # print(k, loss)