# coding: utf-8

# # Quaternion Triple Products and Distance

# by Doug Sweetser, sweetser@alum.mit.edu - please feel free to email

# In this IPython notebook, efforts will be made to understand quaternion triple products and how they are related to distances in space and intervals in space-time as seen in special relativity. Rather than follow a historical story, I will try a more abstract approach. Initialize a few tools.

# In[1]:

get_ipython().run_cell_magic('capture', '', '%matplotlib inline\nimport numpy as np\nimport sympy as sp\nimport matplotlib.pyplot as plt\n\n# To get equations the look like, well, equations, use the following.\nfrom sympy.interactive import printing\nprinting.init_printing(use_latex=True)\nfrom IPython.display import display\n\n# Tools for manipulating quaternions.\nimport Q_tools as qt;')

# ## Spatial Rotations

# Define a triple product function modeled on what it takes to do a spatial rotation, $P R P^*$, where $R$ is a quaternion to be spatially rotated and $P$ is a quaternion parameter to do said rotation.

# In[2]:

def triple_sandwich(r, p=qt.QH([1, 0, 0, 0])):
    """A function that takes 2 quaternions but does a triple product. The default value for P leaves R unchanged."""

    return p.product(r.product(p.conj()))

# In[3]:

t, x, y, z = sp.symbols("t x y z")
s, u, v, w = sp.symbols("s u v w")

R = qt.QH([t, x, y, z])
P = qt.QH([s, u, v, w])
RP_sandwich = triple_sandwich(R, P)
sp.simplify(RP_sandwich.t)

# The first term is just the norm of the parameter $P$ times the scalar value of $R$, how simple! Rotating a value is complicated.

# In[4]:

sp.simplify(RP_sandwich.x)

# Show the interval of $R$ is unchanged up to the norm of the parameter $P$:

# In[5]:

sp.simplify(sp.factor(RP_sandwich.square().t))

# The interval will be invariant so long as the norm of the parameter $P$ is equal to one. A common way to do this is to use sine and cosine functions due to the trig identity $\sin^2(\theta) + \cos^2(\theta) = 1$.

# In[6]:

def triple_trig_z(r, a):
    """A rotation around the z axis only by the double angle of a."""

    return triple_sandwich(r, qt.QH([sp.cos(a), 0, 0, sp.sin(a)]))

def is_quadratic(r):
    """Tests if the the first term of the square of a quaternion is equal to t^2 - x^2 - y^2 - z^2."""

    r2 = r.square()
    simple_r2 = sp.simplify(r2.t)
    it_is = ((simple_r2 == 1.0*t**2 - 1.0*x**2 - 1.0*y**2 - 1.0*z**2)
             or (simple_r2 == t**2 - x**2 - y**2 - z**2))

    if it_is:
        display(t**2 - x**2 - y**2 - z**2)
    else:
        display(simple_r2)

    return it_is

# In[7]:

a = sp.Symbol('a')
display(sp.simplify(triple_trig_z(R, a).t))
display(sp.simplify(triple_trig_z(R, a).x))
display(sp.simplify(triple_trig_z(R, a).y))
display(sp.simplify(triple_trig_z(R, a).z))
is_quadratic(triple_trig_z(R, a))

# An important thing to notice is that rotations work for arbitrarily small values of an angle.

# In[8]:

display(sp.simplify(triple_trig_z(R, 0.01).t))
display(sp.simplify(triple_trig_z(R, 0.01).x))
display(sp.simplify(triple_trig_z(R, 0.01).y))
display(sp.simplify(triple_trig_z(R, 0.01).z))
is_quadratic(triple_trig_z(R, 0.01))

# This is relevant to the fact that the group $SO(3)$ is a compact group. It is easy to visualize the example above: it is a circle in the $xy$ plane with $t$ and $z$ unaltered. Circles are sets of points where the "next" point is an arbitrarily short distance away.

# Can we create a function that can take _any_ quaternion parameter $P$ yet still always generate another member of the group $SO(3)$? This can be done using the inverse of a quaternion which is the conjugate of a quaternion divided by the norm squared. Groups are about binary operations on a set. The binary operation can be a composite function, where the results of one rotation are fed into another.

# In[9]:

def next_rotation(r, p=qt.QH([1, 0, 0, 0])):
    """Generates another member of the rotation group given a quaternion parameter P."""

    return p.product(r.product(p.invert()))

def composite_rotation(r, p1=qt.QH([1, 0, 0, 0]), p2=qt.QH([1, 0, 0, 0])):
    """A composite function of next_rotation."""

    return next_rotation(next_rotation(r, p1), p2)

# In[10]:

display(sp.simplify(composite_rotation(R, qt.QH([s, u, v, w])).t))
display(sp.simplify(composite_rotation(R, qt.QH([s, u, v, w])).x))
is_quadratic(composite_rotation(R, qt.QH([s, u, v, w])))

# The next_rotation function can use any quaternion parameter $P$ as input and create another member of the group. This does not mean that rotations have four degrees of freedom. There is an equivalence relation involved since the product of a quaternion with its inverse has a norm of one. This algebraic constraint means the composite_rotation function has $4-1=3$ degrees of freedom.

# The composite_rotation function could be used to show that there is a real-valued quaternion representation of the compact Lie group $SO(3)$. Since it is well known quaternions can do this, such an effort will be skipped.

# ## Other Triple Products Lead to More Than Just Rotations

# Other triple products are possible. For example, the two quaternions could be on the same side. A number of years ago, a search for a real-valued quaternion function that could do a Lorentz boost turned up this difference between two one-sided triples, $ \frac{1}{2}((P P R)^* - (P^* P^* R)^*)$:

# In[11]:

def triple_2_on_1(r, p=qt.QH([1, 0, 0, 0])):
    """The two are on one side, minus a different two on one side."""

    ppr = p.product(p.product(r)).conj()
    pcpcr = p.conj().product(p.conj().product(r)).conj()
    pd = ppr.dif(pcpcr)
    pd_ave = pd.product(qt.QH([1/2, 0, 0, 0]))
    return pd_ave

# In[12]:

rq_321 = triple_2_on_1(R, P)
display(sp.simplify(rq_321.t))
display(sp.simplify(rq_321.x))
display(sp.simplify(rq_321.y))
display(sp.simplify(rq_321.z))

# If $s=0$, then triple_2_on_1 would contribute nothing.

# Explore the hyperbolic sine and cosines:

# In[13]:

phx = qt.QH([sp.cosh(a), sp.sinh(a), 0, 0])
ppr = triple_2_on_1(R, phx)
display(sp.simplify(ppr.t))

# This is promising for doing a Lorentz boost. There is a direct link between hyperbolic trig functions and the relativistic velocity $\beta$ and stretch factor $\gamma$ of special relativity.

# $$\gamma = \cosh(\alpha)$$
# $$\beta \gamma = \sinh(\alpha)$$

# The trig functions are based on a circle in the plane, while the hyperbolic trig functions start with hyperbolas. The definitions are remarkably similar:

# $$\sin(\alpha) = \frac{e^{i \alpha} - e^{-i \alpha}}{2 i}$$

# $$\cos(\alpha) = \frac{e^{i \alpha} + e^{-i \alpha}}{2 i}$$

# $$\sinh(\alpha) = \frac{e^{\alpha} - e^{-\alpha}}{2}$$

# $$\cosh(\alpha) = \frac{e^{\alpha} + e^{-\alpha}}{2}$$

# The hyperbolic trig functions oddly are "more real", never needing an imaginary factor. The hyperbola of the hyperbolic cosine does touch the unit circle at its minimum, suggesting a solitary link to the trig functions.

# Combine the three triples and test if they do all the work of a Lorentz boost:
# $$\rm{triple-triple}(R, P) \equiv P R P^* + \frac{1}{2}((P P R)^* - (P^* P^* R)^*)$$

# In[14]:

def triple_triple(r, p=qt.QH([1, 0, 0, 0])):
    """Use three triple products for rotations and boosts."""

    # Note: 'qtype' provides a record of what algrabric operations were done to create a quaternion.
    return triple_sandwich(r, p).add(triple_2_on_1(r, p), qtype="triple_triple")

# Can this function do a rotation? If the first value of $P$ is equal to zero, then the two one-sided triple terms, $PPR$, will make no contribution, leaving the triple sandwich $PRP^*$. So long as the norm is equal to unity, then spatial rotations result. Do a rotation:

# In[15]:

jk = qt.QH([0, 0, 3/5, 4/5])
display(sp.simplify(triple_triple(R, jk).t))
display(sp.simplify(triple_triple(R, jk).x))
display(sp.simplify(triple_triple(R, jk).y))
display(sp.simplify(triple_triple(R, jk).z))
is_quadratic(triple_triple(R, jk))

# Something important has changed going from the regular trig functions to these hyperbolic functions for rotations. The requirements that the first term must be zero while the other three terms are normalized to unity means that one cannot go an arbitrarily small distance away and find another transformation. If one wants a product of rotations, those rotations must be at right angles to each other.

# In[16]:

Qi, Qj, Qk = qt.QH([0, 1, 0, 0]), qt.QH([0, 0, 1, 0]), qt.QH([0, 0, 0, 1])
print(triple_triple(triple_triple(R, Qi), Qj))
print(triple_triple(R, Qi.product(Qj)))

# The fact that one cannot find a super close neighbor is a big technical change.

# What is so special about setting the first term equal to zero? Is there a more general form? Perhaps all that is needed is for the first term of the square to be equal to negative one. Test this out:

# In[17]:

minus_1 = qt.QH([2, 2, 1, 0])
print(minus_1.square().t)
display((triple_triple(R, minus_1).t, triple_triple(R, minus_1).x, triple_triple(R, minus_1).y, triple_triple(R, minus_1).z))
is_quadratic(triple_triple(R, minus_1))

# To be honest, this came as a surprise to me. Notice that the value for time changes, so a rotation is getting mixed in with a boost. This sort of mixing of rotations and boosts is known to happen when one does two boosts, one say along $x$, the other along $y$. Now we can say a similar thing is possible for rotations. If there scalar is zero then one gets a pure spatial rotation. When that is not the case, there is a mixture of rotations and boosts.

# Demonstrate that a boost along the $x$ axis works.

# In[18]:

bx = qt.QH([sp.cosh(a), sp.sinh(a), 0, 0])
display(sp.simplify(bx.square().t))
display(sp.simplify(triple_triple(R, bx).t))
display(sp.simplify(triple_triple(R, bx).x))
display(sp.simplify(triple_triple(R, bx).y))
display(sp.simplify(triple_triple(R, bx).z))
is_quadratic(triple_triple(R, bx))

# Perfect. It was this result that began my investigation of triple_triple quaternion products. This is what the boost looks like using gammas and betas: $$(\gamma t - \gamma \beta x, \gamma x - \gamma \beta t, y, z)$$

# The first term of the square of the hyperbolic parameter $P=bx$ is equal to positive one. So long as the triple_triple function is fed a quaternion parameter $P$ whose first term of the square has an absolute value of one, the interval is invariant. That is surprisingly simple.

# Note the double angle in the hyperbolic trig function that appeared earlier for rotations.

# ## Spatial Reflection and Time Reversal

# For a spatial reflection, just one spatial term flips signs. The first term of the square will not be altered. Yet the triple_triple function cannot flip only one sign. It can flip two terms. Thus, using just the triple_triple function one can go from all positive, to two positive-two negative, to all negative terms, but never one or three negative terms starting from an all positive quaternion $R$. The conjugate operator can do odd sign changes. Do a spatial reflection on $x$ only by rotating using $i$ and using the conjugate operator like so:

# In[19]:

x_reflection = triple_triple(R, Qi).conj()
print(x_reflection)
is_quadratic(x_reflection)

# Time reversal also cannot be done using triple_triple. The parameter $P$ is used twice, so its sign is of no consequence for the scalar in $R$. The entire quaternion $R$ must be multiplied by $-1$ then take a conjugate like so:

# In[20]:

t_reversal = triple_triple(R).conj().product(qt.QH([-1, 0, 0, 0], qtype="sign_flip"))
print(t_reversal)
is_quadratic(t_reversal)

# Rotations and boosts do not do the work of time reversal. Time reversal requires different algebraic tricks.

# ## Fixing the Limitations of the Triple_Triple Function

# The triple_triple function must be fed quaternions whose square is either exactly equal to plus or minus one. Create a function that can take in _any_ quaternion as a parameter and generate the next quadratic. The function must be scaled to the square root of the first term of the quaternion parameter $P$ squared. Expand the parameters so both spatial reflections and time reversals can be done.

# If the parameter $P$ is light-like, it cannot be used to do a boost. Feed the triple_triple function a light-like quaternion and it will always return zero. Light-like quaternions can do rotations. The next_rotation function is up to the task.

# In[21]:

def next_quadratic(r, p=qt.QH([1, 0, 0, 0]), conj=False, sign_flip=False):
    """Generates another quadratic using a quaternion parameter p,
    if given any quaternion and whether a conjugate or sign flip is needed."""

    pt_squared = p.square().t

    # Avoid using sp.Abs() so equations can be simplified.
    if isinstance(pt_squared, (int, float)):
        if pt_squared < 0:
            pt_squared *= -1
    else:
        if pt_squared.is_negative:
            pt_squared *= -1

    sqrt_pt_squared = sp.sqrt(pt_squared)

    # A light-like parameter P can rotate but not boost R.
    if sqrt_pt_squared == 0:
        rot_calc = next_rotation(r, p)
    else:
        p_normalized = p.product(qt.QH([1/sqrt_pt_squared, 0, 0, 0]))
        rot_calc = triple_triple(r, p_normalized)

    if conj:
        conj_calc = rot_calc.conj()
    else:
        conj_calc = rot_calc

    if sign_flip:
        sign_calc = conj_calc.product(qt.QH([-1, 0, 0, 0]))
    else:
        sign_calc = conj_calc

    calc_t = sp.simplify(sp.expand(sign_calc.t))
    calc_x = sp.simplify(sp.expand(sign_calc.x))
    calc_y = sp.simplify(sp.expand(sign_calc.y))
    calc_z = sp.simplify(sp.expand(sign_calc.z))

    return qt.QH([calc_t, calc_x, calc_y, calc_z], qtype="L")

# In[22]:

display(sp.simplify(next_quadratic(R, P, True, True).t))
display(sp.simplify(next_quadratic(R, P, True, True).x))
is_quadratic(next_quadratic(R, P, True, True))

# No matter what values are used for the parameter $P$, the next_quadratic function will preserve the interval of $R$. Even a light-like interval works:

# In[23]:

print(next_quadratic(R, qt.QH([s, s, 0, 0])))
is_quadratic(next_quadratic(R, qt.QH([s, s, 0, 0])))

# Notice how the $y$ and $z$ terms flip positions, but the squaring process will put both into their proper spots in the first term of the square.

# ## The Lorentz Group and Functional Composition with the next_quadratic Function

# The Lorentz group is all possible ways to transform an event in space-time yet preserve the quadratic form:
# $$(t, x, y, z) \rightarrow t^2 - x^2 - y^2 - z^2$$
# The elements of the group are the tuples (t, x, y, z) but not the rotation angles, boost velocities, conjugation and sign flips.
#
# A group is defined as a binary operation on a set of elements that has 4 qualities:
# 1. Closure
# 1. An inverse exists
# 1. There is an identity element
# 1. Associative
#
# The next_quadratic function acts on one element of the group. The binary operation is a composite function built from two next_quadratic functions. Take the result of one action of the next_quadratic function, and have that result go into another round of the next_quadratic function.

# In[24]:

def composite_quadratic(r, p1=qt.QH([1, 0, 0, 0]), p2=qt.QH([1, 0, 0, 0]), conj1=False, conj2=False, sign_flip1=False, sign_flip2=False):
    """A composite function for the next_quadratic function."""

    return next_quadratic(next_quadratic(r, p1, conj1, sign_flip1), p2, conj2, sign_flip2)

# In[25]:

print(composite_quadratic(R))
is_quadratic(composite_quadratic(R))
print(composite_quadratic(R, Qi, Qj, True, True, True, False))
is_quadratic(composite_quadratic(R, Qi, Qj, True, True, True, False))
print(composite_quadratic(R, minus_1, Qj, False, True, False, True))
is_quadratic(composite_quadratic(R, minus_1, Qj, False, True, False, True))
print(composite_quadratic(R, bx, P, True, False, True, False))
is_quadratic(composite_quadratic(R, bx, P, True, False, True, False))
print(composite_quadratic(composite_quadratic(R, bx, bx)))
is_quadratic(composite_quadratic(composite_quadratic(R, bx, bx)))

# Each of these composite functions generates exactly the same quadratic as required to be part of the Lorentz group. These five examples argue for closure: every possible choice for what one puts in the composite_quadratic function will have the same quadratic. I don't have the math skills to prove closure (unless one thinks the earlier general case is enough).

# Quaternions are a division algebra. As such, it is reasonable to expect an inverse to exist. Look for one for the $Qi$, $Qk$ parameter case:

# In[26]:

print(composite_quadratic(R, Qi, Qj, True, True, True, False))
print(composite_quadratic(composite_quadratic(R, Qi, Qj, True, True, True, False), Qk))

# Close, but not quite. Add a sign_flip.

# In[27]:

print(composite_quadratic(composite_quadratic(R, Qi, Qj, True, True, True, False), Qk, sign_flip1=True))

# The is back where we started with the quaternion $R$. Again, this is just an example and not a proof. Some inverses are easier to find than others like pure rotations or pure boosts with a rotation or opposite velocity.

# The identity composition was shown to do its fine work in the first composite_quadratic(R) example.

# Composite functions are associative, at least according to wikipedia.

# ## The Difference Between composite_rotation and composite_quadratic

# Both of these composite functions call another function twice, next_rotation and next_quadratic respectively. Both functions do a normalization. The next_rotation normalizes to the norm squared which can be zero if the parameter $P$ is zero, otherwise it is positive. The next_rotation function always does one thing, $P R P^{-1}$. The next_quadratic normalizes to the first term of the square of parameter $P$. That value can be positive, negative, or zero. When the first term of the square is positive or negative, the next_quadratic function treats both cases identically. Three triple quaternion products are used, $P R P^* + \frac{1}{2}((P P R)^* - (P^* P^* R)^*)$. The first term is identical to a rotation so long as the norm is equal to one. Otherwise, it is off just by a scaling factor. The difference happens when it is zero which indicates the properties of light come into play. It is the lightcone that separates time-like events from space-like events. For a time-like value of the parameter $P$, the triple-triple returns zero which is not a member of the group. If one uses the first triple, no matter what its norm of light-like parameter $P$ happens to be, the resulting $R->R'$ remains in the group. The rotation group $SO(3)$ is compact, while the Lorentz group $O(1, 3)$ is not. The change in algebra needed for light-light parameter $P$ may be another way to view this difference.

# ## Degrees of Freedom

# The typical representation of the Lorentz group $O(1, 3)$ says there are six independent variables needed to represent the Lorentz group: three for rotations and three for boosts. Yet when one does two boosts in different directions, it is a mix between a boost and a rotation. This suggests there is no such thing as a completely separate notion of rotations and boosts, that they have a capacity to mix. If true, that decreases the degrees of freedom.
#
# Two spacial rotations will result in spacial rotation:

# In[28]:

print(composite_quadratic(R, qt.QH([0, 1,0,1]), qt.QH([0, 1,1,0])))
is_quadratic(composite_quadratic(R, qt.QH([0, 1,0,1]), qt.QH([0, 1,1,0])))

# Notice that the value of the first squared term is negative. That value gets normalized to negative one in the composite_quadratic function (via the next_quadratic function that gets called twice). What makes these rotations be only spacial is the zero in the first position of the parameter $P$. It is easy enough to look at situations where the first term of the square is negative, and the first term of the parameter is not equal to zero:

# In[29]:

print(composite_quadratic(R, qt.QH([4, 5,0,0])))
is_quadratic(composite_quadratic(R, qt.QH([4, 5,0,0])))

# This is both a boost and a rotation. The boost effect can be seen in the first and second terms where there is a positve and negative term (the negative being the term that "doesn't belong", seeing the $x$ in the first term and $t$ in the second). The rotation appears in the sign flips for $y$ and $z$. If the 4 and 5 are switched, there is no rotation of these terms:

# In[30]:

print(composite_quadratic(R, qt.QH([5, 4,0,0])))

# The first two terms are exactly the same. Now the last two terms don't flip signs because there is no rotation. Both the (4, 5) and (5, 4) parameter composites will have the same first term for the square. This real-valued quaternion representation makes it possible to see.

# At first blush, one looks into the next_quadratic function and sees six degrees of freedom: four for the quaternion parameter $P$, one for the conjugate operator and one for the sign_flip. These last two are needed to generate spatial reflection and time reversal. The quaternion parameter $P$ normalizes to the first term of the square of the quaternion parameter $P$. This means that once three of the values are chosen, then the value of the fourth one is set by this algebraic constraint. The same thing happens with the composite_rotation function defined earlier: a 4D quaternion may go in, but they way it gets normalized means there is an equivalence class to those quaternions that have a norm of one, and thus only 3 degrees of freedom. Representing the Lorentz group with only five degrees of freedom with this real-valued quaternion representation would be an interesting result if it can be rigorously proved.

# ## Comparing the Real 4x4 Matrix Representation of the Lorentz Transformation to composite_quadratic

# There are several canonical examples of the real 4x4 matrix representation of the Lorentz transformation used to generate new members of the Lorentz group. Some transformations are quite simple and direct, like the matrix needed to reverse time and a spatial reflection:

# In[31]:

event_v = sp.Matrix([t, x, y, z])
L_time = sp.Matrix([[-1, 0, 0, 0],[0, 1, 0, 0],[0, 0, 1, 0],[0, 0, 0, 1]])
event_time_reversal = L_time * event_v
display(sp.transpose(event_time_reversal))

# To be in the Lorentz group, the Minkowski metric contraction of both the 4-vectors $(t, x, y, z)$ and $(-t, x, y, z)$ must be the same. This is obvious by inspection, but I wanted to see if I could do the calculation anyway in this notebook so for more complex calculations, it would be clear how to proceed.

# In[32]:

import sympy.tensor.tensor as stt
FlatMetric = stt.TensorIndexType('Lorentz',eps_dim=4,dummy_fmt='L')
m, n = stt.tensor_indices('m, n', FlatMetric)
FlatMetric.data=[1,-1,-1,-1]
p = stt.tensorhead('p',[FlatMetric],[[1]])
q = stt.tensorhead('q',[FlatMetric],[[1]])
p.data = [t, x, y, z]
q.data = [-t, x, y, z]
FlatMetric.metric.data = FlatMetric.data
display((FlatMetric.metric(-m, -n) * p(m) * p(n)).data)
display((FlatMetric.metric(-m, -n) * q(m) * q(n)).data)

# Both event_v and event_time_reversal are in the Lorentz group. The 4x4 real matrix is the tool to transform from one to the other and back again. The same thing exists for the quaternion representation where everything in the chain of tools is a quaternion.

# In[33]:

R_time_reversal = composite_quadratic(R, conj1=True, sign_flip2=True)
print(R_time_reversal)

# There are 4 ways to call composite_quadratic that create the quaternion $(-t, x, y, z)$ that is an element in the Lorentz group:

# In[34]:

print(composite_quadratic(R, conj1=True, sign_flip1=True))
print(composite_quadratic(R, conj1=True, sign_flip2=True))
print(composite_quadratic(R, conj2=True, sign_flip1=True))
print(composite_quadratic(R, conj2=True, sign_flip2=True))

# I don't see that the redundancy for ways to do the transformation is an issue. What matters is an inverse exists:

# In[35]:

print(composite_quadratic(composite_quadratic(R, conj1=True, sign_flip1=True), conj2=True, sign_flip2=True))

# and there is an identity (not many identities):

# In[36]:

print(composite_quadratic(composite_quadratic(R, conj1=True, sign_flip1=True)))

# Another simple case for the 4x4 representation is a spatial reflection:

# In[37]:

L_space_x = sp.Matrix([[1, 0, 0, 0],[0, -1, 0, 0],[0, 0, 1, 0],[0, 0, 0, 1]])
event_space_reflection = L_space_x * event_v
display(sp.transpose(event_space_reflection))
q.data = [event_space_reflection[0], event_space_reflection[1], event_space_reflection[2], event_space_reflection[3]]
display((FlatMetric.metric(-m, -n) * q(m) * q(n)).data)

# Do this again, and one returns to the start, so it is its own inverse:

# In[38]:

display(sp.transpose(L_space_x * (L_space_x * event_v)))

# The same thing happens with the composite_quadratic:

# In[39]:

print(composite_quadratic(R, Qi, conj1=True))
print(composite_quadratic(composite_quadratic(R, Qi, conj1=True), Qi, conj1=True))
is_quadratic(composite_quadratic(R, Qi, conj1=True))

# Let's do a pure boost with a 4x4 real matrix A:

# In[40]:

A = sp.Matrix([[5/4, 3/4, 0, 0],[3/4, 5/4, 0, 0],[0, 0, 1, 0],[0, 0, 0, 1]])
pure_boost = A * event_v
display(sp.transpose(pure_boost))
q.data = [pure_boost[0], pure_boost[1], pure_boost[2], pure_boost[3]]
sp.simplify((FlatMetric.metric(-m, -n) * q(m) * q(n)).data)

# Get the same thing to work for the composite_quadratic.

# In[41]:

Aq = qt.QH([5/4, -3/4, 0, 0])
print(composite_quadratic(R, Aq))
is_quadratic(composite_quadratic(R, Aq))

# So why are the values different? Recall the factor of 2 in the angle that appears in each and every triple product? So give the value a little nudge to acknowledge that factor of 2:

# In[42]:

Aq = qt.QH([12/8, -4/8, 0, 0])
print(composite_quadratic(R, Aq))
is_quadratic(composite_quadratic(R, Aq))

# This is precisely the same Lorentz boost transformation as done before.

# Let's to a rotation with a 4x4 real matrix.

# In[43]:

B = sp.Matrix([[1, 0, 0, 0], [0, 3/5, 4/5, 0],[0, -4/5, 3/5, 0],[0, 0, 0, 1]])
pure_rotation = B * event_v
display(sp.transpose(pure_rotation))
q.data = [pure_rotation[0], pure_rotation[1], pure_rotation[2], pure_rotation[3]]
sp.simplify((FlatMetric.metric(-m, -n) * q(m) * q(n)).data)

# Perfect. Repeat with quaternions, again adjusting the numbers to account for the double angle.

# In[44]:

Bq = qt.QH([0, 1, -2, 0])
print(composite_quadratic(R, Bq))
is_quadratic(composite_quadratic(R, Bq))

# Close, but not quite. The signs for terms 2 and 4 are just plain wrong. Correct them:

# In[45]:

Bq = qt.QH([0, 1, -2, 0])
print(composite_quadratic(R, Bq, Qj))
is_quadratic(composite_quadratic(R, Bq, Qj))

# This quaternion rotation is identical to what was done with the 4x4 real B Lorentz transformation. Nice. I will confess I did have to hunt around like a monkey on a typewriter to find the right combination of signs and factors to make that happen. That only indicates limitations on the author.

# In[46]:

C = sp.Matrix([[5/4, 9/20, 3/5, 0], [3/4, 3/4, 1, 0],[0, -4/5, 3/5, 0],[0, 0, 0, 1]])
rotation_and_boost = C * event_v
display(sp.transpose(rotation_and_boost))
q.data = [rotation_and_boost[0], rotation_and_boost[1], rotation_and_boost[2], rotation_and_boost[3]]
sp.simplify((FlatMetric.metric(-m, -n) * q(m) * q(n)).data)

# The Lorentz transformation accomplished by C is a combination of the previous two. For the quaternion example, use the previous values in the composite_quadratic, one after the other.

# In[47]:

print(composite_quadratic(composite_quadratic(R, Bq, Aq.conj()), Qj))

# All the signs are the same (I am a good monkey at the typewriter).


import numpy as np

import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt

matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42

import numpy as np

def f(x, y):
	# function of two variable
	# Check README file

	return 0.5*np.log(1+100*(np.abs(x-1)**2))+0.5*np.log(1+(100*np.abs(y-1)**2))\
				+ np.log(1+np.abs(x))+np.log(1+np.abs(y))

# creating the grid for plotting
x = np.linspace(-2, 2, 100)
y = np.linspace(-2, 2, 100)
X, Y = np.meshgrid(x, y)
Z = f(X, Y) # obtain the function values over grid

# We need to create 6 plots
# i=0 reserved for contour plot
# i=0 reserved for contour plot
for i in range(6):
	plt.contour(X, Y, Z, 100, cmap='RdBu');
	plt.plot(0, 1, 'bh', fillstyle='none')
	plt.plot(1, 0, 'bh', fillstyle='none')
	plt.plot(1, 1, 'bh', fillstyle='none')
	plt.plot(0, 0, 'bh', fillstyle='none')

	if i>1:
		# to plot with various initializations
		# the array in the file name denote the initialization point

		if i==2:
			p1 = np.loadtxt('results/cocain_trajectory_[2 2].txt')
		elif i ==3:
			p1 = np.loadtxt('results/cocain_trajectory_[ 2 -2].txt')
		elif i==4:
			p1 = np.loadtxt('results/cocain_trajectory_[-2  2].txt')
		else:
			p1 = np.loadtxt('results/cocain_trajectory_[-2 -2].txt')

		plt.plot(p1[:,0],p1[:,1])
		plt.savefig('figures/example_contour_'+str(i)+'.pdf')
		plt.savefig('figures/example_contour_'+str(i)+'.png')
	else:
		# i=0 reserved for contour plot
		plt.savefig('figures/example_contour.pdf')
		plt.savefig('figures/example_contour.png')
	plt.clf()

# The following code is to visualize the loss function
# check the Function surface plot in README file
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import cm
from matplotlib.ticker import LinearLocator, FormatStrFormatter

fig = plt.figure()
ax = fig.gca(projection='3d')
surf = ax.plot_surface(X, Y, Z, rstride=1, cstride=1,
    linewidth=10, antialiased=False)
ax.view_init(73,30)
ax.zaxis.set_major_locator(LinearLocator(4))
ax.zaxis.set_major_formatter(FormatStrFormatter('%.02f'))

ax.set_xlabel('$x_1$')
ax.set_ylabel('$x_2$')
ax.set_zlabel('$z$')

plt.savefig('figures/example_surface.pdf')
plt.savefig('figures/example_surface.png')


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Tue Sep  3 09:37:11 2019

Functions for lab 2

@revised date: 2020-03-27
@author: chenggong
@email: enigne@gmail.com
"""

import numpy as np

def absrelerror(corr=None, approx=None):
    """
    Illustrates the relative and absolute error.
    The program calculates the absolute and relative error.
    For vectors and matrices, numpy.linalg.norm() is used.

    Parameters
    ----------
    corr : float, list, numpy.ndarray, optional
        The exact value(s)
    approx : float, list, numpy.ndarray, optional
        The approximated value(s)

    Returns
    -------
    None
    """

    print('*----------------------------------------------------------*')
    print('This program illustrates the absolute and relative error.')
    print('*----------------------------------------------------------*')

    # Check if the values are given, if not ask to input
    if corr is None:
        corr = float(input('Give the correct, exact number: '))
    if approx is None:
        approx = float(input('Give the approximated, calculated number: '))

    # be default 2-norm/Frobenius-norm is used
    abserror = np.linalg.norm(corr - approx)
    relerror = abserror/np.linalg.norm(corr)

    # Output
    print(f'Absolute error: {abserror}')
    print(f'Relative error: {relerror}')

def ForwardDiff(fx, x, h=0.001):
    """
    ForwardDiff(@fx, x, h);
    Use forward difference to approximatee the derivative of function fx
    in points x, and with step length h
    The function fx must be defined as a function handle with input
    parameter x and the derivative as output parameter

    Parameters
    ----------
    fx : function
        A function defined as fx(x)
    x : float, list, numpy.ndarray
        The point(s) of function fx to compute the derivatives
    h : float, optional
        The step size

    Returns
    -------
    float, list, numpy.ndarray: The numerical derivatives of fx at x with
        the same size as x and the type if from fx()
    """
    return (fx(x+h) - fx(x))/h

def CentralDiff(fx, x, h=0.001):
    """
    CentralDiff(@fx, x, h);
    Use Central difference to approximatee the derivative of function fx
    in points x, and with step length h
    The function fx must be defined as a function handle with input
    parameter x and the derivative as output parameter

    Parameters
    ----------
    fx : function
        A function defined as fx(x)
    x : float, list, numpy.ndarray
        The point(s) of function fx to compute the derivatives
    h : float, optional
        The step size

    Returns
    -------
    float, list, numpy.ndarray: The numerical derivatives of fx at x with
        the same size as x and the type if from fx()
    """
    return (fx(x+h) - fx(x-h))/h*0.5

def FivePointsDiff(fx, x, h=0.001):
    """
    FivePointsDiff(@fx, x, h);
    Use five points difference to approximatee the derivative of function fx
    in points x, and with step length h
    The function fx must be defined as a function handle with input
    parameter x and the derivative as output parameter

    Parameters
    ----------
    fx : function
        A function defined as fx(x)
    x : float, list, numpy.ndarray
        The point(s) of function fx to compute the derivatives
    h : float, optional
        The step size

    Returns
    -------
    float, list, numpy.ndarray: The numerical derivatives of fx at x with
        the same size as x and the type if from fx()
    """
    return (-fx(x+2*h) + 8*fx(x+h) - 8*fx(x-h) + fx(x-2*h)) / (12.0*h)


# Show that 2 layer MLP (with manually chosen weights) can solve the XOR problem
# Based on
# https://github.com/ageron/handson-ml2/blob/master/10_neural_nets_with_keras.ipynb

import numpy as np
import matplotlib.pyplot as plt

import pyprobml_utils as pml

def heaviside(z):
    return (z >= 0).astype(z.dtype)

def sigmoid(z):
    return 1 / (1 + np.exp(-z))

def mlp_xor(x1, x2, activation=heaviside):
    return activation(-activation(x1 + x2 - 1.5) + activation(x1 + x2 - 0.5) - 0.5)

x1s = np.linspace(-0.2, 1.2, 100)
x2s = np.linspace(-0.2, 1.2, 100)
x1, x2 = np.meshgrid(x1s, x2s)

z1 = mlp_xor(x1, x2, activation=heaviside)
z2 = mlp_xor(x1, x2, activation=sigmoid)

#plt.figure(figsize=(10,4))
plt.figure()

#plt.subplot(121)
plt.contourf(x1, x2, z1)
plt.plot([0, 1], [0, 1], "gs", markersize=20)
plt.plot([0, 1], [1, 0], "r^", markersize=20)
plt.title("Activation function: heaviside", fontsize=14)
plt.grid(True)
pml.save_fig("xor-heaviside.pdf")
plt.show()

plt.figure()
#plt.subplot(122)
plt.contourf(x1, x2, z2)
plt.plot([0, 1], [0, 1], "gs", markersize=20)
plt.plot([0, 1], [1, 0], "r^", markersize=20)
plt.title("Activation function: sigmoid", fontsize=14)
plt.grid(True)
pml.save_fig("xor-sigmoid.pdf")
plt.show()


def golden(fun, bound, tot=1e-6):
    """
    一维黄金分割找极小值
    :param fun: 1元函数
    :param bound: [lb,ub]
    :param tot: 最小区间长度
    :return: [x0, minimal_val]
    """
    a, b = bound
    p = 0.382  # 黄金分割 1-p ~= 0.618
    a_next = None
    b_next = None
    cnt = 0
    while b-a > tot:
        cnt+=1
        if a_next is None:
            a1 = a+p*(b-a)
            f_a1 = fun(a1)
            a_next = (a1, f_a1)
        if b_next is None:
            b1 = a+(1-p) * (b-a)
            f_b1 = fun(b1)
            b_next = (b1, f_b1)
        if a_next[1] < b_next[1]:
            b = b_next[0]
            b_next = a_next
            a_next = None
        else:
            a = a_next[0]
            a_next = b_next
            b_next = None
    return [(b+a)/2, fun((b+a)/2),cnt]

if __name__ == '__main__':

    fun = lambda x: x**4 - 14*x**3 + 60*x**2 - 70*x
    ret = golden(fun, [0, 2])
    from scipy.optimize import minimize_scalar
    print('my ans\n', ret)
    res = minimize_scalar(fun,bounds=[0,2],method='brent')
    print('scipy\n',res)


import numpy as np
import matplotlib.pyplot as plot
import math

def jacobi(x):
    x1, x2 = x

    return np.array([[1, math.sin(x2)], [-math.cos(x1), 1]])

def func(x):
    x1, x2 = x

    return np.array([x1 - math.cos(x2) - 1, x2 - math.sin(x1) - 1])

def find_delta_x(f, J):
    return np.linalg.solve(J, -1 * f)

def newthon(x_start, eps):
    x_curr = x_start
    x_next = []
    iters = 0
    is_close = False

    while True:
        print(f'x_{iters} = \n{x_curr}')

        f_x = func(x_curr)
        J_x = jacobi(x_curr)
        print(f'f(x_{iters}) = \n {f_x}')
        print(f'J(x_{iters}) = \n {J_x}')

        x_next = x_curr + find_delta_x(f_x, J_x)

        print(f'x_{iters + 1} = \n{x_next}')

        diff = np.linalg.norm(x_next - x_curr, ord=np.inf)
        print(f'||x_{iters + 1} - x_{iters}|| = {diff}')

        if diff <= eps:
            print(f'Found solution in {iters} steps')
            break

        x_curr = x_next
        iters += 1

    return x_next, iters

def ph(x):
    x1, x2 = x

    return np.array([[1 + math.cos(x2)], [1 + math.sin(x1)]])

def ph_der(x):
    x1, x2 = x

    return np.array([[0, -math.sin(x2)], [math.cos(x1), 0]])

def simple_iters(x_range, phi, phi_deriv, eps):
    x_curr = np.array([[np.mean(x)] for x in x_range])
    x_next = []
    iters = 0
    print(f'x is = {x_range}')
    x_series = [np.linspace(x[0], x[1]) for x in x_range]

    q = max(map(lambda x: np.linalg.norm(phi_deriv(x), ord=np.inf), zip(*x_series)))
    print(f'q = {q}')
    if q >= 1:
        print("q should be < 1")
        exit()

    coeff = q / (1 - q)
    print(f'coeff = {coeff}')
    while True:
        print(f'x_{iters} = \n{x_curr}')

        x_next = phi(x_curr)

        print(f'x_{iters + 1} = \n{x_next}')

        diff = np.linalg.norm(x_next - x_curr, ord=np.inf) * coeff
        print(f'||x_{iters + 1} - x_{iters}|| * coeff = {diff}')

        if diff <= eps:
            print(f'Found solution in {iters} steps')
            break

        x_curr = x_next
        iters += 1

    return x_next, iters

def main():
    # x_series = np.arange(-3.0, 3.1, 0.1)
    # f1_series = [(lambda x2: math.cos(x2) + 1)(x) for x in x_series]
    # f2_series = [(lambda x1: math.sin(x1) + 1)(x) for x in x_series]

    # plot.figure()
    # plot.xlabel('x1')
    # plot.ylabel('x2')
    # plot.plot(x_series, f1_series, 'r')
    # plot.plot(f2_series, x_series,'g')
    # plot.axis('equal')
    # plot.grid(True)
    # plot.show()

    # 1.5 <= x1 <= 2
    # 0.7 <= x2 <= 0.92
#    (1.73, 0.83)
    eps = float(input())
    x_start = np.array([[1.7], [0.75]])

    root_n, iters_n = newthon(x_start, eps)

    x_range = [(1.7, 1.8), (0.75, 0.85)]
    root_si, iters_si = simple_iters(x_range, ph, ph_der, eps)

    print(f'Newthon: root is:\n{root_n}\nfound in {iters_n} iterations')
    print(f'Simple iterations: root is:\n{root_si}\nfound in {iters_si} iterations')

    print(f'Are roots close? {np.allclose(root_n, root_si, atol=eps)}')

main()


import matplotlib.pyplot as plt
from numpy import arange, sign, zeros, log, sqrt, exp
from numpy.linalg import norm
from scipy.stats import norm, t, lognorm

plt.style.use('seaborn')

from ShiftedLNMoments import ShiftedLNMoments

def DiscretizeNormalizeParam(tau, k_, model, par):
    # This function discretizes the one-step normalized pdf when the
    # distribution is parametrically specified
    # INPUTS
    #  tau    :[scalar] projection horizon
    #  k_     :[scalar] coarseness level
    #  model  :[string] specifies the distribution: shiftedLN,.TStudent t.T,Uniform
    #  par    :[struct] model parameters
    # OUTPUTS
    #  xi     :[1 x k_] centers of the bins
    #  f      :[1 x k_] discretized pdf of invariant

    ## Code

    # grid
    a = -norm.ppf(10**(-15),0,sqrt(tau))
    h = 2*a/k_
    xi = arange(-a+h,a+h,h)

    # discretized initial pdf (standardized)
    if model=='shiftedLN':
        m, s,_ = ShiftedLNMoments(par)
        csi = par.c
        mu = par.mu
        sig = sqrt(par.sig2)
        if sign(par.skew)==1:
            M = (m-csi)/s
            f = 1/h*(lognorm.cdf(xi+h/2+M,sig,scale=exp(mu-log(s)))-lognorm.cdf(xi-h/2+M,sig,scale=exp(mu-log(s))))
            f[k_] = 1/h*(lognorm.cdf(-a+h/2+M,sig,scale=exp(mu-log(s)))-lognorm.cdf(-a+M,sig,scale=exp(mu-log(s))) +\
            lognorm.cdf(a+M,sig,scale=exp(mu-log(s)))-lognorm.cdf(a-h/2+M,sig,scale=exp(mu-log(s))))
        elif sign(par.skew)==-1:
            M = (m+csi)/s
            f = 1/h*(lognorm.cdf(-(xi-h/2+M),sig,scale=exp(mu-log(s)))-lognorm.cdf(-(xi+h/2+M),sig,scale=exp(mu-log(s))))
            f[k_-1] = 1/h*(lognorm.cdf(-(-a+M),sig,scale=exp(mu-log(s)))-lognorm.cdf(-(-a+h/2+M),sig,scale=exp(mu-log(s))) +\
            lognorm.cdf(-(a-h/2+M),sig,scale=exp(mu-log(s)))-lognorm.cdf(-(a+M),sig,scale=exp(mu-log(s))))

    elif model=='Student t':
        nu = par
        f = 1/h*(t.cdf(xi+h/2,nu)-t.cdf(xi-h/2,nu))
        f[k_-1] = 1/h*(t.cdf(-a+h/2,nu)-t.cdf(-a,nu) + t.cdf(a,nu)-t.cdf(a-h/2,nu))

    elif model=='Uniform':
        mu = par.mu
        sigma = par.sigma
        f = zeros(k_)
        f[(xi>=-mu/sigma)&(xi<=(1-mu)/sigma)] = sigma
    return xi, f


import math
from numpy import array, abs, sum, median, mean

with open('input.txt') as f:
    lines = f.readlines()

crabSubs = array([ int(i) for i in  lines[0].split(',') ])

print(crabSubs)

# fuel consumption at 1

medianSub = round(median(crabSubs))

print("Median" , medianSub)
print("Part 1 " , sum([ abs(medianSub - i) for i in crabSubs ]))

# fuel consumption incremental by 1 every 1 move

meanSub = mean(crabSubs)
meanSubFloor =  math.floor(meanSub)
meanSubCeil = math.ceil(meanSub)

print("Mean Floor",meanSubFloor)
print("Mean Ceil", meanSubCeil)

def calculateFuel(r):
    return sum([ i for i in range(1,r+1)])

meanSubFuelFloor  = sum([ calculateFuel(i) for i in  abs(array(crabSubs) - meanSubFloor) ])
meanSubFuelCeil  = sum([ calculateFuel(i) for i in  abs(array(crabSubs) - meanSubCeil)  ])

print("Part 2 " , min(meanSubFuelFloor,meanSubFuelCeil))


###############################################################################
# Entropy baseline
#
# Madhavun Candadai
# Dec, 2018
#
# Entropy of a coin flip for different probabilities of HEADS ranging from 0 to
# 1 should give an inverted-U shaped curve
###############################################################################

import numpy as np
import infotheory
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt

# range of probabilties for HEADS in coin flip
p = np.arange(0,1.01,0.01)
entropies = []
for pi in p:
    it = infotheory.InfoTools(1, 0)
    it.set_equal_interval_binning([2], [0], [1])

    # flipping coin 10000 times
    for _ in range(10000):
        if np.random.rand()<pi: it.add_data_point([0])
        else: it.add_data_point([1])

    # estimating entropy
    entropies.append(it.entropy([0]))

plt.figure(figsize=[3,2])
plt.plot(p, entropies)
plt.xlabel('Probability of HEADS')
plt.ylabel('Entropy')
plt.tight_layout()
#plt.savefig('./entropy_baseline.png')
plt.show()


import numpy as np

def MSE(y, yhat):
    return np.mean(np.power(y-yhat, 2))

def dMSE(y, yhat):
    return 2*(yhat-y)/y.size

def MAE(y, yhat):
    return np.sum(np.abs(y-yhat))

def dMAE(y, yhat):
    return 1 if y == yhat else -1

def kl_divergence(y, yhat):

    """
    measures the difference between two probability distributions
    over the same variable.

    Parameters:
    - y : Numpy array
    - yhat : Numpy array

    Returns:
    difference between two probability distribution.

     KL divergence can be calculated as
     the negative sum of probability of each event in P multiplied by
     the log of the probability of the event in Q over the probability of the event

    """
	return sum(y[i] * log2(y[i]/yhat[i]) for i in range(len(y)))

def entropy(y,factor=1e-15):
    """
      measures the performance of a classification model
      whose output is a probability value between 0 and 1

    Parameters:
    - y: Numpy array
    - factor: Optional (To ensure 0 is not returned).

    Returns:
     between 0 to 1
    """
	return -sum([y[i] * log2(y[i]+factor) for i in range(len(y))])

def cross_entropy(y,yhat,mode=None,factor=1e-15):
    """
    calculates loss among two probability vectors.

    Parameters:
    - y: Numpy array
    - yhat: numpy array
    - mode: Optional (mode= kl_divergence then calculate cross entropy using kl_divergence )
    - factor:  Optional (To ensure 0 is not returned).

    Returns:
     between 0 to 1
    """
    if(mode=='Kl_diversion'):
        return entropy(y) + kl_divergence(y, yhat)
	return -sum([y[i]*log2(yhat[i]+factor) for i in range(len(y))])


'''
date: 190509
objective: show how to do simple linear regression with numpy

mini explanation: remember that every pair (x,y) can be explained
by y_i=a*x_i+b. thus, can have:
    y1 = a*x1 + b
    y2 = a*x2 + b
    y3 = a*x3 + b
    ...

otherwise expressed as:
    y = A * k
where A is an Nx2 array lke [x 1], and k is a column vector [a,b]
'''

import numpy as np

# basic case, find a line that goes through all points
x=np.linspace(0,5)
err=(np.random.rand(*x.shape)-0.5)*0.5 # center and scale noise
a=2  # slope
b=-3 # intercept
y = x*a+b+err

arr=np.column_stack((x,np.ones(x.shape)))
pred = np.linalg.lstsq(arr,y,rcond=None)[0]
print('solving for y=a*x+b ======================')
print('true coeffs: {},{}'.format(a,b))
print('pred coeffs:',pred)

# inverse relationship case, find a line with no slope
x=np.linspace(1,5) # don't divide by zero
a=20
y=a/x+err
# to generate arr, linearize the problem. y = a/x = a*u, u=1/x
arr=1/x[:,None] # want an Nx1 matrix, aka column vector
pred = np.linalg.lstsq(arr,y,rcond=None)[0]

print('solving for y=a/x ========================')
print('true coeff:',a)
print('pred coeff:',pred)

# quadratic relationship case, find 3 coeff's
x=np.linspace(0,5)
a=0.5
b=-1
c=3
y=a*x**2+b*x+c+err

arr=np.column_stack((x**2,x,x**0)) # linearly separated values
pred=np.linalg.lstsq(arr,y,rcond=None)[0]

print('solving for y=a*x^2 + b*x + c ============')
print('true coeffs: {},{},{}'.format(a,b,c))
print('pred coeffs:',pred)


from math import factorial
import numpy as np
from matplotlib import pyplot as plt

# Function by https://stackoverflow.com/questions/4941753/is-there-a-math-ncr-function-in-python
import operator as op

def ncr(n, r):
    r = min(r, n - r)
    if r == 0:
        return 1
    numer = reduce(op.mul, xrange(n, n - r, -1))
    denom = reduce(op.mul, xrange(1, r + 1))
    return numer // denom

# print ncr(24, 15)

# Interesting observation, a wave is made when you print ncr
def solution(limit=100):
    c = 0
    for n in range(23, limit + 1):
        for r in range(2, n):
            if ncr(n, r) > 1000000:
                c += 1
                # print ncr(n, r)
    print "Number of values of nCr greater than 1000000: %s" % c

solution()

def list_of_ncr(limit=100):
    ncr_list = []
    for n in range(23, limit + 1):
        for r in range(2, n):
            if ncr(n, r) > 1000000:
                ncr_list.append(ncr(n, r))

    return ncr_list

plt.plot(list_of_ncr()[:100])
plt.ylabel("Pascal numbers > million")
plt.show()


'''
datecreated: 190930
objective: demonstrate rotational matrix. not always easy to figure out
BONUS: demosntrate translation as well in same matrix

general steps:
1. plot a simple rectangle
2. plot a rectangle rotated about origin
3. plot a rectangle rotated about any given center

'''

import numpy as np
import matplotlib.pyplot as plt

# original matrix
orig = np.array([[0,0],[1,0],[1,2],[0,2],[0,0]]) # Nx2

# rotational parameter
xcent = 0.5
ycent = 1.0
angle = 45

# rotated about origin
theta = np.radians(45)
c,s = np.cos(theta),np.sin(theta)
R=np.array([
    [c,-s],
    [s,c]    ]) # 2x2

rot1=orig@R

# rotated about arbitrary point
rot2 = np.column_stack(( orig,np.ones(len(orig)) ))
R2=np.array([
    [c,-s,0],
    [s, c,0],
    [0, 0,1]  ])

rot2[:,0]-=xcent
rot2[:,1]-=ycent
rot2=rot2@R2
rot2[:,0]+=xcent
rot2[:,1]+=ycent

# plot everything
f,p=plt.subplots()
p.set_aspect('equal')
p.set_xlim([-3,3])
p.set_ylim([-3,3])
p.plot([-3,3],[0,0],'k--') # x-axis line
p.plot([0,0],[-3,3],'k--') # y-axis line

p.plot(orig[:,0],orig[:,1],'b') # original, unrotated rectangle
p.plot(rot1[:,0],rot1[:,1],'g') # rotation about origin
p.plot(rot2[:,0],rot2[:,1],'r') # rotation about any point
plt.show()


import numpy as np
from matplotlib import pyplot as plt

N = 1024
limit = 10

t = np.linspace(-limit, limit, N)
# array([-10.        ,  -9.98044966,  -9.96089932, ...,   9.96089932,
#          9.98044966,  10.        ])

dt = t[1] - t[0]  # 0.019550342130987275

y = np.sin(2 * np.pi * 5 * t) + np.sin(2 * np.pi * 1 * t)  # two sine frequencies of 5 Hz and 1 Hz

yhat = np.fft.fft(y)
Y = np.abs(yhat)

min_freq = 0
max_freq = (1.0/dt) / 2  # 25.575
freq = np.linspace(min_freq, max_freq, N//2)

plt.plot(freq, Y[0:N//2])  # unnormalized FFT
plt.plot(freq, Y[0:N//2] * (2/N))  # properly scaled FFT
plt.show()


import numpy as np
from numpy import linalg
import matplotlib.pyplot as plt

mu = None
std = None
def standardize( data):
    """
    Peforms standardization (Z-transform) of data and stores mu and std for use during evaluation
    in global variables
    """
    global  mu, std
    mu = np.mean(data, axis=0)
    std = np.std(data, axis=0)
    data = (data - mu) / (std+1e-8)
    data[:,0] = 1.0
    return data

def fit(x,y, order=1):
    """
    Fits the data after preprocessing and standardization to a polynomial of degree 'order'.
    Program is solved using np.linalg.inv
    """
    d = {}
    d['x' + str(0)] = np.ones([1, len(x)],dtype=np.float)[0]
    for i in np.arange(1, order + 1):
        d['x' + str(i)] = x ** i
    X = np.column_stack(d.values())
    X = standardize(X)
    inter1 = np.matmul(np.transpose(X), X)
    #find inverse
    theta = np.matmul(np.matmul(linalg.inv(inter1), np.transpose(X)), y)
    return theta

def evaluate(saved_heights,theta):
    """
    Performs polynomial regression based on the saved weights after standardizing the data.
    """
    py=[]
    for i in saved_heights:
        r= theta[0]
        for j in range(1, len(theta)):
            r = r + (theta[j] * (pow(i,j)-mu[j])/(std[j]+1e-8))
        py.append(r)
    return py

data = np.loadtxt('whData.dat', dtype=np.object, comments='#', delimiter=None)
# Removing rows with missing weights
saved_row = data[data[:, 0] == '-1', :]

data = data[data[:, 0] != '-1', :]
saved_heights = saved_row[:, 1].astype(np.float)
# read height data into 1D array (i.e. into a matrix)
X = data[:, 1].astype(np.float)

# read weight data into 1D array (i.e. into a vector)
y = data[:, 0].astype(np.float)

d = [1,5,10]
for i in d:

    theta = fit(X,y,i)
    h = np.linspace(155,200, 100)
    #plot_predictedPolyLine(h,y,theta)

    px = evaluate(h,theta)
    y_missing = evaluate(saved_heights,theta)
    print("\nMissing Values:\n(Heights, Weights for d=",i,")\n----------")
    for i in zip(saved_heights, y_missing):
        print(i)
    plt.scatter(saved_heights, y_missing, color='black', label='missing')
    plt.scatter(X,y , color='blue', label='data')
    plt.plot(h, px, color='red', label='MLE fit')
    plt.title('Height vs Weight using Least Squares Method')
    plt.ylim((30,100))
    plt.xlabel('Height')
    plt.ylabel('Weight')
    plt.legend()
    plt.savefig("mle_poly_regr.pdf", facecolor='w', edgecolor='w',
                    papertype=None, format='pdf', transparent=False,
                    bbox_inches='tight', pad_inches=0.1)
    plt.show()


"""
dot

The dot tool returns the dot product of two arrays.

import numpy

A = numpy.array([ 1, 2 ])
B = numpy.array([ 3, 4 ])

print numpy.dot(A, B)       #Output : 11
cross

The cross tool returns the cross product of two arrays.

import numpy

A = numpy.array([ 1, 2 ])
B = numpy.array([ 3, 4 ])

print numpy.cross(A, B)     #Output : -2
Task

You are given two arrays  and . Both have dimensions of X.
Your task is to compute their matrix product.

Input Format

The first line contains the integer .
The next  lines contains  space separated integers of array .
The following  lines contains  space separated integers of array .

Output Format

Print the matrix multiplication of  and .

Sample Input

2
1 2
3 4
1 2
3 4
Sample Output

[[ 7 10]
 [15 22]]
"""

import numpy as np

np.set_printoptions(legacy="1.13")

N = int(input())
arr1 = np.array([list(map(int, input().split())) for _ in range(N)], int)
arr2 = np.array([list(map(int, input().split())) for _ in range(N)], int)
print(np.dot(arr1, arr2))


# Distance formula script, brought to you by PharaohCola13
from numpy import *
import numpy as np

option = int(input("(0) Distance between two points. -- 3D. (1) Use a distance to find the next point. -- 2D (2) Use a origin point, distance, and angle to find the next point. -- 2D\n>> "))

if option == 0:
# The x,y,z componets of the first point
    X1  = float(input("What is the value of x for the first point?\n>> "))
    Y1  = float(input("What is the value of y for the first point?\n>> "))
    Z1  = float(input("What is the value of z for the first point?\n>> "))

# The x,y,z componets of the second point
    X2  = float(input("What is the value of x for the second point?\n>> "))
    Y2  = float(input("What is the value of y for the second point?\n>> "))
    Z2  = float(input("What is the value of z for the second point?\n>> "))

#
    s   = round(np.sqrt((X2-X1)**2 + (Y2-Y1)**2 + (Z2-Z1)**2),5)

    print("The distance between ({0}, {1}, {2}) and ({3}, {4}, {5}) is {6} units" .format(X1,Y1,Z1,X2,Y2,Z2,s))

elif option == 1:
    X1  = float(input("What is the value of x for the first point?\n>> "))
    Y1  = float(input("What is the value of y for the first point?\n>> "))

    s   = float(input("What is the horizontal distance necessary?\n>> "))

    X2 = X1 + s
    Y2 = Y1

    print("The new point is ({0},{1}).".format(X2,Y2))

elif option == 2:
	X1  = float(input("What is the value of x for the first point?\n>> "))
	Y1  = float(input("What is the value of y for the first point?\n>> "))

	s = float(input("What is the distance to the next point?\n>> "))
	theta = float(input("What is the angle?\n>> "))
	theta = (theta /180) * pi

	X2 = round(s * cos(theta), 5) + X1
	Y2 = round(s * sin(theta), 5) + Y1

	print("The new point is({0},{1}).".format(X2,Y2))

# y-y1 = m(x-x1); m=0, dy=0

# x = s*cos(angle)
# y = s*sin(angle)


import matplotlib.pyplot as plt
import numpy as np
from sklearn.datasets import make_regression
from scipy.stats import norm

class SimpleLinearModel(object):
    def __init__(self):
        self.X = None
        self.y = None
        self.xbar = None
        self.ybar = None
        self.b0 = None
        self.b1 = None

    def fit(self, features: np.array, target: np.array):
        """
        fit the linear model
        :param features: features
        :param target: target
        :return:
        """
        self.X = features
        self.y = target

        self.xbar = np.mean(self.X)
        self.ybar = np.mean(self.y)

        self._covariance()
        self._variance()

    def _covariance(self) -> None:
        """ calculate covariance """
        self.b1 = np.sum((self.X - self.xbar) * (self.y - self.ybar)) / np.sum(np.power(self.X - self.xbar, 2))

    def _variance(self) -> None:
        """ calculate variance """
        self.b0 = self.ybar - (self.b1 * self.xbar)

    def predict(self, features) -> np.array:
        """ predict regression line using exiting model """
        return self.b0 + self.b1 * features

    @staticmethod
    def _squared_error(y, yhat) -> np.array:
        """ calculate squared error """
        return sum((yhat - y)**2)

    def _r_squared(self, y, yhat) -> float:
        """ calculate coefficient of determination """
        y_mean = np.mean(y)
        y_line = [y_mean for _ in y]

        se_yhat = self._squared_error(y, yhat)
        se_y_mean = self._squared_error(y, y_line)

        return 1 - (se_yhat / se_y_mean)

    def plot(self, X, y, yhat) -> None:
        """ plot regression line """
        plt.style.use('ggplot')

        r2 = self._r_squared(y, yhat)
        conf = norm.interval(0.95, loc=np.mean(yhat), scale=yhat.std())

        plt.scatter(X, y, color='black')  # actual values
        plt.plot(X, yhat)  # regression line
        plt.fill_between(X.reshape(-1), (yhat+conf[0]), (yhat+conf[1]), color='b', alpha=0.2)

        # Labels
        plt.text(X.min().min(), y.max().max(), '$r^{2}$ = %s' % round(r2, 2))  # r squared
        plt.text(X.min().min(), y.max().max()-10, '95% confidence $\pm$ {:.2f}'.format(abs(conf[0])))  # r squared
        plt.title('Simple Linear Regression')
        plt.ylabel('Target (y)')
        plt.xlabel('Feature (X)')
        plt.show()

if __name__ == '__main__':
    n_samples = 1000
    train_size = int(n_samples * 0.80)

    # generate regression dataset
    X, y = make_regression(n_samples=1000, n_features=1, n_targets=1, random_state=42, noise=10)
    X_train, X_test, y_train, y_test = X[:train_size], X[train_size:], y[:train_size], y[train_size:]

    # Fit regression line
    model = SimpleLinearModel()
    model.fit(X_train.reshape(-1,), y_train)
    predictions = model.predict(X_test.reshape(-1,))
    model.plot(X_test, y_test, predictions)


import numpy as np

def compute_pca(Z, d):
    # Input: the N by D data matrix Z, the number of components d
    # Output: a d by D matrix W_pca, and all eigenvalues of Q

    ### STUDENT TASK ###

    # step1: compute the sample cov. matrix Q
    # YOUR CODE HERE
    N = Z.shape[0]
    Q = Z.T.dot(Z) / N

    # step2: compute the eigenvalues and eigenvectors (see introduction notebook)
    # YOUR CODE HERE
    eigvalues, eigvectors = np.linalg.eig(Q)
    eigvalues = eigvalues.real
    eigvectors = eigvectors.real
    # step3: Sort the eigenvectors by decreasing eigenvalues, choose the d largest eigenvalues, form W_pca
    # YOUR CODE HERE
    idx = eigvalues.argsort()[::-1]
    eigvalues = eigvalues[idx]
    eigvectors = eigvectors[:, idx]
    W_pca = eigvectors[:, :d].T

    return W_pca, eigvalues


print("Méthode Crout ou LU ")

"""
n=int(input("Quelle est la dimension de l'inconnu ? "))
M=[]
for i in range(n):
    M.append([])
    for j in range(n):
        M[i].append(float(input("Entrer l'élément A["+str(i+1)+","+str(j+1)+"]= ")))

B=[]
for i in range(n):
    #B.append([])
    B[i].append(float(input("Entrer l'élément B["+str(i+1)+"]= ")))

"""

n=4
M=[[1,1,1,1],[2,-1,1,2],[1,-3,1,-1],[5,0,1,0]]
#B=[[10],[9],[-4],[22]]
B=[10,9,-4,22]

import numpy as np
M=12*60*np.matrix(M)
#B=np.matrix(B)
B=12*60*np.array(B)

from numpy import matlib as mat

U=mat.zeros((n, n))
L=12*mat.eye(n, k=0)
#print("U= {0}".format(U))
#print("L= {0}".format(L))

for i in range(n):
    #print(i)
    for j in range(i,n,1):
        U[i,j]=M[i,j]
        for k in range(0,i,1):
            U[i,j]=U[i,j]-L[i,k]*U[k,j]
        U[i,j]=U[i,j]/L[i,i]
    #print(U)
    for j in range(i+1,n,1):
        L[j,i]=M[j,i]
        for k in range(0,i,1):
            L[j,i]=L[j,i]-L[j,k]*U[k,i]
        L[j,i]=L[j,i]/U[i,i]
    #print(L)

Y=B.copy()
Y[0]=Y[0]/L[0,0]
for i in range(1,n):
    for j in range(0,i):
        Y[i]=Y[i]-L[i,j]*Y[j]
    Y[i]=Y[i]/L[i,i]

X=Y.copy()
X[n-1]=X[n-1]/U[n-1,n-1]
for i in range(2,n+1):
    for j in range(1,i):
        X[n-i]=X[n-i]-U[n-i,n-j]*X[n-j]
    X[n-i]=X[n-i]/U[n-i,n-i]

print(M/(12*60))
print(Y)
X=np.transpose(X)
print(X)

print("M= {0}".format(M))
print("L= {0}".format(L))
print("U= {0}".format(U))
print("B= {0}".format(B))
#print("LU-M= {0}".format(L*U-M))


# For generating our heightmap
import math
import numpy as np

# For generating our palette
import colorsys

# For saving the result
import voxbox.util
import voxbox.magicavoxel

"""
Generates an animated heightmap from overlapping sine functions.
"""
def generate_waves_heightmap(width, height, timestep):

    freq = 0.1 # Controls spacing between waves.
    result = np.zeros([height, width])

    # Input timestep is in the range 0.0 to 1.0. Map this to zero to 2*pi and
    # pass through sine function to give a vertical scaling factor which to
    # 0.0 to 1.0 to -1.0 and back to 0.0.
    vertical_scale = math.sin(timestep * math.pi * 2.0)

    # For each pixel in the 2D heightmap
    for row in range(0, height):
        for col in range(0, width):

            # Generate two overlapping sine waves as our base output
            result[row][col] = math.sin(row * freq) * math.sin(col * freq)

            # Scale according to the timestep to animate it.
            result[row][col] *= vertical_scale

            # Output is in the range -1.0 to 1.0, scale to range 0.0 to 1.0
            result[row][col] += 1.0
            result[row][col] *= 0.5

    return result

def generate_rainbow_colourmap():

    # Create an empty palette
    palette = np.zeros((256, 4))

    for index in range(0, 256):

        rgb = colorsys.hsv_to_rgb(index / 255.0, 1.0, 1.0) # Get an RGB value
        palette[index] = np.array(rgb + (1.0,)) # Append 1.0 for alpha

    # Our palette is in the 0.0 to 1.0 floating point range.
    # Convert to 8-bit unsigned integers for MagicaVoxel
    palette *= 255.0
    palette = palette.astype(np.uint8)

    return palette

# Define the size of the volume
col_count = 126 # Width
row_count = 126 # Height
plane_count = 64 # Depth
frame_count = 30

palette = generate_rainbow_colourmap()

# Start with an empty list of volumes
volume_list = []

# Later we will choose a material based on the height (plane) of the voxel
palette_offset_per_plane = (len(palette) - 1.0) / plane_count

# For each voxel in the volume
for frame in range(0, frame_count):

    print("Generating frame {} of {}...".format(frame + 1, frame_count))

    volume = np.zeros((plane_count, row_count, col_count), dtype=np.uint8)

    # Create a simple heightmap (could also load something from disk)
    time_step = frame / frame_count
    heightmap = generate_waves_heightmap(col_count, row_count, time_step)

    for plane in range(0, plane_count):
        for row in range(0, row_count):
            for col in range(0, col_count):

                # Get the height from the heightmap, and
                # scale to the height of the volume
                height = heightmap[row][col]
                height *= plane_count

                # If the current voxel is below the
                # heightmap then set it to be solid.
                if plane <= height:
                    volume[plane][row][col] = palette_offset_per_plane * plane

    # Add the new frame (volume) to the list
    volume_list.append(volume)

# Save the volume to disk as a MagicaVoxel file.
print("Saving results...")
filename = "waves.vox"
voxbox.magicavoxel.write(volume_list, filename, palette)
print("Done.")

# Open the file in the default app, which should be MagicaVoxel. Could instead
# try to run MagicaVoxel directly but then we need to know where it is.
voxbox.util.open_in_default_app(filename)


from math import log
from numpy import exp

def logit(u):
    return log(u / (1 - u))

def inverse_logit(v):
    return 1 / (1 + exp(-v))

def make_transform_func_with_lower_bound(lower_bound, func_type):
    assert func_type in ["direct", "inverse", "inverse_derivative"]
    if func_type == "direct":

        def func(x):
            return log(x - lower_bound)

    elif func_type == "inverse":

        def func(y):
            return exp(y) + lower_bound

    elif func_type == "inverse_derivative":

        def func(y):
            return exp(y)

    return func

def make_transform_func_with_upper_bound(upper_bound, func_type):
    assert func_type in ["direct", "inverse", "inverse_derivative"]
    if func_type == "direct":

        def func(x):
            return log(upper_bound - x)

    elif func_type == "inverse":

        def func(y):
            return upper_bound - exp(y)

    elif func_type == "inverse_derivative":

        def func(y):
            return exp(y)

    return func

def make_transform_func_with_two_bounds(lower_bound, upper_bound, func_type):
    assert func_type in ["direct", "inverse", "inverse_derivative"]
    if func_type == "direct":

        def func(x):
            return logit((x - lower_bound) / (upper_bound - lower_bound))

    elif func_type == "inverse":

        def func(y):
            # prevent overflow when the parameter gets close to its bounds (i.e. transformed param's norm gets too big)
            if y < -10:
                x = lower_bound
            elif y > 10:
                x = upper_bound
            else:
                x = lower_bound + (upper_bound - lower_bound) * inverse_logit(y)
            return x

    elif func_type == "inverse_derivative":

        def func(y):
            inv_logit = inverse_logit(y)
            return (upper_bound - lower_bound) * inv_logit * (1 - inv_logit)

    return func


def unit_vector_N(u_BC, u_AB):
    # Calculates unit normal vector which is perpendicular to plane ABC
    import numpy as np

    cross_product = np.cross(u_BC, u_AB)
    norm_u_N = np.linalg.norm(cross_product)
    u_N = cross_product / norm_u_N
    return u_N


from __future__ import annotations
from typing import NoReturn
from ...base import BaseEstimator
import numpy as np
from numpy.linalg import pinv
from IMLearn.metrics.loss_functions import mean_square_error

class LinearRegression(BaseEstimator):
    """
    Linear Regression Estimator

    Solving Ordinary Least Squares optimization problem
    """

    def __init__(self, include_intercept: bool = True) -> LinearRegression:
        """
        Instantiate a linear regression estimator

        Parameters
        ----------
        include_intercept: bool, default=True
            Should fitted model include an intercept or not

        Attributes
        ----------
        include_intercept_: bool
            Should fitted model include an intercept or not

        coefs_: ndarray of shape (n_features,) or (n_features+1,)
            Coefficients vector fitted by linear regression. To be set in
            `LinearRegression.fit` function.
        """
        super().__init__()
        self.include_intercept_, self.coefs_ = include_intercept, None

    def _fit(self, X: np.ndarray, y: np.ndarray) -> NoReturn:
        """
        Fit Least Squares model to given samples

        #my notes - We want to minimize the square error -
        the principle of learning is ERM
        The algorithm used to minimize RSS is by SVM decomposition

        Parameters
        ----------
        X : ndarray of shape (n_samples, n_features)
            Input data to fit an estimator for

        y : ndarray of shape (n_samples, )
            Responses of input data to fit to

        Notes
        -----
        Fits model with or without an intercept depending on value of `self.include_intercept_`
        """
        # when there is an intercept included, to make the equation linear
        # (not affine) we add a zero-th coordinate with value 1
        if self.include_intercept_:
            X = np.insert(X, 0, np.ones(np.shape(X)[0]), axis=1)
        # pinv X- Compute the (Moore-Penrose) pseudo-inverse of a matrix.
        # Calculate the generalized inverse of a matrix using its
        # singular-value decomposition (SVD) and including all
        # large singular values.
        self.coefs_ = np.linalg.pinv(X) @ y

    def _predict(self, X: np.ndarray) -> np.ndarray:
        """
        Predict responses for given samples using fitted estimator

        Parameters
        ----------
        X : ndarray of shape (n_samples, n_features)
            Input data to predict responses for

        Returns
        -------
        responses : ndarray of shape (n_samples, )
            Predicted responses of given samples
        """

        # we get samples and using the model we found (w hat) the result
        # is calculated
        # checks the intercept so the dimensions of the multiplication would
        # be ok
        if self.include_intercept_:
            # = np.insert(X, 0, np.ones(np.shape(X)[0]), axis=1)
            return (X @ self.coefs_[1:]) + self.coefs_[0]
        # X multiply w hat = y
        return X @ self.coefs_

    def _loss(self, X: np.ndarray, y: np.ndarray) -> float:
        """
        Evaluate performance under MSE loss function

        Parameters
        ----------
        X : ndarray of shape (n_samples, n_features)
            Test samples

        y : ndarray of shape (n_samples, )
            True labels of test samples

        Returns
        -------
        loss : float
            Performance under MSE loss function
        """
        # using the function I have implemented
        return mean_square_error(self._predict(X), y)


# -*- coding: utf-8 -*-
"""Lab 3.

Gradient descent
"""

import numpy as np

def calculate_mse(b, A, x):
    """Calculate the mean squared error for vector e."""
    # computing the prediction
    y_pred = np.dot(A, x)
    # subtract the differences
    diff = np.subtract(y_pred, b)
    # return MSE
    return np.square(diff).mean() / 2

def compute_gradient(b, A, x):
    """Compute the gradient."""
    # computing the prediction
    h = np.dot(A, x)
    #  compute gradient
    grad = 2 * np.dot(A.T, (h - b)) / len(b)
    return grad

def gradient_descent(b, A, initial_x, max_iters, gamma):
    """Gradient descent algorithm."""
    # Define parameters to store x and objective func. values
    xs = [initial_x]
    objectives = []
    x = initial_x
    for n_iter in range(max_iters):
        # ***************************************************
        # compute gradient and objective function
        # ***************************************************
        grad = compute_gradient(b, A, x)
        obj = calculate_mse(b, A, x)
        # ***************************************************
        # update x by a gradient descent step
        # ***************************************************
        x = x - (gamma * grad)

        # store x and objective function value
        xs.append(x)
        objectives.append(obj)
        print("Gradient Descent({bi}/{ti}): objective={l}".format(
              bi=n_iter, ti=max_iters - 1, l=obj))

    return objectives, xs


"""
David Silver's Reinforcement Learning - lecture 3
Dynamic programming
small grid world example

policy evaluation for 3 different methods
1. direct matrix operation
2. synchronous iteration
3. asynchronous iteration
"""
import numpy as np
from matplotlib import pyplot as plt
from random import shuffle

# reward vector
R = np.ones(15) * -1
R[0] = 0

# discount factor
gam = 0.9

# state transition probability matrix
P = np.zeros([15, 15])
# P = np.full([15, 15], np.nan) # alternative
# setup P
P[0, 0] = 1
dx = [-1, 0, 1, 0]
dy = [0, 1, 0, -1]
for i in range(1, 15):
    x = i // 4  # from
    y = i % 4
    for d in range(0, 4):
        xx = x + dx[d]  # to
        yy = y + dy[d]
        if xx in range(0, 4) and yy in range(0, 4):
            ii = i + 4 * dx[d] + dy[d]
            P[i, ii % 15] = .25
        else:
            P[i, i] += .25

# method 1: direct matrix operation
A = np.eye(15) - gam * P
temp = np.linalg.solve(A[1:, 1:], R[1:])
v1 = np.zeros(15)
v1[1:] = temp

print(np.append(v1, 0).reshape(4, 4))  # pretty output

# method 2: synchronous iteration
v2 = np.zeros(15)
i = 0
while True:
    i += 1
    v2 = R + gam * P @ v2
    diff = np.linalg.norm(v2 - v1)
    if diff < 1e-3:
        break

print("iteration time: " + str(i))
print("error: " + str(np.linalg.norm(v2 - v1)))
print(np.append(v2, 0).reshape(4, 4))

eig = np.sort(abs(np.linalg.eigvals(P)))
plt.plot(eig, 'o')
plt.grid(True)
plt.show()

print(eig[-2])  # is too close to 1

# method 3: asynchronous iteration
v3 = np.zeros(15)
i = 0
while True:
    i += 1
    r = list(range(15))
    shuffle(r)
    for j in r:
        v3[j] = R[j] + gam * P[j] @ v3
    diff = np.linalg.norm(v3 - v1)
    if diff < 1e-3:
        break

print("iteration time: " + str(i))
print("error: " + str(np.linalg.norm(v3 - v1)))
print(np.append(v3, 0).reshape(4, 4))

# wild card
np.append(P[6], np.nan).reshape(4, 4)

# todo: make this to jupiter notebook


# -*- coding: utf-8 -*-

import numpy as np

def cart2pol(x,y):
    rho=np.sqrt(x**2+y**2)
    phi=np.arctan2(y,x)
    phi=phi*(180/np.pi)
    return(rho,phi)

def pol2cart(rho, phi):
    phi=phi*(np.pi/180)
    x=rho*np.cos(phi)
    y=rho*np.sin(phi)
    return(x,y)


###APPROXMIATING LEGENDRE POLYNOMIAL EIGENVALS via MATRIX CALCULUS & GRAM-SCHMIDT

#WIKIPEDIA: "Legendre polynomials (named after Adrien-Marie Legendre, who discovered them in 1782) are a system of complete and orthogonal polynomials,
#with a vast number of mathematical properties, and numerous applications. They can be defined in many ways, and the various definitions highlight
#different aspects as well as suggest generalizations and connections to different mathematical structures and physical and numerical applications.

#Package Imports
import numpy as np
import math as m
import matplotlib.pylab as plt
%matplotlib inline

#Legendre Equation done in Python:

#Construct a matrix that approximates the deriv

#Define range and fineness of function
xmin = (-1)
xmax = (1)
nn = 300
dx = (xmax-xmin)/nn
x = np.linspace(xmin,xmax,nn,True)

#Define $d/dx$
d_x = np.zeros([nn,nn],dtype=float)
d_x[0,0]=-1.0/dx
d_x[0,1]=1.0/dx
d_x[-1,-1]=1.0/dx
d_x[-1,-2]=-1.0/dx

for i in range(1,nn-1) :
    d_x[i,i-1]=-0.5/dx
    d_x[i,i+1]=0.5/dx

#Define the function we want to differentiate
fx = np.array([x_i for x_i in x])

#Finally, we compute the derivative
dfx_dx = d_x.dot(fx)

#Verify code still correctly computes deriv of f(x) = x

#Plot, f(x) is red, f'(x) is blue
plt.figure()
plt.title("f(x), df(x)/dx vs x")
plt.ylabel('f(x),df(x)/dx')
plt.xlabel('x')
plt.plot(x,fx,'rs')
plt.plot(x,dfx_dx,'gs')

#Repeat for Various polynomials

#i) f(x) = x^2
fx = [x_i**2 for x_i in x]
dfx_dx = d_x.dot(fx)

#plt.figure()
#plt.ylabel('f(x),df(x)/dx')
#plt.xlabel('x')
#plt.plot(x,fx,'rs')
#plt.plot(x,dfx_dx,'bo')

#ii) f(x) = exp(x)
fx = [np.exp(x_i) for x_i in x]
dfx_dx = d_x.dot(fx)

#plt.figure()
#plt.ylabel('f(x),df(x)/dx')
#plt.xlabel('x')
#plt.plot(x,fx,'rs')
#plt.plot(x,dfx_dx,'bo')

#iii) f(x) = sin(2pix)
fx = np.array([np.sin(2*np.pi*x_i) for x_i in x])
dfx_dx = d_x.dot(fx)

#plt.figure()
#plt.ylabel('f(x),df(x)/dx')
#plt.xlabel('x')
#plt.plot(x,fx,'rs')
#plt.plot(x,dfx_dx,'bo')

# Write down the algebraic expression for a finite diff approx:

#Define $d/dx$
d_x = np.zeros([nn,nn],dtype=float)
d_x[0,0]=-1.0/dx
d_x[0,1]=1.0/dx
d_x[-1,-1]=1.0/dx
d_x[-1,-2]=-1.0/dx

for i in range(1,nn-1) :
    d_x[i,i-1]=-0.5/dx
    d_x[i,i+1]=0.5/dx

#Take derivative of d_x to get d2_x
d2_x = d_x.dot(d_x)

#Check that d2_x takes the 2nd derivative by plotting

#try f(x) = x^2
fx = [x_i**2 for x_i in x]
d2fx_dx = d2_x.dot(fx)
plt.figure()
plt.ylabel('f(x),d^2f(x)/dx^2')
plt.xlabel('x')
plt.xlim(-.98,.98)
plt.plot(x,fx, linewidth = 5, color = 'red')
plt.plot(x,d2fx_dx, linewidth = 5, color = 'green')

#try f(x) = x^3
fx = [x_i**3 for x_i in x]
d2fx_dx = d2_x.dot(fx)
plt.figure()
plt.ylabel('f(x),d^2f(x)/dx^2')
plt.xlabel('x')
plt.plot(x,fx, linewidth = 5, color = 'red')
plt.plot(x,d2fx_dx,linewidth =5, color = 'green' )
plt.xlim(-.98,.98)

#Therefore d2_x works as a second derivative function

#Construct matrices to represent multiplication by functions

#i) (1-x^2)
C1 = np.zeros([nn,nn],dtype = float)
for i in range(0,nn):
    C1[i,i]  = 1 - x[i]**2

#ii) (2x)
C2 = np.zeros([nn,nn], dtype = float)

for i in range(0,nn):
    C2[i,i] = -2*x[i]

#iii) Plot C1 and C2 x f(x) = 1 to check
fx = np.array([1 for x_i in x])
fx = np.zeros([nn,1],dtype = float)
for i in range(0,nn):
    fx[i,0] = 1

C1_fx = C1.dot(fx)
C2_fx = C2.dot(fx)

#Check functions by plotting

#i) (1-x^2)
plt.figure()
plt.title('(1-x^2) vs x')
plt.ylabel('(1-x^2)')
plt.xlabel('x')
plt.plot(x,C1_fx, linewidth = 5, color= 'green')

#ii) (-2x)
plt.figure()
plt.title('-2x vs x')
plt.ylabel('(-2x)')
plt.xlabel('x')
plt.plot(x,C2_fx, linewidth = 5, color = 'red')

Define a matrix L that approximates the Legendre Equation
L =  C1.dot(d2_x) + C2.dot(d_x)

#Check
print(L)
print(L.shape)

#L is a matrix, made up of d_x,d2_x,C1 and C2

#L acting on f(x) = x
fx = np.array([x_i for x_i in x])
L_fx = L.dot(fx)

#transpose of L acting on f(x) = x
fx = np.array([x_i for x_i in x])
L_trans = L.transpose()
L_trans_fx = L_trans.dot(fx)

#plotting
plt.figure()

plt.title('L_fx,L_trans_fx vs x')
plt.ylabel('L_fx')
plt.xlabel('x')
plt.xlim(-0.95,0.95)
plt.ylim(-2,2)
plt.plot(x,L_fx, color= 'green')
plt.plot(x,L_trans_fx, color= 'red')

Compare L_fx and L_trans_fx for various functs

#i) f(x) = x^3
fx = np.array([x_i**3 for x_i in x])
L_fx = L.dot(fx)

#transpose of L acting on f(x) = x^3
L_trans = L.transpose()
L_trans_fx = L_trans.dot(fx)

#plotting
#plt.figure()
#plt.xlim(-.95,.95)
#plt.ylim(-3,3)
#plt.title('L_fx,L_trans_fx vs x')
#plt.ylabel('L_fx')
#plt.xlabel('x')
#plt.plot(x,L_fx, color= 'purple')
#plt.plot(x,L_trans_fx, color= 'yellow')

#ii) f(x) = exp(x)
fx = np.array([np.exp(x_i) for x_i in x])
L_fx = L.dot(fx)

#transpose of L acting on f(x) = exp(x)
L_trans = L.transpose()
L_trans_fx = L_trans.dot(fx)

#plotting
#plt.figure()
#plt.xlim(-.95,.95)
#plt.title('L_fx,L_trans_fx vs x')
#plt.ylabel('L_fx')
#plt.xlabel('x')
#plt.ylim(-3,3)
#plt.plot(x,L_fx, color= 'purple')
#plt.plot(x,L_trans_fx, color= 'yellow')

#iii) f(x) = sin(x)
fx = np.array([np.sin(x_i) for x_i in x])
L_fx = L.dot(fx)

#transpose of L acting on f(x) = sin(x)
L_trans = L.transpose()
L_trans_fx = L_trans.dot(fx)

#plotting
#plt.figure()
#plt.xlim(-.95,.95)
#plt.ylim(-3,3)
#plt.title('L_fx,L_trans_fx vs x')
#plt.ylabel('L_fx')
#plt.xlabel('x')
#plt.plot(x,L_fx, color= 'purple')
#plt.plot(x,L_trans_fx, color= 'yellow')

#Calculating the eigenvalues of L

#import numpy's lin alg package to use for calculations
import numpy.linalg
import scipy.linalg

#i) Calculate eigenvalues of real, symmetric matrix L
e_vals, e_vecs = scipy.linalg.eigh(L)

#ii) Arranging eigenvalues in decreasing order
sorted_evals = sorted(e_vals, reverse = True)
n = np.arange(0,300,1)
n_n = (-1)*n
n_2n = (n+1)*n_n

#plotting
plt.figure()
plt.title('Lambda_n vs n and -n(n+1) vs n')
plt.ylabel('Lambda_n')
plt.xlabel('n')
plt.scatter(n,sorted_evals, color = 'red')
plt.scatter(n,n_2n, color = 'green')

#Finding the eigenvectors of L

#From above:
e_vals, e_vecs = scipy.linalg.eig(L)

#Find poistion of (almost) 0 value in list so as to find corresponding eigenvec

#Convert np array to list and then round in order to use index func to find position of "0" eigenvec"
e_val_list = e_vals.tolist()

#As index(0) doesn't work due to the value being not quite 0, the value had to be found using brute force
#print(e_val_list)
a = e_val_list.index(-7.213923634679477e-13+0j)

#Thus the eigenvec corresponding to the 0 eigenval is
P_0 = e_vecs[a]
#Checking eigenvec
L_P_0 = L.dot(P_0)

#Plotting
plt.figure()
plt.title('L_P_0 vs x ')
plt.ylabel('L_P_0')
plt.xlabel('x')
plt.plot(x,L_P_0, color= 'green')

#Other eigenvectors via Gram-Schmidt

#i) Trial function (DNS):

t_1  = np.array([x_i for x_i in x])

##ii) Putative eigenvector (DNS):|P_1>, via Gram-Schmidt

#Gram-Schmidt projecting |P_0> orthogonally onto the line spanned by t
a = np.inner(P_0,t_1)
b = np.inner(t_1,t_1)
P_1 = t_1*(a*(b))

#iii) Plotting (Submit):

#L acting on P_1
L_P1 = L.dot(P_1)

plt.figure()
plt.title('t_1,P_1 and L_P_1 vs x ')
plt.xlabel('x')
plt.plot(x,t_1, color = 'red')
plt.plot(x,P_1, color = 'blue')
plt.plot(x,L_P1, color= 'g')

#Is P_1 an eigenvector of L?

#For P_1 to be a an eigenvector of L  L|P_1> = lambda|P_1>

#Ideally should find a matrix of one constant (the eigenval)

e_val_1 = P_1/L

#Thus the corresponding e_val is 0.03006337

e_val_1 = 0.03006337

#i) x^2
t_2  = np.array([x_i**2 for x_i in x])

#Gram-Schmidt projecting |P_1> orthogonally onto the line spanned by t_2
a = np.inner(P_0,t_2)
b = np.inner(t_2,t_2)
P_2 = t_2*(a*(b))

#ii) x^3
t_3 = np.array([x_i**3 for x_i in x])

#Gram-Schmidt projecting |P_2> orthogonally onto the line spanned by t_3
a = np.inner(P_0,t_3)
b = np.inner(t_3,t_3)
P_3 = t_3*(a*(b))

#iii) x^3
t_4 = np.array([x_i**4 for x_i in x])

#Gram-Schmidt projecting |P_2> orthogonally onto the line spanned by t_3
a = np.inner(P_0,t_3)
b = np.inner(t_3,t_3)
P_4 = t_3*(a*(b))

#Plotting

#i) |P_2>
L_P_2 = L.dot(P_2)
plt.figure()
plt.title('t_2,P_2 and L_P_2 vs x ')
plt.xlabel('x')
plt.plot(x,t_2, color = 'r')
plt.plot(x,P_2, color = 'b')
plt.plot(x,L_P_2, color= 'g')

#ii) |P_3>
L_P_3 = L.dot(P_3)
plt.figure()
plt.title('t_3,P_3 and L_P_3 vs x ')
plt.xlabel('x')
plt.plot(x,t_3, color = 'r')
plt.plot(x,P_3, color = 'b')
plt.plot(x,L_P_3, color= 'g')

#ii) |P_4>
L_P_4 = L.dot(P_4)
plt.figure()
plt.title('t_4,P_4 and L_P_4 vs x ')
plt.xlabel('x')
plt.plot(x,t_4, color = 'r')
plt.plot(x,P_4, color = 'b')
plt.plot(x,L_P_4, color= 'g')

#Finding Eigenvalues

#i) |P_2>

#For P_2 to be a an eigenvector of L  L|P_2> = lambda|P_2>
#Ideally should find a matrix of one constant (the eigenval)

e_val_2 = P_2/L
#print(e_val_2)

#Thus the corresponding e_val is 0.16272140
e_val_2 = 0.16272140

#ii) |P_3>

#For P_3 to be a an eigenvector of L  L|P_3> = lambda|P_3>
#Ideally should find a matrix of one constant (the eigenval)

e_val_3 = P_3/L
#print(e_val_3)

#Thus the corresponding e_val is 0.01425235
e_val_3 = 0.01425235

#iii) |P_4>
#For P_4 to be a an eigenvector of L  L|P_4> = lambda|P_3>
#Ideally should find a matrix of one constant (the eigenval)

e_val_4 = P_4/L
#print(e_val_4)

#Thus the corresponding e_val is 0.00717384
e_val_4 = 0.00717384


import time
from IPython import display
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
import seaborn as sns
from sklearn.datasets import load_iris
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import make_blobs

def get_data_iris():
	X, Y = load_iris(True)
	X = pd.DataFrame(X, columns = ['SEPAL_LENGTH', 'SEPAL_WIDTH', 'PETAL_LENGTH', 'PETAL_WIDTH'])
	Y = pd.Series(Y, name = 'SPECIES')
	return X, Y

def plot_pair_plots(X, Y):
	data = pd.concat((X,Y),axis=1)
	sns.pairplot(data,hue="SPECIES")

def super_simple_classifier_plot(X, Y):
	data = pd.concat((X,Y),axis=1)
	data.loc[data["SPECIES"]!=0, "SPECIES"] = 1
	sns.lmplot("SEPAL_LENGTH", "PETAL_WIDTH",data=data, hue="SPECIES", fit_reg=False)
	plt.plot(np.linspace(4,8,1000), [0.8]*1000, 'r-')

def linear_separation_plot(X, Y):
	data = pd.concat((X,Y),axis=1)
	xx, yy = np.mgrid[4:8:.01, 0:2.8:.01]
	grid = np.c_[xx.ravel(), yy.ravel()]
	for species in [2,1,0]:
		labels = data.loc[data.SPECIES != species,'SPECIES'].unique()
		clf = LogisticRegression(random_state=0).fit(data.loc[data.SPECIES != species,['SEPAL_LENGTH','PETAL_WIDTH']],
			data.loc[data.SPECIES != species,'SPECIES'])
		probs = clf.predict_proba(grid)[:, 1].reshape(xx.shape)
		f, ax = plt.subplots(figsize=(8, 6))
		contour = ax.contourf(xx, yy, probs, 25, cmap="RdBu",
		                      vmin=0, vmax=1)
		ax_c = f.colorbar(contour)

		ax_c.set_label("$P(Species = %0.1i)$"%labels[1])
		ax_c.set_ticks([0, .25, .5, .75, 1])

		ax.scatter(data.loc[data.SPECIES != species,'SEPAL_LENGTH'], data.loc[data.SPECIES != species,'PETAL_WIDTH'],
			c=data.loc[data.SPECIES != species,'SPECIES'],
			s=50,cmap="RdBu", vmin=-.2, vmax=1.2,edgecolor="white", linewidth=1)
		ax.set(aspect="equal",
		       xlim=(4, 8), ylim=(0, 2.8),
		       xlabel="SEPAL_LENGTH", ylabel="PETAL_WIDTH", title="Logistic Regression Classification Between Species %0.1i and %0.1i" % (labels[0], labels[1]))
		plt.show()

def predict_probability_point(X, Y, point):
	data = pd.concat((X,Y),axis=1)
	a, b = point
	clf = LogisticRegression(random_state=0).fit(data.loc[data.SPECIES != 2,['SEPAL_LENGTH','PETAL_WIDTH']],
			data.loc[data.SPECIES != 2,'SPECIES'])
	probs = clf.predict_proba(np.array([[a,b]]))[:, 1]
	print("The probability of point (%0.1f,%0.1f) belonging to Species 1 is %0.2f" % (a, b, probs))

def gradient_descent_classification_plot(X, Y, iter):
	data = pd.concat((X,Y),axis=1)
	xx, yy = np.mgrid[4:8:.01, 0:2.8:.01]
	grid = np.c_[xx.ravel(), yy.ravel()]
	species = 2
	labels = data.loc[data.SPECIES != species,'SPECIES'].unique()
	clf = LogisticRegression(max_iter=1, warm_start = True, solver = 'sag',random_state=0).fit(data.loc[data.SPECIES != species,['SEPAL_LENGTH','PETAL_WIDTH']],
			data.loc[data.SPECIES != species,'SPECIES'])
	probs = clf.predict_proba(grid)[:, 1].reshape(xx.shape)
	f, ax = plt.subplots(figsize=(8, 6))

	for i in range(iter):
		display.clear_output(wait=True)
		time.sleep(0.1)
		contour = ax.contourf(xx, yy, probs, 25, cmap="RdBu",
		vmin=0, vmax=1)

		ax.scatter(data.loc[data.SPECIES != species,'SEPAL_LENGTH'], data.loc[data.SPECIES != species,'PETAL_WIDTH'],
		c=data.loc[data.SPECIES != species,'SPECIES'],
		s=50,cmap="RdBu", vmin=-.2, vmax=1.2,edgecolor="white", linewidth=1)
		ax.set(aspect="equal",
		       xlim=(4, 8), ylim=(0, 2.8),
		       xlabel="SEPAL_LENGTH", ylabel="PETAL_WIDTH", title="Logistic Regression Classification Between Species %0.1i and %0.1i \n (iteration %0.1i)" % (labels[0], labels[1], i+1))
		if(i==0):
			ax_c = f.colorbar(contour)
			ax_c.set_label("$P(Species = %0.1i)$"%labels[1])
			ax_c.set_ticks([0, .25, .5, .75, 1])
		display.display(plt.gcf());
		clf.fit(data.loc[data.SPECIES != species,['SEPAL_LENGTH','PETAL_WIDTH']],
		data.loc[data.SPECIES != species,'SPECIES'])
		probs = clf.predict_proba(grid)[:, 1].reshape(xx.shape)
	plt.clf();

def plot_cross_entropy():
	x = np.linspace(0,1,1000)
	cost_one = []
	cost_zero = []
	for i in x:
		cost_one.append(logloss(1,i))
		cost_zero.append(logloss(0,i))
	f, axs = plt.subplots(nrows=1,ncols=2)
	ax = axs[0]
	ax.set(xlabel="$\hat{p}$", ylabel="Cross-Entropy", title="Cross-Entropy Loss for y = 0")
	ax.plot(x,cost_zero)
	ax = axs[1]
	ax.set(xlabel="$\hat{p}$", ylabel="Cross-Entropy", title="Cross-Entropy Loss for y = 1")
	ax.plot(x,cost_one)
	plt.tight_layout()
	plt.show()

def logloss(true_label, predicted, eps=1e-15):
  p = np.clip(predicted, eps, 1 - eps)
  if true_label == 1:
    return -np.log(p)
  else:
    return -np.log(1 - p)

def OvR_plot(X, Y):
	data = pd.concat((X,Y),axis=1)
	xx, yy = np.mgrid[4:8:.01, 0:2.8:.01]
	grid = np.c_[xx.ravel(), yy.ravel()]
	fc = np.zeros((3,xx.shape[0], xx.shape[1]))
	labels = [0,1]
	for species in [0,1,2]:
		new_data = data.copy()
		new_data.loc[new_data.SPECIES == species, "SPECIES"] = 3
		new_data.loc[~np.isin(new_data.SPECIES,3), "SPECIES"] = 0
		clf = LogisticRegression(random_state=0).fit(new_data[['SEPAL_LENGTH','PETAL_WIDTH']],
			new_data['SPECIES'])
		probs = clf.predict_proba(grid)[:, 1].reshape(xx.shape)
		f, ax = plt.subplots(figsize=(8, 6))
		contour = ax.contourf(xx, yy, probs, 25, cmap="RdBu",
		                      vmin=0, vmax=1)
		ax_c = f.colorbar(contour)

		ax_c.set_label("$P(Species = %0.1i)$"%species)
		ax_c.set_ticks([0, .25, .5, .75, 1])

		ax.scatter(new_data['SEPAL_LENGTH'], new_data['PETAL_WIDTH'],
			c=new_data['SPECIES'],
			s=50,cmap="RdBu", vmin=-.2, vmax=1.2,edgecolor="white", linewidth=1)
		ax.set(aspect="equal",
		       xlim=(4, 8), ylim=(0, 2.8),
		       xlabel="SEPAL_LENGTH", ylabel="PETAL_WIDTH", title="Logistic Regression Classification Between Species %0.1i and the remaining ones" % (species))
		fc[species] = probs
		plt.show()
	probs = np.argmax(fc,axis=0)
	f, ax = plt.subplots(figsize=(8, 6))
	ax_c.set_ticks([0, .25, .5, .75, 1])

	ax.set(aspect="equal",
		       xlim=(4, 8), ylim=(0, 2.8),
		       xlabel="SEPAL_LENGTH", ylabel="PETAL_WIDTH", title="Logistic Regression Classification with OvR")
	contour = ax.contourf(xx, yy, probs, 25, cmap="RdBu",
		                      vmin=0, vmax=1)
	ax.scatter(data['SEPAL_LENGTH'], data['PETAL_WIDTH'],
			c=data['SPECIES'],
			s=50,cmap="RdBu", vmin=-.2, vmax=1.2,edgecolor="white", linewidth=1)
	plt.show()

def multinomial_plot(X, Y):
	data = pd.concat((X,Y),axis=1)
	xx, yy = np.mgrid[4:8:.01, 0:2.8:.01]
	grid = np.c_[xx.ravel(), yy.ravel()]
	clf = LogisticRegression(random_state=0, multi_class='multinomial', solver='lbfgs').fit(data[['SEPAL_LENGTH','PETAL_WIDTH']],
			data['SPECIES'])
	probs = clf.predict(grid).reshape(xx.shape)
	f, ax = plt.subplots(figsize=(8, 6))
	contour = ax.contourf(xx, yy, probs, 25, cmap="RdBu",
	                      vmin=0, vmax=1)
	ax.scatter(data['SEPAL_LENGTH'], data['PETAL_WIDTH'],
		c=data['SPECIES'],
		s=50,cmap="RdBu", vmin=-.2, vmax=1.2,edgecolor="white", linewidth=1)
	ax.set(aspect="equal",
	       xlim=(4, 8), ylim=(0, 2.8),
	       xlabel="SEPAL_LENGTH", ylabel="PETAL_WIDTH", title="Multinomial Logistic Regression Classification")
	plt.show()

def OvR_vs_multinomial_plot():
	# Authors: Tom Dupre la Tour <tom.dupre-la-tour@m4x.org>
	# License: BSD 3 clause
	# make 3-class dataset for classification
	centers = [[-5, 0], [0, 1.5], [5, -1]]
	X, y = make_blobs(n_samples=1000, centers=centers, random_state=40)
	transformation = [[0.4, 0.2], [-0.4, 1.2]]
	X = np.dot(X, transformation)

	for multi_class in ('multinomial', 'ovr'):
	    clf = LogisticRegression(solver='sag', max_iter=100, random_state=42,
	                             multi_class=multi_class).fit(X, y)

	    # print the training scores
	    print("training score : %.3f (%s)" % (clf.score(X, y), multi_class))

	    # create a mesh to plot in
	    h = .02  # step size in the mesh
	    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
	    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
	    xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
	                         np.arange(y_min, y_max, h))

	    # Plot the decision boundary. For that, we will assign a color to each
	    # point in the mesh [x_min, x_max]x[y_min, y_max].
	    Z = clf.predict(np.c_[xx.ravel(), yy.ravel()])
	    # Put the result into a color plot
	    Z = Z.reshape(xx.shape)
	    plt.figure()
	    plt.contourf(xx, yy, Z, cmap=plt.cm.Paired)
	    plt.title("Decision surface of LogisticRegression (%s)" % multi_class)
	    plt.axis('tight')

	    # Plot also the training points
	    colors = "bry"
	    for i, color in zip(clf.classes_, colors):
	        idx = np.where(y == i)
	        plt.scatter(X[idx, 0], X[idx, 1], c=color, cmap=plt.cm.Paired,
	                    edgecolor='black', s=20)

	    # Plot the three one-against-all classifiers
	    xmin, xmax = plt.xlim()
	    ymin, ymax = plt.ylim()
	    coef = clf.coef_
	    intercept = clf.intercept_

	    def plot_hyperplane(c, color):
	        def line(x0):
	            return (-(x0 * coef[c, 0]) - intercept[c]) / coef[c, 1]
	        plt.plot([xmin, xmax], [line(xmin), line(xmax)],
	                 ls="--", color=color)

	    for i, color in zip(clf.classes_, colors):
	        plot_hyperplane(i, color)

	plt.show()


#!/usr/bin/env python3

# Solution to Project Euler problem 81

import numpy as np

def read_file(filename):
    matrix = []
    with open(filename) as file:
        for line in file:
            matrix.append([int(x) for x in line.split(',')])
    return np.asarray(matrix)

def find_min_path(matrix):
    cost_matrix = np.zeros(shape=(len(matrix), len(matrix)))
    cost_matrix[0][0] = matrix[0][0]

    # Start with finding cost of first row and first column
    for i in range(1,len(matrix)):
        cost_matrix[0][i] = cost_matrix[0][i-1] + matrix[0][i]
        cost_matrix[i][0] = cost_matrix[i-1][0] + matrix[i][0]

    # Then loop rest of the two matrixes...
    for i in range(1, len(matrix)):
        for j in range(1, len(matrix)):
            cost_matrix[i][j] = min(cost_matrix[i-1][j], cost_matrix[i][j-1]) + matrix[i][j]

    return cost_matrix

def solve():
    matrix = read_file('p081_matrix.txt')
    return int(find_min_path(matrix)[-1][-1])

if __name__ == "__main__":
    print(solve())


# used for manipulating directory paths
import os

# Scientific and vector computation for python
import numpy as np

# Plotting library
from matplotlib import pyplot
from mpl_toolkits.mplot3d import Axes3D  # needed to plot 3-D surfaces

def computeCost(X, y, theta):
    """
    Compute cost for linear regression. Computes the cost of using theta as the
    parameter for linear regression to fit the data points in X and y.

    Parameters
    ----------
    X : array_like
        The input dataset of shape (m x n+1), where m is the number of examples,
        and n is the number of features. We assume a vector of one's already
        appended to the features so we have n+1 columns.

    y : array_like
        The values of the function at each data point. This is a vector of
        shape (m, ).

    theta : array_like
        The parameters for the regression function. This is a vector of
        shape (n+1, ).

    Returns
    -------
    J : float
        The value of the regression cost function.

    Instructions
    ------------
    Compute the cost of a particular choice of theta.
    You should set J to the cost.
    """

    # initialize some useful values
    m = y.size  # number of training examples

    # You need to return the following variables correctly
    J = 0

    # Add a column of ones to X. The numpy function stack joins arrays along a given axis.
    # The first axis (axis=0) refers to rows (training examples)
    # and second axis (axis=1) refers to columns (features).
    X = np.stack([np.ones(m), X], axis=1)

    for i in range(0, m):
        J += ((theta[0] + theta[1] * X[i][1]) - y[i]) ** 2
        # for more than one feature I´d simply implement it with a second for loop

    return J * (1/(2 * m))

if __name__ == '__main__':
    data = np.loadtxt(os.path.join("/home/thelichking/Desktop/ml-coursera-python-assignments/Exercise1/Data",
                                   "ex1data1.txt"), delimiter=',')

    data_test = [[], []]
    X, y = data[:, 0], data[:, 1]

    J = computeCost(X, y, theta=np.array([0.0, 0.0]))
    print('With theta = [0, 0] \nCost computed = %.2f' % J)
    print('Expected cost value (approximately) 32.07\n')

    # further testing of the cost function
    J = computeCost(X, y, theta=np.array([-1, 2]))
    print('With theta = [-1, 2]\nCost computed = %.2f' % J)
    print('Expected cost value (approximately) 54.24')


import numpy as np
import cv2

# Y' = 0.299 R + 0.587 G + 0.114 B
def rgb_to_gray(X_rgb):
    X_gray = []
    for sample in X_rgb:
        grayscale = np.dot(sample[...,:3], [0.299, 0.587, 0.114])
        X_gray.append(grayscale)

    # Turn into numpy ararry
    X_gray = np.asarray(X_gray)
    X_gray = np.expand_dims(X_gray, axis=3)
    return X_gray

#(pixel - 128)/ 128 is a quick way to approximately normalize the data and can be used in this project.
def naive_normalization(X):
    return (X - 128)/128

def cv2_gray_n_normalize(img):
    """
    Using the embedded function in open cv to convert rgb into gray
    Using the n-mean(n)/stdev(n)
    """
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    gray = (gray - np.mean(gray)) / (np.std(gray) + 1e-8) # normalization
    gray = np.dstack([gray]).astype(np.float32)
    return gray