'''
In this assignment you will implement one or more algorithms for the all-pairs shortest-path problem. Here are data files describing three graphs:

The first line indicates the number of vertices and edges, respectively. Each subsequent line describes an edge (the first two numbers are its tail and head, respectively)
and its length (the third number). NOTE: some of the edge lengths are negative. NOTE: These graphs may or may not have negative-cost cycles.

Your task is to compute the "shortest shortest path". Precisely, you must first identify which, if any, of the three graphs have no negative cycles. For each such graph,
you should compute all-pairs shortest paths and remember the smallest one (i.e., compute \min_{u,v \in V} d(u,v)min u,v ∈ V d(u,v), where d(u,v)d(u,v) denotes the shortest-path distance from uu to vv).

If each of the three graphs has a negative-cost cycle, then enter "NULL" in the box below. If exactly one graph has no negative-cost cycles, then enter the length of its shortest shortest
path in the box below. If two or more of the graphs have no negative-cost cycles, then enter the smallest of the lengths of their shortest shortest paths in the box below.

OPTIONAL: You can use whatever algorithm you like to solve this question. If you have extra time, try comparing the performance of different all-pairs shortest-path algorithms!

OPTIONAL: Here is a bigger data set to play with.
'''

import os, math
from collections import defaultdict
from heapq import heappop, heappush

import numpy as np
from tqdm import tqdm

data_dir = os.getcwd() + '\\data\\illuminated\\shortest-paths\\'

submission_file_paths = [data_dir + sub for sub in ['g1.txt', 'g2.txt', 'g3.txt']]
big_submission_file_path = data_dir + 'large.txt'

test_file_path = data_dir + 'problem18.8test1.txt'

def bellman_ford(graph, graph_inverted):
    """
    Implementation of the Bellman-Ford algorithm to find the single-source shortest path for a graph possibly containing
    negative edges.
    """

    print('creating modified graph by adding artificial start vertex s (id=0) with zero-edge to each vertex')

    s = 0
    graph_ = defaultdict(set, graph)
    graph_inverted_ = defaultdict(set, graph_inverted)

    for vertex in graph.keys():
        graph_[s].add(vertex)
        graph_inverted_[vertex].add((0, s))

    n = len(graph_)
    A = np.full(n, np.inf)
    A[s] = 0

    # we don't need to compute the full DP matrix, only the last 2 columns
    arrays_are_equal = False
    for _ in tqdm(range(1, n + 1)):
        A_next = np.full(n, np.inf)
        for v in graph.keys():
            case_1 = A[v]

            if v in graph_inverted:
                # v has incoming edges from vertices w: compute path to v with at most i-1 edges from all ws
                case_2 = min(A[w] + c for (c, w) in graph_inverted[v])
            else:
                case_2 = math.inf

            A_next[v] = min(case_1, case_2)

        arrays_are_equal = np.array_equal(A, A_next)

        if arrays_are_equal:
            print('early stopping because no change in previous iteration')
            return A[1:]

        A = A_next

    if not arrays_are_equal:
        print('negative cycle detected in additional iteration')
        return None

    print('no negative cycle detected')
    return A[1:]

def dijkstra(graph, s, t):
    """
    Heap-based implementation of Dijkstra's algorithms for SSSP for a given source vertex s and target vertex t
    :param graph: graph as dictionary mapping the edges for each vertex as follows:
                    source_vertex -> {(cost, target_vertex}
    :param s: start vertex
    :param t: target vertex
    :return: the cost and path of the minimal path from s to t if there is one, else infinity
    """
    q, seen, mins = [(0, s, ())], set(), {s: 0}
    while q:
        (cost, v1, path) = heappop(q)
        if v1 not in seen:
            seen.add(v1)
            path = (v1, path)
            if v1 == t:
                return cost, path

            for c, v2 in graph.get(v1, ()):
                if v2 in seen:
                    continue
                prev = mins.get(v2, None)
                next = cost + c
                if prev is None or next < prev:
                    mins[v2] = next
                    heappush(q, (next, v2, path))

    return float("inf")

def johnson(filename):

    print(f'processing {filename}')

    with open(filename) as f:
        lines = f.readlines()

    n_vertices, n_edges = map(int, lines[0].split())

    print(f'number of vertices: {n_vertices}')
    print(f'number of edges: {n_edges}')

    # create graph as dict: source -> (cost, target)
    graph = defaultdict(set)
    graph_inverted = defaultdict(set)  # precompute incoming edges for performance speedup

    for line in lines[1:]:
        u, v, c = map(int, line.split())
        graph[u].add((c, v))
        graph_inverted[v].add((c, u))

    assert len(graph.keys()) == n_vertices, 'length of vertices does not match'
    assert len(list(edge for edges in graph.values() for edge in edges)) == n_edges, 'length of edges does not match'

    print('running Bellman-Ford once to get single-source shortest path for a graph with possibly negative edges')

    A = bellman_ford(graph, graph_inverted)

    if A is None:
        # negative cost cycle detected
        return math.inf

    return min(A)

for s in submission_file_paths:
    print(f'{s}: {johnson(s)}')


import matplotlib.pyplot as plt
import numpy as np
from scipy import signal
from scipy.io import wavfile
from scipy.fftpack import rfft, irfft, rfftfreq
from ipywidgets import widgets
from IPython.display import display

#
# Fourier Series Code
#

#  funtions for fourier approximation
def f1(x,xMin,xMax):
    ''' f1(x) = x
    '''
    return x

def f2(x,xMin,xMax):
    ''' Square wave
               ~ -1, -1 < x < 0
        f2(x)= |
               ~  1,  0 < x < 1
    '''
    return signal.square(2*x)

def f3(x,xMin,xMax):
    ''' Cubic
    '''

    return 5*x*(x+1.)*(x-1.)

def example_4b(x,xMin,xMax):
    T = np.abs(xMax-xMin)
    f = np.zeros(len(x))
    for i in range(len(x)):
        xc = x[i]-np.rint(x[i]/T)*T
        if xc < -0.5*np.pi or xc > 0.5*np.pi:
            f[i] = -1.
        else:
            f[i] = 1.
    return f

def example_4c(x,xMin,xMax):
    T = np.abs(xMax-xMin)
    f = np.zeros(len(x))
    for i in range(len(x)):
        f[i] = x[i]-np.rint(x[i]/T)*T
    return f

def example_5a_sine(x,xMin,xMax):
    T = np.abs(xMax-xMin)
    f = np.zeros(len(x))
    for i in range(len(x)):
        xc = x[i]-np.rint(x[i]/T)*T
        f[i] = xc
    return f

def example_5a_cosine(x,xMin,xMax):
    T = np.abs(xMax-xMin)
    f = np.zeros(len(x))
    for i in range(len(x)):
        xc = x[i]-np.rint(x[i]/T)*T
        if xc < 0.:
            f[i] = -1.*xc
        else:
            f[i] = 1.*xc
    return f

def example_5b_sine(x,xMin,xMax):
    T = np.abs(xMax-xMin)
    f = np.zeros(len(x))
    for i in range(len(x)):
        xc = x[i]-np.rint(x[i]/T)*T
        if xc < 0.:
            f[i] = -1.
        else:
            f[i] = 1.
    return f

def example_5b_cosine(x,xMin,xMax):
    T = np.abs(xMax-xMin)
    f = np.zeros(len(x))
    for i in range(len(x)):
        xc = x[i]-np.rint(x[i]/T)*T
        if xc < 1. and xc > -1.:
            f[i] = 1.
        else:
            f[i] = 0.
    return f

def produceFourierSeries(x, nMax, xMin, xMax, f):
    """ Produces Fourier Series approximation of a function

        ----------

        Parameters

        ----------

        x: array_like
            sample values

        nMax: int
            number of terms

        xMin: float
            lower bound on window/approximation

        xMax: float
            upper bound on window/approximation

        f: function
            function to be approximated

    """
    T = xMax - xMin

    # need new range over the actual range of the approximation (not the plotting range)
    # so integration is correct
    xFuncPeriod = np.arange(xMin, xMax, 0.001)

    # set initial state of series to be the a0 constant
    series = np.zeros(len(x))
    series += (1./T)*np.trapz(f(xFuncPeriod,xMin,xMax), x=xFuncPeriod)

    prev = None

    for i in range(nMax):
        n = i+1 # set n term number based on loop counter
        an = (2./T)*np.trapz(f(xFuncPeriod,xMin,xMax)*np.cos(2*n*np.pi*xFuncPeriod*(1/T)), x=xFuncPeriod)
        bn = (2./T)*np.trapz(f(xFuncPeriod,xMin,xMax)*np.sin(2*n*np.pi*xFuncPeriod*(1/T)), x=xFuncPeriod)

        prev = np.add(an * np.cos(2*n*np.pi*x*(1/T)), bn * np.sin(2*n*np.pi*x*(1/T)))
        series = np.add(series, prev)

    return series, prev

def fourierMain(function, nMax, showPrevTerm):
    """
        Main function for calling produceFourierSeries and plotting

        ----------

        Parameters

        ----------

        function: string
            function key

        nMax: int
            number of terms

        showPrevTerm: bool
            true if most recent term should be displayed. False otherwise
    """

    # window upper and lower
#     xMin = -np.pi
#     xMax = np.pi

    functions = {'Linear':[f1, -np.pi, np.pi],'Square Wave':[f2, -np.pi/2, np.pi/2],'Cubic':[f3, -np.pi, np.pi],\
                '4B':[example_4b,-np.pi,np.pi],'4C':[example_4c,-np.pi,np.pi],\
                '5A_Sine':[example_5a_sine,-1.,1.],'5A_Cosine':[example_5a_cosine,-1.,1.],
                '5B_Sine':[example_5b_sine,-1.,1.],'5B_Cosine':[example_5b_cosine,-2.,2.]}

    f, xMin, xMax = functions[function]
    x = np.arange(xMin, xMax, 0.001)
    fx = f(x,xMin,xMax)
    fMin = np.min(fx)
    fMax = np.max(fx)

    _, ax = plt.subplots(figsize=(16,10))

    seriesApprox, prev = produceFourierSeries(x, nMax, xMin, xMax, f)

    if nMax > 0:
        label_tmp = 'Partial Sums, N={}'
    else:
        label_tmp = 'a0'

    ax.plot(x, seriesApprox, label=label_tmp.format(nMax), color='C0')
    # plot prev term if necessary. Don't plot if no prev term. (Shouldnt be possible with slider lim anyway)
    if (nMax > 0) and showPrevTerm: ax.plot(x, prev, label='Latest Term', color='g')
    ax.plot(x, fx, label='Function over Fundamental Range', color='C1')

    xbuffer = np.abs(xMax-xMin)/5.
    ybuffer = np.abs(fMax-fMin)/5.

    ax.set_xlabel('x')
    ax.set_ylabel('f(x)')
    ax.set_xlim(xMin-xbuffer,xMax+xbuffer)
    ax.set_ylim(fMin-ybuffer,fMax+ybuffer)
    ax.axhline(0,ls='-',c='k',lw=0.5)
    ax.axvline(0,ls='-',c='k',lw=0.5)
    ax.legend()
    plt.show()

def FourierSeries():
    """
        Main function called by notebook
    """
    terms_sldr = widgets.IntSlider(value=0, description='Terms', min=0, max=100, display=False, continuous_update=False)
    func_drp = widgets.Dropdown(options=['Linear','Square Wave','Cubic','4B','4C','5A_Sine','5A_Cosine'\
                                        ,'5B_Sine','5B_Cosine'],description='Function')
    prevTerm_check = widgets.Checkbox(value=False, description='Show most recent term')
    return widgets.VBox([widgets.HBox([terms_sldr, func_drp, prevTerm_check]), widgets.interactive_output(fourierMain, {'function':func_drp, 'nMax':terms_sldr, 'showPrevTerm':prevTerm_check})])

#
# Signal Processing Code
#

# some useful parameters

# min and max frequency for genwaves
minf = 1
maxf = 5

# file names and descriptor in dropdown
files = {'Guitar A string':'anote.wav', 'Bell C6':'c3Bell.wav', 'Synth':'generatedAudio.wav'}
path = "sound/"
# both these values depend on the exact .wav file you have.
# Must be right for the frequency readout to be correct but will still give general shape
sampFreq = 44100. # sample frequency, MUST be a float
bitDepth = 16

def genWaves(noWaves, seed):
    """ produces amplitude and angular velocity of waves and places them in an array noWaves long with [a, w, s]
        where:

        ----------

        Parameters

        ----------

        noWaves: int
            number of waves to produce

        seed: int
            seed of random number generator

        ----------

        Returns

        ----------

        array_like
            noWaves by 3 with, a = amplitude, w = angular velocity s = horizontal shift

    """
    np.random.seed(seed)

    waves = []
    for _ in range(noWaves):
        f = np.random.uniform(minf,maxf)
        a = np.random.uniform()/(f) # crude downward bias on the amplitude of high frequencies
        s = np.random.uniform(0, 2*np.pi)

        waves.append([a, f, s])
    return waves

def plotWaves(waves):
    """ Main plotting function for RandomWave. Also sums waves to produce signal

        ----------

        Parameters

        ----------

        waves: array_like
            noWavesx3 array that contains amplitude, angular velocity and horizontal shift of each wave
    """
    _, (rawAx, compAx, ampAx) = plt.subplots(3,figsize=(16,10))
    x = np.arange(0,2*np.pi,0.01)
    out = 0
    for wave in waves:
        a, f, s = wave
        w = 2*np.pi*f
        out += a*np.sin(w*x-s)

        compAx.plot(x, a*np.sin(w*x-s), ls="--",linewidth=0.5)

    rawAx.plot(x, out, linewidth=2)
    freqAmp = np.array(waves).T
    order = np.argsort(freqAmp[1])
    freq = freqAmp[1][order]
    amp = freqAmp[0][order]
    ampAx.bar(freq, amp, width=0.1)

    compAx.set_title("Broken into Components")
    rawAx.set_title("Raw Signal")
    ampAx.set_xlim(minf,maxf)
    ampAx.set_xlabel("Frequency")
    ampAx.set_ylabel("Amplitude")
    ampAx.set_title("Frequency vs. Amplitude of Components")

    compAx.xaxis.set_ticklabels([])
    compAx.yaxis.set_ticklabels([]) # Removing all trace of axis labels because it looks prettier
    rawAx.xaxis.set_ticklabels([])
    rawAx.yaxis.set_ticklabels([])

    compAx.yaxis.set_ticks([])
    rawAx.xaxis.set_ticks([])
    compAx.xaxis.set_ticks([])
    rawAx.yaxis.set_ticks([])
    plt.show()

def runWaves(noWaves, seed, filterRange=[minf, maxf]):
    """ Filters waves from genWaves
        ----------

        Parameters

        ----------

        noWaves: int
            number of waves to produce

        seed: int
            seed for random number generator

        filterRange: array_like
            pair of min and max filter values
    """
    filterMin, filterMax = filterRange
    waves = genWaves(noWaves, seed)
    # filter frequency outside allowed range
    filteredWaves = [wave for wave in waves if((wave[1] < filterMin) or wave[1] > filterMax)]
    if len(filteredWaves) > 1: plotWaves(filteredWaves)

def RandomWave():
    """ Main function called by notebook
    """
    noWaves_sldr = widgets.IntSlider(value=10, min=2, max=20, step=1, description='No. Waves', continuous_update=False)
    seed_Text = widgets.IntText(123, description='Seed')
    filter_sldr = widgets.FloatRangeSlider(value=[minf, minf],
                                           min=minf,
                                           max=maxf,
                                           description="Filter Range",
                                           continuous_update=False)
    return widgets.VBox([
        widgets.HBox([noWaves_sldr,filter_sldr,seed_Text]),
        widgets.interactive_output(runWaves, {'noWaves':noWaves_sldr, 'seed':seed_Text, 'filterRange':filter_sldr})])

##########

# Fourier Analysis

##########

def readWav(fileName):
    """
        Reads data from .wav

        ----------

        Parameters

        ----------

        fileName: str
            name of file or path

        ----------

        Returns

        ----------

        Raw audio data

    """

    _, data = wavfile.read(path+fileName) # load the data

    audioData = data.T
    # check if wav is stereo
    if len(data.shape)>1:
        audioData = audioData[0]
    return audioData

def processSignal(audioData):
    """
        Performs actual transform and returns:
        raw wav data
        transformed signal
        frequencies of transformed data

        ----------

        Parameters

        ----------

        audioData: array_like
            raw audio data

        ----------

        Returns

        ----------

        audioData: array_like
            Raw audio data

        tranformed: array_like
            frequency domain data

        freqs: array_like
            frequencies

    """
    tranformed = (rfft(audioData)) # perform transform

    freqs = rfftfreq(len(tranformed), d=1/sampFreq)

    return [audioData, tranformed, freqs]

def FourierAnalysis():
    """
        Called in notebook. Returns fft'd and raw data from bell and guitar .wavs.
        Calls the processSignal.

        for each file returns: [audio, tranformed, freqs]

    """
    # this is where I got the files from:
    #       guitar: https://freesound.org/people/casualdave/sounds/44729/
    #       bell  : https://freewavesamples.com/steel-bell-c6
    #       I made synth

    out = []

    for _, value in files.items():
        out.append(processSignal(readWav(value)))

    return out

def MusicNote(audioData, freqDom, freqs):
    """
        Produces UI componants for user. Drives plotting of PlotFourierAnalysis

        ----------

        Parameters

        ----------

        audioData: array_like
            raw audio data

        freqDom: array_like
            transformed frequency domain data

        freqs: array_like
            frequencies
    """
    xlim_sldr = widgets.IntRangeSlider(value=[0, 22.5e3], min=0, max=22.5e3, step=1000, continuous_update=False, description='Ax. lim')

    return widgets.VBox([widgets.interactive_output(PlotSignal, {
            'signal':widgets.fixed(audioData),
            'amps':widgets.fixed(freqDom),
            'freqs':widgets.fixed(freqs),
            'FALim':xlim_sldr}),
        widgets.HBox([xlim_sldr])])

def FilterFreqs(audioData, freqDom, freqs):
    """
        Produces UI componants for user. Drives plotting of filtered signal (FilterFreqs)

        ----------

        Parameters

        ----------

        audioData: array_like
            raw audio data

        freqDom: array_like
            transformed frequnecy domain data

        freqs: array_like
        frequencies
    """
    filter_sldr = widgets.IntRangeSlider(value=[0,0], min=0, max=23e3, step=1000, continuous_update=False, description='Freq Band')
    xlim_sldr = widgets.IntRangeSlider(value=[0, 22.5e3], min=0, max=22.5e3, step=1000, continuous_update=False, description='Ax. lim')
    export_btn = widgets.ToggleButton(value=False, description='Export to .wav')

    display(widgets.VBox([widgets.interactive_output(FilterBand, {
            'audioData':widgets.fixed(audioData),
            'freqDom':widgets.fixed(freqDom),
            'freqs':widgets.fixed(freqs),
            'filtFreq':filter_sldr,
            'FALim':xlim_sldr,
            'export':export_btn
            }),
        widgets.HBox([xlim_sldr, filter_sldr, export_btn])]))

def FilterBand(audioData, freqDom, freqs, filtFreq, FALim, export):
    """
        Sets amplitudes in transform to 0 that are outside frequency range. Performs inverse tranform
        Calls PlotSignal

        ----------

        Parameters

        ----------

        audioData: array_like
            raw audio data

        processed: array_like
            fourier transformed data

        freqs: array_like
            frequencies for filtering and inversion

        filtFreq: array_like
            pair of frequencies representing filter band

        FALim: array_like
            pair of values representing x axis limits for analysis plot
    """
    # list comprehension alternative. Slower.
    # filteredTrans = [0 if (freqs[i] >= filtFreq[0]) and (freqs[i] <= filtFreq[1]) else freqDom[i] for i in range(len(freqDom))]

    # to store filtered transformed data
    # filteredTrans = np.zeros(len(freqDom))
    filteredTrans = freqDom.copy()
    for i in range(len(freqDom)):
        if ((freqs[i] >= filtFreq[0]) and (freqs[i] <= filtFreq[1])):
            filteredTrans[i] = 0

        if freqs[i] > filtFreq[1]: break

    # invert to filtered signal
    filteredSignal = irfft(filteredTrans)
    # slice out negatives
    filteredSignal = (filteredSignal[:int(len(filteredSignal)/2)])

    if export:
        exportWav(filteredSignal)

    PlotSignal(filteredSignal, filteredTrans, freqs, FALim)

def PlotSignal(signal, amps, freqs, FALim):
    """
        Inverts and plots filterd signal.
        2 figures, freq and time domains.
        Called by FilterBand and MusicNote

        ----------

        Parameters

        ----------

        signal: array_like
            audio data to be plotted on top plot ()

        amps: array_like
            amplitude data for fourier analysis graph

        freqs: array_like
            freqencies for analysis graph

        FALim: array_like
            Fourier analysis plot x limits
    """

    _, (ax1, ax2) = plt.subplots(2, figsize=(16,8))

    ax1.plot(signal, linewidth=0.1)
    ax1.set_title('Signal')
    ax1.set_xlim(0)
    ax1.set_ylim(-35e3, 35e3)

    # commenting scale on x because samples arent super useful. Can convert to seconds.
    ax1.xaxis.set_ticklabels([])
    ax1.xaxis.set_ticks([])

    ax2.set_title('Fourier Analysis of Signal')
    ax2.plot(freqs, np.abs(amps), 'r', linewidth=0.5)
    ax2.set_yscale('log')
    ax2.set_xlim(*FALim)
    ax2.set_xlabel('Frequency (Hz)')
    ax2.set_ylabel('Amplitude')
    ax2.set_ylim(1e-3, 1e9)

    plt.show()

def exportWav(audioData):
    """ Function for exporting signal to wav

        ----------

        Parameters

        ----------

        audioData: array_like
            discrete audio signal
    """
    audioData = np.asarray(audioData, dtype=np.int16)

    wavfile.write(path+'exportedAudio.wav',int(sampFreq), audioData)

##########

# Even odd

##########

def oddSquare(x):
    ''' Square wave
               ~ -1, -1 < x < 0
        f2(x)= |
               ~  1,  0 < x < 1
    '''
    return signal.square(2*x)

def evenSquare(x):
    ''' Square wave
               ~ -1, -1 < x < 0
        f2(x)= |
               ~  1,  0 < x < 1
    '''
    return signal.square(2*(x + np.pi/4))

def cos2(x):
    return np.cos(2*x)

def sin2(x):
    return np.sin(2*x)

def quad(x):
    return x**2

def EvenOdd():
    funcs = {"cos(x)":np.cos, "cos(2x)":cos2,"cos(2x)":cos2,"sin(x)":np.sin, "x^2":quad, "Odd Square":oddSquare, "Even Square":evenSquare}
    f1_drop = widgets.Dropdown(options=funcs)
    f2_drop = widgets.Dropdown(options=funcs)
    sym_check = widgets.Checkbox(value=False, description="Check Symmetry")
    area_check = widgets.Checkbox(value=False, description="Integration Region")

    display(
        widgets.VBox([
            widgets.HBox([
                f1_drop,
                f2_drop,
                sym_check,
                area_check
                ]),
            widgets.interactive_output(symmetryCheck, {'f1':f1_drop, 'f2':f2_drop, "showSym":sym_check, "showArea":area_check})
        ])
    )

def symmetryCheck(f1, f2, showSym, showArea):

    a_slider = widgets.FloatSlider(value=1, min=0, max=np.pi, step=0.1, continuous_update=False)

    if not showSym:
        displayEvenOdd(f1, f2, showArea)
    else:
        display(
            widgets.VBox([
                a_slider,
                widgets.interactive_output(displayEvenOdd, {"f1":widgets.fixed(f1), "f2":widgets.fixed(f2), "showArea":widgets.fixed(showArea), "a":a_slider})
            ])
        )

def displayEvenOdd(f1, f2, showArea, a=None):

    x = np.linspace(-np.pi, np.pi,200)

    fig, (ax1, ax2, ax3) = plt.subplots(3, 1, figsize=(16, 8))
    fig.tight_layout()

    ax1.plot(x, f1(x), color='C0')
    ax1.set_title('Function 1')
    ax2.plot(x, f2(x), color='C0')
    ax2.set_title('Function 2')
    result = np.multiply(f1(x), f2(x))
    ax3.plot(x, result, color='C3')
    ax3.set_title('Function 1 * Function 2')

    if a is not None:
        ax1.plot(a, f1(a), marker=".", markersize="15", color="C4", label="f({})".format(a))
        ax1.plot(-1*a, f1(a), marker=".", markersize="15", color="C6", label="f(-{})".format(a))

        ax2.plot(a, f2(a), marker=".", markersize="15", color="C4", label="f({})".format(a))
        ax2.plot(-1*a, f2(a), marker=".", markersize="15", color="C6", label="f(-{})".format(a))

        ax3.plot(a, f1(a)*f2(a), marker=".", markersize="15", color="C4", label="f({})".format(a))
        ax3.plot(-1*a, f1(a)*f2(a), marker=".", markersize="15", color="C6", label="f(-{})".format(a))

        ax1.legend()
        ax2.legend()

    if showArea:

        ax1.fill_between(x[:round(len(x)/2)], f1(x[:round(len(x)/2)]), np.zeros(round(len(x)/2)), color='C0', alpha=0.1)
        ax1.fill_between(x[round(len(x)/2):], f1(x[round(len(x)/2):]), np.zeros(round(len(x)/2)), color='C3', alpha=0.1)

        ax2.fill_between(x[:round(len(x)/2)], f2(x[:round(len(x)/2)]), np.zeros(round(len(x)/2)), color='C0', alpha=0.1)
        ax2.fill_between(x[round(len(x)/2):], f2(x[round(len(x)/2):]), np.zeros(round(len(x)/2)), color='C3', alpha=0.1)

        ax3.fill_between(x[:round(len(x)/2)], result[:round(len(x)/2)], np.zeros(round(len(x)/2)), color='C0', alpha=0.1)
        ax3.fill_between(x[round(len(x)/2):], result[round(len(x)/2):], np.zeros(round(len(x)/2)), color='C3', alpha=0.1)

    ax1.axvline(0, color='k', linewidth=0.5)
    ax2.axvline(0, color='k', linewidth=0.5)
    ax3.axvline(0, color='k', linewidth=0.5)

    ax1.axhline(0, color='k', linewidth=0.5)
    ax2.axhline(0, color='k', linewidth=0.5)
    ax3.axhline(0, color='k', linewidth=0.5)

    ax1.set_xlim(-np.pi, np.pi)
    ax2.set_xlim(-np.pi, np.pi)
    ax3.set_xlim(-np.pi, np.pi)


import numpy as np
import matplotlib.pyplot as plt

def norm_histogram(hist):
    """
    takes a histogram of counts and creates a histogram of probabilities

    :param hist: list
    :return: list
    """

    j = 0;
    hist_sum = 0;

    hist_new = [];

    while j < len(hist):
        hist_sum =  hist_sum + hist[j];
        j += 1;

    for i in hist:
        p = i / hist_sum
        hist_new.append(p);

    return(hist_new)

    pass

def compute_j(histo, width):
    """
    takes histogram of counts, uses norm_histogram to convert to probabilties, it then calculates compute_j for one bin width

    :param histo: list
    :param width: float
    :return: float
    """

    sq_sum = 0;
    m = 0;
    k = 0;

    hist_p = norm_histogram(histo)

    hist_sq =[0] * len(hist_p)

    while k < len(hist_p):
        hist_sq[k] = hist_p[k] ** 2;
        sq_sum += (hist_sq[k]);
        m += histo[k];
        k += 1;

    j = 2/((m - 1) * width) - (((m + 1)/((m-1)*width))*(sq_sum))
    return(j);
    pass

def sweep_n(data, minimum, maximum, min_bins, max_bins):
    """
    find the optimal bin
    calculate compute_j for a full sweep [min_bins to max_bins]
    please make sure max_bins is included in your sweep

    :param data: list
    :param minimum: int
    :param maximum: int
    :param min_bins: int
    :param max_bins: int
    :return: list
    """
    optimal_hist = []

    b = min_bins

    while b <= max_bins:
        j = compute_j(plt.hist(data, b, (minimum,maximum))[0], (maximum-minimum)/b);

        optimal_hist.append(j)

        b += 1;

    return(optimal_hist)

    pass

def find_min(l):
    """
    generic function that takes a list of numbers and returns smallest number in that list its index.
    return optimal value and the index of the optimal value as a tuple.

    :param l: list
    :return: tuple
    """

    list_min = min(l);

    min_index = l.index(list_min);

    min_tuple = (list_min,min_index);

    return(min_tuple)

    pass

if __name__ == '__main__':
    data = np.loadtxt('input.txt')  # reads data from input.txt
    lo = min(data)
    hi = max(data)
    bin_l = 1
    bin_h = 100
    js = sweep_n(data, lo, hi, bin_l, bin_h)
    """
    the values bin_l and bin_h represent the lower and higher bound of the range of bins.
    They will change when we test your code and you should be mindful of that.
    """
    print(find_min(js))


'''
Source codes for Python Machine Learning By Example 3rd Edition (Packt Publishing)
Chapter 4 Predicting Online Ads Click-through with Tree-Based Algorithms
Author: Yuxi (Hayden) Liu (yuxi.liu.ece@gmail.com)
'''

import matplotlib.pyplot as plt
import numpy as np

# Plot Gini Impurity in binary case
pos_fraction = np.linspace(0.00, 1.00, 1000)
gini = 1 - pos_fraction**2 - (1-pos_fraction)**2
plt.plot(pos_fraction, gini)
plt.xlabel('Positive fraction')
plt.ylabel('Gini Impurity')
plt.ylim(0, 1)
# plt.show()

# Given labels of a data set, the Gini Impurity calculation function
def gini_impurity(labels):
    # When the set is empty, it is also pure
    if not labels:
        return 0
    # Count the occurrences of each label
    counts = np.unique(labels, return_counts=True)[1]
    fractions = counts / float(len(labels))
    return 1 - np.sum(fractions ** 2)

print(f'{gini_impurity([1, 1, 0, 1, 0]):.4f}')
print(f'{gini_impurity([1, 1, 0, 1, 0, 0]):.4f}')
print(f'{gini_impurity([1, 1, 1, 1]):.4f}')

# Plot entropy in binary case
pos_fraction = np.linspace(0.00, 1.00, 1000)
ent = - (pos_fraction * np.log2(pos_fraction) + (1 - pos_fraction) * np.log2(1 - pos_fraction))
plt.plot(pos_fraction, ent)
plt.xlabel('Positive fraction')
plt.ylabel('Entropy')
plt.ylim(0, 1)
# plt.show()

# Given labels of a data set, the entropy calculation function
def entropy(labels):
    if not labels:
        return 0
    counts = np.unique(labels, return_counts=True)[1]
    fractions = counts / float(len(labels))
    return - np.sum(fractions * np.log2(fractions))

print(f'{entropy([1, 1, 0, 1, 0]):.4f}')
print(f'{entropy([1, 1, 0, 1, 0, 0]):.4f}')
print(f'{entropy([1, 1, 1, 1]):.4f}')

# def information_gain(y, mask, func=entropy):
#     s1 = np.sum(mask)
#     s2 = mask.size - s1
#     if (s1 == 0 | s2 == 0): return 0
#     return func(y) - s1 / float(s1 + s2) * func(y[mask]) - s2 / float(s1 + s2) * func(y[np.logical_not(mask)])

criterion_function = {'gini': gini_impurity, 'entropy': entropy}
def weighted_impurity(groups, criterion='gini'):
    """
    Calculate weighted impurity of children after a split
    @param groups: list of children, and a child consists a list of class labels
    @param criterion: metric to measure the quality of a split, 'gini' for Gini Impurity or 'entropy' for Information Gain
    @return: float, weighted impurity
    """
    total = sum(len(group) for group in groups)
    weighted_sum = 0.0
    for group in groups:
        weighted_sum += len(group) / float(total) * criterion_function[criterion](group)
    return weighted_sum

children_1 = [[1, 0, 1], [0, 1]]
children_2 = [[1, 1], [0, 0, 1]]
print(f"Entropy of #1 split: {weighted_impurity(children_1, 'entropy'):.4f}")
print(f"Entropy of #2 split: {weighted_impurity(children_2, 'entropy'):.4f}")

def gini_impurity_np(labels):
    # When the set is empty, it is also pure
    if labels.size == 0:
        return 0
    # Count the occurrences of each label
    counts = np.unique(labels, return_counts=True)[1]
    fractions = counts / float(len(labels))
    return 1 - np.sum(fractions ** 2)

def entropy_np(labels):
    # When the set is empty, it is also pure
    if labels.size == 0:
        return 0
    counts = np.unique(labels, return_counts=True)[1]
    fractions = counts / float(len(labels))
    return - np.sum(fractions * np.log2(fractions))

criterion_function_np = {'gini': gini_impurity_np, 'entropy': entropy_np}
def weighted_impurity(groups, criterion='gini'):
    """
    Calculate weighted impurity of children after a split
    @param groups: list of children, and a child consists a list of class labels
    @param criterion: metric to measure the quality of a split, 'gini' for Gini Impurity or 'entropy' for Information Gain
    @return: float, weighted impurity
    """
    total = sum(len(group) for group in groups)
    weighted_sum = 0.0
    for group in groups:
        weighted_sum += len(group) / float(total) * criterion_function_np[criterion](group)
    return weighted_sum

def split_node(X, y, index, value):
    """
    Split data set X, y based on a feature and a value
    @param X: numpy.ndarray, dataset feature
    @param y: numpy.ndarray, dataset target
    @param index: int, index of the feature used for splitting
    @param value: value of the feature used for splitting
    @return: list, list: left and right child, a child is in the format of [X, y]
    """
    x_index = X[:, index]
    # if this feature is numerical
    if X[0, index].dtype.kind in ['i', 'f']:
        mask = x_index >= value
    # if this feature is categorical
    else:
        mask = x_index == value
    # split into left and right child
    left = [X[~mask, :], y[~mask]]
    right = [X[mask, :], y[mask]]
    return left, right

def get_best_split(X, y, criterion):
    """
    Obtain the best splitting point and resulting children for the data set X, y
    @param X: numpy.ndarray, dataset feature
    @param y: numpy.ndarray, dataset target
    @param criterion: gini or entropy
    @return: dict {index: index of the feature, value: feature value, children: left and right children}
    """
    best_index, best_value, best_score, children = None, None, 1, None
    for index in range(len(X[0])):
        for value in np.sort(np.unique(X[:, index])):
            groups = split_node(X, y, index, value)
            impurity = weighted_impurity([groups[0][1], groups[1][1]], criterion)
            if impurity < best_score:
                best_index, best_value, best_score, children = index, value, impurity, groups
    return {'index': best_index, 'value': best_value, 'children': children}

def get_leaf(labels):
    # Obtain the leaf as the majority of the labels
    return np.bincount(labels).argmax()

def split(node, max_depth, min_size, depth, criterion):
    """
    Split children of a node to construct new nodes or assign them terminals
    @param node: dict, with children info
    @param max_depth: int, maximal depth of the tree
    @param min_size: int, minimal samples required to further split a child
    @param depth: int, current depth of the node
    @param criterion: gini or entropy
    """
    left, right = node['children']
    del (node['children'])
    if left[1].size == 0:
        node['right'] = get_leaf(right[1])
        return
    if right[1].size == 0:
        node['left'] = get_leaf(left[1])
        return
    # Check if the current depth exceeds the maximal depth
    if depth >= max_depth:
        node['left'], node['right'] = get_leaf(left[1]), get_leaf(right[1])
        return
    # Check if the left child has enough samples
    if left[1].size <= min_size:
        node['left'] = get_leaf(left[1])
    else:
        # It has enough samples, we further split it
        result = get_best_split(left[0], left[1], criterion)
        result_left, result_right = result['children']
        if result_left[1].size == 0:
            node['left'] = get_leaf(result_right[1])
        elif result_right[1].size == 0:
            node['left'] = get_leaf(result_left[1])
        else:
            node['left'] = result
            split(node['left'], max_depth, min_size, depth + 1, criterion)
    # Check if the right child has enough samples
    if right[1].size <= min_size:
        node['right'] = get_leaf(right[1])
    else:
        # It has enough samples, we further split it
        result = get_best_split(right[0], right[1], criterion)
        result_left, result_right = result['children']
        if result_left[1].size == 0:
            node['right'] = get_leaf(result_right[1])
        elif result_right[1].size == 0:
            node['right'] = get_leaf(result_left[1])
        else:
            node['right'] = result
            split(node['right'], max_depth, min_size, depth + 1, criterion)

def train_tree(X_train, y_train, max_depth, min_size, criterion='gini'):
    """
    Construction of a tree starts here
    @param X_train: list of training samples (feature)
    @param y_train: list of training samples (target)
    @param max_depth: int, maximal depth of the tree
    @param min_size: int, minimal samples required to further split a child
    @param criterion: gini or entropy
    """
    X = np.array(X_train)
    y = np.array(y_train)
    root = get_best_split(X, y, criterion)
    split(root, max_depth, min_size, 1, criterion)
    return root

CONDITION = {'numerical': {'yes': '>=', 'no': '<'},
             'categorical': {'yes': 'is', 'no': 'is not'}}
def visualize_tree(node, depth=0):
    if isinstance(node, dict):
        if node['value'].dtype.kind in ['i', 'f']:
            condition = CONDITION['numerical']
        else:
            condition = CONDITION['categorical']
        print('{}|- X{} {} {}'.format(depth * '  ', node['index'] + 1, condition['no'], node['value']))
        if 'left' in node:
            visualize_tree(node['left'], depth + 1)
        print('{}|- X{} {} {}'.format(depth * '  ', node['index'] + 1, condition['yes'], node['value']))
        if 'right' in node:
            visualize_tree(node['right'], depth + 1)
    else:
        print(f"{depth * '  '}[{node}]")

X_train = [['tech', 'professional'],
           ['fashion', 'student'],
           ['fashion', 'professional'],
           ['sports', 'student'],
           ['tech', 'student'],
           ['tech', 'retired'],
           ['sports', 'professional']]

y_train = [1,
           0,
           0,
           0,
           1,
           0,
           1]

tree = train_tree(X_train, y_train, 2, 2)
visualize_tree(tree)

X_train_n = [[6, 7],
           [2, 4],
           [7, 2],
           [3, 6],
           [4, 7],
           [5, 2],
           [1, 6],
           [2, 0],
           [6, 3],
           [4, 1]]

y_train_n = [0,
           0,
           0,
           0,
           0,
           1,
           1,
           1,
           1,
           1]

tree = train_tree(X_train_n, y_train_n, 2, 2)
visualize_tree(tree)

from sklearn.tree import DecisionTreeClassifier
tree_sk = DecisionTreeClassifier(criterion='gini', max_depth=2, min_samples_split=2)
tree_sk.fit(X_train_n, y_train_n)

from sklearn.tree import export_graphviz
export_graphviz(tree_sk, out_file='tree.dot', feature_names=['X1', 'X2'], impurity=False, filled=True, class_names=['0', '1'])


'''created 9/25/16 by Charles Franzen

Notes
-----
This module implements:

* three matrix multiplication algorithms:
    -naive iterative
    -naive recursive
    -Strassen recursive

* two QR decompositions:
    -Givens rotations
    -Householder reflections

* two Hessenberg algorithms:
    -upper Hessenberg reduction
    -QR decomp of upper Hessenberg matrices

* the QR algorithm for real eigenvalues, both with and without Hessenberg
  reduction.

This was created as part of a final project for DSCI6001 at GalvanizeU.
'''

import numpy as np
import numpy.linalg as LA
import time

# matrix multiplication

def naive_mat_mul(A, B):
    '''
    Naive iterative method of matrix multiplication

    Inputs- A, B (np.array): two matrices of the same dimensions 2^n x 2^n

    Output- (np.array): a matrix of dimensions 2^n x 2^n
    '''
    # get matrix dimensions
    n = A.shape[0]
    # initialize result matrix to all zeros
    C = np.zeros((n, n), dtype=np.complex_)
    # iterate through rows and columns to build up dot products
    for i in range(n):
        for l in range(n):
            for k in range(n):
                C[i, l] += A[i, k] * B[k, l]
    return C

def recursive_mat_mul(A, B):
    '''
    Naive recursive method of matrix multiplication

    Inputs- A, B (np.array): two matrices of the same dimensions 2^n x 2^n

    Output- (np.array): a matrix of dimensions 2^n x 2^n
    '''
    # get matrix dimensions
    n = A.shape[0]
    # base case: 1x1 matrices
    if n == 1:
        return A * B
    # recursive case
    else:
        # get dimensions of sub-matrices
        i = int(n / 2)
        # initialize result matrix
        C = np.zeros((n, n), dtype=np.complex_)

        # calculate each submatrix as an addition of two matrix products
        # note that this requires 8 matrix multiplications
        C[:i, :i] = recursive_mat_mul(A[:i, :i], B[:i, :i])\
            + recursive_mat_mul(A[:i, i:], B[i:, :i])

        C[:i, i:] = recursive_mat_mul(A[:i, :i], B[:i, i:])\
            + recursive_mat_mul(A[:i, i:], B[i:, i:])

        C[i:, :i] = recursive_mat_mul(A[i:, :i], B[:i, :i])\
            + recursive_mat_mul(A[i:, i:], B[i:, :i])

        C[i:, i:] = recursive_mat_mul(A[i:, :i], B[:i, i:])\
            + recursive_mat_mul(A[i:, i:], B[i:, i:])

        return C

def strassen_mat_mul(A, B):
    '''
    Strassen's recursive method of matrix multiplication

    Inputs- A, B (np.array): two matrices of the same dimensions 2^n x 2^n

    Output- (np.array): a matrix of dimensions 2^n x 2^n
    '''
    # get matrix dimensions
    n = A.shape[0]
    # base case: 1x1 matrices
    if n == 1:
        return A * B
    # recursive case
    else:
        # get dimensions of sub-matrices
        i = int(n / 2)
        # initialize result matrix
        C = np.zeros((n, n), dtype=np.complex_)

        # calculate intermediate matrices.
        # note that this requires only 7 matrix multiplications
        M1 = strassen_mat_mul(A[:i, :i] + A[i:, i:], B[:i, :i] + B[i:, i:])
        M2 = strassen_mat_mul(A[i:, :i] + A[i:, i:], B[:i, :i])
        M3 = strassen_mat_mul(A[:i, :i], B[:i, i:] - B[i:, i:])
        M4 = strassen_mat_mul(A[i:, i:], B[i:, :i] - B[:i, :i])
        M5 = strassen_mat_mul(A[:i, :i] + A[:i, i:], B[i:, i:])
        M6 = strassen_mat_mul(A[i:, :i] - A[:i, :i], B[:i, :i] + B[:i, i:])
        M7 = strassen_mat_mul(A[:i, i:] - A[i:, i:], B[i:, :i] + B[i:, i:])

        # calculate sub-matrices by addition of the intermediate matrices
        C[:i, :i] = M1 + M4 - M5 + M7
        C[:i, i:] = M3 + M5
        C[i:, :i] = M2 + M4
        C[i:, i:] = M1 - M2 + M3 + M6

        return C

def mat_time(n, func, number=1000, repeats=3):
    '''
    Records running times for matrix multiplications

    Inputs- n (int): size of test matrices. must be a power of 2
            func (function): matrix multiplication function to be used
            num_trials (int): number of trials for the experiment
            repeats (int): number of experiments

    Output- list: times for each trial
    '''
    # create random n x n matrices
    A = np.random.randint(0, n**2, (n, n))
    B = np.random.randint(0, n**2, (n, n))

    def _timer(A, B):
        t1 = time.time()
        func(A, B)
        t2 = time.time()
        return t2 - t1
    # run all trials
    trials = [np.mean([_timer(A, B) for _ in range(number)])
              for _ in range(repeats)]
    return trials

# QR decomp and the QR algorithm

def _givens(a, b):
    '''
    Subroutine to calculate a rotation matrix g: [gamma, -sigma]
                                                 [sigma, gamma]
    such that g*[a] = [r]
                [b]   [0]

    This is a linear transformation that zeros the lower left element of
    a matrix M

    Inputs- a, b (float): values from the matrix to be decomposed

    Outputs- gamma, sigma (float): scalars used to construct a givens rotation
                                   matrix
    '''
    if np.abs(b) >= np.abs(a):
        tau = a / b
        sigma = 1 / np.sqrt(1 + tau**2)
        gamma = sigma * tau
    else:
        tau = b / a
        gamma = 1 / np.sqrt(1 + tau**2)
        sigma = gamma * tau
    return gamma, sigma

def givens_QR(A):
    '''
    Uses givens rotations to perform QR decomposition of A

    Input- A (np.array): the matrix to be decomposed

    Outputs- Q, R (np.array): an orthonormal matrix Q and an upper triangular
                              matrix R such that Q*R=A
    '''
    # get dimensions of a
    n, m = A.shape
    # initialize Q
    Q = np.identity(n)
    # initialize R
    R = np.copy(A)
    # for n columns
    for j in range(n):
        # for rows j-m
        for i in range(m - 1, j, -1):
            # calculate rotation matrix
            c, s = _givens(R[i - 1, j], R[i, j])
            # broadcast to an nxn matrix G
            G = np.identity(n)
            G[i - 1, i - 1] = c
            G[i, i] = c
            G[i, i - 1] = s
            G[i - 1, i] = -s
            # these matrices zero the lower diagonal elements of R
            # update R
            R = np.dot(G.T, R.copy())
            # update Q
            Q = np.dot(Q.copy(), G)
    return Q, R

def _givens_mat(c, s, n, i):
    '''
    Subroutine to return a givens rotation matrix

    Inputs- c, s (int): parameters for the givens matrix
            n (int): n_rows of the matrix
            i (int): index of the diagonal submatrix into which c and s should
                     be broadcast

    Output- G (np.array): givens rotation matrix
    '''
    # initialize matrix
    G = np.identity(n,)
    # assign values to the ith 2x2 diagonal submatrix
    G[i, i] = c
    G[i + 1, i + 1] = c
    G[i + 1, i] = s
    G[i, i + 1] = -s
    return G

def house_QR(A):
    '''
    Uses householder reflections to perform QR decomposition of A

    Input- A (np.array): the matrix to be decomposed

    Outputs- Q, R (np.array): an orthonormal matrix Q and an upper triangular
                              matrix R such that Q*R=A
    '''
    # get dimension of A
    n = len(A)
    # initialize Q
    Q = np.identity(n)
    # initialize R
    R = A.copy()
    # for n-1 columns
    for i in range(n - 1):
        # initialize householder tranformation
        H = np.identity(n)
        # embed reflection matrix
        H[i:, i:] = _house_mat(R[i:, i])
        # update R and Q
        Q = np.dot(Q, H)
        R = np.dot(H, R)
    return Q, R

def _house_mat(x):
    '''
    subroutine to calculate a householder reflection to zero all but the first
    entry of a column of a matrix

    Input- x (vector): vector to be reflected

    Output- H (matrix): householder transformation matrix. usually embedded into
                        an identity matrix
    '''
    # calculate normal to the hyperplane of reflection
    v = x / (x[0] + np.copysign(np.linalg.norm(x), x[0]))
    v[0] = 1
    # initialize H
    H = np.identity(x.shape[0])
    # calculate tranformation matrix
    H -= (2 / np.dot(v, v)) * np.outer(v, v)
    return H

def hessenberg(A, return_Q=True):
    '''
    Uses householder reflections to transform a matrix A into upper hessberg form

    Input- A (np.array): square matrix to be transformed

    Outputs- Q, H (np.array): orthonormal matrix Q and upper hessberg matrix H
                              such that Q*H=A
    '''
    # get dimension of A
    n = len(A)
    # initialize H
    R = A.copy()
    # initialize Q
    if return_Q:
        Q = np.identity(n)
    # perform n-2 householder reflections
    for i in range(n - 2):
        H = np.identity(n)
        # perform the reflection to zero all but the first 2 entries of the
        # column
        H[i + 1:, i + 1:] = _house_mat(R[i + 1:, i])
        R = H.dot(R).dot(H.T)
        if return_Q:
            Q = np.dot(Q, H)
    if return_Q:
        return Q, R
    else:
        return R

def hessenberg_QR(H):
    '''
    Uses givens rotations to perform QR decomposition on an upper hessberg matrix

    Input- H (np.array): square matrix in upper hessberg form

    Outputs- R (np.array): upper triangular matrix of the QR decomposition
             Gs (list): list of givens rotation matrices used to produce R
    '''
    # initialize R
    R = H.copy()
    # get dimension of H
    n = len(H)
    # initialize Gs
    Gs = []
    # perform n-1 givens rotations on the lower subdiagonal
    for i in range(n - 1):
        # get givens params
        c, s = _givens(R[i, i], R[i + 1, i])
        # get givens transformation matrix
        G = _givens_mat(c, s, n, i)
        # update R
        R = np.dot(G.T, R.copy())
        # update Gs
        Gs.append(G)
    return R, Gs

def eig(A, iterations=100):
    '''
    Uses givens rotations to perform the QR algorithm for finding real eigenvalues

    Inputs- A (np.array): matrix for which to find eigenvalues
            iterations (int): desired number of iterations

    Output- np.array: sorted array of real eigenvalues
    '''
    for _ in range(iterations):
        # calculate QR decomp
        Q, R = givens_QR(A)
        # compose new A
        A = R.dot(Q)
    # eigenvalues converge on the diagonal of A
    return np.sort(np.diag(A))

def hessenberg_eig(A, iterations=100):
    '''
    Converts a matrix into upper hessberg form, then uses givens rotations to
    perform the QR algorithm for finding real eigenvalues

    Inputs- A (np.array): matrix for which to find eigenvalues
            iterations (int): desired number of iterations

    Output- np.array: sorted array of real eigenvalues
    '''
    # convert A to upper hessberg form
    H = hessenberg(A, return_Q=False)
    for _ in range(iterations):
        # QR decompostion of H
        H, G = hessenberg_QR(H)
        # since Q is the product of all the transformations in G, R*Q can be
        # calculated as the product of R and all the transformations in G
        # this is guaranteed to return another upper hessberg matrix, for which
        # the QR decomposition can be computed again using the fast hessberg_QR
        # method
        for g in G:
            H = np.dot(H, g)
    # eigenvalues converge on the diagonal of H
    return np.sort(np.diag(H))

def eig_time(n, func, number=1000, repeats=3, A=None):
    '''
    Records running times for eigenvalue algorithms

    Inputs- n (int): size of test matrix
            func (function): eigenvalue function to be used
            num_trials (int): number of trials for the experiment
            repeats (int): number of experiments

    Output- list: times for each trial
    '''
    if A is None:
        # create random n x n matrix
        A = np.random.randint(0, n, (n, n))
        A = A.dot(A.T)

    def _timer():
        t1 = time.time()
        func(A)
        t2 = time.time()
        return t2 - t1
    # run all trials
    trials = [np.mean([_timer() for _ in range(number)])
              for _ in range(repeats)]
    return trials


# -*- coding: utf-8 -*-
"""
这是一个pip打包的例子

@author: lowinli

@email: lowinli@outlook.com
"""

import numpy as np

def softmax(x):
    """对向量进行softmax处理

    这只是对自动api文档的一个例子

    Args:
        x (int): 输入向量

    Returns:
        np.array: softmax结果

    Examples:
        >>> array = softmax(np.array([1, 1]))
        >>> array
        np.array([0.5, 0.5])

    References:
        softmax维基百科  https://en.wikipedia.org/wiki/Softmax_function
    """
    return np.exp(x) / np.sum(np.exp(x), axis=0)


import numpy as np

MAX_ITERS = 100
DOUBLE_PRECISION_THRESHOLD = 1e-15

def euler(y_n, t_n, f, h):
    return y_n + h*f(t_n, y_n)

def trapezoidal_rule(y_n, t_n, f, h, y_nn=None):
    if y_nn is None: # we should solve nonlinear system then
        raise Exception('Not implemented')

    t_nn = t_n + h
    return y_n + h/2. * (f(t_n, y_n) + f(t_nn, y_nn))

def heun(y_n, t_n, f, h):
    # euler is predictor, trapezoidal is corrector
    y_pred = euler(y_n, t_n, f, h)
    return trapezoidal_rule(y_n, t_n, f, h, y_pred)

def rk4(y_n, t_n, f, h):
    k_1 = h*f(t_n, y_n)
    k_2 = h*f(t_n + h/2., y_n + k_1/2.)
    k_3 = h*f(t_n + h/2., y_n + k_2/2.)
    k_4 = h*f(t_n + h, y_n + k_3)
    return y_n + 1/6. * (k_1 + 2*k_2 + 2*k_3 + k_4)

def ab4(y_4n, t_n, f, h):
    # Adams-Bashforth 4-step scheme
    # y_n == y_4n[3], y_{n-1} == y_4n[2], ..., y_{n-3} == y_4n[0]
    return y_4n[3] + h/24. * (55*f(t_n, y_4n[3]) - 59*f(t_n - h, y_4n[2]) + 37*f(t_n - 2*h, y_4n[1]) - 9*f(t_n - 3*h, y_4n[0]))

def am4(y_3n, t_n, f, h, y_nn=None):
    # Adams-Moulton 3-step scheme
    # y_n == y_3n[2], y_{n-1} == y_3n[1], y_{n-2} == y_3n[0]
    if y_nn is None:
        raise Exception('Not implemented')

    return y_3n[2] + h/24. * (9*f(t_n + h, y_nn) + 19*f(t_n, y_3n[2]) - 5*f(t_n - h, y_3n[1]) + f(t_n - 2*h, y_3n[0]))

def am_predictor_corrector(y_4n, t_n, f, h):
    # Apply predictor
    y_nn = ab4(y_4n, t_n, f, h)

    # Use inner iteration for implicit method (corrector) to converge
    for k in range(MAX_ITERS):
        y_nn_prev = y_nn
        y_nn = am4(y_4n[1:], t_n, f, h, y_nn=y_nn_prev)
        if np.abs(y_nn_prev - y_nn) < DOUBLE_PRECISION_THRESHOLD:

            print('Iteration converged at k = {} with error = {}'.format(k, np.abs(y_nn_prev - y_nn)))
            return y_nn

    return y_nn


"""
Euclidean algorithm (gcd).
"""
from sage.all import *

def euclidean_algorithm(f, g):
    """
    Euclidean Algorithm.

    Returns one gcd of two elements f, g belonging to an euclidean domain.

    Parameters
    ----------
    f : the first element.
    g : the second element.

    Returns
    -------
    A gcd of f and g.
    """

    if f.parent() is not g.parent():
        raise ValueError("Arguments should belong to the same ring")

    domain = f.parent()

    if not domain.is_euclidean_domain():
        raise ValueError("Arguments should belong to an euclidean domain")

    a, b = f, g

    while b != domain.zero():
        _, rem = a.quo_rem(b)
        a, b = b, rem

    return a

def main():
    """ Execute the examples. """
    print euclidean_algorithm(ZZ(126), ZZ(35))  # Expected 7
    print euclidean_algorithm(ZZ(112242), ZZ(989712))  # Expected 6
    print euclidean_algorithm(ZZ(33461), ZZ(236322))  # Expected 1

    R = PolynomialRing(QQ, 'x')
    print euclidean_algorithm(R('x^2 - 1'), R('x + 1'))  # Expected x + 1
    print euclidean_algorithm(R('x + 1'), R('x + 2'))  # Expected 1 (or -1)

if __name__ == '__main__':
    main()


# Sebastian Raschka 2014-2020
# mlxtend Machine Learning Library Extensions
#
# Functions for different linear algebra operations.
# Author: Sebastian Raschka <sebastianraschka.com>
#
# License: BSD 3 clause

import numpy as np

def vectorspace_orthonormalization(ary,  # method='gram-schmidt',
                                   eps=1e-13):
    """Transforms a set of column vectors to a orthonormal basis.

    Given a set of orthogonal vectors, this functions converts such
    column vectors, arranged in a matrix, into orthonormal basis
    vectors.

    Parameters
    ----------
    ary : array-like, shape=[num_vectors, num_vectors]
        An orthogonal set of vectors (arranged as columns in a matrix)

    eps : float (default: 1e-13)
        A small tolerance value to determine whether
        the vector norm is zero or not.

    Returns
    ----------
    arr : array-like, shape=[num_vectors, num_vectors]
        An orthonormal set of vectors (arranged as columns)

    """

    # Gram-Schmidt Process
    #  1) col i for i = 0: keep & normalize:
    #  2) col i for i > 1:
    #   2a) project column i+1 onto i
    #   2b) subtract column i from projection vector
    #   2c) Normalize if linearly independent,
    #       and set to zero otherwise

    arr = ary.astype(np.float).copy()

    for i in range(arr.shape[1]):
        for j in range(i):
            # 2a) & 2b)
            arr[:, i] -= np.dot(arr[:, i], arr[:, j]) * arr[:, j]
        # 2c
        tmp = np.linalg.norm(arr[:, i])
        is_linearly_indepedent = tmp > eps
        if is_linearly_indepedent:
            arr[:, i] /= tmp
        else:
            arr[:, i] = np.zeros(arr[:, i].shape)

    # elif method == 'qr-factorization':
    #    Q, R = np.linalg.qr(ary)
    #    arr = Q

    # QR factorization is not used here because of non-useful
    # results in cases of linear dependence

    # else:
    #     raise ValueError("Method must be 'gram-schmidt'"
    #                      "or 'qr-factorization'")

    return arr

def vectorspace_dimensionality(ary):
    """Computes the hyper-volume spanned by a vector set

    Parameters
    ----------
    ary : array-like, shape=[num_vectors, num_vectors]
        An orthogonal set of vectors (arranged as columns in a matrix)

    Returns
    ----------
    dimensions : int
        An integer indicating the "dimensionality" hyper-volume spanned by
        the vector set

    """
    # Note that since the vectors of
    # an orthonormal vectoset have unit length or are zero,
    # the sum of the individual
    # norms equals the dimensionality of that vector space
    return int(np.sum(np.linalg.norm(
        vectorspace_orthonormalization(ary), axis=0)))


''' x = gaussPivot(a,b,tol=1.0e-9).
Resuelve [a]{x} = {b} por eliminacion de Gauss
con pivoteo parcial escalado
'''
# A  = np.array([[3.,-13., 9., 3.],[-6., 4., 1., -18.],[6., -2., 2., 4.],[12., -8., 6, 10.]])
# B  = np.array([-19., -34., 16., 26.])

# AA = np.array([[0.143, 0.357, 2.01],[-1.31, 0.911, 1.99],[11.2, -4.30, -0.605]])
# BB = np.array([-5.17, -5.46, 4.42])
import numpy as np
import sys

def gaussPivot(A,B,tol=1.0e-12):
    n = len(B)
    a = np.copy(A)
    b = np.copy(B)

    # Vector de escala
    s = np.zeros(n)
    for i in range(n):
        s[i] = max(abs(a[i,:]))

    # Vector de permutaciones
    vp = np.array(range(n))

    for k in range(0,n-1): # Ciclo sobre las columnas

        # Intercambio de renglones si es necesario
        p = np.argmax(abs(a[k:n,k])/s[k:n]) + k
        if abs(a[p,k]) < tol:
           err('La matriz es singular')
        if p != k:
           swapRensAGP(a,k,p)  #Matriz de coeficientes
           swapRensAGP(b,k,p)  #Vector independiente
           swapRensAGP(s,k,p)  #Vector de escala
           swapRensAGP(vp,k,p) #Vector de permutaciones
           print 'Intercambio:',k,p
        # Eliminacion
        for i in range(k+1,n):
            if a[i,k] != 0.0:
               lam = a[i,k]/a[k,k]
               a[i,k:n] = a [i,k:n] - lam*a[k,k:n]
               b[i] = b[i] - lam*b[k]

    if abs(a[n-1,n-1]) < tol:
         err('La matriz es singular')

      # Sustitucion regresiva
    for k in range(n-1,-1,-1):
        b[k] = (b[k] - np.dot(a[k,k+1:n],b[k+1:n]))/a[k,k]
    return b,vp

def swapRensAGP(v,i,j):
    if len(v.shape) == 1:
       v[i],v[j] = v[j],v[i]
    else:
      temp = v[i].copy()
      v[i] = v[j]
      v[j] = temp

def err(string):
    print string
    raw_input('Oprime return para salir')
    sys.exit()


"""
Author: Jacob Dachenhaus, jdachenh@purdue.edu
Assignment: 10.4 - Sin Cos
Date: 12/06/2021

Description:
    Program that calculates and plots the y=sin(x) and y=cos(x)
    functions from x=[0, pi/2].

Contributors:
    None

My contributor(s) helped me:
    [x] understand the assignment expectations without
        telling me how they will approach it.
    [x] understand different ways to think about a solution
        without helping me plan my solution.
    [x] think through the meaning of a specific error or
        bug present in my code without looking at my code.
    Note that if you helped somebody else with their code, you
    have to list that person as a contributor.

Academic Integrity Statement:
    I have not used source code obtained from any unauthorized
    source, either modified or unmodified; nor have I provided
    another student access to my code.  The project I am
    submitting is my own original work.
"""

import matplotlib.pyplot as plt
import numpy as np

def main():
    # Calculate the x and y values at each step
    step = 0.001
    x = np.arange(0, 2 * np.pi + step, step) # [start, stop)
    y1 = np.sin(x)
    y2 = np.cos(x)

    ##
    #  Make the plot
    ##
    fig, ax = plt.subplots()

    ax.plot(x, y1, color='r')
    ax.plot(x, y2, color='b')

    ##
    #  Customize the plot
    ##

    # Format the axes
    for spine in ('top', 'right'):
        ax.spines[spine].set_visible(False)
    for spine in ('bottom', 'left'):
        ax.spines[spine].set_position('zero')

    # Set ticks and labels
    ax.set_xticks([np.pi / 2, np.pi, 3 * np.pi / 2, 2 * np.pi])
    ax.set_xticklabels(['$\\pi/2$', '$\\pi$', '$3\\pi/2$', '$2\\pi$'])

    ax.set_yticks([-1, 1])

    ##
    #  Display the plot
    ##
    plt.show()

if __name__ == '__main__':
    main()


from function import f
from function import g
import numpy as np
import math

def fixedPoint (xi,tol,max_iter):
    res = {}
    f_xi = f(xi)
    g_xi = g(xi)
    return_list = []
    return_list.append({
            'iter':0,
            'xi': xi,
            'g(xi)':g_xi,
            'f(xi)': f_xi,
            'error':'NA'
            })
    count = 1
    error = tol + 1
    while error > tol and count <= max_iter:
        xn = g_xi
        g_xi = g(xn)
        f_xi = f(xn)
        error = abs(xn-xi)
        xi = xn
        row = {
            'iter' : count,
            'xi': xi,
            'g(xi)':g_xi,
            'f(xi)': f_xi,
            'error': error
            }
        return_list.append(row)
        if(f_xi == 0):
            res["iters"] = return_list
            res["status"] = 'Root found! ;)'
            res["error"] = False
            return res
        elif(error < tol):
            res["iters"] = return_list
            res["status"] = 'Err lower than tolerance! :)'
            res["error"] = False
            return res
        elif(count >= max_iter):
            res["iters"] = return_list
            res["status"] = 'Overpassed max iteration! :('
            res["error"] = True
            return res
        count = count + 1

    return {"iters" : return_list}


'''
02 - Hierarchies of stocks

In chapter 1, you used k-means clustering to  cluster  companies  according  to  their
stock price movements. Now, you'll perform  hierarchical  clustering of the  companies.
You are given a NumPy array of price movements  movements, where the  rows  correspond
to companies, and a list of the company names companies. SciPy hierarchical  clustering
doesn't fit into a sklearn pipeline, so you'll need to use the normalize() function from
sklearn.preprocessing instead of Normalizer.

linkage and dendrogram have already been imported from scipy.cluster.hierarchy, and PyPlot
has been imported as plt.

Instructions:

- Import normalize from sklearn.preprocessing.

- Rescale the price movements for each stock by using the normalize() function on movements.

- Apply the linkage() function to normalized_movements, using 'complete' linkage, to calculate
  the hierarchical clustering. Assign the result to mergings.

- Plot a dendrogram of the hierarchical clustering, using the list companies of company names as
  the labels. In addition, specify the leaf_rotation=90, and leaf_font_size=6 keyword arguments as
  you did in the previous exercise.
'''
# Import normalize
from sklearn.preprocessing import normalize

# Normalize the movements: normalized_movements
normalized_movements = normalize(movements)

# Calculate the linkage: mergings
mergings = linkage(normalized_movements, method='complete')

# Plot the dendrogram
dendrogram(mergings,
          labels=companies,
          leaf_rotation=90,
          leaf_font_size=6)
plt.show()

27_Unsupervised Learning in Python/02_visualization-with-hierarchical-clustering-and-t-sne/02_hierarchies-of-stocks.py


import numpy as np
import matplotlib.pyplot as plt
from sklearn.decomposition import PCA

def create_corr_data(corr = 0.8, data_min = 10, data_max = 80):

    # Set the Random Seed
    np.random.seed(seed = 0)

    # Set the x and y range for the data
    data_x_range = np.array([data_min, data_max])
    data_y_range = np.array([data_min, data_max])

    # Set the average of the x and y postions of the data points
    means = [data_x_range.mean(), data_y_range.mean()]

    # Set the standard deviation of the x and y positions of the data points
    stds = [data_x_range.std() / 3, data_y_range.std() / 3]

    # Construct the Covariance matrix
    covs = [[stds[0]**2, stds[0]*stds[1]*corr],
            [stds[0]*stds[1]*corr, stds[1]**2]]

    # Return the correlated data with 1000 data points
    return np.random.multivariate_normal(means, covs, 1000)

def plot_data(X, corr=None):

    # Make a Scatter plot of the data
    plt.scatter(X[:,0], X[:,1],color = 'white', alpha = 0.5, linewidth = 0)
    ax = plt.gca()
    ax.set_facecolor('lightslategray')
    plt.grid()
    plt.xlabel('$X$', fontsize = 20)
    plt.ylabel('$Y$', fontsize = 20)

    # Put legend on plot if correlation is provided
    if corr:
        props = dict(boxstyle = 'round', facecolor = 'wheat', alpha = 0.5)
        textstr = 'Corr(X,Y) = %.2f'%(corr)
        plt.text(0.05, 0.91, textstr, fontsize = 20, horizontalalignment = 'left',
                verticalalignment = 'top', bbox = props, transform = ax.transAxes)

    plt.show()

def mean_normalize_data(X):

    # Average of the values in each column of X
    ave_cols = X.mean(axis = 0)

    # Standard Deviation of the values in each column of X
    std_cols = X.std(axis = 0)

    # Return the mean normalized X
    return (X - ave_cols) / std_cols

def rotate_data(X, angle = 45.0):

    # Convert the angle from degrees to radians
    theta = (angle / 180.0) * np.pi

    #Create the 2D rotation matrix
    rotMatrix = np.array([[np.cos(theta), -np.sin(theta)], [np.sin(theta),  np.cos(theta)]])

    # Return the rotated X
    return np.matmul(X, rotMatrix)

def get_pca_comp(X, n_cpm = 2):

    # Perform PCA for the given number of components
    pca = PCA(n_components = n_cpm)
    pca.fit(X)

    # Return the principal components
    return pca.components_

def get_proj_pts(X, pca_components):

    # Rotate X to get the min and max values
    X_rot = rotate_data(X)

    # Create and empty array to hold our projected points
    proj_pts = np.array([])

    for i in range(2):

        # Get the indices of the min and max data points
        min_idx = X_rot[:,i].argmin()
        max_idx = X_rot[:,i].argmax()

        # Select the x and y coordinates of the points to be projected
        q1 = np.array([X[min_idx,0], X[min_idx,1]])
        q2 = np.array([X[max_idx,0], X[max_idx,1]])

        # Set a point equal to the origin
        p0 = np.array([0,0])

        # Set a point that is in the direction of the ith eigenvector
        p1 = pca_components[i]

        # Create the A matrices
        a1 = np.array([[-q1[0]*(p1[0]-p0[0]) - q1[1]*(p1[1]-p0[1])], [-p0[1]*(p1[0]-p0[0]) + p0[0]*(p1[1]-p0[1])]])
        a2 = np.array([[-q2[0]*(p1[0]-p0[0]) - q2[1]*(p1[1]-p0[1])], [-p0[1]*(p1[0]-p0[0]) + p0[0]*(p1[1]-p0[1])]])

        # Create the B matrix
        b = np.array([[p1[0] - p0[0], p1[1] - p0[1]], [p0[1] - p1[1], p1[0] - p0[0]]])

        # Solve the linear equations Bx = A
        proj_pt1 = - np.linalg.solve(b, a1)
        proj_pt2 = - np.linalg.solve(b, a2)

        # Append the values of the projected points
        proj_pts = np.append(proj_pts, proj_pt1)
        proj_pts = np.append(proj_pts, proj_pt2)

    return proj_pts

def plot_data_with_pca_comp(corr = 0.8, data_min = 10, data_max = 80):

    # Create correlated data
    X = create_corr_data(corr, data_min, data_max)

    # Mean normalize X
    X_norm = mean_normalize_data(X)

    # Get the principal components
    pca_components = get_pca_comp(X_norm)

    # Get the projected points onto the principal components
    proj_pts = get_proj_pts(X_norm, pca_components)

    # Set the first and second components
    first_pc = pca_components[0]
    second_pc = pca_components[1]

    # Plot the correlated data
    plt.scatter(X_norm[:,0], X_norm[:,1], color = 'white', alpha = 0.5, linewidth = 0)
    ax = plt.gca()
    ax.set_facecolor('lightslategray')
    plt.grid()

    # Plot the first principal component (eigenvector) as a line
    x_vals = np.linspace(proj_pts[1], proj_pts[3], 10)
    y_vals = (first_pc[1] / first_pc[0]) * x_vals
    plt.plot(x_vals, y_vals, color = 'm', linewidth = 2, label = '$1^{st}$ PC')

    # Plot the second principal component (eigenvector) as a line
    x_vals = np.linspace(-proj_pts[5], -proj_pts[7], 10)
    y_vals = (second_pc[1] / second_pc[0]) * x_vals
    plt.plot(x_vals, y_vals, color = 'c', linewidth = 2, label = '$2^{nd}$ PC')
    plt.legend()

#     # Plot projected points
#     plt.scatter(proj_pts[0],proj_pts[1], color='r')
#     plt.scatter(proj_pts[2],proj_pts[3], color='c')
#     plt.scatter(proj_pts[4],proj_pts[5], color='r')
#     plt.scatter(proj_pts[6],proj_pts[7], color='c')

    # Plot legend
    props = dict(boxstyle = 'round', facecolor = 'wheat', alpha = 0.5)
    textstr = 'Corr(X,Y) = %.2f'%(corr)
    plt.text(0.05, 0.91, textstr, fontsize = 20, horizontalalignment = 'left',
            verticalalignment = 'top', bbox = props, transform = ax.transAxes)

    # Set the axis to equal or square
    plt.axis('equal')
#     plt.axis('square')

    plt.show()


import numpy as np
import pandas as pd
import scipy.sparse as sps
import matplotlib.pyplot as plt

from mlhub.pkg import mlask, mlcat
from IPython.display import display
from collections import Counter

from relm.mechanisms import LaplaceMechanism

mlcat("Differentially Private Release Mechanism", """\
This demo is based on the Jupyter Notebook from the RelM
package on github.

RelM can be readily utilised for the differentially private
release of data. In our demo database the records indicate the age
group of each patient who received a COVID-19 test on 9 March 2020.
Each patient is classified as belonging to one of eight age groups:
0-9, 10-19, 20-29, 30-39, 40-49, 50-59, 60-69, and 70+.  One common
way to summarise this kind of data is with a histogram.  That is, to
report the number of patients that were classified as belonging to
each age group.

For this demonstration we will create a histogram for the actual data
and then a histogram for differentially private data.

The data is first loaded from a csv file. It simply consists of two
columns, the first is the date and the scond is the age group.""")

# Read the raw data.

data = pd.read_csv("pcr_testing_age_group_2020-03-09.csv")

mlask(True, True)

# Compute the exact query responses.

exact_counts = data["age_group"].value_counts().sort_index()
values = exact_counts.values

mlcat("Data Sample", """\
Here's a random sample of some of the records:
""")

print(data.sample(10))

mlask(True, True)

mlcat("Laplace Mechanism", """\
The Laplace mechanism can be used to produce a differentially private
histogram that summarises the data without compromising the privacy of
the patients whose data comprise the database. To do so, Laplace noise
is added to the count for each age group and the noisy counts are
released instead of the exact counts.

The noise that is added results in perturbed values that are real
numbers rather than integers, and so if the results we are expecting
are integers, the results can be rounded without loss of privacy. We
do that here for our peturbed values.""")

# Create a differentially private release mechanism

epsilon = 0.1
mechanism = LaplaceMechanism(epsilon=epsilon, sensitivity=1.0)
perturbed_counts = mechanism.release(values=values.astype(np.float))
perturbed_counts = perturbed_counts.astype(np.int64)

mlask(True, True)

mlcat("Choosing Epsilon", f"""\
The magnitude of the differences between the exact counts
and perturbed counts depends only on the value of the privacy parameter,
epsilon. Smaller values of epsilon yield larger perturbations. Larger
perturbations yeild lower utility.

To understand this compare the actual and peturbed values below. If
the value of epsilon is not too small, then we expect that the two
histograms will look similar.

For our purposes we have chosen epsilon as {epsilon} resulting in the
following peturbation.
""")

# Extract the set of possible age groups.

age_groups = np.sort(data["age_group"].unique())

# Reformat the age group names for nicer display.

age_ranges = np.array([a.lstrip("AgeGroup_") for a in age_groups])

# Create a dataframe with both exact and perturbed counts.

column_names = ["Age Group", "Exact Counts", "Perturbed Counts"]
column_values = [age_ranges, values, perturbed_counts]
table = {k: v for (k, v) in zip(column_names, column_values)}
df = pd.DataFrame(table)

# Display as a table.

print(df)

mlask(True, True)

mlcat("Visualising the Perturbations", """\
The two histograms show that the peturbed values remain consistent
with the true values, whilst ensuring privacy.
""")

# Plot the two histograms as bar graphs.

df.plot(x="Age Group", title="Test Counts by Age Group", kind="bar", rot=0)
plt.show()

exit()

mlcat("Geometric Mechanism", """\

TODO

In this example, all of the exact counts are integers. That is because
they are the result of so-called counting queries. The perturbed counts
produced by the Laplace mechanism are real-valued. In some applications,
e.g. when some downstream processing assumes it will receive
integer-valued data, we may need the perturbed counts to be integers.
One way to achieve this is by simply rounding the outputs of the Laplace
mechanism to the nearest integer. Because this differentially private
release mechanisms are not affected by this kind of post-processing,
doing so will not affect any privacy guarantees.

Alternatively, we could use the geometric mechanism to compute the
permuted counts. The geometric mechanism is simply a discrete version of
the Laplace mechanism and it produces integer valued perturbations.
""")
mlask()

mlcat("", """Basic Usage
""")
mlask()

mlcat("", """
# Create a differentially private release mechanism
from relm.mechanisms import GeometricMechanism
mechanism = GeometricMechanism(epsilon=0.1, sensitivity=1.0)
perturbed_counts = mechanism.release(values=values)
""")
mlask()
# Create a differentially private release mechanism
from relm.mechanisms import GeometricMechanism
mechanism = GeometricMechanism(epsilon=0.1, sensitivity=1.0)
perturbed_counts = mechanism.release(values=values)
mlcat("", """Visualising the Results
As with the Laplace mechanism, we can plot the exact histogram alongside
the differentially private histogram to get an idea if we have used too
small a value for epsilon.
""")
mlask()

mlcat("", """
# Create a dataframe with both exact and perturbed counts
column_values = [age_ranges, values, perturbed_counts]
table = {k: v for (k, v) in zip(column_names, column_values)}
df = pd.DataFrame(table)

# Display the two histograms as a table
display(df.style.set_caption("Test Counts by Age Group"))

# Plot the two histograms as bar graphs
df.plot(x="Age Group", title="Test Counts by Age Group", kind="bar", rot=0)
plt.show()
""")
mlask()
# Create a dataframe with both exact and perturbed counts
column_values = [age_ranges, values, perturbed_counts]
table = {k: v for (k, v) in zip(column_names, column_values)}
df = pd.DataFrame(table)

# Display the two histograms as a table
display(df.style.set_caption("Test Counts by Age Group"))

# Plot the two histograms as bar graphs
df.plot(x="Age Group", title="Test Counts by Age Group", kind="bar", rot=0)
plt.show()
mlcat("", """Exponential Mechanism
""")
mlask()

mlcat("", """Basic Usage
The ExponentialMechanism does not lend itself to vectorised queries as
easily as the LaplaceMechanism or GeometricMechanism. So, to produce a
histogram query that is comparable to those discussed above we wrap the
query releases in a loop and compute them one at a time.
""")
mlask()

mlcat("", """
# Create a differentially private release mechanism
from relm.mechanisms import ExponentialMechanism

output_range = np.arange(2**10)
utility_function = lambda x: -abs(output_range - x)

perturbed_counts = np.empty(len(values), dtype=np.int)
for i, value in enumerate(values.astype(np.float)):
    mechanism = ExponentialMechanism(epsilon=0.1,
                                     utility_function=utility_function,
                                     sensitivity=1.0,
                                     output_range=output_range)

    perturbed_counts[i] = mechanism.release(values=value)
""")
mlask()
# Create a differentially private release mechanism
from relm.mechanisms import ExponentialMechanism

output_range = np.arange(2**10)
utility_function = lambda x: -abs(output_range - x)

perturbed_counts = np.empty(len(values), dtype=np.int)
for i, value in enumerate(values.astype(np.float)):
    mechanism = ExponentialMechanism(epsilon=0.1,
                                     utility_function=utility_function,
                                     sensitivity=1.0,
                                     output_range=output_range)

    perturbed_counts[i] = mechanism.release(values=value)
mlcat("", """Visualising the Results
""")
mlask()

mlcat("", """
# Create a dataframe with both exact and perturbed counts
column_values = [age_ranges, values, perturbed_counts]
table = {k: v for (k, v) in zip(column_names, column_values)}
df = pd.DataFrame(table)

# Display the two histograms as a table
display(df.style.set_caption("Test Counts by Age Group"))

# Plot the two histograms as bar graphs
df.plot(x="Age Group", title="Test Counts by Age Group", kind="bar", rot=0)
plt.show()
""")
mlask()
# Create a dataframe with both exact and perturbed counts
column_values = [age_ranges, values, perturbed_counts]
table = {k: v for (k, v) in zip(column_names, column_values)}
df = pd.DataFrame(table)

# Display the two histograms as a table
display(df.style.set_caption("Test Counts by Age Group"))

# Plot the two histograms as bar graphs
df.plot(x="Age Group", title="Test Counts by Age Group", kind="bar", rot=0)
plt.show()
mlcat("", """Sparse Mechanisms
We currently have four mechanisms that take advantage of sparsity to
answer more queries about the data for a given privacy budget. All of
these mechanisms compare noisy query responses to a noisy threshold
value. If a noisy response does not exceed the noisy threshold, then the
mechanism reports only that the value did not exceed the threshold.
Otherwise, the mechanism reports that the value exceeded the threshold.
Furthermore, in the latter case some mechanisms release more information
about the underlying exact count. This extra information is computed
using some other differentially private mechanism and therefore imposes
some additional privacy costs.
""")
mlask()

mlcat("", """Data Wrangling
All three of our mechanims share an input format. We require a sequence
of exact query responses and a threshold value to which these responses
will be compared.
""")
mlask()

mlcat("", """
# Read the raw data
fp = '20200811_QLD_dummy_dataset_individual_v2.xlsx'
data = pd.read_excel(fp)

# Limit our attention to the onset date column
data.drop(list(data.columns[1:]), axis=1, inplace=True)

# Remove data with no onset date listed
mask = data['ONSET_DATE'].notna()
data = data[mask]

# Compute the exact query responses
queries = [(pd.Timestamp('2020-01-01')  + i*pd.Timedelta('1d'),) for i in range(366)]
exact_counts = dict.fromkeys(queries, 0)
exact_counts.update(data.value_counts())

dates, values = zip(*sorted(exact_counts.items()))
values = np.array(values, dtype=np.float64)
""")
mlask()
# Read the raw data
fp = '20200811_QLD_dummy_dataset_individual_v2.xlsx'
data = pd.read_excel(fp)

# Limit our attention to the onset date column
data.drop(list(data.columns[1:]), axis=1, inplace=True)

# Remove data with no onset date listed
mask = data['ONSET_DATE'].notna()
data = data[mask]

# Compute the exact query responses
queries = [(pd.Timestamp('2020-01-01')  + i*pd.Timedelta('1d'),) for i in range(366)]
exact_counts = dict.fromkeys(queries, 0)
exact_counts.update(data.value_counts())

dates, values = zip(*sorted(exact_counts.items()))
values = np.array(values, dtype=np.float64)
mlcat("", """Above Threshold
The simplest of the sparse release mechanisms simply reports the index
of the first query response that exceeds the specified threshold.
""")
mlask()

mlcat("", """Basic Usage
""")
mlask()

mlcat("", """
from relm.mechanisms import AboveThreshold
mechanism = AboveThreshold(epsilon=1.0, sensitivity=1.0, threshold=100)
index = mechanism.release(values)
""")
mlask()
from relm.mechanisms import AboveThreshold
mechanism = AboveThreshold(epsilon=1.0, sensitivity=1.0, threshold=100)
index = mechanism.release(values)
mlcat("", """Empirical Distribution
Because AboveThreshold returns a single index, we can run many
experiments and plot a histogram of the results of those experiments to
get an empirical estimate of the distribution of the mechanism’s output.
""")
mlask()

mlcat("", """
TRIALS = 2**10
threshold = 100
results = np.zeros(TRIALS)
for i in range(TRIALS):
    mechanism = AboveThreshold(epsilon=1.0, sensitivity=1.0, threshold=threshold)
    index = mechanism.release(values)
    results[i] = index
""")
mlask()
TRIALS = 2**10
threshold = 100
results = np.zeros(TRIALS)
for i in range(TRIALS):
    mechanism = AboveThreshold(epsilon=1.0, sensitivity=1.0, threshold=threshold)
    index = mechanism.release(values)
    results[i] = index
mlcat("", """Visualising the Results
""")
mlask()

mlcat("", """
print("The index of the first exact count that exceeds the threshold is: %i\n" % np.argmax(values >= threshold))

histogram = dict.fromkeys(np.arange(min(results), max(results)), 0)
histogram.update(Counter(results))

plt.xlabel("Index")
plt.ylabel("Count")
plt.title("Distribution of AboveThreshold outputs")
plt.plot(list(histogram.keys()), list(histogram.values()))
plt.axvline(x=np.argmax(values >= threshold), color='orange')
""")
mlask()
print("The index of the first exact count that exceeds the threshold is: %i\n" % np.argmax(values >= threshold))

histogram = dict.fromkeys(np.arange(min(results), max(results)), 0)
histogram.update(Counter(results))

plt.xlabel("Index")
plt.ylabel("Count")
plt.title("Distribution of AboveThreshold outputs")
plt.plot(list(histogram.keys()), list(histogram.values()))
plt.axvline(x=np.argmax(values >= threshold), color='orange')
mlcat("", """Sparse Indicator
The SparseIndicator is a straightforward extension of the AboveThreshold
mechanism. Here, we find the indices of several values that exceeds the
specified threshold. The number of indices that this mechanism will
return is controlled by the cutoff parameter.
""")
mlask()

mlcat("", """Basic Usage
""")
mlask()

mlcat("", """
from relm.mechanisms import SparseIndicator
mechanism = SparseIndicator(epsilon=1.0, sensitivity=1.0, threshold=100, cutoff=3)
indices = mechanism.release(values)
""")
mlask()
from relm.mechanisms import SparseIndicator
mechanism = SparseIndicator(epsilon=1.0, sensitivity=1.0, threshold=100, cutoff=3)
indices = mechanism.release(values)
mlcat("", """Visualising the Results
""")
mlask()

mlcat("", """
TRIALS = 16
cutoff = 4
threshold = 100
indices = np.empty((TRIALS, cutoff), dtype=np.int)
for i in range(TRIALS):
    mechanism = SparseIndicator(epsilon=1.0, sensitivity=1.0, threshold=threshold, cutoff=cutoff)
    indices[i] = mechanism.release(values)

df = pd.DataFrame(indices,
                  columns=["Hit %i" % (j+1) for j in range(cutoff)],
                  index=["Trial %i" % i for i in range(TRIALS)])
display(df)
""")
mlask()
TRIALS = 16
cutoff = 4
threshold = 100
indices = np.empty((TRIALS, cutoff), dtype=np.int)
for i in range(TRIALS):
    mechanism = SparseIndicator(epsilon=1.0, sensitivity=1.0, threshold=threshold, cutoff=cutoff)
    indices[i] = mechanism.release(values)

df = pd.DataFrame(indices,
                  columns=["Hit %i" % (j+1) for j in range(cutoff)],
                  index=["Trial %i" % i for i in range(TRIALS)])
display(df)
mlcat("", """Sparse Numeric
The SparseNumeric mechanism returns perturbed values alongside the
indices of the values that exceeded the threshold.
""")
mlask()

mlcat("", """Basic Usage
""")
mlask()

mlcat("", """
from relm.mechanisms import SparseNumeric
mechanism = SparseNumeric(epsilon=1.0, sensitivity=1.0, threshold=100, cutoff=3)
indices, perturbed_values = mechanism.release(values)
""")
mlask()
from relm.mechanisms import SparseNumeric
mechanism = SparseNumeric(epsilon=1.0, sensitivity=1.0, threshold=100, cutoff=3)
indices, perturbed_values = mechanism.release(values)
mlcat("", """Visualising the Results
Notice that in these experiemnts we have set epsilon=4.0. This is a
larger value than we use in the other sparse mechanisms. Because the
SparseNumeric mechanism releases more information about the underlying
exact query responses than does SparseIndices, for example, it consumes
the available privacy budget more quickly. To achieve comparable utility
with respect the the indices returned by the two mechanisms, we
therefore need to a larger value for epsilon.
""")
mlask()

mlcat("", """
TRIALS = 2**4
cutoff = 3
threshold = 100
indices = np.empty(shape=(TRIALS, cutoff), dtype=np.int)
perturbed_values = np.empty(shape=(TRIALS, cutoff), dtype=np.float)
for i in range(TRIALS):
    mechanism = SparseNumeric(epsilon=4.0, sensitivity=1.0, threshold=threshold, cutoff=cutoff)
    indices[i], perturbed_values[i] = mechanism.release(values)

hit_names = ["Hit %i" % (j+1) for j in range(cutoff)]
value_names = ["Value %i" % (j+1) for j in range(cutoff)]
column_pairs = zip(hit_names, value_names)
column_names = [val for pair in column_pairs for val in pair]
value_pairs = zip(indices.transpose(), perturbed_values.transpose())
column_values = [val for pair in value_pairs for val in pair]
table = {k: v for (k, v) in zip(column_names, column_values)}
df = pd.DataFrame(table, index=["Trial %i" % i for i in range(TRIALS)])
display(df)
""")
mlask()
TRIALS = 2**4
cutoff = 3
threshold = 100
indices = np.empty(shape=(TRIALS, cutoff), dtype=np.int)
perturbed_values = np.empty(shape=(TRIALS, cutoff), dtype=np.float)
for i in range(TRIALS):
    mechanism = SparseNumeric(epsilon=4.0, sensitivity=1.0, threshold=threshold, cutoff=cutoff)
    indices[i], perturbed_values[i] = mechanism.release(values)

hit_names = ["Hit %i" % (j+1) for j in range(cutoff)]
value_names = ["Value %i" % (j+1) for j in range(cutoff)]
column_pairs = zip(hit_names, value_names)
column_names = [val for pair in column_pairs for val in pair]
value_pairs = zip(indices.transpose(), perturbed_values.transpose())
column_values = [val for pair in value_pairs for val in pair]
table = {k: v for (k, v) in zip(column_names, column_values)}
df = pd.DataFrame(table, index=["Trial %i" % i for i in range(TRIALS)])
display(df)
mlcat("", """Report Noisy Max
""")
mlask()

mlcat("", """Basic Usage
""")
mlask()

mlcat("", """
from relm.mechanisms import ReportNoisyMax
mechanism = ReportNoisyMax(epsilon=1.0)
index, perturbed_value = mechanism.release(values)
""")
mlask()
from relm.mechanisms import ReportNoisyMax
mechanism = ReportNoisyMax(epsilon=1.0)
index, perturbed_value = mechanism.release(values)
mlcat("", """Empirical Distribution
Because ReportNoisyMax returns a single index and a single value, we can
run many experiments and plot a histogram of the results of those
experiments to get an empirical estimate of the distribution of the
mechanism’s output.
""")
mlask()

mlcat("", """
TRIALS = 2**10
indices = np.zeros(TRIALS)
perturbed_values = np.zeros(TRIALS)
for i in range(TRIALS):
    mechanism = ReportNoisyMax(epsilon=0.1)
    indices[i], perturbed_values[i] = mechanism.release(values)
""")
mlask()
TRIALS = 2**10
indices = np.zeros(TRIALS)
perturbed_values = np.zeros(TRIALS)
for i in range(TRIALS):
    mechanism = ReportNoisyMax(epsilon=0.1)
    indices[i], perturbed_values[i] = mechanism.release(values)
mlcat("", """Visualising the Results
The ReportNoisyMax mechanism resturns two values, an index and a
perturbed query response. We analyze the distribution of each output
individually.
""")
mlask()

mlcat("", """Indices
""")
mlask()

mlcat("", """
print("The index of the greatest exact count is: %i\n" % np.argmax(values))

histogram = dict.fromkeys(np.arange(min(indices), max(indices)), 0)
histogram.update(Counter(indices))

plt.xlabel("Index")
plt.ylabel("Count")
plt.title("Distribution of ReportNoisyMax output indices")
plt.plot(list(histogram.keys()), list(histogram.values()))
plt.axvline(x=np.argmax(values), color='orange')
""")
mlask()
print("The index of the greatest exact count is: %i\n" % np.argmax(values))

histogram = dict.fromkeys(np.arange(min(indices), max(indices)), 0)
histogram.update(Counter(indices))

plt.xlabel("Index")
plt.ylabel("Count")
plt.title("Distribution of ReportNoisyMax output indices")
plt.plot(list(histogram.keys()), list(histogram.values()))
plt.axvline(x=np.argmax(values), color='orange')
mlcat("", """Perturbed Values
""")
mlask()

mlcat("", """
print("The greatest exact count is: %i\n" % np.max(values))

#histogram = dict.fromkeys(np.arange(min(results), max(results)), 0)
#histogram.update(Counter(results))

plt.hist(perturbed_values, bins=128)

plt.xlabel("Perturbed Value")
plt.ylabel("Count")
plt.title("Distribution of ReportNoisyMax output values")
plt.axvline(x=np.max(values), color='orange')
""")
mlask()
print("The greatest exact count is: %i\n" % np.max(values))

#histogram = dict.fromkeys(np.arange(min(results), max(results)), 0)
#histogram.update(Counter(results))

plt.hist(perturbed_values, bins=128)

plt.xlabel("Perturbed Value")
plt.ylabel("Count")
plt.title("Distribution of ReportNoisyMax output values")
plt.axvline(x=np.max(values), color='orange')
mlcat("", """Correlated Noise Mechanisms
""")
mlask()

mlcat("", """Data Wrangling
""")
mlask()

mlcat("", """
# Generate some synthetic data
data = pd.DataFrame()
test_results = np.random.choice(["Negative", "Positive", "N/A"], size=100000, p=[0.55,0.35,0.1])
data["Test Result"] = test_results

# Compute a histogram representation of the data
from relm.histogram import Histogram
hist = Histogram(data)
real_database = hist.get_db()
db_size = real_database.size
db_l1_norm = real_database.sum()

# Specify the queries to be answered
queries = [[{"Test Result": "Negative"}, {"Test Result": "Positive"}],
           [{"Test Result": "Positive"}, {"Test Result": "N/A"}],
           [{"Test Result": "Negative"}, {"Test Result": "N/A"}]]
num_queries = len(queries)
queries = sps.vstack([hist.get_query_vector(q) for q in queries])

# Compute the exact query responses
values = (queries @ real_database)/real_database.sum()
""")
mlask()
# Generate some synthetic data
data = pd.DataFrame()
test_results = np.random.choice(["Negative", "Positive", "N/A"], size=100000, p=[0.55,0.35,0.1])
data["Test Result"] = test_results

# Compute a histogram representation of the data
from relm.histogram import Histogram
hist = Histogram(data)
real_database = hist.get_db()
db_size = real_database.size
db_l1_norm = real_database.sum()

# Specify the queries to be answered
queries = [[{"Test Result": "Negative"}, {"Test Result": "Positive"}],
           [{"Test Result": "Positive"}, {"Test Result": "N/A"}],
           [{"Test Result": "Negative"}, {"Test Result": "N/A"}]]
num_queries = len(queries)
queries = sps.vstack([hist.get_query_vector(q) for q in queries])

# Compute the exact query responses
values = (queries @ real_database)/real_database.sum()
mlcat("", """SmallDB Mechanism
""")
mlask()

mlcat("", """Basic Usage
""")
mlask()

mlcat("", """
from relm.mechanisms import SmallDB
mechanism = SmallDB(epsilon=0.01, alpha=0.1)
synthetic_database = mechanism.release(values, queries, db_size, db_l1_norm)
""")
mlask()
from relm.mechanisms import SmallDB
mechanism = SmallDB(epsilon=0.01, alpha=0.1)
synthetic_database = mechanism.release(values, queries, db_size, db_l1_norm)
mlcat("", """Visualising the Results
""")
mlask()

mlcat("", """
TRIALS = 2**4
synthetic_responses = np.empty((TRIALS, queries.shape[0]))
for i in range(TRIALS):
    mechanism = SmallDB(epsilon=0.01, alpha=0.1)
    synthetic_database = mechanism.release(values, queries, db_size, db_l1_norm)
    synthetic_responses[i] = (queries @ synthetic_database) / synthetic_database.sum()

df = pd.DataFrame(data=np.row_stack((values, synthetic_responses)),
                  columns=["Query %i" % i for i in range(queries.shape[0])],
                  index=["Exact Responses"] + ["TRIAL %i" % i for i in range(TRIALS)])

display(df)
""")
mlask()
TRIALS = 2**4
synthetic_responses = np.empty((TRIALS, queries.shape[0]))
for i in range(TRIALS):
    mechanism = SmallDB(epsilon=0.01, alpha=0.1)
    synthetic_database = mechanism.release(values, queries, db_size, db_l1_norm)
    synthetic_responses[i] = (queries @ synthetic_database) / synthetic_database.sum()

df = pd.DataFrame(data=np.row_stack((values, synthetic_responses)),
                  columns=["Query %i" % i for i in range(queries.shape[0])],
                  index=["Exact Responses"] + ["TRIAL %i" % i for i in range(TRIALS)])

display(df)
mlcat("", """Online Multiplicative Weights Mechanism
""")
mlask()

mlcat("", """
from scipy.special import comb

# Generate some synthetic data
data = pd.DataFrame()
db_size = 8
probs = np.random.random(db_size)
probs /= probs.sum()
print(probs)
test_results = np.random.choice(np.arange(db_size), size=2**22, p=probs)
data["Test Result"] = test_results

# Compute a histogram representation of the data
from relm.histogram import Histogram
hist = Histogram(data)
real_database = hist.get_db()
db_size = real_database.size
db_l1_norm = real_database.sum()

# Specify the queries to be answered
num_ones = 5
q_size = comb(db_size, num_ones, exact=True)
queries = [{"Test Result": k for k in np.random.choice(np.arange(db_size), size=num_ones, replace=False)} for i in range(1024)]
num_queries = len(queries)
queries = sps.vstack([hist.get_query_vector(q) for q in queries])

# Compute the exact query responses
values = (queries @ real_database)/real_database.sum()
""")
mlask()
from scipy.special import comb

# Generate some synthetic data
data = pd.DataFrame()
db_size = 8
probs = np.random.random(db_size)
probs /= probs.sum()
print(probs)
test_results = np.random.choice(np.arange(db_size), size=2**22, p=probs)
data["Test Result"] = test_results

# Compute a histogram representation of the data
from relm.histogram import Histogram
hist = Histogram(data)
real_database = hist.get_db()
db_size = real_database.size
db_l1_norm = real_database.sum()

# Specify the queries to be answered
num_ones = 5
q_size = comb(db_size, num_ones, exact=True)
queries = [{"Test Result": k for k in np.random.choice(np.arange(db_size), size=num_ones, replace=False)} for i in range(1024)]
num_queries = len(queries)
queries = sps.vstack([hist.get_query_vector(q) for q in queries])

# Compute the exact query responses
values = (queries @ real_database)/real_database.sum()
mlcat("", """Basic Usage
""")
mlask()

mlcat("", """
from relm.mechanisms import PrivateMultiplicativeWeights
mechanism = PrivateMultiplicativeWeights(epsilon=1.0, alpha=0.15, beta=0.01, q_size=q_size, db_size=db_size, db_l1_norm=db_l1_norm)
dp_responses = mechanism.release(values, queries)
""")
mlask()
from relm.mechanisms import PrivateMultiplicativeWeights
mechanism = PrivateMultiplicativeWeights(epsilon=1.0, alpha=0.15, beta=0.01, q_size=q_size, db_size=db_size, db_l1_norm=db_l1_norm)
dp_responses = mechanism.release(values, queries)
mlcat("", """
print(values)
print(dp_responses)
""")
mlask()
print(values)
print(dp_responses)
mlcat("", """
print(mechanism.db_l1_norm)
print(mechanism.cutoff)
print(mechanism.q_size)
print(mechanism.beta)
print(mechanism.est_data)
print(mechanism.threshold)
""")
mlask()
print(mechanism.db_l1_norm)
print(mechanism.cutoff)
print(mechanism.q_size)
print(mechanism.beta)
print(mechanism.est_data)
print(mechanism.threshold)
mlcat("", """
print(mechanism.alpha)
temp = 36*np.log(mechanism.db_size)
temp *= np.log(mechanism.q_size) + np.log(32*mechanism.db_size/(mechanism.alpha**2 * mechanism.beta))
temp /= mechanism.epsilon * mechanism.db_l1_norm * mechanism.alpha**2
print(temp)
""")
mlask()
print(mechanism.alpha)
temp = 36*np.log(mechanism.db_size)
temp *= np.log(mechanism.q_size) + np.log(32*mechanism.db_size/(mechanism.alpha**2 * mechanism.beta))
temp /= mechanism.epsilon * mechanism.db_l1_norm * mechanism.alpha**2
print(temp)
mlcat("", """
np.max(np.abs(values - dp_responses))
""")
mlask()
np.max(np.abs(values - dp_responses))
mlcat("", """

""")
mlask()

mlcat("", """

""")
mlask()

mlcat("", """

""")
mlask()