#to do
#generalise the names of paramater files and output files say as per glacier id
# avoid writing external file as least as possible

# utility to convert GSLIB Geo-EAS files to a pandas DataFrame for use with Python methods
import os
import subprocess
import numpy as np
import pandas as pd
import random as rand
import matplotlib.pyplot as plt

def GSLIB2Dataframe(data_file):
    colArray = []
    with open(data_file) as myfile:   # read first two lines
        head = [next(myfile) for x in range(2)]
        line2 = head[1].split()
        ncol = int(line2[0])
        for icol in range(0, ncol):
            head = [next(myfile) for x in range(1)]
            colArray.append(head[0].split()[0])
        data = np.loadtxt(myfile, skiprows = 0)
        df = pd.DataFrame(data)
        df.columns = colArray
        return df

# utility to convert pandas DataFrame to a GSLIB Geo-EAS file for use with GSLIB methods
def Dataframe2GSLIB(data_file,df):
    colArray = []
    colArray = df.columns
    ncol = len(df.columns)
    nrow = len(df.index)
    file_out = open(data_file, "w")
    file_out.write(data_file + '\n')
    file_out.write(str(ncol) + '\n')
    for icol in range(0, ncol):
        file_out.write(df.columns[icol]  + '\n')

    for irow in range(0, nrow):
        for icol in range(0, ncol):
            file_out.write(str(df.iloc[irow,icol])+ ' ')
        file_out.write('\n')

    file_out.close()

# utility to convert GSLIB Geo-EAS files to a numpy ndarray for use with Python methods
def GSLIB2ndarray(data_file,kcol,nx,ny):

    colArray = []
    if ny > 1:
        array = np.ndarray(shape=(ny,nx),dtype=float,order='F')
    else:
        array = np.zeros(nx)

    with open(data_file) as myfile:   # read first two lines
        head = [next(myfile) for x in range(2)]
        line2 = head[1].split()
        ncol = int(line2[0])          # get the number of columns
        for icol in range(0, ncol):   # read over the column names
            head = [next(myfile) for x in range(1)]
            if icol == kcol:
                col_name = head[0].split()[0]
        for iy in range(0,ny):
            for ix in range(0,nx):
                head = [next(myfile) for x in range(1)]
                array[ny-1-iy][ix] = head[0].split()[kcol]
    return array,col_name

# utility to convert numpy ndarray to a GSLIB Geo-EAS file for use with GSLIB methods
def ndarray2GSLIB(array,data_file,col_name):
    file_out = open(data_file, "w")
    file_out.write(data_file + '\n')
    file_out.write('1 \n')
    file_out.write(col_name  + '\n')
    if array.ndim == 2:
        ny = (array.shape[0])
        nx = (array.shape[1])
        ncol = 1
        for iy in range(0, ny):
            for ix in range(0, nx):
                file_out.write(str(array[ny-1-iy,ix])+ '\n')
    elif array.ndim == 1:
        nx = len(array)
        for ix in range(0, nx):
            file_out.write(str(array[ix])+ '\n')
    else:
        Print("Error: must use a 2D array")
        file_out.close()
        return
    file_out.close()

# histogram, reimplemented in Python of GSLIB hist with MatPlotLib methods
def hist(array,xmin,xmax,log,cumul,bins,weights,xlabel,title):
    plt.figure(figsize=(8,6))
    cs = plt.hist(array, alpha = 0.2, color = 'red', edgecolor = 'black', bins=bins, range = [xmin,xmax], weights = weights, log = log, cumulative = cumul)
    plt.title(title)
    plt.xlabel(xlabel); plt.ylabel('Frequency')
    plt.show()
    return

# pixel plot, reimplemention in Python of GSLIB pixelplt with MatPlotLib methods
def pixelplt(array,xmin,xmax,ymin,ymax,step,vmin,vmax,title,xlabel,ylabel,vlabel,cmap):
    xx, yy = np.meshgrid(np.arange(xmin, xmax, step),np.arange(ymax, ymin, -1*step))
    plt.figure(figsize=(8,6))
    im = plt.contourf(xx,yy,array,cmap=cmap,vmin=vmin,vmax=vmax,levels=np.linspace(vmin,vmax,100))
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    cbar = plt.colorbar(im,orientation = 'vertical',ticks=np.linspace(vmin,vmax,10))
    cbar.set_label(vlabel, rotation=270, labelpad=20)
    plt.show()
    return im

# location map, reimplemention in Python of GSLIB locmap with MatPlotLib methods
def locmap(df,xcol,ycol,vcol,xmin,xmax,ymin,ymax,vmin,vmax,title,xlabel,ylabel,vlabel,cmap):
    ixy = 0
    plt.figure(figsize=(8,6))
    im = plt.scatter(df[xcol],df[ycol],s=None, c=df[vcol], marker=None, cmap=cmap, norm=None, vmin=vmin, vmax=vmax, alpha=0.8, linewidths=0.8, verts=None, edgecolors="black")
    plt.title(title)
    plt.xlim(xmin,xmax)
    plt.ylim(ymin,ymax)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    cbar = plt.colorbar(im, orientation = 'vertical',ticks=np.linspace(vmin,vmax,10))
    cbar.set_label(vlabel, rotation=270, labelpad=20)
    plt.show()
    return im

def vargplt(lag,gamma,npair,vtype,name,xmin,xmax,ymin,ymax,sill,title,cmap):
    plt.figure(figsize=(8,6))
    marker = ["o","v","s","h","^",">","<"]
    if vtype==0:
        im = plt.scatter(lag,gamma,s=None, c=npair, marker=None, label = name,cmap=cmap, norm=None, alpha=0.8, linewidths=0.8, verts=None, edgecolors="black")
    else:
        plt.plot(lag,gamma,'C3',lw=3,c='black')
    ixy = 0
    plt.title(title)
    plt.xlim(xmin,xmax)
    plt.ylim(ymin,ymax)
    plt.xlabel('Lag Distance (m)')
    plt.ylabel('Variogram')
    plt.arrow(0,sill,xmax,0,width=0.002,color='red',head_length=0.0,head_width=0.0)
    plt.legend(loc = 'lower right')
    cbar = plt.colorbar(im, orientation = 'vertical')
    cbar.set_label('Number of Pairs', rotation=270, labelpad=20)
    plt.show()
    return im

def vargplts(lag,gamma,npair,vtype,name,xmin,xmax,ymin,ymax,sill,title,cmap):

    plt.figure(figsize=(8,6))
    marker = ["o","v","s","h","^",">","<"]
    nvar = lag.shape[0]
    for ivar in range(0, nvar):
        if vtype[ivar]==0:
            im = plt.scatter(lag[ivar],gamma[ivar],s=None,label = name[ivar],c=npair[ivar], marker=marker[ivar], cmap=cmap, norm=None, alpha=0.8, linewidths=0.8, verts=None, edgecolors="black")
        else:
            plt.plot(lag[ivar],gamma[ivar], 'C3', lw=3,c='black')
    ixy = 0
    plt.title(title)
    plt.xlim(xmin,xmax)
    plt.ylim(ymin,ymax)
    plt.xlabel('Lag Distance (m)')
    plt.ylabel('Variogram')
    plt.arrow(0,sill,xmax,0,width=0.002,color='red',head_length=0.0,head_width=0.0)
    plt.legend(loc = 'lower right')
    cbar = plt.colorbar(im, orientation = 'vertical')
    cbar.set_label('Number of Pairs', rotation=270, labelpad=20)
    plt.show()
    return im

# regular grid variogram, 2D wrapper for gam from GSLIB (.exe must be in working directory)
def gam_2d(array,nx,ny,hsiz,nlag,xlag,ylag,bstand):

    lag = []; gamma = []; npair = []

    ndarray2GSLIB(array,"gam_out.dat","gam.dat")

    file = open("gam.par", "w")
    file.write("                  Parameters for GAM                                       \n")
    file.write("                  ******************                                       \n")
    file.write("                                                                           \n")
    file.write("START OF PARAMETERS:                                                       \n")
    file.write("gam_out.dat           -file with data                                      \n")
    file.write("1   1   0             -   number of variables, column numbers              \n")
    file.write("-1.0e21     1.0e21    -   trimming limits                                  \n")
    file.write("gam.out               -file for variogram output                           \n")
    file.write("1                     -grid or realization number                          \n")
    file.write(str(nx) + " 0.0 " + str(hsiz) + "  -nx, xmn, xsiz                           \n")
    file.write(str(ny) + " 0.0 " + str(hsiz) + "  -ny, ymn, ysiz                           \n")
    file.write(" 1   0.5   1.0        -nz, zmn, zsiz                                       \n")
    file.write("1 " + str(nlag) + "   -number of directions, number of lags                \n")
    file.write(str(xlag) + " " + str(ylag) + " 0 -ixd(1),iyd(1),izd(1)                     \n")
    file.write("1                     -standardize sill? (0=no, 1=yes)                     \n")
    file.write("1                     -number of variograms                                \n")
    file.write("1   1   1             -tail variable, head variable, variogram type        \n")
    file.close()
    cmd = ['gam,', 'gam.par']
    subprocess.call(cmd)
    reading = True
    with open("gam.out") as myfile:
        head = [next(myfile) for x in range(1)] # skip the first line
        iline = 0
        while reading:
            try:
                head = [next(myfile) for x in range(1)]
                lag.append(np.float64(head[0].split()[1]))
                gamma.append(np.float64(head[0].split()[2]))
                npair.append(np.float64(head[0].split()[3]))
                iline = iline + 1
            except StopIteration:
                reading = False

    return(lag,gamma,npair)

# irregular grid variogram, 2D wrapper for gam from GSLIB (.exe must be in working directory)
def gamv_2d(df,xcol,ycol,vcol,nlag,lagdist,azi,atol,bstand,bandh,bandv):

    lag = []; gamma = []; npair = []

    df_ext = pd.DataFrame({'X':df[xcol],'Y':df[ycol],'Z':df[vcol]})
    Dataframe2GSLIB("gamv_out.dat",df_ext)

    file = open("gamv.par", "w")

    file.write("                  Parameters for GAMV                                      \n")
    file.write("                  *******************                                      \n")
    file.write("                                                                           \n")
    file.write("START OF PARAMETERS:                                                       \n")
    file.write("gamv_out.dat                    -file with data                            \n")
    file.write("1   2   0                         -   columns for X, Y, Z coordinates      \n")
    file.write("1   3   0                         -   number of variables,col numbers      \n")
    file.write("-1.0e21     1.0e21                -   trimming limits                      \n")
    file.write("gamv.out                          -file for variogram output               \n")
    file.write(str(nlag) + "                      -number of lags                          \n")
    file.write(str(lagdist) + "                       -lag separation distance                 \n")
    file.write(str(lagdist*0.5) + "                   -lag tolerance                           \n")
    file.write("1                                 -number of directions                    \n")
    file.write(str(azi) + " " + str(atol) + " " + str(bandh) + " 0.0  0 " + str(bandv) + "  -azm,atol,bandh,dip,dtol,bandv \n")
    file.write(str(bstand) + "                    -standardize sills? (0=no, 1=yes)        \n")
    file.write("1                                 -number of variograms                    \n")
    file.write("1   1   1                         -tail var., head var., variogram type    \n")
    file.close()
    cmd=['gamv','gamv.par']
    subprocess.call(cmd)
    reading = True
    with open("gamv.out") as myfile:
        head = [next(myfile) for x in range(1)] # skip the first line
        iline = 0
        while reading:
            try:
                head = [next(myfile) for x in range(1)]
                lag.append(np.float64(head[0].split()[1]))
                gamma.append(np.float64(head[0].split()[2]))
                npair.append(np.float64(head[0].split()[3]))
                iline = iline + 1
            except StopIteration:
                reading = False

    return(lag,gamma,npair)

# irregular spaced data, 2D wrapper for varmap from GSLIB (.exe must be in working directory)
def varmapv_2d(df,xcol,ycol,vcol,nx,ny,lagdist,minpairs,vmax,bstand,title,vlabel):

    lag = []; gamma = []; npair = []

    df_ext = pd.DataFrame({'X':df[xcol],'Y':df[ycol],'Z':rand_sample[vcol]})
    Dataframe2GSLIB("varmap_out.dat",df_ext)

    file = open("varmap.par", "w")

    file.write("              Parameters for VARMAP                                        \n")
    file.write("              *********************                                        \n")
    file.write("                                                                           \n")
    file.write("START OF PARAMETERS:                                                       \n")
    file.write("varmap_out.dat          -file with data                                    \n")
    file.write("1   3                        -   number of variables: column numbers       \n")
    file.write("-1.0e21     1.0e21           -   trimming limits                           \n")
    file.write("0                            -1=regular grid, 0=scattered values           \n")
    file.write(" 50   50    1                -if =1: nx,     ny,   nz                      \n")
    file.write("1.0  1.0  1.0                -       xsiz, ysiz, zsiz                      \n")
    file.write("1   2   0                    -if =0: columns for x,y, z coordinates        \n")
    file.write("varmap.out                   -file for variogram output                    \n")
    file.write(str(nx) + " " + str(ny) + " 0 " + "-nxlag, nylag, nzlag                     \n")
    file.write(str(lagdist) + " " + str(lagdist) + " 1.0              -dxlag, dylag, dzlag \n")
    file.write(str(minpairs) + "             -minimum number of pairs                      \n")
    file.write(str(bstand) + "               -standardize sill? (0=no, 1=yes)              \n")
    file.write("1                            -number of variograms                         \n")
    file.write("1   1   1                    -tail, head, variogram type                   \n")
    file.close()
    cmd=['varmap','varmap.par']
    subprocess.call(cmd)
    nnx = nx*2+1; nny = ny*2+1
    varmap, name = GSLIB2ndarray("varmap.out",0,nnx,nny)

    xmax = ((float(nx)+0.5)*lagdist); xmin = -1*xmax;
    ymax = ((float(ny)+0.5)*lagdist); ymin = -1*ymax;
    pixelplt(varmap,xmin,xmax,ymin,ymax,lagdist,0,vmax,title,'X','Y',vlabel,cmap)
    return(varmap)

# regular spaced data, 2D wrapper for varmap from GSLIB (.exe must be in working directory)
def varmap_2d(array,nx,ny,hsiz,nlagx,nlagy,minpairs,vmax,bstand,title,vlabel):

    ndarray2GSLIB(array,"varmap_out.dat","gam.dat")

    file = open("varmap.par", "w")

    file.write("              Parameters for VARMAP                                        \n")
    file.write("              *********************                                        \n")
    file.write("                                                                           \n")
    file.write("START OF PARAMETERS:                                                       \n")
    file.write("varmap_out.dat          -file with data                                    \n")
    file.write("1   1                        -   number of variables: column numbers       \n")
    file.write("-1.0e21     1.0e21           -   trimming limits                           \n")
    file.write("1                            -1=regular grid, 0=scattered values           \n")
    file.write(str(nx) + " " + str(ny) + " 1  -if =1: nx,     ny,   nz                     \n")
    file.write(str(hsiz) + " " + str(hsiz) + " 1.0  - xsiz, ysiz, zsiz                     \n")
    file.write("1   2   0                    -if =0: columns for x,y, z coordinates        \n")
    file.write("varmap.out                   -file for variogram output                    \n")
    file.write(str(nlagx) + " " + str(nlagy) + " 0 " + "-nxlag, nylag, nzlag               \n")
    file.write(str(hsiz) + " " + str(hsiz) + " 1.0              -dxlag, dylag, dzlag       \n")
    file.write(str(minpairs) + "             -minimum number of pairs                      \n")
    file.write(str(bstand) + "               -standardize sill? (0=no, 1=yes)              \n")
    file.write("1                            -number of variograms                         \n")
    file.write("1   1   1                    -tail, head, variogram type                   \n")
    file.close()
    cmd =['varmap','varmap.par']
    subprocess.call(cmd)
    nnx = nlagx*2+1; nny = nlagy*2+1
    varmap, name = GSLIB2ndarray("varmap.out",0,nnx,nny)

    xmax = ((float(nlagx)+0.5)*hsiz); xmin = -1*xmax;
    ymax = ((float(nlagy)+0.5)*hsiz); ymin = -1*ymax;
    pixelplt(varmap,xmin,xmax,ymin,ymax,hsiz,0,vmax,title,'X','Y',vlabel,cmap)
    return(varmap)

# variogram model, 2D wrapper for vmodel from GSLIB (.exe must be in working directory)
def vmodel_2d(nlag,step,azi,nug,nst,tstr1,c1,azi1,rmaj1,rmin1,tstr2=1,c2=0,azi2=0,rmaj2=0,rmin2=0):

    lag = []; gamma = []

    file = open("vmodel.par", "w")
    file.write("                                                                           \n")
    file.write("                  Parameters for VMODEL                                    \n")
    file.write("                  *********************                                    \n")
    file.write("                                                                           \n")
    file.write("START OF PARAMETERS:                                                       \n")
    file.write("vmodel.var                   -file for variogram output                    \n")
    file.write("1 " + str(nlag) + "          -number of directions and lags                \n")
    file.write(str(azi) + " 0.0 " + str(step) + " -azm, dip, lag distance                  \n")
    file.write(str(nst) + " " + str(nug) + " -nst, nugget effect                           \n")
    file.write(str(tstr1) + " " + str(c1) + " " + str(azi1) + " 0.0   0.0   0.0 -it,cc,ang1,ang2,ang3 \n")
    file.write(str(rmaj1) + " " + str(rmin1) + " 0.0 -a_hmax, a_hmin, a_vert               \n")
    file.write(str(tstr2) + " " + str(c2) + " " + str(azi2) + " 0.0   0.0   0.0 -it,cc,ang1,ang2,ang3 \n")
    file.write(str(rmaj2) + " " + str(rmin2) + " 0.0 -a_hmax, a_hmin, a_vert               \n")
    file.close()
    cmd = ['vmodel', 'vmodel.par']
    subprocess.call(cmd)
    reading = True
    with open("vmodel.var") as myfile:
        head = [next(myfile) for x in range(1)] # skip the first line
        iline = 0
        while reading:
            try:
                head = [next(myfile) for x in range(1)]
                lag.append(float(head[0].split()[1]))
                gamma.append(float(head[0].split()[2]))
                iline = iline + 1
            except StopIteration:
                reading = False

    return(lag,gamma)

# pixelplt with location map superimposed, reimplementation in Python of a MOD from GSLIB with MatPlotLib methods
def locpix(array,xmin,xmax,ymin,ymax,step,vmin,vmax,df,xcol,ycol,vcol,title,xlabel,ylabel,vlabel,cmap):
    xx, yy = np.meshgrid(np.arange(xmin, xmax, step),np.arange(ymax, ymin, -1*step))
    plt.figure(figsize=(8,6))
    cs = plt.contourf(xx, yy, array, cmap=cmap,vmin=vmin, vmax=vmax, levels=np.linspace(vmin,vmax,100))
    im = plt.scatter(df[xcol],df[ycol],s=None, c=df[vcol], marker=None, cmap=cmap, norm=None, vmin=vmin, vmax=vmax, alpha=0.8, linewidths=0.8, verts=None, edgecolors="black")
    plt.xlim(xmin,xmax-step)
    plt.ylim(ymin+step,ymax)
    plt.title(title)
    plt.xlabel(xlabel)
    plt.ylabel(ylabel)
    cbar = plt.colorbar(orientation = 'vertical',ticks=np.linspace(vmin,vmax,10))
    cbar.set_label(vlabel, rotation=270, labelpad=20)
    plt.show()
    return cs

# affine distribution correction reimplemented in Python with numpy methods
def affine(array,tmean,tstdev):
    if array.ndim != 2:
        Print("Error: must use a 2D array")
        return
    nx = array.shape[0]
    ny = array.shape[1]
    mean = np.average(array)
    stdev = np.std(array)
    for iy in range(0,ny):
        for ix in range(0,nx):
             array[ix,iy]= (tstdev/stdev)*(array[ix,iy] - mean) + tmean
    return(array)

# normal score transform, wrapper for nscore from GSLIB (.exe must be in working directory)(not used in this demo)
def nscore(x,nobs):
    file = 'nscore_out.dat'
    ndarray2GSLIB(x,"nscore.dat","value")

    file = open("nscore.par", "w")
    file.write("                  Parameters for NSCORE                                    \n")
    file.write("                  *********************                                    \n")
    file.write("                                                                           \n")
    file.write("START OF PARAMETERS:                                                       \n")
    file.write("nscore.dat           -file with data                                       \n")
    file.write("1   0                    -  columns for variable and weight                \n")
    file.write("-1.0e21   1.0e21         -  trimming limits                                \n")
    file.write("0                        -1=transform according to specified ref. dist.    \n")
    file.write("../histsmth/histsmth.out -  file with reference dist.                      \n")
    file.write("1   2                    -  columns for variable and weight                \n")
    file.write("nscore.out               -file for output                                  \n")
    file.write("nscore.trn               -file for output transformation table             \n")
    file.close()
    cmd = ['nscore', 'nscore.par']
    subprocess.call(cmd)
    file_in = 'nscore.out'
    ny=1#the returned file contains only transformed values, so ny=1 (its 1d)
    nx = nobs
    #y,name = GSLIB2ndarray('nscore.out',1,nx,ny)
    y = GSLIB2Dataframe('nscore.out')
    return(y)

# cell-based declustering, 2D wrapper for declus from GSLIB (.exe must be in working directory)
def declus(df,xcol,ycol,vcol,cmin,cmax,cnum,bmin):
    nrow = len(df)
    weights = []
    file = 'declus_out.dat'
    file_out = open(file, "w")
    file_out.write('declus_out.dat' + '\n')
    file_out.write('3' + '\n')
    file_out.write('x' + '\n')
    file_out.write('y' + '\n')
    file_out.write('value' + '\n')
    for irow in range(0, nrow):
        file_out.write(str(df.iloc[irow][xcol])+' '+str(df.iloc[irow][ycol])+' '+str(df.iloc[irow][vcol])+' \n')
    file_out.close()

    file = open("declus.par", "w")
    file.write("                  Parameters for DECLUS                                    \n")
    file.write("                  *********************                                    \n")
    file.write("                                                                           \n")
    file.write("START OF PARAMETERS:                                                       \n")
    file.write("declus_out.dat           -file with data                                   \n")
    file.write("1   2   0   3               -  columns for X, Y, Z, and variable           \n")
    file.write("-1.0e21     1.0e21          -  trimming limits                             \n")
    file.write("declus.sum                  -file for summary output                       \n")
    file.write("declus.out                  -file for output with data & weights           \n")
    file.write("1.0   1.0                   -Y and Z cell anisotropy (Ysize=size*Yanis)    \n")
    file.write(str(bmin) + "                -0=look for minimum declustered mean (1=max)   \n")
    file.write(str(cnum) + " " + str(cmin) + " " + str(cmax) + " -number of cell sizes, min size, max size      \n")
    file.write("5                           -number of origin offsets                      \n")
    file.close()
    cmd=['declus', 'declus.par']
    subprocess.call(cmd)
    df = GSLIB2Dataframe("declus.out")
    for irow in range(0, nrow):
        weights.append(df.iloc[irow,3])

    return(weights)

# sequential Gaussian simulation, 2D unconditional wrapper for sgsim from GSLIB (.exe must be in working directory)
def GSLIB_sgsim_2d_uncond(nreal,nx,ny,hsiz,seed,hrange1,hrange2,azi,output_file):

    hmn = hsiz * 0.5
    hctab = int(hrange1/hsiz)*2 + 1

    sim_array = np.random.rand(nx,ny)

    file = open("sgsim.par", "w")
    file.write("              Parameters for SGSIM                                         \n")
    file.write("              ********************                                         \n")
    file.write("                                                                           \n")
    file.write("START OF PARAMETER:                                                        \n")
    file.write("none                          -file with data                              \n")
    file.write("1  2  0  3  5  0              -  columns for X,Y,Z,vr,wt,sec.var.          \n")
    file.write("-1.0e21 1.0e21                -  trimming limits                           \n")
    file.write("0                             -transform the data (0=no, 1=yes)            \n")
    file.write("none.trn                      -  file for output trans table               \n")
    file.write("1                             -  consider ref. dist (0=no, 1=yes)          \n")
    file.write("none.dat                      -  file with ref. dist distribution          \n")
    file.write("1  0                          -  columns for vr and wt                     \n")
    file.write("-4.0    4.0                   -  zmin,zmax(tail extrapolation)             \n")
    file.write("1      -4.0                   -  lower tail option, parameter              \n")
    file.write("1       4.0                   -  upper tail option, parameter              \n")
    file.write("0                             -debugging level: 0,1,2,3                    \n")
    file.write("nonw.dbg                      -file for debugging output                   \n")
    file.write(str(output_file) + "           -file for simulation output                  \n")
    file.write(str(nreal) + "                 -number of realizations to generate          \n")
    file.write(str(nx) + " " + str(hmn) + " " + str(hsiz) + "                              \n")
    file.write(str(ny) + " " + str(hmn) + " " + str(hsiz) + "                              \n")
    file.write("1 0.0 1.0                     - nz zmn zsiz                                \n")
    file.write(str(seed) + "                  -random number seed                          \n")
    file.write("0     8                       -min and max original data for sim           \n")
    file.write("12                            -number of simulated nodes to use            \n")
    file.write("0                             -assign data to nodes (0=no, 1=yes)          \n")
    file.write("1     3                       -multiple grid search (0=no, 1=yes),num      \n")
    file.write("0                             -maximum data per octant (0=not used)        \n")
    file.write(str(hrange1) + " " + str(hrange2) + " 1.0 -maximum search  (hmax,hmin,vert) \n")
    file.write(str(azi) + "   0.0   0.0       -angles for search ellipsoid                 \n")
    file.write(str(hctab) + " " + str(hctab) + " 1 -size of covariance lookup table        \n")
    file.write("0     0.60   1.0              -ktype: 0=SK,1=OK,2=LVM,3=EXDR,4=COLC        \n")
    file.write("none.dat                      -  file with LVM, EXDR, or COLC variable     \n")
    file.write("4                             -  column for secondary variable             \n")
    file.write("1    0.0                      -nst, nugget effect                          \n")
    file.write("1    1.0 " + str(azi) + " 0.0 0.0 -it,cc,ang1,ang2,ang3                    \n")
    file.write(" " + str(hrange1) + " " + str(hrange2) + " 1.0 -a_hmax, a_hmin, a_vert     \n")
    file.close()
    cmd = ['sgsim','sgsim.par']
    subprocess.call(cmd)
    sim_array = GSLIB2ndarray(output_file,0,nx,ny)
    return(sim_array)

# extract regular spaced samples from a model
def regular_sample(array,xmin,xmax,ymin,ymax,step,mx,my,name):
    x = []; y = []; v = []; iix = 0; iiy = 0;
    xx, yy = np.meshgrid(np.arange(xmin, xmax, step),np.arange(ymax, ymin, -1*step))
    iiy = 0
    for iy in range(0,ny):
        if iiy >= my:
            iix = 0
            for ix in range(0,nx):
                if iix >= mx:
                    x.append(xx[iy,ix]);y.append(yy[iy,ix]); v.append(array[iy,ix])
                    iix = 0; iiy = 0
                iix = iix + 1
        iiy = iiy + 1
    df = pd.DataFrame(np.c_[x,y,v],columns=['X', 'Y', name])
    return(df)

# extract random set of samples from a model
def random_sample(array,xmin,xmax,ymin,ymax,step,nsamp,name):
    x = []; y = []; v = []; iix = 0; iiy = 0;
    xx, yy = np.meshgrid(np.arange(xmin, xmax, step),np.arange(ymax-1, ymin-1, -1*step))
    ny = xx.shape[0]
    nx = xx.shape[1]
    sample_index = rand.sample(range((nx)*(ny)), nsamp)
    for isamp in range(0,nsamp):
        iy = int(sample_index[isamp]/ny)
        ix = sample_index[isamp] - iy*nx
        x.append(xx[iy,ix])
        y.append(yy[iy,ix])
        v.append(array[iy,ix])
    df = pd.DataFrame(np.c_[x,y,v],columns=['X', 'Y', name])
    return(df)
def GSLIB_sgsim_2d_cond(df,xcol,ycol,vcol,nreal,xmn,ymn,nx,ny,res,seed,hrange1,hrange2,azi,output_file):
    df_ext = pd.DataFrame({'X':df[xcol],'Y':df[ycol],'Z':df[vcol]})
    Dataframe2GSLIB("sgsim.dat",df_ext)
    #hmn = hsiz * 0.5
    hctab = int(hrange1/res)*2 + 1

    sim_array = np.random.rand(nx,ny)
    #hrange2=0.1
    file = open("sgsim.par", "w")
    file.write("              Parameters for SGSIM                                         \n")
    file.write("              ********************                                         \n")
    file.write("                                                                           \n")
    file.write("START OF PARAMETER:                                                        \n")
    file.write("sgsim.dat                     -file with data                              \n")
    file.write("1  2  0  3  0  0              -  columns for X,Y,Z,vr,wt,sec.var.          \n")
    file.write("-1.0e21 1.0e21                -  trimming limits                           \n")
    file.write("1                             -transform the data (0=no, 1=yes)            \n")
    file.write("sgsim.trn                     -  file for output trans table               \n")
    file.write("0                             -  consider ref. dist (0=no, 1=yes)          \n")
    file.write("none.dat                      -  file with ref. dist distribution          \n")
    file.write("1  0                          -  columns for vr and wt                     \n")
    file.write("-4.0    4.0                   -  zmin,zmax(tail extrapolation)             \n")
    file.write("1      -4.0                   -  lower tail option, parameter              \n")
    file.write("1       4.0                   -  upper tail option, parameter              \n")
    file.write("1                             -debugging level: 0,1,2,3                    \n")
    file.write("sgsim.dbg                     -file for debugging output                   \n")
    file.write(str(output_file) + "           -file for simulation output                  \n")
    file.write(str(nreal) + "                 -number of realizations to generate          \n")
    file.write(str(nx) + " " + str(xmn) + " " + str(res) + "                               \n")
    file.write(str(ny) + " " + str(ymn) + " " + str(res) + "                               \n")
    file.write("1 0.0 1.0                     - nz zmn zsiz                                \n")
    file.write(str(seed) + "                  -random number seed                          \n")
    file.write("0     8                       -min and max original data for sim           \n")
    file.write("12                            -number of simulated nodes to use            \n")
    file.write("0                             -assign data to nodes (0=no, 1=yes)          \n")
    file.write("1     3                       -multiple grid search (0=no, 1=yes),num      \n")
    file.write("0                             -maximum data per octant (0=not used)        \n")
    file.write(str(hrange1) + " " + str(hrange2) + " 1.0 -maximum search  (hmax,hmin,vert) \n")
    file.write(str(azi) + "   0.0   0.0       -angles for search ellipsoid                 \n")
    file.write(str(hctab) + " " + str(hctab) + " 1 -size of covariance lookup table        \n")
    file.write("0     0.60   1.0              -ktype: 0=SK,1=OK,2=LVM,3=EXDR,4=COLC        \n")
    file.write("none.dat                      -  file with LVM, EXDR, or COLC variable     \n")
    file.write("4                             -  column for secondary variable             \n")
    file.write("1    0.0                      -nst, nugget effect                          \n")
    file.write("1    1.0 " + str(azi) + " 0.0 0.0 -it,cc,ang1,ang2,ang3                    \n")
    file.write(" " + str(hrange1) + " " + str(hrange2) + " 1.0 -a_hmax, a_hmin, a_vert     \n")
    file.close()
    cmd = ['sgsim','sgsim.par']
    subprocess.call(cmd)
    sim_array = GSLIB2ndarray(output_file,0,nx,ny)
    return(sim_array)

if __name__ == "__main__":
    main()


import time
import os
from tkinter import LEFT
from traceback import print_tb
from unicodedata import name
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import keras
from keras.models import Model, Sequential
from keras.layers import Input, LSTM, Lambda, Layer
from keras.optimizers import Adam, RMSprop, Adadelta
import keras.backend as K
from scipy.interpolate import UnivariateSpline
from scipy.stats import zscore
import torch

def GetTrainData(TrainDataFile):
    # 时序数据训练集
    TimeSeriesTrainData = []
    # 用户草图训练集
    SkTrainData = []
    # 相似度数据
    SimData = []
    # 用户评分
    df = pd.read_csv(TrainDataFile)
    # global X_train,Y_train,df,max_seq_length
    for rowId, row in df.iterrows():
        # 生成sketch数据
        tmp = row["Sketch"].split(";")
        TempSketch = []
        for i in range(0,len(tmp)):
            # 将str转化为对应的array
            t = tmp[i].replace('(','')
            t = t.replace(')','')
            t = t.strip()
            t = t.strip("\n")
            t = t.strip("\t")
            # 判断是否有空字符 print(t.isspace())  print('tmp:',t)
            tmparr = t.split(',')
            #print('1:',float(tmparr[0]))
            #print('2:',float(tmparr[1]))
            arraytmp = np.array([float(tmparr[0]),float(tmparr[1])])
            #print(arraytmp)
            #print('shepe:',arraytmp.shape)
            #print(np.array(tmp[i]).shape) #()
            #TempSketch.append(float(np.array(tmp[i])))
            TempSketch.append(arraytmp)
        SkTrainData.append(TempSketch)
        #print('SkTrainData Type:', type(SkTrainData))

        # 生成时间数据
        tmp = row["MatchSeries"].split(";")
        TempTimeSeries = []
        for i in range(0,len(tmp)):
            TempTimeSeries.append(float(tmp[i]))
            #TempTimeSeries.append(tmp[i])
        TimeSeriesTrainData.append(TempTimeSeries)
        # print('TimeData Type:', type(TimeSeriesTrainData)) # type= list

        # 取出相似度
        SimData.append(float(row["Sim"]))
    print("SketchTrainData Length: ", len(SkTrainData))
    print("TimeSeriesTrainData Length: ", len(TimeSeriesTrainData))
    print("SimData Length: ", len(SimData))
    return SkTrainData, TimeSeriesTrainData, SimData

def TrainDataPreprocess(SketchTrainData, TimeSeriesTrainData):
    NewSketchData = []
    for i in range(len(SketchTrainData)):
        tmpSketchData = []
        for j in range(1, len(SketchTrainData[i])):
            tmpSketchData.append(SketchTrainData[i][j] - SketchTrainData[i][j-1])
        NewSketchData.append(tmpSketchData)

    NewTimeSeriesData = []
    for i in range(len(TimeSeriesTrainData)):
        tmpTimeSeriesData = []
        for j in range(1, len(TimeSeriesTrainData[i])):
            tmpTimeSeriesData.append(TimeSeriesTrainData[i][j] - TimeSeriesTrainData[i][j-1])
        NewTimeSeriesData.append(tmpTimeSeriesData)
    # 测试
    return NewSketchData, NewTimeSeriesData

def TrainDataPreprocessZScore(SketchTrainData, TimeSeriesTrainData):
    NewSketchData = []
    for i in range(len(SketchTrainData)):
        NewSketchData.append(zscore(SketchTrainData[i]))
    NewTimeSeriesData = []
    for i in range(len(TimeSeriesTrainData)):
        NewTimeSeriesData.append(zscore(TimeSeriesTrainData[i]))
    # 测试
    return NewSketchData, NewTimeSeriesData

def exponent_neg_manhattan_distance(left, right):
    '''Helper function for the similarity estimate of the LSTMs outputs'''
    return K.exp(-K.sum(K.abs(left-right), axis=-1, keepdims=True))

def euclidean_distance(left, right):
    return K.sqrt(K.sum(K.square(left - right), axis=-1, keepdims=True))

def euclidean_distance_output_shape(shape):
    shape1, shape2 = shape
    return (shape1[0] , 1)

class SaveWeight(keras.callbacks.Callback):
    def on_epoch_end(self, epoch, logs=None):
        if epoch % 50 == 0:
            self.model.save("./config/FINALLSTMV3-{}-.h5".format(epoch))

# 创建contrastive_loss
def contrastive_loss(y_true, y_pred):
    margin = 1
    square_pred = K.square(y_pred)
    margin_square = K.square(K.maximum(margin - y_pred, 0))

    return K.mean(y_true * square_pred + (1 - y_true) * margin_square)

def TrainNetwork(SketchData, TimeSeriesData, SimData, FileName):
    #batch_size = len(SketchData)
    batch_size = 32
    hunits = 16 # 单元内部隐藏层的大小
    n_epoch = 10 # epoch个数
    gradient_clipping_norm = 1.05
    # keras.optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)是keras的一种优化器，以在训练初期加快训练
    # amsgrad: 布尔型，是否使用AMSGrad变体
    adam = Adam(learning_rate=0.001)

    shared_model = LSTM(hunits)

    left_input = Input(shape=(len(TimeSeriesData[0]),2), dtype='float')
    #right_input = Input(shape=(len(SketchData[0]),1), dtype='float')
    right_input = Input(shape=(len(SketchData[0]),2), dtype='float')
    left_output = shared_model(left_input)
    right_output = shared_model(right_input)
    malstm_distance = Lambda(function=lambda x: euclidean_distance(x[0], x[1]), output_shape=lambda x: (x[0][0], 1))([left_output, right_output])
    # 定义输入和输出
    # model = Model(inputs=[left_input, right_input], outputs=[malstm_distance])
    model = Model(inputs=[left_input, right_input], outputs=malstm_distance)
    # Adadelta optimizer, with gradient clipping by norm

    optimizer = Adadelta(clipnorm=gradient_clipping_norm)
    # optimizer = RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)
    # 评价函数用于评估当前训练模型的性能，当模型编译后（compile），评价函数应该作为metrics的参数来输入。
    # 评价函数和损失函数相似，只不过评价函数的结果不会用于训练过程中。我们可以传递已有的评价函数名称，或者传递一个自定义的 Theano/TensorFlow 函数来使用。
    # optimizer = Adadelta(clipnorm=gradient_clipping_norm)
    # rms = RMSprop(lr=0.001, rho=0.9, epsilon=None, decay=0.0)
    # model.compile(loss='mean_squared_error', optimizer=adam, metrics=['accuracy']) # 损失函数为均方误差
    rms = RMSprop()
    model.compile(loss=contrastive_loss, optimizer=rms, metrics=['accuracy'])
    training_start_time = time.time()
    # malstm_trained = model.fit([np.array(TimeSeriesData).reshape(-1, len(TimeSeriesData[0]), 1), np.array(SketchData).reshape(-1, len(SketchData[0]), 1)], np.array(SimData), batch_size=batch_size, epochs=n_epoch,validation_split=0.3, callbacks=[MySaver])
    TimeUse = torch.tensor(np.array(TimeSeriesData).reshape(-1, len(TimeSeriesData[0]), 1))
    TimeUse = TimeUse.expand(14100,len(TimeSeriesData[0]),2)
    TimeUse = TimeUse.numpy()
    malstm_trained = model.fit([TimeUse, np.array(SketchData).reshape(-1,len(SketchTrainData[0]),2)], np.array(SimData), batch_size=batch_size, epochs=n_epoch,validation_split=0.2)

    training_end_time = time.time()

    print("Training time finished.\n%d epochs in %12.2f" % (n_epoch, training_end_time - training_start_time))
    model.save('./config/' + FileName)

    # plot accuracy
    plt.plot(malstm_trained.history['loss'])
    plt.plot(malstm_trained.history['val_loss'])
    plt.title('model train vs validation loss')
    plt.ylabel('loss')
    plt.xlabel('epoch')
    plt.legend(['train', 'validation'], loc='upper right')
    plt.show()

    print(str(malstm_trained.history['val_accuracy'][-1])[:6] +
          "(max: " + str(max(malstm_trained.history['val_accuracy']))[:6] + ")")
    print("Done.")

path = "../TrainData1028" # 训练数据文件夹
files = os.listdir(path)
# 时序数据训练集
TimeTrainData = []
# 用户草图训练集
SketchTrainData = []
# 相似度数据
SimTrainData = []
for file in files:
    position = path + '\\' + file
    TmpSketch = []
    TmpTime = []
    TmpSim = []
    TmpSketch, TmpTime,TmpSim = GetTrainData(position)
    SketchTrainData.append(TmpSketch)
    TimeTrainData.append(TmpTime)
    SimTrainData.append(TmpSim)
# SketchTrainData = all_length_dataset
H5FileName = "FinalLSTMV1.h5"
#TrainNetwork(SketchTrainData, TimeTrainData, SimTrainData, H5FileName)
print(len(SimTrainData))

TrainDataFile = "../TrainDataSet/TrainDada.csv"
SketchTrainData, TimeSeriesTrainData, SimData = GetTrainData(TrainDataFile)
H5FileName = "FinalLSTM_V3.h5"
TrainNetwork(SketchTrainData, TimeSeriesTrainData, SimData, H5FileName)


import bs4
import urllib.request as req

url = "https://www.ptt.cc/bbs/movie/index.html"
request = req.Request(url, headers={
    "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36"
})

with req.urlopen(request) as response:
    data = response.read().decode("utf-8")

root = bs4.BeautifulSoup(data, "html.parser")

def movie():
    global root
    result = ""
    page_count = 0

    while page_count < 3:
        titles = root.find_all("div", class_="title")
        tweets = root.find_all("div", class_="nrec")
        for title, tweet in zip(titles, tweets):

            if title.a is not None:
                title_text = title.a.string
                title_link = title.a["href"]
                title_url = "https://www.ptt.cc" + title_link
                title_request = req.Request(title_url, headers={
                    "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36"
                })

                with req.urlopen(title_request) as title_response:
                    title_data = title_response.read().decode("utf-8")

                title_root = bs4.BeautifulSoup(title_data, "html.parser")
                date_element = title_root.find(
                    "span", class_="article-meta-tag", string="時間")

                if date_element is not None:
                    date = date_element.find_next_sibling("span").string
                else:
                    date = "Not found"

                tweet_num = tweet.string

                if tweet_num is not None:
                    tweet_num = tweet.string
                else:
                    tweet_num = "0"

                result += f"{title_text},{tweet_num},{date}\n"
                print(result)

        pre_link_element = root.find("a", class_="btn wide", string="‹ 上頁")

        if pre_link_element is not None:
            pre_link = pre_link_element["href"]
            pre_url = "https://www.ptt.cc" + pre_link
            pre_request = req.Request(pre_url, headers={
                "User-Agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Mobile Safari/537.36"
            })

            with req.urlopen(pre_request) as pre_response:
                pre_data = pre_response.read().decode("utf-8")
                root = bs4.BeautifulSoup(pre_data, "html.parser")
                page_count += 1
        else:
            print("Can't find the link to the previous page")
            break

    with open("movie.txt", "w", encoding="utf-8") as file:
        file.write(result)

movie()


import numpy as numpy
import matplotlib.pyplot as plt
import matplotlib
import math
import scipy
def vcol(vett):
    return vett.reshape(vett.size, 1)

def vrow(vett):
    return vett.reshape(1, vett.size)

def empirical_mean(D):
    return vcol(D.mean(1))

def empirical_covariance(D, mu):
    n = numpy.shape(D)[1]
    DC = D - vcol(mu)
    C = 1 / n * numpy.dot(DC, numpy.transpose(DC))
    return C
def znorm(DTR, DTE):
    mu_DTR = vcol(DTR.mean(1))
    std_DTR = vcol(DTR.std(1))

    DTR_z = (DTR - mu_DTR) / std_DTR
    DTE_z = (DTE - mu_DTR) / std_DTR
    return DTR_z, DTE_z

def gaussianize_features(DTR, TO_GAUSS):
    P = []
    for dIdx in range(DTR.shape[0]):
        DT = vcol(TO_GAUSS[dIdx, :])
        X = DTR[dIdx, :] < DT
        R = (X.sum(1) + 1) / (DTR.shape[1] + 2)
        P.append(scipy.stats.norm.ppf(R))
    return numpy.vstack(P)

def shuffle_dataset(D, L):
    numpy.random.seed(0)
    idx = numpy.random.permutation(D.shape[1])
    return D[:, idx], L[idx]
def load(name):
    try:
        file = open(name, "r")
    except FileNotFoundError:
        exit(-1)

    Dlist = []
    listLabel = []
    for row in file:
        line = row.rstrip().split(",")
        singleLine = line[0:10]
        label = line[-1]
        Dlist.append(singleLine)
        listLabel.append(label)

    numpyArr = numpy.array(Dlist, dtype=float)
    labelpy = numpy.array(listLabel, dtype=int)
    D, L = shuffle_dataset(numpyArr.T, labelpy)
    return D.T, L

def bayes_error_plot(pArray, scores, labels, minCost=False, th=None):
    y = []
    for p in pArray:
        pi = 1.0 / (1.0 + numpy.exp(-p))
        if minCost:
            y.append(compute_min_DCF(scores, labels, pi, 1, 1))
        else:
            y.append(compute_act_DCF(scores, labels, pi, 1, 1, th))
    return numpy.array(y)

def bayes_error_min_act_plot(D, LTE,  ylim):
    p = numpy.linspace(-3, 3, 21)
    plt.plot(p, bayes_error_plot(p, D, LTE, minCost=False), color='r', label='actDCF')
    plt.plot(p, bayes_error_plot(p, D, LTE, minCost=True), color='b', label='minDCF')
    plt.ylim(0, ylim)
    plt.ylabel('DCF')
    plt.legend(loc = 'upper left')
    plt.show()

def bayes_error_min_act_plot_compare(mvg, qlog,svm,gmm, mvg_l, qlog_l,svm_l, gmm_l, ylim):
    p = numpy.linspace(-3, 3, 21)
    plt.plot(p, bayes_error_plot(p, mvg, mvg_l, minCost=False), color='r', label='MVG Full - actDCF' )
    plt.plot(p, bayes_error_plot(p, mvg, mvg_l, minCost=True), color='r', label='MVG Full - minDCF', linestyle='dashed')
    plt.plot(p, bayes_error_plot(p, qlog, qlog_l, minCost=False), color='b', label='Q-Log-Reg - actDCF')
    plt.plot(p, bayes_error_plot(p, qlog, qlog_l, minCost=True), color='b', label='Q-Log-Reg - minDCF', linestyle='dashed')
    plt.plot(p, bayes_error_plot(p, svm, svm_l, minCost=False), color='y', label='SVM RBF - actDCF')
    plt.plot(p, bayes_error_plot(p, svm, svm_l, minCost=True), color='y', label='SVM RBF - minDCF', linestyle='dashed')
    plt.plot(p, bayes_error_plot(p, gmm, gmm_l, minCost=False), color='c', label='GMM - actDCF')
    plt.plot(p, bayes_error_plot(p, gmm, gmm_l, minCost=True), color='c', label='GMM - minDCF', linestyle='dashed')

    plt.ylim(0, ylim)
    plt.ylabel('DCF')
    plt.legend(loc = 'upper left')
    plt.show()

def compute_correlation(X, Y):
    x_sum = numpy.sum(X)
    y_sum = numpy.sum(Y)

    x2_sum = numpy.sum(X ** 2)
    y2_sum = numpy.sum(Y ** 2)

    sum_cross_prod = numpy.sum(X * Y.T)

    n = X.shape[0]
    numerator = n * sum_cross_prod - x_sum * y_sum
    denominator = numpy.sqrt((n * x2_sum - x_sum ** 2) * (n * y2_sum - y_sum ** 2))

    corr = numerator / denominator
    return corr

def computeCovDiag(M, muc):

    cov = numpy.dot((M-muc),(M-muc).T)/M.shape[1]
    diagCov = cov*numpy.identity(cov.shape[0])
    return (diagCov)

def confusion_matrix(Lpred, LTE, k=2):
    # k = number of classes
    conf = numpy.zeros((k, k))
    for i in range(k):
        for j in range(k):
            conf[i][j] = ((Lpred == i) * (LTE == j)).sum()
    return conf

def assign_labels(scores, pi, Cfn, Cfp, th=None):
    if th is None:
        th = -numpy.log(pi * Cfn) + numpy.log((1 - pi) * Cfp)
    P = scores > th
    return numpy.int32(P)

def confusion_matrix_binary(Lpred, LTE):
    C = numpy.zeros((2, 2))
    C[0, 0] = ((Lpred == 0) * (LTE == 0)).sum()
    C[0, 1] = ((Lpred == 0) * (LTE == 1)).sum()
    C[1, 0] = ((Lpred == 1) * (LTE == 0)).sum()
    C[1, 1] = ((Lpred == 1) * (LTE == 1)).sum()
    return C

def compute_emp_Bayes_binary(CM, pi, Cfn, Cfp):
    fnr = CM[0, 1] / (CM[0, 1] + CM[1, 1])
    fpr = CM[1, 0] / (CM[0, 0] + CM[1, 0])
    return pi * Cfn * fnr + (1 - pi) * Cfp * fpr

def compute_normalized_emp_Bayes(CM, pi, Cfn, Cfp):
    empBayes = compute_emp_Bayes_binary(CM, pi, Cfn, Cfp)
    return empBayes / min(pi * Cfn, (1 - pi) * Cfp)

def compute_act_DCF(scores, labels, pi, Cfn, Cfp, th=None):
    Pred = assign_labels(scores, pi, Cfn, Cfp, th=th)
    CM = confusion_matrix_binary(Pred, labels)
    return compute_normalized_emp_Bayes(CM, pi, Cfn, Cfp)

def compute_min_DCF(scores, labels, pi, Cfn, Cfp):
    t = numpy.array(scores)
    t.sort()
    dcfList = []
    for _th in t:
        dcfList.append(compute_act_DCF(scores, labels, pi, Cfn, Cfp, th=_th))
    return min(dcfList)


#!/usr/bin/env python3
"""remtool.py

Usage:
  remtool.py ls [PATH]
  remtool.py put [-f] [-c | --clear] FILE [FOLDER]
  remtool.py show PATH
  remtool.py (-h | --help)
  remtool.py --version

Options:
  -f            Force overwrite if file already exists
  -c --clear    If forcing overwrite, clear annotations on files
  -h --help     Show this screen
  --version     Show version
"""
import configparser
import json
import os
import shutil
import subprocess
import sys
import tempfile
import uuid
from collections import deque
from dataclasses import dataclass
from docopt_ng import docopt
from glob import glob
from pathlib import Path, PurePath
from time import time

import pprint
pp_ = pprint.PrettyPrinter(indent=2)
pp = pp_.pprint

SCRIPTPATH = os.path.dirname(os.path.realpath(__file__))

# read config file
config = configparser.ConfigParser()
config.read(f"{SCRIPTPATH}/remtool.cfg")
CONFIG = {}
CONFIG['SSH_HOSTNAME'] = config['main']['reMarkableHostname']

# color codes for pretty output
colors = {
    'BLUE': '\033[0;34m',
    'BOLDBLUE': '\033[1;34m',
    'CYAN': '\033[0;36m',
    'BOLDCYAN': '\033[1;36m',
    'GREEN': '\033[0;32m',
    'BOLDGREEN': '\033[1;32m',
    'BOLDYELLOW': '\033[1;33m',
    'RESET': '\033[0m',
}

def colored(c, text):
    return colors[c] + text + colors['RESET']

class reMarkable:
    def __init__(self, hostname: str):
        self.hostname = hostname

        # get content tree and metadata from reMarkable
        self.ct = ContentTree(self._get_metadata())

    def put(self, file: str, folder: str='', force_overwrite: bool=False,
            clear_annotations: bool=False):
        # assemble params into a target path
        filename = Path(file)
        folderpath = Path(folder)
        path = folderpath / filename.stem

        # check if destination folder exists
        parent = self.ct.get_node_by_path(folder)
        if parent is None:
            print(f"Folder '{folder}' does not exist.")
            return

        # check source for valid file type
        source_type = filename.suffix.lstrip('.').lower()
        if source_type not in ['pdf', 'epub']:
            # this is a simple file extension check that could be improved
            # in the future
            raise RuntimeError('Filetype not supported. '
                               '(Valid filetypes: pdf, epub)')

        print(f"{colored('CYAN', 'Copying')} "
              f"{colored('BOLDCYAN', str(filename))}"
              f"{colored('CYAN', '...')}")

        # check if file already exists at this target path
        target = self.ct.get_node_by_path(path)
        if target is None:
            # generate minimum required metadata
            now = str(int(time()*1000))
            new_meta = Metadata(deleted=False,
                                last_modified=now,
                                last_opened=now,
                                last_opened_page=0,
                                metadata_modified=False,
                                modified=False,
                                parent=parent.uuid,
                                pinned=False,
                                synced=False,
                                type_='DocumentType',
                                version=0,
                                visible_name=filename.stem)
            new_node = Node(uuid=uuid.uuid4(), metadata=new_meta,
                            filetype=source_type)
            parent.add_child(new_node)

            # render node as files on disk and send them to the device
            with tempfile.TemporaryDirectory(prefix='remtool_') as tempdir:
                new_node.render_to_disk(tempdir)
                shutil.copy(filename,
                            f"{tempdir}/{new_node.uuid}{filename.suffix}")
                rendered = glob(f"{tempdir}/*")
                self._scp(rendered, '.local/share/remarkable/xochitl')
                self._ssh('systemctl restart xochitl')
            print(colored('GREEN', 'Success.'))
        else:
            print(colored('BOLDYELLOW',
                          f"{target.nice_type().capitalize()} already "
                          "exists at"), path)

            # prompt user to confirm overwrite if -f isn't set
            if not force_overwrite:
                res = input(colored('BOLDYELLOW',
                                    "Overwrite? [y/N]: ")).lower()
                if res != 'y':
                    print('Canceled.')
                    return
            else:
                print(colored('BOLDYELLOW', 'Forcing overwrite...'))

            # copy file over existing one on device
            with tempfile.TemporaryDirectory(prefix='remtool_') as tempdir:
                # if this is an epub, delete pre-rendered pdf on device first
                if target.filetype == 'epub':
                    self._ssh(f"rm .local/share/remarkable/xochitl/"
                              f"{target.uuid}.pdf")
                # delete thumbnails
                self._ssh(f"rm .local/share/remarkable/xochitl/"
                          f"{target.uuid}.thumbnails/*")
                # delete annotation files, if option is set
                if clear_annotations:
                    print(colored('BOLDYELLOW', 'Clearing annotations...'))
                    self._ssh(f"rm .local/share/remarkable/xochitl/"
                              f"{target.uuid}/*")
                # now we can do the actual file copy
                shutil.copy(filename,
                            f"{tempdir}/{target.uuid}.{target.filetype}")
                rendered = glob(f"{tempdir}/*")
                self._scp(rendered, '.local/share/remarkable/xochitl')
                self._ssh('systemctl restart xochitl')
            print(colored('GREEN', 'Success.'))

    def show(self, path: str):
        target = self.ct.get_node_by_path(path)
        if target is None:
            print(f"Path '{path}' not found.")
            return

        print('path:', target.path)
        print('uuid:', target.uuid)
        if not target.is_folder():
            print('filetype:', target.filetype)
        print()
        print('metadata:')
        print(target.metadata)

    def ls(self, path: str=None):
        if path is None:
            path = ''

        target = self.ct.get_node_by_path(path)
        if target is None:
            print(f"Path '{path}' not found.")
            return

        if not target.is_folder():
            print(f"Path '{path}' is not a folder.")
            return

        for child in target.children:
            print(child.path)

    def _ssh(self, cmd: str, pipe_in: bool=False):
        args = ['ssh', f"root@{self.hostname}"]
        if pipe_in:
            args.insert(1, '-T')  # disable pseudo-terminal allocation
        else:
            args.append(cmd)

        if pipe_in:
            sp = subprocess.Popen(args,
                                  stdin=subprocess.PIPE,
                                  stdout=subprocess.PIPE,
                                  stderr=subprocess.PIPE,
                                  encoding='utf-8')
            result, err = sp.communicate(cmd)
            if sp.returncode > 0:
                print(f"ERROR: {err.strip()}")
                sys.exit(1)
        else:
            try:
                out = subprocess.run(args, capture_output=True, check=True,
                                     encoding='utf-8')
                result = out.stdout
            except subprocess.CalledProcessError as e:
                print(f"ERROR: {e}")
                sys.exit(1)
        return result

    def _scp(self, source: list, dest: str):
        args = ['scp', '-r']
        args.extend(source)
        args.append(f"root@{self.hostname}:{dest}")

        try:
            subprocess.run(args, capture_output=True, check=True,
                           encoding='utf-8')
        except subprocess.CalledProcessError as e:
            print(f"ERROR: {e}")
            sys.exit(1)
        return True

    def _get_metadata(self):
        cmd = '''
shopt -s nullglob
metafiles=(.local/share/remarkable/xochitl/*.metadata)
numfiles=${#metafiles[@]}

filecount=1
echo '['
for file in ${metafiles[@]}
do
    uuid="${file%.*}"
    tmp=`grep fileType $uuid.content`
    if [[ $tmp =~ :\ \\"(.+)\\", ]]; then
        filetype=${BASH_REMATCH[1]}
    else
        filetype=""
    fi
    echo '{"filename": "'$file'", "metadata": '
    cat $file
    echo ', "filetype": "'$filetype'"}'
    if [[ $filecount -lt $numfiles ]]; then
        echo ','
    fi
    filecount=$(($filecount + 1))
done
echo ']'
        '''.strip()
        raw_meta = self._ssh(cmd, pipe_in=True)
        json_meta = json.loads(raw_meta)
        return json_meta

@dataclass
class Metadata:
    deleted: bool
    last_modified: str
    last_opened: str
    last_opened_page: int
    metadata_modified: bool
    modified: bool
    parent: str
    pinned: bool
    synced: bool
    type_: str
    version: int
    visible_name: str

    def as_dict(self):
        out = {}
        for f_ in Metadata.__dataclass_fields__:
            if f_ == 'type_':
                field = 'type'
            elif f_ == 'visible_name':
                field = 'visibleName'
            elif f_ == 'metadata_modified':
                field = 'metadatamodified'
            elif f_ == 'last_modified':
                field = 'lastModified'
            elif f_ == 'last_opened':
                field = 'lastOpened'
            elif f_ == 'last_opened_page':
                field = 'lastOpenedPage'
            else:
                field = f_
            out[field] = getattr(self, f_)
        return out

    def __str__(self):
        """ Convert metadata into a human-readable string """
        width = max([len(fname) for fname in Metadata.__dataclass_fields__])+1
        out = ''
        for f_ in Metadata.__dataclass_fields__:
            if f_ == 'type_':
                field = 'type'
            else:
                field = f_
            out += f"{field.ljust(width)}: {getattr(self, f_)}\n"
        return out.strip()

class Node:
    def __init__(self, uuid: str, metadata: Metadata=None, filetype: str=None,
                 children: list=None):
        self.uuid = uuid
        self.metadata = metadata
        self.filetype = filetype
        self.path = ''
        self.content = {}
        if children is None:
            self.children = []

    def add_child(self, node):
        if self.metadata is None:
            if node.is_folder():
                node.path = f"{node.metadata.visible_name}/"
            else:
                node.path = f"{node.metadata.visible_name}"
        else:
            node.path = f"{self.path}{node.metadata.visible_name}"
        self.children.append(node)

    def is_folder(self):
        if self.metadata is None:
            return True
        else:
            return self.metadata.type_ == 'CollectionType'

    def nice_type(self):
        if self.metadata is None:
            return 'root folder'
        elif self.metadata.type_ == 'DocumentType':
            return 'file'
        elif self.metadata.type_ == 'CollectionType':
            return 'folder'

    def render_to_disk(self, tempdir):
        filestem = f"{tempdir}/{self.uuid}"
        metafile = f"{filestem}.metadata"
        contentfile = f"{filestem}.content"

        with open(f"{metafile}", 'w') as METAFILE:
            json.dump(self.metadata.as_dict(), METAFILE, indent=4)

        if not self.content:
            content = self._default_content()
        with open(f"{contentfile}", 'w') as CONTENTFILE:
            json.dump(content, CONTENTFILE, indent=4)

        if not self.is_folder():
            os.makedirs(f"{filestem}")
            os.makedirs(f"{filestem}.thumbnails")

    def _default_content(self):
        if self.is_folder():
            return {}

        content = {}
        if self.filetype == 'pdf':
            content = {
                'extraMetadata': {
                    'LastFinelinerv2Color': 'Black',
                    'LastFinelinerv2Size': '2',
                    'LastPen': 'Finelinerv2',
                    'LastTool': 'Finelinerv2'
                },
                'zoomMode': 'fitToHeight'
            }
        elif self.filetype == 'epub':
            content = {
                'extraMetadata': {
                    'LastPen': 'Highlighterv2',
                    'LastTool': 'Highlighterv2',
                    'LastHighlighterv2Color': 'HighlighterYellow'
                },
                'coverPageNumber': 0,
                'fontName': 'Noto Serif',
                'lineHeight': 100,
                'margins': 125,
                'textAlignment': 'justify',
                'textScale': 1
            }
        return content

    def __repr__(self):
        if self.uuid == '':
            uuid = 'root'
        else:
            uuid = self.uuid
        out = f"Node(path={self.path} uuid={uuid} filetype={self.filetype}) ["
        if self.children:
            out += '\n'
        for child in self.children:
            out += f"  {child.path}\n"
        out += ']'
        return out

class ContentTree:
    def __init__(self, metadata: str):
        self.tree = Node(uuid='')
        self._build_tree(metadata)

    def get_node_by_uuid(self, uuid: str, root_node: Node=None):
        if root_node is None:
            root_node = self.tree

        if root_node.uuid == uuid:
            return root_node

        for node in root_node.children:
            result = self.get_node_by_uuid(uuid, node)
            if result is not None:
                return result

        return None

    def get_node_by_path(self, path: str, root_node: Node=None):
        if root_node is None:
            root_node = self.tree

        root_path = PurePath(root_node.path)
        ppath = PurePath(path)
        if str(root_path) == str(ppath):
            return root_node

        for node in root_node.children:
            result = self.get_node_by_path(path, node)
            if result is not None:
                return result

        return None

    def _build_tree(self, metadata):
        item_queue = deque(metadata)

        while item_queue:
            item = item_queue.popleft()

            if 'deleted' not in item['metadata']:
                item['metadata']['deleted'] = False

            if (item['metadata']['deleted']
                    or item['metadata']['parent'] == 'trash'):
                continue

            uuid = Path(item['filename']).stem

            # check for missing metadata keys
            # certain versions (3.2) now omit certain metadata keys when
            # new notebooks are created
            if 'lastOpened' not in item['metadata']:
                item['metadata']['lastOpened'] = ''
            if 'lastOpenedPage' not in item['metadata']:
                item['metadata']['lastOpenedPage'] = 0
            if 'modified' not in item['metadata']:
                item['metadata']['modified'] = False
            if 'metadatamodified' not in item['metadata']:
                item['metadata']['metadatamodified'] = False
            if 'synced' not in item['metadata']:
                item['metadata']['synced'] = False
            if 'version' not in item['metadata']:
                item['metadata']['version'] = 0

            meta = Metadata(item['metadata']['deleted'],
                            item['metadata']['lastModified'],
                            item['metadata']['lastOpened'],
                            item['metadata']['lastOpenedPage'],
                            item['metadata']['metadatamodified'],
                            item['metadata']['modified'],
                            item['metadata']['parent'],
                            item['metadata']['pinned'],
                            item['metadata']['synced'],
                            item['metadata']['type'],
                            item['metadata']['version'],
                            item['metadata']['visibleName'])

            if item['metadata']['parent'] == '':
                # item has no parent, i.e. it's on the root level
                self.tree.add_child(Node(uuid=uuid, metadata=meta,
                                         filetype=item['filetype']))
            else:
                parent = self.get_node_by_uuid(item['metadata']['parent'])
                if parent is None:
                    # item _should_ have a parent, but we haven't seen it yet
                    # so, send the item back to the end of the queue
                    item_queue.append(item)
                else:
                    # item has a parent that we know about, so add it
                    parent.add_child(Node(uuid=uuid, metadata=meta,
                                          filetype=item['filetype']))
        return

if __name__ == "__main__":
    args = docopt(__doc__, default_help=True, version='remtool 0.2')

    reM = reMarkable(CONFIG['SSH_HOSTNAME'])

    if args['put']:
        reM.put(args['FILE'], args['FOLDER'], args['-f'], args['--clear'])
    elif args['ls']:
        reM.ls(args['PATH'])
    elif args['show']:
        reM.show(args['PATH'])


from pathlib import (
    Path,
)
from typing import (
    IO,
    Dict,
    Iterable,
    Optional,
    Tuple,
    Union,
    cast,
)

import numpy

from wasm._utils.decorators import (
    to_tuple,
)
from wasm.datatypes import (
    DataSegment,
    ElementSegment,
    FunctionAddress,
    FunctionInstance,
    GlobalAddress,
    HostFunction,
    Import,
    MemoryAddress,
    Module,
    ModuleInstance,
    Store,
    TableAddress,
)
from wasm.exceptions import (
    InvalidModule,
    MalformedModule,
    ParseError,
    Unlinkable,
    ValidationError,
)
from wasm.parsers import (
    parse_module,
)
from wasm.typing import (
    TValue,
)
from wasm.validation import (
    validate_external_type_match,
    validate_module,
)

from .configuration import (
    Configuration,
)
from .instructions import (
    InstructionSequence,
)
from .stack import (
    Frame,
)

TAddress = Union[FunctionAddress, TableAddress, MemoryAddress, GlobalAddress]

@to_tuple
def _get_import_addresses(runtime: 'Runtime',
                          imports: Tuple[Import, ...]) -> Iterable[TAddress]:
    """
    Helper function to normalize the descriptors from a set of Imports to their
    addresses.
    """
    for import_ in imports:
        if not runtime.has_module(import_.module_name):
            raise Unlinkable(f"Runtime has no known module named '{import_.module_name}'")
        module = runtime.get_module(import_.module_name)
        for export in module.exports:
            if export.name == import_.as_name:
                yield export.value
                break
        else:
            raise Unlinkable(
                f"No export found with name '{import_.as_name}'"
            )

@to_tuple
def _initialize_globals(store: Store,
                        module: Module,
                        globals_addresses: Tuple[GlobalAddress, ...],
                        ) -> Iterable[TValue]:
    """
    Helper function for running the initialization code for the Globals
    to compute their initial values.
    """
    module_instances = ModuleInstance(
        types=(),
        func_addrs=(),
        memory_addrs=(),
        table_addrs=(),
        global_addrs=globals_addresses,
        exports=(),
    )

    for global_ in module.globals:
        config = Configuration(store=store)
        frame = Frame(
            module=module_instances,
            locals=[],
            instructions=InstructionSequence(global_.init),
            arity=1,
        )
        config.push_frame(frame)
        result = config.execute()
        if len(result) != 1:
            raise Exception("Invariant: globals initialization returned empty result")
        yield result[0]

@to_tuple
def _compute_table_offsets(store: Store,
                           elements: Tuple[ElementSegment, ...],
                           module_instance: ModuleInstance) -> Iterable[numpy.uint32]:
    """
    Helper function for running the initialization code for the ElementSegment
    objects to compute the table indices
    """
    for element_segment in elements:
        frame = Frame(
            module=module_instance,
            locals=[],
            instructions=InstructionSequence(element_segment.offset),
            arity=1,
        )
        config = Configuration(store=store)
        config.push_frame(frame)
        result = config.execute()
        if len(result) != 1:
            raise Exception("Invariant: element segment offset returned empty result")
        offset = numpy.uint32(cast(int, result[0]))

        table_address = module_instance.table_addrs[element_segment.table_idx]
        table_instance = store.tables[table_address]

        if offset + len(element_segment.init) > len(table_instance.elem):
            raise Unlinkable(
                f"Computed element segment offset exceeds table size: {offset} "
                f"+ {len(element_segment.init)} > {len(table_instance.elem)}"
            )
        yield offset

@to_tuple
def _compute_data_offsets(store: Store,
                          datas: Tuple[DataSegment, ...],
                          module_instance: ModuleInstance) -> Iterable[numpy.uint32]:
    """
    Helper function for running the initialization code for the DataSegment
    objects to compute the memory offsets.
    """
    for data_segment in datas:
        frame = Frame(
            module=module_instance,
            locals=[],
            instructions=InstructionSequence(data_segment.offset),
            arity=1,
        )
        config = Configuration(store=store)
        config.push_frame(frame)
        result = config.execute()
        if len(result) != 1:
            raise Exception("Invariant: data segment offset returned empty result")
        offset = numpy.uint32(cast(int, result[0]))

        memory_address = module_instance.memory_addrs[data_segment.memory_idx]
        memory_instance = store.mems[memory_address]

        if offset + len(data_segment.init) > len(memory_instance.data):
            raise Unlinkable(
                f"Computed data segment offset exceeds memory size: {offset} "
                f"+ {len(data_segment.init)} > {len(memory_instance.data)}"
            )
        yield offset

class Runtime:
    """
    Represents the *runtime* for the py-wasm Web Assembly interpreter.
    """
    store: Store
    modules: Dict[str, ModuleInstance]

    def __init__(self):
        self.store = Store()
        self.modules = {}

    def register_module(self, name: str, module: ModuleInstance) -> None:
        """
        Register a module under the provided name in this runtime.  This makes
        it available for other modules to import.
        """
        self.modules[name] = module

    def has_module(self, name: str) -> bool:
        """
        Return boolean whether a module is present in this runtime.
        """
        return name in self.modules

    def get_module(self, name: str) -> ModuleInstance:
        """
        Return the module instance registered for the provided name from this
        runtime.
        """
        return self.modules[name]

    def _get_import_addresses(self, imports: Tuple[Import, ...]) -> Tuple[TAddress, ...]:
        return _get_import_addresses(self, imports)

    def load_module(self, file_path: Path) -> Module:
        """
        Load a Web Assembly module from its binary source file.
        """
        if file_path.suffix != ".wasm":
            raise Exception("Unsupported file type: {file_path.suffix}")

        with file_path.open("rb") as wasm_file:
            try:
                module = parse_module(wasm_file)
            except ParseError as err:
                raise MalformedModule from err

        try:
            validate_module(module)
        except ValidationError as err:
            raise InvalidModule from err

        return module

    def load_buffer(self, buffer: IO) -> Module:
        """
        Load a Web Assembly module from its binary source file.
        """
        try:
            module = parse_module(buffer)
        except ParseError as err:
            raise MalformedModule from err

        try:
            validate_module(module)
        except ValidationError as err:
            raise InvalidModule from err

        return module

    def instantiate_module(self,
                           module: Module,
                           ) -> Tuple[ModuleInstance, Optional[Tuple[TValue, ...]]]:
        """
        Instantiate a Web Assembly module into this runtime environment.
        """
        # Ensure the module is valid
        try:
            module_import_types, module_export_types = validate_module(module)
        except ValidationError as err:
            raise InvalidModule from err

        # Gather all of the addresses for the external types for this module.
        all_import_addresses = self._get_import_addresses(module.imports)

        # Validate all of the addresses are known to the store.
        for address in all_import_addresses:
            try:
                self.store.validate_address(address)
            except ValidationError as err:
                raise InvalidModule from err

        # Gather the types for each of the values referenced by the addresses
        # of the module imports.
        store_import_types = tuple(
            self.store.get_type_for_address(address)
            for address in all_import_addresses
        )
        if len(module_import_types) != len(store_import_types):
            # TODO: This validation may be superfluous as both of these values
            # are generated from `module.imports` and the generation process
            # **should** give strong guarantees that the resulting list is the
            # same length as `module.imports`.
            raise Unlinkable(
                f"Mismatched number of import types: {len(module_import_types)} != "
                f"{len(store_import_types)}"
            )

        # Ensure that the module's internal types for it's imports match the
        # types found in the store.
        for module_type, store_type in zip(module_import_types, store_import_types):
            try:
                validate_external_type_match(store_type, module_type)
            except ValidationError as err:
                raise Unlinkable from err

        global_addresses = GlobalAddress.filter(all_import_addresses)
        global_values = _initialize_globals(self.store, module, global_addresses)

        module_instance = self.store.allocate_module(module, all_import_addresses, global_values)

        element_segment_offsets = _compute_table_offsets(
            self.store,
            module.elem,
            module_instance,
        )
        data_segment_offsets = _compute_data_offsets(
            self.store,
            module.data,
            module_instance,
        )

        for offset, element_segment in zip(element_segment_offsets, module.elem):
            for idx, function_idx in enumerate(element_segment.init):
                function_address = module_instance.func_addrs[function_idx]
                table_address = module_instance.table_addrs[element_segment.table_idx]
                table_instance = self.store.tables[table_address]
                table_instance.elem[offset + idx] = function_address

        for offset, data_segment in zip(data_segment_offsets, module.data):
            memory_address = module_instance.memory_addrs[data_segment.memory_idx]
            memory_instance = self.store.mems[memory_address]
            data_length = len(data_segment.init)
            memory_instance.data[offset:offset + data_length] = data_segment.init

        result: Optional[Tuple[TValue, ...]]

        if module.start is not None:
            function_address = module_instance.func_addrs[module.start.function_idx]
            result = self.invoke_function(function_address)
        else:
            result = None

        return module_instance, result

    def get_configuration(self) -> Configuration:
        return Configuration(self.store)

    def invoke_function(self,
                        function_address: FunctionAddress,
                        function_args: Tuple[TValue, ...] = None,) -> Tuple[TValue, ...]:
        """
        Invoke a function denoted by the function address with the provided arguments.
        """
        try:
            function = self.store.funcs[function_address]
        except IndexError:
            raise TypeError(
                f"Function address out of range: {function_address} >= "
                f"{len(self.store.funcs)}"
            )
        if function_args is None:
            function_args = tuple()

        for arg, valtype in zip(function_args, function.type.params):
            valtype.validate_arg(arg)

        config = self.get_configuration()

        if isinstance(function, FunctionInstance):
            from wasm.logic.control import _setup_function_invocation
            _setup_function_invocation(config, function_address, function_args)
            ret = config.execute()
            return ret
        elif isinstance(function, HostFunction):
            ret = function.hostcode(config, function_args)
            return ret
        else:
            raise Exception(f"Invariant: unknown function type: {type(function)}")


import argparse, random, sys

# Define topology variables
MAX_MINER_PEERS=3
ANCHOR_PEERS=3
MIN_PORT = 7000
MAX_PORT = 7499
USED_PORTS = []
ANCHOR_MINERS = []
MINERS = []
CLIENTS = []

# Parse command-line arguments
parser = argparse.ArgumentParser(description='Generate blockchain demo topology.')
parser.add_argument('--miners', dest='num_miners', help='total number of miners in the network',
                    type=int, required=True)
parser.add_argument('--clients', dest='num_clients', help='total number of clients in the network',
                    type=int, required=True)
parser.add_argument('--malicious-miners', dest='num_mal_miners', help='total number of malicious miners in the network',
                    type=int, required=True)
parser.add_argument('--traffic', dest='traffic', help='indicates if trafic generation is on or off',
					type=int, required=True)
args = parser.parse_args()

# Sanity check for minimal amount of anchor-miners and number of malicious miners
if args.num_miners < ANCHOR_PEERS:
    print('Number of miners must be greater or equal than %d' % ANCHOR_PEERS)
    sys.exit(1)

if args.num_mal_miners >= args.num_miners:
    print('Number of malicious miners must be smaller than miners')
    sys.exit(1)

# Define anchor-miners
while len(USED_PORTS) < ANCHOR_PEERS:
    anchor_port=random.randrange(MIN_PORT, MAX_PORT)
    if anchor_port not in USED_PORTS:
        node = {'ID': 'AM%d' % anchor_port, 'port': anchor_port, 'role': 'anchor-miner', 'malicious': False, 'peers': []}
        ANCHOR_MINERS.append(node)
        USED_PORTS.append(anchor_port)

# Define miners
while len(USED_PORTS) < args.num_miners:
    port=random.randrange(MIN_PORT, MAX_PORT)
    if port not in USED_PORTS:
        node = {'ID': 'M%d' % port, 'port': port, 'role': 'miner', 'malicious': False, 'peers': []}
        MINERS.append(node)
        USED_PORTS.append(port)

# Define clientd
while len(USED_PORTS) < args.num_miners + args.num_clients:
    port=random.randrange(MIN_PORT, MAX_PORT)
    if port not in USED_PORTS:
        node = {'ID': 'C%d' % port, 'port': port, 'role': 'client', 'peers': []}
        CLIENTS.append(node)
        USED_PORTS.append(port)

# Fully connect anchor-miners together
for i in ANCHOR_MINERS:
    for j in ANCHOR_MINERS:
        if i != j and j['port'] not in i['peers']:
            i['peers'].append(j['port'])

# Connect miners to at least 1 anchor-miner and other miners
for i in MINERS:
    i['peers'].append(random.choice(ANCHOR_MINERS)['port'])
    # Find remaining peers from concatenation of anchors-miners and miners MINUS the current miner
    sample = random.sample(ANCHOR_MINERS + list(filter(lambda x: x['port'] != i['port'], MINERS)) , MAX_MINER_PEERS-1)
    for j in sample:
        i['peers'].append(j['port'])
        j['peers'].append(i['port']) # Bidirectional relationship

# Connect clients to a random anchor
for i in CLIENTS:
    am = random.choice(ANCHOR_MINERS)
    i['peers'].append(am['port'])
    am['peers'].append(i['port']) # Bidirectional relationship

# Randomly select malicious miners
for i in random.sample(MINERS+ANCHOR_MINERS, args.num_mal_miners):
    i['malicious'] = True

# Connect clients to non malicious miner if they are tied to one
for i in CLIENTS:
    for j in i['peers']:
        # j is a port
        peer = None
        # Find peer entity
        for h in MINERS+ANCHOR_MINERS:
            if h['port'] == j:
                peer = h
                break
        if peer['malicious'] == False:
            break
        # Find another peer that isn't malicious
        newPeer = None
        while True:
            miner = random.choice(MINERS+ANCHOR_MINERS)
            if miner['malicious'] == False:
                newPeer = miner
                break
        i['peers'].append(newPeer['port'])
        newPeer['peers'].append(i['port'])
        break

# Create all docker-compose client services and vis.js content
services = ''
visjs_vertices = ''
visjs_edges = ''
ui_port_counter = 8000
for i in CLIENTS:
    services += '%s:\n  image: log735-chat:latest\n  container_name: node-%s\n  environment:\n\
    - PEERS=%s\n    - TRAFFIC=%s\n    - ROLE=client\n    - PORT=%s\n  ports:\n    - \'%s:8000\'\n  networks:\n    - blockchain\n' \
                % (i['ID'], i['port'], " ".join(str(x) for x in i['peers']), args.traffic, i['port'], ui_port_counter)
    visjs_vertices += '        {id: %s, label: \'%s\'},\n' % (i['port'], i['ID'])
    ui_port_counter += 1
    for x in i['peers']:
        visjs_edges += '        {from: %s, to: %s},\n' % (i['port'], x)

for i in ANCHOR_MINERS:
    services += '%s:\n  image: log735:latest\n  container_name: node-%s\n  environment:\n\
    - PEERS=%s\n    - ROLE=miner\n    - PORT=%s\n    - MALICIOUS=%s\n  networks:\n    - blockchain\n' \
                % (i['ID'], i['port'], " ".join(str(x) for x in i['peers']), i['port'], i['malicious'])
    visjs_vertices += '        {id: %s, label: \'%s\'},\n' % (i['port'], i['ID'])
    for x in i['peers']:
        visjs_edges += '        {from: %s, to: %s},\n' % (i['port'], x)

for i in MINERS:
    services += '%s:\n  image: log735:latest\n  container_name: node-%s\n  environment:\n\
    - PEERS=%s\n    - ROLE=miner\n    - PORT=%s\n    - MALICIOUS=%s\n  networks:\n    - blockchain\n' \
                % (i['ID'], i['port'], " ".join(str(x) for x in i['peers']), i['port'], i['malicious'])
    visjs_vertices += '        {id: %s, label: \'%s\'},\n' % (i['port'], i['ID'])
    for x in i['peers']:
        visjs_edges += '        {from: %s, to: %s},\n' % (i['port'], x)

# Write docker-compose.yaml
with open('docker-compose.template', 'r') as f:
    content = f.read()

content = content.replace('%SERVICES%', ''.join('  '+line for line in services.splitlines(True)))

with open('docker-compose.yaml', 'w+') as f:
    f.write(content)

# Write index.html
with open('../webapp/scripts/index.html.template', 'r') as f:
    content = f.read()

content = content.replace('%VERTICES%', ''.join(''+line for line in visjs_vertices[:-2].splitlines(True)))
content = content.replace('%EDGES%', ''.join(''+line for line in visjs_edges[:-2].splitlines(True)))

with open('../webapp/frontend/index.html', 'w+') as f:
    f.write(content)


r"""usb-host-py - Simple host for a USB device.
"""

from timeit import default_timer
import usb.core
import usb.util

# Copied from usb-device.h
_BULK_TRANSFER_LENGTH = 1024

_INVALID_EP_ADDRESS = 0xff
_epbulk_in_address = _INVALID_EP_ADDRESS
_epbulk_in_mps = 0
_epbulk_out_address = _INVALID_EP_ADDRESS
_epbulk_out_mps = 0

def _print_device_list():
    print(usb.core.show_devices())

def _get_end_point_addresses(device):
    global _epbulk_in_address, _epbulk_in_mps, _epbulk_out_address, _epbulk_out_mps

    configuration = device[0]
    if configuration.bNumInterfaces != 1:
        print("unexpected number of interfaces {}".format(configuration.bNumInterfaces))
        return None

    interface = configuration[(0, 0)]
    if interface.bAlternateSetting != 0:
        print("unexpected number of alternate settings {}".format(interface.bAlternateSetting))
        return None
    if interface.bNumEndpoints != 2:
        print("unexpected number of endpoint {}".format(interface.bNumEndpoints))
        return None

    for endpoint in interface:
        print(
            "bEndpointAddress 0x{:x} bmAttributes 0x{:x} wMaxPacketSize 0x{:x}"
            .format(endpoint.bEndpointAddress, endpoint.bmAttributes, endpoint.wMaxPacketSize)
            )

        is_in = usb.util.endpoint_direction(endpoint.bEndpointAddress) == usb.util.ENDPOINT_IN
        is_bulk = usb.util.endpoint_type(endpoint.bmAttributes) == usb.util.ENDPOINT_TYPE_BULK
        if is_in and is_bulk:
            if _epbulk_in_address != _INVALID_EP_ADDRESS:
                print("Found multiple bulk IN endpoints")
                return None
            _epbulk_in_address = endpoint.bEndpointAddress
            _epbulk_in_mps = endpoint.wMaxPacketSize

        is_out = usb.util.endpoint_direction(endpoint.bEndpointAddress) == usb.util.ENDPOINT_IN
        if is_out and is_bulk:
            if _epbulk_out_address != _INVALID_EP_ADDRESS:
                print("Found multiple bulk IN endpoints")
                return None
            _epbulk_out_address = endpoint.bEndpointAddress
            _epbulk_out_mps = endpoint.wMaxPacketSize

    return device

def _open_device(id_vendor, id_product):
    device = usb.core.find(idVendor=id_vendor, idProduct=id_product)
    if device is not None:
        print("found device with idVendor 0x{:x} idProduct 0x{:x}".format(id_vendor, id_product))
        device.set_configuration()
        return _get_end_point_addresses(device)

    print(
        "failed to find device with idVendor 0x{:x} idProduct 0x{:x}"
        .format(id_vendor, id_product)
        )

    return None

def _check_configuration_value(device):
    configuration = device.get_active_configuration()

    if configuration.bConfigurationValue == 1:
        return True

    print(
        "'libusb_get_configuration' succeeded, configuration value is {}"
        .format(configuration.bConfigurationValue)
        )

    return False

def _control_transfer_out(device):
    data = "some data\0"
    bytes_transferred = device.ctrl_transfer(
        bmRequestType
            = usb.util.CTRL_OUT
            | usb.util.CTRL_TYPE_VENDOR
            | usb.util.CTRL_RECIPIENT_DEVICE,
        bRequest=0,
        wValue=0,
        wIndex=0,
        data_or_wLength=data
        )
    print("bytes_transferred {}".format(bytes_transferred))
    if bytes_transferred != len(data):
        print("bytes_transferred {}, expected {}".format(bytes_transferred, len(data)))
        return False

    return True

def _control_transfer_in(device):
    data = device.ctrl_transfer(
        bmRequestType
            = usb.util.CTRL_IN
            | usb.util.CTRL_TYPE_VENDOR
            | usb.util.CTRL_RECIPIENT_DEVICE,
        bRequest=0,
        wValue=0,
        wIndex=0,
        data_or_wLength=13
        )
    if len(data) != 13:
        return False

    return True

def _bulk_transfer_in(device):
    global _epbulk_in_address, _epbulk_in_mps

    assert _epbulk_in_address != _INVALID_EP_ADDRESS
    assert _epbulk_in_mps != 0

    data = device.read(_epbulk_in_address, _BULK_TRANSFER_LENGTH)

    if len(data) != _BULK_TRANSFER_LENGTH:
        print(
            "Number of bytes actually transferred not the same as the requested length, transferred {}, length {}"
            .format(len(data), _BULK_TRANSFER_LENGTH)
            )
        return False

    return True

def _repeat_bulk_in_transfer(device):
    number_of_bulk_in_repeats = 10000
    print("perform {} bulk in transfers\n".format(number_of_bulk_in_repeats))

    start = default_timer()
    for _ in range(number_of_bulk_in_repeats):
        if not _bulk_transfer_in(device):
            return False

    stop = default_timer()
    duration_us = (stop - start) * 1000000

    print("duration_us {} us".format(round(duration_us)))
    throughput_bytes_per_us = (_BULK_TRANSFER_LENGTH * number_of_bulk_in_repeats) / duration_us
    throughput_bytes_per_s = throughput_bytes_per_us * 100000
    throughput_megabytes_per_s = throughput_bytes_per_s / 100000
    throughput_megabits_per_s = throughput_megabytes_per_s * 8
    print("throughput MB/s {}".format(throughput_megabytes_per_s))
    print("throughput Mbit/s {}".format(throughput_megabits_per_s))
    print("completed {} bulk in transfers".format(number_of_bulk_in_repeats))

    return True

def _itoa(data):
    r"simulate std::itoa and avoid overflow exception"
    initialiser = 2
    for idx, _ in enumerate(data):
        data[idx] = initialiser
        initialiser += 1
        if initialiser == 256:
            initialiser = 0

def _bulk_transfer_out(device):
    global _epbulk_out_address, _epbulk_out_mps

    assert _epbulk_out_address != _INVALID_EP_ADDRESS
    assert _epbulk_out_mps != 0

    data = bytearray(_BULK_TRANSFER_LENGTH)
    _itoa(data)

    length = len(data)
    transferred = device.write(_epbulk_out_address, data)
    if transferred != length:
        print(
            "Number of bytes actually transferred not the same as the requested length,"
            " transferred {}, length {}"
            .format(transferred, length)
            )
        return False

    return True

def _do_something_with_device(device):
    assert device is not None

    if not _check_configuration_value(device):
        return

    # Control endpoint, i.e. endpoint 0, available regardless of whether or not interface is claimed
    if not _control_transfer_out(device):
        return
    if not _control_transfer_in(device):
        return

    # Interface claiming should be managed automatically

    if not _repeat_bulk_in_transfer(device):
        return
    _bulk_transfer_out(device)

def _main():
    print("usb-host")

    devices = list(usb.core.find(find_all=True))
    print("number_of_devices {}".format(len(devices)))
    _print_device_list()

    device = _open_device(0x1f00, 0x2012)

    _do_something_with_device(device)

if __name__ == "__main__":
    _main()


from django.urls import path

from .views import *

urlpatterns = [
    path('', bibl_main, name="main"),
    path('add/', bibl_add, name="add"),
    path('change/', bibl_change, name="change"),
    path('api/v1/ablist/', AddressBookAPIView.as_view(), name="ablist")
]


import os
import zipfile
from shutil import rmtree

def unzip(file, dest):
    zip = zipfile.ZipFile(file)
    if os.path.isdir(dest):
        rmtree(dest)
    os.mkdir(dest)
    zip.extractall(dest)

    zip.close()


# Standard
import sqlite3
import os
from pathlib import Path

class transactions:
    """Intended use is with an insert function into the transation table.

    This class is passed into the db_banner.database function.
    """
    def __init__(self, id, account, date, description, amount, misc):
        self.id = id
        self.account = account
        self.date = date
        self.description = description
        self.amount = amount
        self.misc = misc

class tags:
    """Not currently in use. Intended use is with an insert function into the tags table.
    """
    def __init__(self, id, tag):
        self.id = id
        self.tag = tag

class tags_link:
    """Not currently in use. Intended use is with an insert function into the tags_link table.
    """
    def __init__(self, id, id_tag, id_transaction):
        self.id = id
        self.id_tag = id_tag
        self.id_transaction = id_transaction

class database(object):
    """Handles all database connectsion, inputs, and outputs

    Class constructor initiates sqlite3 database connection. If used in WITH statement
    the connection will cleanly close after the statement is finished. If there are
    uncommitted transactions they will be rolled back prior to connection closure.
    """
    def __init__(self):
        __DB_LOCATION = (
            Path.home()
            / "Documents"
            / "GitHub"
            / "_appdata"
            / "cli_money_tool"
            / "accounts.db"
        )
        if os.path.exists(__DB_LOCATION):
            self.__db_connection = sqlite3.connect(str(__DB_LOCATION))
            self.cur = self.__db_connection.cursor()
        else:
            Path(
                Path.home() / "Documents" / "GitHub" / "_appdata" / "cli_money_tool"
            ).mkdir(parents=True, exist_ok=True)
            self.__db_connection = sqlite3.connect(str(__DB_LOCATION))
            self.cur = self.__db_connection.cursor()

    def __del__(self):
        self.__db_connection.close()

    def __enter__(self):
        return self

    def __exit__(self, ext_type, exc_value, traceback):
        self.cur.close()
        if isinstance(exc_value, Exception):
            self.__db_connection.rollback()
        else:
            self.__db_connection.commit()
        self.__db_connection.close()

    def execute(self, new_data: str) -> tuple:
        """Executes an valid SQL statement passed through as a string.

        Arugments:
            new_data (string): Valid SQL statement
        """
        return self.cur.execute(new_data)

    def executemany(self, many_new_data: str) -> None:
        """Not currently in use.
        """
        self.cur.executemany("REPLACE INTO jobs VALUES(?, ?, ?, ?)", many_new_data)

    def insert(self, record):
        """Inserts a transaction record. Designed for use with the transaction class.

        Arguments:
            record (transaction class): class or dictionary containing the following values:

                - id
                - account
                - date
                - description
                - amount
                - misc
        """
        record.id = self.cur.execute("""SELECT MAX(id) FROM transactions""").fetchone()[0]
        record.id = record.id + 1 if record.id else 1
        self.cur.execute(
            f"""INSERT INTO transactions
                             VALUES (
                                        "{record.id}",
                                        "{record.account}",
                                        "{record.date}",
                                        "{record.description}",
                                        "{record.amount}",
                                        "{record.misc}"
                        )"""
        )

    def create_tables(self):
        """This function confirms the existence of or creates the path, database, and tables.

        Can be used by calling the function directly, but is designed to by used by install.py, which is called by the install.bat file.
        """
        if (
            Path.home()
            / "Documents"
            / "GitHub"
            / "_appdata"
            / "cli_money_tool"
            / "accounts.db"
        ):
            pass
        else:
            Path(Path.cwd() / ".." / "_appdata" / "cli_money_tool").mkdir(
                parents=True, exist_ok=True
            )

        """create a database table if it does not exist already"""
        self.cur.execute(
            """CREATE TABLE IF NOT EXISTS
                                "transactions" (
                                    "id"            INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                                    "account"    	TEXT NOT NULL,
                                    "date"	        TEXT NOT NULL,
                                    "description"	TEXT,
                                    "amount"	    INTEGER NOT NULL,
                                    "misc"          INTEGER
                        )"""
        )
        self.cur.execute(
            """CREATE TABLE IF NOT EXISTS
                                "tags_link" (
                                    "id"	          INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                                    "id_tag"	      INTEGER NOT NULL,
                                    "id_transaction"  INTEGER NOT NULL
                        )"""
        )
        self.cur.execute(
            """CREATE TABLE IF NOT EXISTS
                                "tags" (
                                    "id"	INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE,
                                    "tag"	TEXT NOT NULL
                        )"""
        )

    def select_all_transactions(self) -> tuple:
        """Returns all transactions

        Returns:
            tuple: contains all existing transactions in transaction table.
        """
        return self.cur.execute("""SELECT * FROM transactions""").fetchall()

    def sum_transactions_all(self, account: str) -> int:
        """Sums all transactions for specified account.

        Arguments:
            account (string): selected account/institution

        Returns:
            tuple: contains single value of rounded (2 decimal paces) result.
        """
        all_amounts = self.cur.execute(
            f"""SELECT amount FROM transactions
                                           WHERE account = '{account}'
                                        """
        ).fetchall()
        calc = 0
        for i in all_amounts:
            calc += float(i[0])
        return round(calc, 2)

    def commit(self):
        """Use after any other database class function to commit changes.

        This function is separated from initial transactions to enable the __exit__ function to rollback changes in the case that errors are encountered.
        """
        self.__db_connection.commit()