# coding: utf-8

# In[1]:

import os
import pandas as pd
import matplotlib.pyplot as plt
import numpy as np
from scipy.signal import welch
from detect_peaks import detect_peaks
from scipy.stats import kurtosis
from scipy.stats import skew
import matplotlib.backends.backend_pdf
from sklearn import model_selection
from sklearn.metrics import classification_report
from datetime import datetime

# In[2]:

#Import Classifiers
#More info at https://scikit-learn.org/stable/supervised_learning.html#supervised-learning
#Commented Out Classifiers refer to models that gave an error

#Linear Models
from sklearn.linear_model import ARDRegression
from sklearn.linear_model import BayesianRidge
from sklearn.linear_model import ElasticNet
from sklearn.linear_model import ElasticNetCV
from sklearn.linear_model import Hinge
#from sklearn.linear_model import Huber #Gave an error
from sklearn.linear_model import HuberRegressor
from sklearn.linear_model import Lars
from sklearn.linear_model import LarsCV
from sklearn.linear_model import Lasso
from sklearn.linear_model import LassoCV
from sklearn.linear_model import LassoLars
from sklearn.linear_model import LassoLarsCV
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import Log
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import LogisticRegressionCV
from sklearn.linear_model import ModifiedHuber
from sklearn.linear_model import MultiTaskElasticNet
from sklearn.linear_model import MultiTaskElasticNetCV
from sklearn.linear_model import MultiTaskLasso
from sklearn.linear_model import MultiTaskLassoCV
from sklearn.linear_model import OrthogonalMatchingPursuit
from sklearn.linear_model import OrthogonalMatchingPursuitCV
from sklearn.linear_model import PassiveAggressiveClassifier
from sklearn.linear_model import PassiveAggressiveRegressor
from sklearn.linear_model import Perceptron
from sklearn.linear_model import RANSACRegressor
#from sklearn.linear_model import RandomizedLasso #deprecated in 0.19 and will be removed in 0.21.
#from sklearn.linear_model import RandomizedLogisticRegression #deprecated in 0.19 and will be removed in 0.21.
from sklearn.linear_model import Ridge
from sklearn.linear_model import RidgeCV
from sklearn.linear_model import RidgeClassifier
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import SGDRegressor
from sklearn.linear_model import SquaredLoss
from sklearn.linear_model import TheilSenRegressor

from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import Pipeline

# Linear and Quadratic Discriminant Analysis
from sklearn.discriminant_analysis import BaseEstimator
from sklearn.discriminant_analysis import ClassifierMixin
from sklearn.discriminant_analysis import LinearClassifierMixin
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.discriminant_analysis import StandardScaler
from sklearn.discriminant_analysis import TransformerMixin

#Kernel Ridge Regression
from sklearn.kernel_ridge import BaseEstimator
from sklearn.kernel_ridge import KernelRidge
from sklearn.kernel_ridge import RegressorMixin

#Support Vector Machines
from sklearn.svm import LinearSVC
from sklearn.svm import LinearSVR
from sklearn.svm import NuSVC
from sklearn.svm import NuSVR
from sklearn.svm import OneClassSVM
from sklearn.svm import SVC
from sklearn.svm import SVR

#Stochastic Gradient Descent
from sklearn.linear_model import SGDClassifier
from sklearn.linear_model import SGDRegressor

#Nearest Neighbors
from sklearn.neighbors import BallTree
#from sklearn.neighbors import DistanceMetric #DistanceMetric is an abstract class
from sklearn.neighbors import KDTree
from sklearn.neighbors import KNeighborsClassifier
from sklearn.neighbors import KNeighborsRegressor
from sklearn.neighbors import KernelDensity
#from sklearn.neighbors import LSHForest #deprecated in 0.19. It will be removed in version 0.21.
from sklearn.neighbors import LocalOutlierFactor
from sklearn.neighbors import NearestCentroid
from sklearn.neighbors import NearestNeighbors
from sklearn.neighbors import RadiusNeighborsClassifier
from sklearn.neighbors import RadiusNeighborsRegressor

#Gaussian Processes
#from sklearn.gaussian_process import GaussianProcess #deprecated in version 0.18 and will be removed in 0.20.
from sklearn.gaussian_process import GaussianProcessRegressor
from sklearn.gaussian_process import GaussianProcessClassifier

#Cross Decomposition
from sklearn.cross_decomposition import CCA
from sklearn.cross_decomposition import PLSCanonical
from sklearn.cross_decomposition import PLSRegression
from sklearn.cross_decomposition import PLSSVD

#Naive Bayes
#from sklearn.naive_bayes import ABCMeta #Gave an error
#from sklearn.naive_bayes import BaseDiscreteNB #Can't instantiate abstract class BaseNB with abstract methods _joint_log_likelihood
from sklearn.naive_bayes import BaseEstimator
#from sklearn.naive_bayes import BaseNB #Can't instantiate abstract class BaseNB with abstract methods _joint_log_likelihood
from sklearn.naive_bayes import BernoulliNB
from sklearn.naive_bayes import ClassifierMixin
from sklearn.naive_bayes import GaussianNB
from sklearn.naive_bayes import LabelBinarizer
from sklearn.naive_bayes import MultinomialNB

#Decision Trees
from sklearn.tree import DecisionTreeClassifier
from sklearn.tree import DecisionTreeRegressor
from sklearn.tree import ExtraTreeClassifier

#Ensemble Methods
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import AdaBoostRegressor
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import BaggingRegressor
#from sklearn.ensemble import BaseEnsemble #Can't instantiate abstract class BaseEnsemble with abstract methods __init__
from sklearn.ensemble import ExtraTreesClassifier
from sklearn.ensemble import ExtraTreesRegressor
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.ensemble import IsolationForest
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn.ensemble import RandomTreesEmbedding
from sklearn.ensemble import VotingClassifier

#Multiclass and multilabel algorithms
from sklearn.multiclass import BaseEstimator
from sklearn.multiclass import ClassifierMixin
from sklearn.multiclass import LabelBinarizer
from sklearn.multiclass import MetaEstimatorMixin
from sklearn.multiclass import OneVsOneClassifier
from sklearn.multiclass import OneVsRestClassifier
from sklearn.multiclass import OutputCodeClassifier
from sklearn.multiclass import Parallel
#from sklearn.multioutput import ABCMeta #Gave an error
from sklearn.multioutput import BaseEstimator
from sklearn.multioutput import ClassifierChain
from sklearn.multioutput import ClassifierMixin
from sklearn.multioutput import MetaEstimatorMixin
from sklearn.multioutput import MultiOutputClassifier
#from sklearn.multioutput import MultiOutputEstimator #Can't instantiate abstract class MultiOutputEstimator with abstract methods __init__
from sklearn.multioutput import MultiOutputRegressor
from sklearn.multioutput import Parallel
from sklearn.multioutput import RegressorMixin

#Semi-Suportvised
from sklearn.semi_supervised import LabelPropagation
from sklearn.semi_supervised import LabelSpreading
from sklearn.semi_supervised import label_propagation

#Isotonic Regression
from sklearn.isotonic import BaseEstimator
from sklearn.isotonic import IsotonicRegression
from sklearn.isotonic import RegressorMixin
from sklearn.isotonic import TransformerMixin

# Neural network models (supervised)
from sklearn.neural_network import BernoulliRBM
from sklearn.neural_network import MLPClassifier
from sklearn.neural_network import MLPRegressor

# In[3]:

def GenerateIMSDictionary(FileOfInterest,TrainingDataFile,HomeDirectory):
    """
    returns UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)

    GenerateIMSDictionary(
        FileOfInterest - File name a single IMS file
        TrainingDataFile - File name of TrainingDataFile
        HomeDirectory - Directory with training file
    )

    Example:
        n = 2000 / 60
        N = 16
        Bd = 0.331*254
        Pd = 2.815*254
        phi = 15.17 * np.pi / 180
        SampleFrequency = 20000
        FileName = FileOfInterest
        data = getIMSB1XData(FileName)
        HomeDirectory = HomeDirectory
        directory = os.listdir(HomeDirectory)
        TrainingDataFile = TrainingDataFile
        UserInput = UserInputs2WorkingForm(n,N,Bd,Pd,phi,SampleFrequency,FileName,data,HomeDirectory,directory,TrainingDataFile)

    This is the same as: GenerateIMSDictionary('2003.10.22.12.06.24',"TrainingData.csv",os.getcwd())

    The Purpose of this function is quickly hardcode parameters that would be taken from the GUI.
    """

    #Hardcoded Bearing Info
    n = 2000 / 60    #Shaft rotational speed [Hz], n
    N = 16  #No. of rolling elements [-], N
    Bd = 0.331*254 #Diameter of a rolling element [mm], Bd
    Pd = 2.815*254 #Pitch diameter [mm], Pd
    phi = 15.17 * np.pi / 180 #Contact angle [rad], Phi
    SampleFrequency = 20000 #Sampling Frequency
    FileName = FileOfInterest #Filename in string format
    data = getIMSB1XData(FileName) #Get the bearing 1: X direction from the filename
    HomeDirectory = HomeDirectory #Homedirectory
    directory = os.listdir(HomeDirectory) #Files HomeDirectory
    TrainingDataFile = TrainingDataFile #String of File Name that is the Training file csv
    UserInput = UserInputs2WorkingForm(n,N,Bd,Pd,phi,SampleFrequency,FileName,data,HomeDirectory,directory,TrainingDataFile)

    return UserInput

# In[4]:

def UserInputs2WorkingForm(n,N,Bd,Pd,phi,SampleFrequency,FileName,RawData,HomeDirectory,directory,TrainingDataFile):
    """
    Returns a dictionary of relevant file information

    UserInputs2WorkingForm(
        n - Shaft rotational speed [Hz], n
        N - No. of rolling elements [-], N
        Bd - Diameter of a rolling element [mm], Bd
        Pd - Pitch diameter [mm], Pd
        phi - Contact angle [rad], Phi
        SampleFrequecy - SampleFrequency,
        FileOfInterest - Title of CSV containing the raw data
        HomeDirectory - Location of the Current Directory
        directory - Directory of the FileOfInterest
        TrainingDataFile - Title of CSV containg the training data for the machine learning
        )

    This functions serves to take all relevant motor characteristics and puts them in a
    dictionary.
    This dictionary will serve as the building blocks for the rest of the functions.
    """
    #Get Extra Info
    NumberOfSamples = len(RawData)
    dt = 1/SampleFrequency
    Tmax = dt*NumberOfSamples

    #Arrange
    x = {
        'n': n, #Shaft rotational speed [Hz], n
        'N': N, #No. of rolling elements [-], N
        'Bd': Bd, #Diameter of a rolling element [mm], Bd
        'Pd': Pd, #Pitch diameter [mm], Pd
        'Phi': phi, #Contact angle [rad], Phi
        'Sampling Frequency': SampleFrequency,
        'Time of Sampling': Tmax,
        'Number of Samples': NumberOfSamples,
        'File of Interest': FileName,
        'HomeDirectory': HomeDirectory,
        'Working Directory': directory,
        'TrainingFileName': TrainingDataFile,
        'Signal Data of Interest': RawData
    }
    return x

# In[5]:

def getIMSB1XData(FileOfInterest):
    """
    Returns an 1-D array of IMS data

    Subfunction for UserInputs2WorkingForm
    GetData(
    FileOfInterest - Title of CSV containing the raw data
    )
    This function reads the IMS bearing dataset for set 1 that was taken from:
    http://data-acoustics.com/measurements/bearing-faults/bearing-4/
    """
    #Get Data
    data = pd.read_table(FileOfInterest,header = None)
    data.columns = ['b1x','b1y','b2x','b2y','b3x','b3y','b4x','b4y']
    return np.transpose(data.values[:,0])

# In[6]:

def getValuesFromRawData(filename):
    """

    """

    dataset = pd.read_csv(filename, header = None, index_col = False)
    dataset.rename(columns={0: "Time", 1: "Value"}, inplace = True)

    #Prepare Raw Data
    i = 0
    time = []
    amp = []
    while i < len(dataset['Time'].values):
        i += 1

        if dataset['Time'].values[i] == 0 and dataset['Value'].values[i] == 0:
            break
        else:
            time.append(dataset['Time'].values[i])
            amp.append(dataset['Value'].values[i])

    return time,amp

# In[7]:

def ReplaceSignalDataofInterest(Data,UserInput,filename):
    """
    Returns a dictionary of relevant file information

    UserInputs2WorkingForm(
        Data - This is the actual data
        UserInput - UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    This functions replaces the signal data of interest value of a dictionary.
    This function was created to "cheat" the system and allow the generation of training data
    by manually inputting the actual data. This helps becuase right now the GetData() function
    only gets B1X column for simplicity.
    """

    x = UserInput.copy()
    x['Signal Data of Interest'] = Data
    x['File of Interest'] = filename

    return x

# In[8]:

def BearingInfomation(UserInput):
    """
    Returns a dictionary with Bearing Characteristic Frequencies

    BearingInfomation(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    This functions calculates the bearing characteristic frequencies
    """
    #Get Needed Info
    n = UserInput['n']
    N = UserInput['N']
    Bd = UserInput['Bd']
    Pd = UserInput['Pd']
    phi = UserInput['Phi']

    #Calculate Bearing Frequncies
    xx = Bd/Pd*np.cos(phi)
    BPFI = (N/2)*(1 + xx)*n
    BPFO = (N/2)*(1 - xx)*n
    BSF = (Pd/(2*Bd))*(1-(xx)**2)*n
    FTF= (1/2)*(1 - xx)*n

    #Arrange
    x = {
        "BPFI": BPFI,
        "BPFO": BPFO,
        "BSF":  BSF,
        "FTF":  FTF
    }
    return x

# In[9]:

def RemoveDCOffset(UserInput):
    """
    Returns a modified dictionary

    RemoveDCOffset(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    This functions removes the dc bias from the signal in the UserInput dictionary
    """
    temp = UserInput.copy()
    temp["Signal Data of Interest"] = temp["Signal Data of Interest"] - np.mean(temp["Signal Data of Interest"])
    return temp

# In[10]:

def FourierTransform(UserInput):
    """
    Returns a dictionary

    FourierTransform(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
    )

    The functions perfroms fast fourier transform on the UserInput Signal
    Data of Interest
    """

    #Get Needed Info
    sig = UserInput['Signal Data of Interest']
    NumberOfSamples = UserInput['Number of Samples']
    Tmax = UserInput['Time of Sampling']

    #Fourier Transform
    frq = np.arange(NumberOfSamples)/(Tmax)# two sides frequency range
    frq = frq[range(int(NumberOfSamples/(2)))] # one side frequency range
    Y = abs(np.fft.fft(sig))/NumberOfSamples # fft computing and normalization
    Y = Y[range(int(NumberOfSamples/2))]

    #Arrange
    x = {
        "Frequency":frq,
        "Freq. Amp.": Y
        }
    return x

# In[11]:

def get_psd_values(UserInput):
    """
    Returns a dictionary

    get_psd_values(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
    )

    The functions perfroms power spectrum density on the UserInput Signal
    Data of Interest
    """
    #Get Needed Info
    sig = UserInput['Signal Data of Interest']
    SamplingFrequency = UserInput['Sampling Frequency']

    #Perfrom psd
    frq, psd_values = welch(sig, fs=SamplingFrequency)

    #Arrange
    x = {
        "Frequency":frq,
        "PSD": psd_values
        }
    return x

# In[12]:

def autocorr(x):
    """
    Taken from:
    https://ipython-books.github.io/103-computing-the-autocorrelation-of-a-time-series/

    Returns the autocorrelation of the signal x

    autocorr(
        x - signal of interest
        )

    This functions performs correlation
    """
    result = np.correlate(x, x, mode='full')
    return result[len(result)//2:]

# In[13]:

def get_autocorr_values(UserInput):
    """
    Modified from:
    https://ipython-books.github.io/103-computing-the-autocorrelation-of-a-time-series/

    Returns a dictionary

    get_autocorr_values(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
    )

    The functions perfroms autocorrelation on the UserInput Signal
    Data of Interest
    """
    #Get needed info
    sig = UserInput['Signal Data of Interest']
    Tmax = UserInput['Time of Sampling']
    N = UserInput['Number of Samples']

    #Call correlation function
    autocorr_values = autocorr(sig)

    #Arrange
    x_values = np.array([Tmax * jj for jj in range(0, N)])
    x = {
        "X Values":x_values,
        "Autocorr Values": autocorr_values
        }
    return x

# In[14]:

def TimeDomainInformation(UserInput):
    """
    Returns a dictionary with Time Domain Characteristics

    TimeDomainInformation(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    This functions calculates the Time Domain Characteristics
    """
    #Get Needed Info
    sig = UserInput['Signal Data of Interest']

    #Arrange
    x = {
        "RMS": np.mean(sig**2),
        "STD": np.std(sig),
        "Mean": np.mean(sig),
        "Max": np.max(sig),
        "Min": np.min(sig),
        "Peak-to-Peak": (np.max(sig) - np.min(sig)),
        "Max ABS": np.max(abs(sig)),
        "Kurtosis": kurtosis(sig),
        "Skew": skew(sig),
    }

    return x

# In[15]:

def Magnitude(Y):
    """

    """
    mag = 0
    for i in range(0,len(Y)):
        mag = mag + Y[i]**2
    mag = mag ** 0.5
    return mag

# In[16]:

def Normalize(UserInput):
    """

    """

    x = UserInput.copy()
    mag = Magnitude(x['Signal Data of Interest'])
    x['Signal Data of Interest'] = x['Signal Data of Interest'] / mag

    return x

# In[17]:

def PosMagnitude(Y):
    """

    """
    len(Y)
    mag = 0
    for i in range(0,len(Y)):
        if Y[i] > 0:
            mag = mag + Y[i]**2
    mag = mag ** 0.5
    return mag

# In[53]:

def plotPeaks(X,Y,xlabel,ylabel,Title):
    """

    """
    #Set Parameters
    Ymag = PosMagnitude(Y)
    Ynew = Y/Ymag
    min_peak_height = .04
    threshold = 0.15*np.std(Ynew)

    #Get indices of peak
    peak = detect_peaks(Ynew,edge = 'rising',mph = min_peak_height, mpd = 5, threshold = threshold )
    fig = plt.figure()
    plt.plot(X,Y)

    for i in peak:
       plt.scatter(X[i],Y[i], c= 'r', marker='*',s = 80)

    plt.xlabel(xlabel, fontsize=12)
    plt.ylabel(ylabel, fontsize=12)
    plt.title(Title)
    plt.grid(True)
    plt.show()

    return fig

# In[19]:

def GetSortedPeak(X,Y):
    """
    SubFunction for FrequencyDomainInformation

    Returns Amplitude of Y, Loctation

    GetSortedPeak(
        X - Independent Variable
        Y - Dependent Variable
        )

    Uses detect_peaks function taken from Github:
    __author__ = "Marcos Duarte, https://github.com/demotu/BMC"

    Get the indices of relevant peaks
    Then Returns the amplitude,location of the relevant peaks
    """
    #Original
    """
    #Set Parameters
    min_peak_height = 0.1 * np.nanmax(Y) #Original
    threshold = 0.05 * np.nanmax(Y) #Original

    #Get indices of peak
    peak = detect_peaks(Y,edge = 'rising',mph = min_peak_height, mpd = 2, threshold = threshold ) #Original
    """
    #NEW
    #Set Parameters
    Ymag = PosMagnitude(Y)
    Ynew = Y/Ymag
    min_peak_height = .04
    threshold = 0.15*np.std(Ynew)

    #Get indices of peak
    peak = detect_peaks(Ynew,edge = 'rising',mph = min_peak_height, mpd = 5, threshold = threshold )

    #Get values corresponding to indices
    m = []
    mm = []
    for i in peak:
        m.append(Y[i])
        mm.append(X[i])

    #Sort arcording to the amplitude
    mmm = np.argsort(m)
    n = []
    nn = []
    for i in mmm:
        n.append(m[i])
        nn.append(mm[i])

    #Sort in Descending Amplitdue while keeping locations matched
    n  = n[::-1] #amplitude
    nn = nn[::-1] #location

    #Arrange
    return n, nn

# In[20]:

def FrequencyDomainInformation(UserInput):
    """
    Returns a dictionary with Frequency Domain Characteristics
    Top 5 frequncy and amplitudes for:
    fft
    psd
    correlation

    FrequencyDomainInformation(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )
    """
    #Call FFT, PSD, and Correlation Values
    x1 = FourierTransform(UserInput)
    x2 = get_psd_values(UserInput)
    x3 = get_autocorr_values(UserInput)
    FTamp,FTfreq = GetSortedPeak(x1['Frequency'],x1['Freq. Amp.'])
    PSDamp,PSDfreq = GetSortedPeak(x2['Frequency'],x2['PSD'])
    Cor,CorTime = GetSortedPeak(x3['X Values'],x3['Autocorr Values'])

    #Originally -999
    #Now 77777
    #Take Care of Empty Values
    while len(FTamp) <= 5:
        FTamp.append(['77777'])
    while len(FTfreq) <= 5:
        FTfreq.append(['77777'])
    while len(PSDamp) <= 5:
        PSDamp.append(['77777'])
    while len(PSDfreq) <= 5:
        PSDfreq.append(['77777'])
    while len(Cor) <= 5:
        Cor.append(['77777'])
    while len(CorTime) <= 5:
        CorTime.append(['77777'])

    #Arrange
    x = {
        "FFT Frq @ Peak 1": FTfreq[0],
        "FFT Frq @ Peak 2": FTfreq[1],
        "FFT Frq @ Peak 3": FTfreq[2],
        "FFT Frq @ Peak 4": FTfreq[3],
        "FFT Frq @ Peak 5": FTfreq[4],
        "FFT Amp @ Peak 1": FTamp[0],
        "FFT Amp @ Peak 2": FTamp[1],
        "FFT Amp @ Peak 3": FTamp[2],
        "FFT Amp @ Peak 4": FTamp[3],
        "FFT Amp @ Peak 5": FTamp[4],
        "PSD Frq @ Peak 1": PSDfreq[0],
        "PSD Frq @ Peak 2": PSDfreq[1],
        "PSD Frq @ Peak 3": PSDfreq[2],
        "PSD Frq @ Peak 4": PSDfreq[3],
        "PSD Frq @ Peak 5": PSDfreq[4],
        "PSD Amp @ Peak 1": PSDamp[0],
        "PSD Amp @ Peak 2": PSDamp[1],
        "PSD Amp @ Peak 3": PSDamp[2],
        "PSD Amp @ Peak 4": PSDamp[3],
        "PSD Amp @ Peak 5": PSDamp[4],
        "Autocorrelate Time @ Peak 1": CorTime[0],
        "Autocorrelate Time @ Peak 2": CorTime[1],
        "Autocorrelate Time @ Peak 3": CorTime[2],
        "Autocorrelate Time @ Peak 4": CorTime[3],
        "Autocorrelate Time @ Peak 5": CorTime[4],
        "Autocorrelate @ Peak 1": Cor[0],
        "Autocorrelate @ Peak 2": Cor[1],
        "Autocorrelate @ Peak 3": Cor[2],
        "Autocorrelate @ Peak 4": Cor[3],
        "Autocorrelate @ Peak 5": Cor[4]
    }
    return x

# In[21]:

def getAbsoluteTime(file):
    """
    Subfunction for StateInformation

    Returns the "magnitude" of the time stamp

    getAbsolutTime(
        file - file name that has bearing information within it
        )

    This function computes the magnitude of time when the IMS data was taken
    """
    #Get needed info
    year   = int(file[0:4])
    month  = int(file[5:7])
    day    = int(file[8:10])
    hour   = int(file[11:13])
    minute = int(file[14:16])
    second = int(file[17:19])

    #Compute starting from the 10 month
    #in seconds don't include years taking 10 as the start month
    x = second + 60*minute + 60*60*hour + 24*60*60*day + 31*24*60*60*(month - 10)
    return x

# In[23]:

"""
http://mkalikatzarakis.eu/wp-content/uploads/2018/12/IMS_dset.html
Previous work done on this dataset states that seven different states of health were observed:

Early (initial run-in of the bearings)
Normal
Suspect (the health seems to be deteriorating)
Imminent failure (for bearings 1 and 2, which didn’t actually fail, but were severely worn out)
Inner race failure (bearing 3)
Rolling element failure (bearing 4)
Stage 2 failure (bearing 4)
For the first test (the one we are working on), the following labels have been proposed per file:

Bearing 1
early: 2003.10.22.12.06.24 - 2013.10.23.09.14.13
suspect: 2013.10.23.09.24.13 - 2003.11.08.12.11.44 (bearing 1 was in suspicious health from the beginning, but showed some self-healing effects)
normal: 2003.11.08.12.21.44 - 2003.11.19.21.06.07
suspect: 2003.11.19.21.16.07 - 2003.11.24.20.47.32
imminent failure: 2003.11.24.20.57.32 - 2003.11.25.23.39.56

Bearing 2
early: 2003.10.22.12.06.24 - 2003.11.01.21.41.44
normal: 2003.11.01.21.51.44 - 2003.11.24.01.01.24
suspect: 2003.11.24.01.11.24 - 2003.11.25.10.47.32
imminent failure: 2003.11.25.10.57.32 - 2003.11.25.23.39.56

Bearing 3
early: 2003.10.22.12.06.24 - 2003.11.01.21.41.44
normal: 2003.11.01.21.51.44 - 2003.11.22.09.16.56
suspect: 2003.11.22.09.26.56 - 2003.11.25.10.47.32
Inner race failure: 2003.11.25.10.57.32 - 2003.11.25.23.39.56

Bearing 4
early: 2003.10.22.12.06.24 - 2003.10.29.21.39.46
normal: 2003.10.29.21.49.46 - 2003.11.15.05.08.46
suspect: 2003.11.15.05.18.46 - 2003.11.18.19.12.30
Rolling element failure: 2003.11.19.09.06.09 - 2003.11.22.17.36.56
Stage 2 failure: 2003.11.22.17.46.56 - 2003.11.25.23.39.56
"""

def StateInformation(UserInput,BearingNum):
    """
    Returns a Dictionary of a Bearing State

    StateInformation(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        BearingNum - Bearing Num to know which failure type
        )

    This function is used to generate a known outcome for the training data.
    This function is only intended to aid in generating the training data.
    """
    #Get Needed Info
    file = UserInput['File of Interest']

    #Comput time for comarison
    absolutetime = getAbsoluteTime(file)

    #Transitions according to the above comments
    #Bearing 1 transitions
    b1e2s  = getAbsoluteTime("2013.10.23.09.14.13")
    b1s2n  = getAbsoluteTime("2003.11.08.12.11.44")
    b1n2s  = getAbsoluteTime("2003.11.19.21.06.07")
    b1s2i  = getAbsoluteTime("2003.11.24.20.47.32")

    #Bearing 2 transitions
    b2e2n  = getAbsoluteTime("2003.11.01.21.41.44")
    b2n2s  = getAbsoluteTime("2003.11.24.01.01.24")
    b2s2i  = getAbsoluteTime("2003.11.25.10.47.32")

    #Bearing 3 transitions
    b3e2n  = getAbsoluteTime("2003.11.01.21.41.44")
    b3n2s  = getAbsoluteTime("2003.11.22.09.16.56")
    b3s2irf  = getAbsoluteTime("2003.11.25.10.47.32")

    #Bearing 4 transitions
    b4e2n  = getAbsoluteTime("2003.10.29.21.39.46")
    b4n2s  = getAbsoluteTime("2003.11.15.05.08.46")
    b4s2r  = getAbsoluteTime("2003.11.18.19.12.30")
    b4r2f  = getAbsoluteTime("2003.11.22.17.36.56")

    #Get state / output error if no state possible
    m = "ERROR"
    if BearingNum == 1:
        if absolutetime   <= b1e2s:
            m = "Early"
        elif absolutetime <= b1s2n:
            m = "Suspect"
        elif absolutetime <= b1n2s:
            m = "Normal"
        elif absolutetime <= b1s2i:
            m = "Suspect"
        elif absolutetime > b1s2i:
            m = "Imminent Failure"
    elif BearingNum == 2:
        if absolutetime   <= b2e2n:
            m = "Early"
        elif absolutetime <= b2n2s:
            m = "Normal"
        elif absolutetime <= b2s2i:
            m = "Suspect"
        elif absolutetime > b2s2i:
            m = "Imminent Failure"
    elif BearingNum == 3:
        if absolutetime   <= b3e2n:
            m = "Early"
        elif absolutetime <= b3n2s:
            m = "Normal"
        elif absolutetime <= b3s2irf:
            m = "Suspect"
        elif absolutetime >= b3s2irf:
            m = "Inner Race Failure"
    elif BearingNum == 4:
        if absolutetime   <= b4e2n:
            m = "Early"
        elif absolutetime <= b4n2s:
            m = "Normal"
        elif absolutetime <= b4s2r:
            m = "Suspect"
        elif absolutetime <= b4r2f:
            m = "Rolling Element Failure"
        elif absolutetime > b4r2f:
            m = "Stage 2 Failure"
    else:
        m = "ERROR"

    #NOT in the original model
    State2Int = StateDict()

    #Arrange
    x = {
        "State": State2Int[m]
    }
    return x

# In[24]:

def MotorInformation(UserInput):
    """
    Returns a Dictionary containg motor characteristics used in the IMS dataset

    MotorInformation(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    Only valid for IMS dataset
    """
    x = {
        "Motor Type AC(1)-DC(0)": 1,
        "Shaft Speed [Hz]": 2000/60
    }
    return x

# In[25]:

def getCompleteDataFrame(UserInput,BearingNum):
    """
    Returns a Dataframe for sample

    getCompleteDataFrame(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        BearingNum - Bearing Num to know which failure type
        )

    This function is used to generate a known outcome for the training data.
    This function is only intended to aid in generating the training data.
    """
    #Call specific function order for consistency
    UserInput1 = UserInput.copy()
    UserInput2 = RemoveDCOffset(UserInput1)
    UserInput3 = Normalize(UserInput2)
    BearingInfo = BearingInfomation(UserInput3)
    TimeDomainInfo = TimeDomainInformation(UserInput3)
    FrequecyDomainInfo = FrequencyDomainInformation(UserInput3)
    StateInfo = StateInformation(UserInput3,BearingNum)
    MotorInfo = MotorInformation(UserInput3)

    #Arrange
    Features = {**StateInfo,**MotorInfo,**BearingInfo,**TimeDomainInfo,**FrequecyDomainInfo}
    Features = pd.DataFrame(Features, index=[0])
    return Features

# In[26]:

def getTESTDataFrame(UserInput):
    """
    Returns a Dataframe that does not need the state

    getTESTDataFrame(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    This function generates a dataframe without knowing its state
    """
    #Call specific function order for consistency
    UserInput1 = UserInput.copy()
    UserInput2 = RemoveDCOffset(UserInput1)
    UserInput3 = Normalize(UserInput2)
    BearingInfo = BearingInfomation(UserInput3)
    TimeDomainInfo = TimeDomainInformation(UserInput3)
    FrequecyDomainInfo = FrequencyDomainInformation(UserInput3)
    MotorInfo = MotorInformation(UserInput3)

    #Arrange (with no state info)
    Features = {**MotorInfo,**BearingInfo,**TimeDomainInfo,**FrequecyDomainInfo}
    Features = pd.DataFrame(Features, index=[0])
    return Features

# In[27]:

def getTESTMatrix(UserInput):
    """

    """
    Features = getTESTDataFrame(UserInput)
    return Features.values[:,0:(Features.shape[1]-1)]

# In[28]:

def getPlot(X,Y,xlabel,ylabel,Title):
    """
    Subfunction of getGraphs
    Returns a figure

    getPlot(
        X - Data for independent variable
        Y - Data for dependent variable
        xlabel - X-axis label
        ylabel - Y-axis label
        Title - Title of figure
        )

    Performs plt.plot
    """

    #Plot
    fig = plt.figure()
    plt.plot(X,Y,c = np.random.rand(3,))
    plt.xlabel(xlabel, fontsize=12)
    plt.ylabel(ylabel, fontsize=12)
    plt.title(Title)
    plt.grid(True)

    return fig

# In[29]:

def getGraphs(UserInput):
    """
    Returns an array of figures

    getGraphs(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    This function generates a figures for:
    Raw time series
    Time series with no DC offset
    FFT
    PSD
    Correlation
    """
    #Create time series array
    t = np.arange(0,UserInput['Time of Sampling'],1/UserInput['Sampling Frequency'])

    #Perform FFT, PSD, Correlation, DC Offset
    UserInput1 = RemoveDCOffset(UserInput)
    UserInput2 = Normalize(UserInput1)
    x1 = FourierTransform(UserInput2)
    x2 = get_psd_values(UserInput2)
    x3 = get_autocorr_values(UserInput2)

    #Get Figures
    figs = []
    figs.append(getPlot(t,UserInput['Signal Data of Interest'],"time (s)","Amplitude","Raw Data"))
    figs.append(getPlot(t,UserInput1['Signal Data of Interest'],"time (s)","Amplitude","Raw Data w/ Removed DC Offset"))
    figs.append(getPlot(t,UserInput2['Signal Data of Interest'],"time (s)","Amplitude","Normalized Raw Data"))
    figs.append(getPlot(x1['Frequency'],x1['Freq. Amp.'],'Frequency [Hz]',"Normalized Amplitude","FFT"))
    figs.append(getPlot(x2['Frequency'],x2['PSD'],'Frequency [Hz]','Normalized PSD [V**2 / Hz]',"PSD"))
    figs.append(getPlot(x3['X Values'],x3['Autocorr Values'],'time delay [s]',"Normalized Autocorrelation amplitude","Autocorrelation"))

    return figs

# In[30]:

def getBarPlot(X,Y,xlabel,Title):
    """
    Subfunction of getGraphs
    Returns a figure

    getBarPlot(
        X - Data for independent variable
        Y - Data for dependent variable
        xlabel - X-axis label

        Title - Title of figure
        )

    Performs plt.barh
    """
    #Bar plot
    fig = plt.figure()
    y_pos = np.arange(len(Y))
    plt.barh(y_pos, X, align='center')
    plt.xlabel(xlabel, fontsize=12)
    plt.yticks(y_pos, Y)
    plt.title(Title)
    plt.grid(True)
    return fig

# In[31]:

def truncate(f, n):
    '''https://stackoverflow.com/questions/783897/truncating-floats-in-python/51172324#51172324'''
    '''Truncates/pads a float f to n decimal places without rounding'''
    s = '{}'.format(f)
    if 'e' in s or 'E' in s:
        return '{0:.{1}f}'.format(f, n)
    i, p, d = s.partition('.')
    return '.'.join([i, (d+'0'*n)[:n]])

# In[32]:

def GetSplitTrainingData(UserInput, seed = 6):
    """
    Returns an X_train, X_test, Y_train, Y_test

    getGraphs(UserInput)
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        seed - random number for splitting of test and trainig (default = 6)
        )

    This returns the training and test sets
    """

    #Find training file name and read it
    for file in UserInput['Working Directory']:
        if file == UserInput['TrainingFileName']:
            dataset = pd.read_csv(file,header = 0,index_col = 0)

    X = dataset.values[:,1:(dataset.shape[1]-1)]
    Y = dataset.values[:,0]
    validation_size = 0.20
    seed = seed
    X_train, X_test, Y_train, Y_test = model_selection.train_test_split(X, Y, test_size=validation_size, random_state=seed)

    return X_train, X_test, Y_train, Y_test

# In[33]:

def GetTrainingData(UserInput):
    """
    Returns X_train, Y_train

    getGraphs(UserInput)
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    This returns the training and test sets
    """

    #Find training file name and read it
    for file in UserInput['Working Directory']:
        if file == UserInput['TrainingFileName']:
            dataset = pd.read_csv(file,header = 0,index_col = 0)

    X_train = dataset.values[:,1:(dataset.shape[1]-1)]
    Y_train = dataset.values[:,0]

    return X_train, Y_train, dataset

# In[34]:

def GetTESTDataFrameNames(UserInput):
    """
    Returns an array of strings

    GetTESTDataFrameNames(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    This returns the names of each column of the training file
    """
    for file in UserInput['Working Directory']:
        if file == UserInput['TrainingFileName']:
            dataset = pd.read_csv(file,header = 0,index_col = 0)
    names = []
    for x in dataset.columns:
        names.append(x)
    return names

# In[35]:

def TrainModel(X_train,Y_train):
    """
    Returns a classifier that has been fit

    TrainModel(
        X_train - Training Data
        Y_train - Results of Training Data for supervised learning
        )

    Currently only fits RandomForestClassifier
    """
    classifier = RandomForestClassifier(min_samples_split= 10 ,n_estimators = 200)
    classifier = classifier.fit(X_train, Y_train)
    return classifier

# In[36]:

def get_key(value,dictionary):
    """

    Modified from:
    https://www.geeksforgeeks.org/python-get-key-from-value-in-dictionary/
    """
    result = []
    for x in value:
        for key, val in dictionary.items():
             if val == x:
                result.append( key )

    return result

# In[37]:

def PredictModel(classifier,X_test):
    """
    Returns a prediction in integer form, string form

    PredictModel(
        classifier - fitted classifier
        X_test - data to be tested
        )
    """
    State2Int = StateDict()
    Y_test_pred = classifier.predict(X_test)
    Y_test_pred_string = get_key(Y_test_pred,State2Int)
    return Y_test_pred, Y_test_pred_string

# In[38]:

def PredictProbModel(classifier,X_test):
    """
    Returns a prediction probability (out of 100 not 1)

    PredictModel(
        classifier - fitted classifier
        X_test - data to be tested
        )
    """
    Y_test_pred_proba = classifier.predict_proba(X_test)

    return Y_test_pred_proba*100

# In[39]:

def GetUserInputNames(UserInput):
    """
    Returns an array of strings

    GetUserInputNames(
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    This returns the names of the columns in UserInput
    """
    names = []
    for x in UserInput:
        names.append(x)
    return names

# In[40]:

def FeatureComparison(models,X_train, X_test, Y_train, Y_test,UserInput):
    """
    returns:
        results - an array of classification results
        string - an array of classifier names and classification results
        string1 - an array of classifiers that do not have feature importance graphs
        time - an array of time stamps recording how long a fit/prediction/classification
                result took for an figure
        fig - an array of figures containing feature importance for classifiers

    FeatureComparison(
        models - an array of classifiers for testing
        X_train - Training Data
        X_test - Data to be fitted
        Y_train - Results of Training Data for supervised learning
        Y_test - results of the fitted data
        UserInput - Dictionary of relevant info (see UserInputs2WorkingForm)
        )

    The function usefullness lies in its ability to test mass quantities of classifiers
    """
    results = []
    string = []
    string1 = []
    fig = []
    time = []
    for ModelName in models:

        before = datetime.now()

        #Fit Model
        error = False
        try:
            CTest = models[ModelName].fit(X_train, Y_train)
        except:
            print()
            error = True

        #Classification report
        if not error:
            try:
                Y_pred = CTest.predict(X_test)
                temporary = classification_report(Y_test,Y_pred)
                results.append(temporary)
                #CTest.score(X_test, Y_test))) #optional metric
                string.append("{} has the following results: \n\n {} \n\n".format(ModelName,temporary))
            except:
                try:
                    Y_pred = CTest.predict(X_test)
                    temporary = classification_report(Y_test,np.round_(Y_pred))
                    results.append(temporary)
                    #CTest.score(X_test, Y_test))) #optional metric
                    string.append("{} has the following results: \n\n {} \n\n".format(ModelName,temporary))
                except:
                    string.append("{} failed during classification_report.".format(ModelName))
        else:
            string.append("{} failed during the fit.".format(ModelName))

        #Time fitting and scoring took
        end = datetime.now()
        time.append("{} took {} time".format(ModelName,(end-before)))

        #Try figures
        try:
            m = CTest.feature_importances_
            m1 = GetTESTDataFrameNames(UserInput)
            Z = [x for _,x in sorted(zip(m,m1))]
            Z1 = sorted(m)
            fig.append(getBarPlot(Z1[-10:],Z[-10:],"Relative Importance",ModelName))
        except:
            string1.append("{} has no feature importance".format(ModelName))

    return results,string,string1,time,fig

# In[41]:

def GenerateComparisonResultFiles(results,string,string1,time,fig,str1 = "Graphs.pdf",                                  str2 = "Time.txt",str3 = "NoGraphs.txt",str4 = "Scoring.txt"):
    """
    returns true value upon completion

    GenerateComparisonResultFiles(
        results - an array of classification results
        string - an array of classifier names and classification results
        string1 - an array of classifiers that do not have feature importance graphs
        time - an array of time stamps recording how long a fit/prediction/classification
                result took for an figure
        fig - an array of figures containing feature importance for classifiers
        str1 - title of pdf file relating to figs (Default: "Graphs.pdf")
        str2 = title of txt file relating to time (Default: Time.txt")
        str3 = title of txt file relating to string1 (Default: "NoGraphs.txt")
        str4 = title of txt file relating to string (Default: "Scoring.txt")
        )

    The function generates 4 files.
    This function is to be used after FeatureComparison()
    """
    #PDF of figures
    pdf = matplotlib.backends.backend_pdf.PdfPages(str1)
    i = 0
    for figure in fig:
        pdf.savefig( fig[i],dpi=300, bbox_inches = "tight")
        i += 1
    pdf.close()

    #.txt of Time
    if not(not time):
        with open(str2, 'w') as writeFile:
            for i in np.arange(len(time)):
                writeFile.write("%(t)s\n" % {"t":time[i]})
    writeFile.close()

    #.txt of NoGraphs
    if not(not string1):
        with open(str3, 'w') as writeFile:
            for i in np.arange(len(string1)):
                writeFile.write("%(t)s\n" % {"t":string1[i]})
    writeFile.close()

    #.txt Scoring Results
    if not(not string):
        with open(str4, 'w') as writeFile:
            for i in np.arange(len(string)):
                writeFile.write("%(t)s\n" % {"t":string[i]})
    writeFile.close()

    return True

# In[42]:

def SignalGenerator(t):
    """
    returns numpy.ndarray

    SignalGenerator(
        t - numpy.ndarray
        )

    The file generates a random sine wave combination with white noise
    given an input time series
    """
    #Signal generator for practice
    noise1 = np.random.randn(len(t))                # white noise 1
    noise2 = np.random.randn(len(t))                # white noise 2
    noise3 = np.random.randn(len(t))                # white noise 3
    phase  = np.random.randn(3)                     #radians
    frequency1 = np.random.randint(1,1000)          #Hz
    frequency2 = np.random.randint(1,1000)          #Hz
    frequency3 = np.random.randint(1,1000)          #Hz
    mag = np.random.randn(3)
    base1 = mag[0] * np.sin(2 * np.pi * frequency1 * t + phase[0] ) + noise1  #base signal
    base2 = mag[1] * np.sin(2 * np.pi * frequency2 * t + phase[1] ) + noise2  #base signal
    base3 = mag[2] * np.sin(2 * np.pi * frequency3 * t + phase[2] ) + noise3  #base signal
    return base1 + base2 + base3
    #End of practice signal generator

# In[43]:

def GenerateTrainingFile(string):
    """

    """

    #Hard Coded file and changing directories
    HomeDirectory = "/Users/tbryan/Desktop/9 2019 Fall/ECEN 403/Programming/ProgramsForDemo/Final"
    os.chdir(HomeDirectory)
    os.chdir('Data')
    os.chdir('IMS')
    directory = [sorted(os.listdir('1st_test')),sorted(os.listdir('2nd_test')),sorted(os.listdir('3rd_test/txt'))]

    #directory.remove(".DS_Store")
    os.chdir('1st_test')
    #IMSDictionary = GenerateIMSDictionary('2003.10.22.12.06.24',"DELETE.csv",os.getcwd())
    IMSDictionary = GenerateIMSDictionary('2003.10.22.12.06.24'," ",os.getcwd())

    m1 = []
    m2 = []
    m3 = []
    m4 = []
    m1y = []
    m2y = []
    m3y = []
    m4y = []

    for j in range(0,1):
        i = 0
        while i < len(directory[j]):
        #while i < 20:
            if directory[j][i] != ".DS_Store":
                data = pd.read_table(directory[j][i],header = None)
                DF = ReplaceSignalDataofInterest(np.transpose(data.values[:,0]),IMSDictionary,directory[j][i])
                B1X = getCompleteDataFrame(DF,1)
                DF = ReplaceSignalDataofInterest(np.transpose(data.values[:,1]),IMSDictionary,directory[j][i])
                B1Y = getCompleteDataFrame(DF,1)
                DF = ReplaceSignalDataofInterest(np.transpose(data.values[:,2]),IMSDictionary,directory[j][i])
                B2X = getCompleteDataFrame(DF,2)
                DF = ReplaceSignalDataofInterest(np.transpose(data.values[:,3]),IMSDictionary,directory[j][i])
                B2Y = getCompleteDataFrame(DF,2)
                DF = ReplaceSignalDataofInterest(np.transpose(data.values[:,4]),IMSDictionary,directory[j][i])
                B3X = getCompleteDataFrame(DF,3)
                DF = ReplaceSignalDataofInterest(np.transpose(data.values[:,5]),IMSDictionary,directory[j][i])
                B3Y = getCompleteDataFrame(DF,3)
                DF = ReplaceSignalDataofInterest(np.transpose(data.values[:,6]),IMSDictionary,directory[j][i])
                B4X = getCompleteDataFrame(DF,4)
                DF = ReplaceSignalDataofInterest(np.transpose(data.values[:,7]),IMSDictionary,directory[j][i])
                B4Y = getCompleteDataFrame(DF,4)
                ColumnTitle = B1X.columns
                m1.append(B1X.values[0,:])
                m1y.append(B1Y.values[0,:])
                m2.append(B2X.values[0,:])
                m2y.append(B2Y.values[0,:])
                m3.append(B3X.values[0,:])
                m3y.append(B3Y.values[0,:])
                m4.append(B4X.values[0,:])
                m4y.append(B4Y.values[0,:])
                i += 1
            else:
                i += 1

        B1X = pd.DataFrame(m1,columns = ColumnTitle)
        B1Y = pd.DataFrame(m1y,columns = ColumnTitle)
        B2X = pd.DataFrame(m2,columns = ColumnTitle)
        B2Y = pd.DataFrame(m2y,columns = ColumnTitle)
        B3X = pd.DataFrame(m3,columns = ColumnTitle)
        B3Y = pd.DataFrame(m3y,columns = ColumnTitle)
        B4X = pd.DataFrame(m4,columns = ColumnTitle)
        B4Y = pd.DataFrame(m4y,columns = ColumnTitle)
        TD = B1X
        TD = TD.append(B1Y)
        TD = TD.append(B2X)
        TD = TD.append(B2Y)
        TD = TD.append(B3X)
        TD = TD.append(B3Y)
        TD = TD.append(B4X)
        TD = TD.append(B4Y)

    os.chdir(HomeDirectory)
    TD.to_csv(string)

    return True

# In[44]:

def GetAllModelsForComparison():
    """
    Returns a an array of all possible ML classifiers with "default" settings

    GetAllModelsForComparison()

    """
    #Arrange
    models = {
        'ARDRegression': ARDRegression(),
        'BayesianRidge': BayesianRidge(),
        'ElasticNet': ElasticNet(),
        'ElasticNetCV': ElasticNetCV(),
        'Hinge': Hinge(),
        #'Huber': Huber(), #Gave an error
        'HuberRegressor': HuberRegressor(),
        'Lars': Lars(),
        'LarsCV': LarsCV(),
        'Lasso': Lasso(),
        'LassoCV': LassoCV(),
        'LassoLars': LassoLars(),
        'LassoLarsCV': LassoLarsCV(),
        'LinearRegression': LinearRegression(),
        'Log': Log(),
        'LogisticRegression': LogisticRegression(),
        'LogisticRegressionCV': LogisticRegressionCV(),
        'ModifiedHuber': ModifiedHuber(),
        'MultiTaskElasticNet': MultiTaskElasticNet(),
        'MultiTaskElasticNetCV': MultiTaskElasticNetCV(),
        'MultiTaskLasso': MultiTaskLasso(),
        'MultiTaskLassoCV': MultiTaskLassoCV(),
        'OrthogonalMatchingPursuit': OrthogonalMatchingPursuit(),
        'OrthogonalMatchingPursuitCV': OrthogonalMatchingPursuitCV(),
        'PassiveAggressiveClassifier': PassiveAggressiveClassifier(),
        'PassiveAggressiveRegressor': PassiveAggressiveRegressor(),
        'Perceptron': Perceptron(),
        'RANSACRegressor': RANSACRegressor(),
        #'RandomizedLasso': RandomizedLasso(), #deprecated in 0.19 and will be removed in 0.21.
        #'RandomizedLogisticRegression': RandomizedLogisticRegression(), #deprecated in 0.19 and will be removed in 0.21.
        'Ridge': Ridge(),
        'RidgeCV': RidgeCV(),
        'RidgeClassifier': RidgeClassifier(),
        'SGDClassifier': SGDClassifier(),
        'SGDRegressor': SGDRegressor(),
        'SquaredLoss': SquaredLoss(),
        'TheilSenRegressor': TheilSenRegressor(),
        'BaseEstimator': BaseEstimator(),
        'ClassifierMixin': ClassifierMixin(),
        'LinearClassifierMixin': LinearClassifierMixin(),
        'LinearDiscriminantAnalysis': LinearDiscriminantAnalysis(),
        'QuadraticDiscriminantAnalysis': QuadraticDiscriminantAnalysis(),
        'StandardScaler': StandardScaler(),
        'TransformerMixin': TransformerMixin(),
        'BaseEstimator': BaseEstimator(),
        'KernelRidge': KernelRidge(),
        'RegressorMixin': RegressorMixin(),
        'LinearSVC': LinearSVC(),
        'LinearSVR': LinearSVR(),
        'NuSVC': NuSVC(),
        'NuSVR': NuSVR(),
        'OneClassSVM': OneClassSVM(),
        'SVC': SVC(),
        'SVR': SVR(),
        'SGDClassifier': SGDClassifier(),
        'SGDRegressor': SGDRegressor(),
        'BallTree': BallTree(X_train), #Needs Data
        #'DistanceMetric': DistanceMetric(), #DistanceMetric is an abstract class
        'KDTree': KDTree(X_train), #Needs Data
        'KNeighborsClassifier': KNeighborsClassifier(),
        'KNeighborsRegressor': KNeighborsRegressor(),
        'KernelDensity': KernelDensity(),
        #'LSHForest': LSHForest(), #deprecated in 0.19. It will be removed in version 0.21.
        'LocalOutlierFactor': LocalOutlierFactor(),
        'NearestCentroid': NearestCentroid(),
        'NearestNeighbors': NearestNeighbors(),
        'RadiusNeighborsClassifier': RadiusNeighborsClassifier(),
        'RadiusNeighborsRegressor': RadiusNeighborsRegressor(),
        #'GaussianProcess': GaussianProcess(), #deprecated in version 0.18 and will be removed in 0.20.
        'GaussianProcessRegressor': GaussianProcessRegressor(),
        'GaussianProcessClassifier': GaussianProcessClassifier(),
        'CCA': CCA(),
        'PLSCanonical': PLSCanonical(),
        'PLSRegression': PLSRegression(),
        'PLSSVD': PLSSVD(),
        #'ABCMeta': ABCMeta(), #Gave an error
        #'BaseDiscreteNB': BaseDiscreteNB(), #Can't instantiate abstract class BaseNB with abstract methods _joint_log_likelihood
        'BaseEstimator': BaseEstimator(),
        #'BaseNB': BaseNB(), #Can't instantiate abstract class BaseNB with abstract methods _joint_log_likelihood
        'BernoulliNB': BernoulliNB(),
        'ClassifierMixin': ClassifierMixin(),
        'GaussianNB': GaussianNB(),
        'LabelBinarizer': LabelBinarizer(),
        'MultinomialNB': MultinomialNB(),
        'DecisionTreeClassifier': DecisionTreeClassifier(),
        'DecisionTreeRegressor': DecisionTreeRegressor(),
        'ExtraTreeClassifier': ExtraTreeClassifier(),
        'AdaBoostClassifier': AdaBoostClassifier(),
        'AdaBoostRegressor': AdaBoostRegressor(),
        'BaggingClassifier': BaggingClassifier(),
        'BaggingRegressor': BaggingRegressor(),
        #'BaseEnsemble': BaseEnsemble(), #Can't instantiate abstract class BaseEnsemble with abstract methods __init__
        'ExtraTreesClassifier': ExtraTreesClassifier(),
        'ExtraTreesRegressor': ExtraTreesRegressor(),
        'GradientBoostingClassifier': GradientBoostingClassifier(),
        'GradientBoostingRegressor': GradientBoostingRegressor(),
        'IsolationForest': IsolationForest(),
        'RandomForestClassifier': RandomForestClassifier(),
        'RandomForestRegressor': RandomForestRegressor(),
        'RandomTreesEmbedding': RandomTreesEmbedding(),
        'VotingClassifier': VotingClassifier(estimators=[('LR', LogisticRegression()), \
                                    ('RFC', RandomForestClassifier()), ('GBC', GradientBoostingClassifier())]), #Needs estimator
        'BaseEstimator': BaseEstimator(),
        'ClassifierMixin': ClassifierMixin(),
        'LabelBinarizer': LabelBinarizer(),
        'MetaEstimatorMixin': MetaEstimatorMixin(),
        'OneVsOneClassifier': OneVsOneClassifier(estimator=[('LR', LogisticRegression()), \
                                    ('RFC', RandomForestClassifier()), ('GBC', GradientBoostingClassifier())]), #Needs estimator
        'OneVsRestClassifier': OneVsRestClassifier(estimator=[('LR', LogisticRegression()), \
                                    ('RFC', RandomForestClassifier()), ('GBC', GradientBoostingClassifier())]), #Needs estimator
        'OutputCodeClassifier': OutputCodeClassifier(estimator=[('LR', LogisticRegression()), \
                                    ('RFC', RandomForestClassifier()), ('GBC', GradientBoostingClassifier())]), #Needs estimator
        'Parallel': Parallel(),
        #'ABCMeta': ABCMeta(), #Gave an error
        'BaseEstimator': BaseEstimator(),
        'ClassifierChain': ClassifierChain(base_estimator=[('LR', LogisticRegression()), \
                                    ('RFC', RandomForestClassifier()), ('GBC', GradientBoostingClassifier())]), #Needs estimator
        'ClassifierMixin': ClassifierMixin(),
        'MetaEstimatorMixin': MetaEstimatorMixin(),
        'MultiOutputClassifier': MultiOutputClassifier(estimator=[('LR', LogisticRegression()), \
                                    ('RFC', RandomForestClassifier()), ('GBC', GradientBoostingClassifier())]), #Needs estimator
        #'MultiOutputEstimator': MultiOutputEstimator(), #Can't instantiate abstract class MultiOutputEstimator with abstract methods __init__
        'MultiOutputRegressor': MultiOutputRegressor(estimator=[('LR', LogisticRegression()), \
                                    ('RFC', RandomForestClassifier()), ('GBC', GradientBoostingClassifier())]), #Needs estimator
        'Parallel': Parallel(),
        'RegressorMixin': RegressorMixin(),
        'LabelPropagation': LabelPropagation(),
        'LabelSpreading': LabelSpreading(),
        'BaseEstimator': BaseEstimator(),
        'IsotonicRegression': IsotonicRegression(),
        'RegressorMixin': RegressorMixin(),
        'TransformerMixin': TransformerMixin(),
        'BernoulliRBM': BernoulliRBM(),
        'MLPClassifier': MLPClassifier(),
        'MLPRegressor': MLPRegressor()
        }
    return models

# In[45]:

def GetFinalEightModelsForComparison(MinSampleSplit=2):
    """
    Returns a dictionary of the 8 best possible ML classifiers with ability to change the
    min_samples_split setting

    GetFinalEightModelsForComparison(
        MinSampleSplit - integer to change min_samples_split (default = 2)
        )

    """
    #Arrange
    models = {
        'DecisionTreeClassifier': DecisionTreeClassifier(min_samples_split=MinSampleSplit),
        'DecisionTreeRegressor': DecisionTreeRegressor(min_samples_split=MinSampleSplit),
        'ExtraTreeClassifier': ExtraTreeClassifier(min_samples_split=MinSampleSplit),
        'ExtraTreesClassifier': ExtraTreesClassifier(min_samples_split=MinSampleSplit),
        'ExtraTreesRegressor': ExtraTreesRegressor(min_samples_split=MinSampleSplit),
        'GradientBoostingClassifier': GradientBoostingClassifier(min_samples_split=MinSampleSplit),
        'RandomForestClassifier': RandomForestClassifier(min_samples_split=MinSampleSplit),
        'RandomForestRegressor': RandomForestRegressor(min_samples_split=MinSampleSplit)

        }
    return models

# In[46]:

def GetOnlyTwoModelsForComparison():
    """
    Returns a dictionary of just TWO possible ML classifiers with "default" settings

    GetOnlyTwoModelsForComparison()

    """
    #Arrange
    models = {
        'RandomForestClassifier': RandomForestClassifier(),
        'ExtraTreesClassifier': ExtraTreesClassifier()
    }

    return models

# In[47]:

def GetFinalModelForComparison(MinSampleSplit=2, Nestimators = 10):
    """
    Returns a dictionary of the the final ML classifier with ability to change the
    min_samples_split and n_estimators setting

    GetFinalModelForComparison(MinSampleSplit=2, Nestimators = 10)
    """
    #Arrange
    models = {
        'RandomForestClassifier': RandomForestClassifier(min_samples_split=MinSampleSplit,n_estimators = Nestimators),

        }
    return models

# In[22]:

def StateDict():
    """
    Returns a dictionary "Key": Value

    StateDict()

    The key is the more verbose describtion of the output state of the training data
    The value is the int that is defined to relate to the corresponding key.
    """

    State2Int = {
        "Early": 0,
        "Suspect": 1,
        "Normal": 2,
        "Imminent Failure": 3,
        "Inner Race Failure": 4,
        "Rolling Element Failure": 5,
        "Stage 2 Failure": 6,
        "ERROR": 77777
    }

    return State2Int

# In[48]:

def GetReducedFeatureArraysFromDataFrame(dataset):
    """

    """
    ReducedFeatureTrainingData = {
        '2':  dataset[['RMS','FTF']],
        '3':  dataset[['RMS','FTF','Max ABS']],
        '4':  dataset[['RMS','FTF','Max ABS','Skew']],
        '5':  dataset[['RMS','FTF','Max ABS','Skew','PSD Frq @ Peak 5']],
        '6':  dataset[['RMS','FTF','Max ABS','Skew','PSD Frq @ Peak 5','Autocorrelate Time @ Peak 5']],
        '7':  dataset[['RMS','FTF','Max ABS','Skew','PSD Frq @ Peak 5','Autocorrelate Time @ Peak 5','Min']],
        '8':  dataset[['RMS','FTF','Max ABS','Skew','PSD Frq @ Peak 5','Autocorrelate Time @ Peak 5','Min','PSD Frq @ Peak 1']],
        '9':  dataset[['RMS','FTF','Max ABS','Skew','PSD Frq @ Peak 5','Autocorrelate Time @ Peak 5','Min','PSD Frq @ Peak 1','FFT Frq @ Peak 1']],
        '10': dataset[['RMS','FTF','Max ABS','Skew','PSD Frq @ Peak 5','Autocorrelate Time @ Peak 5','Min','PSD Frq @ Peak 1','FFT Frq @ Peak 1','PSD Frq @ Peak 2']],
        '3a': dataset[['RMS','Max ABS','Skew']],
        '4a': dataset[['RMS','Max ABS','Skew','Max']],
        '5a': dataset[['RMS','Max ABS','Skew','Max','Min']],
        'all': dataset.drop(columns=['State'])
    }

    return ReducedFeatureTrainingData, dataset[['State']]

# In[49]:

def GetFinalReducedFeatureArraysFromDataFrame(dataset):
    """

    """
    ReducedFeatureTrainingData = {
        '10': dataset[['RMS','FTF','Max ABS','Skew','PSD Frq @ Peak 5','Autocorrelate Time @ Peak 5','Min','PSD Frq @ Peak 1','FFT Frq @ Peak 1','PSD Frq @ Peak 2']],
    }

    return ReducedFeatureTrainingData, dataset[['State']]

# In[50]:

def Get10FeaturesFromDataFrame(df):
    """

    """
    x = df[['RMS','FTF','Max ABS','Skew','PSD Frq @ Peak 5','Autocorrelate Time @ Peak 5','Min','PSD Frq @ Peak 1','FFT Frq @ Peak 1','PSD Frq @ Peak 2']]

    return x

# In[51]:

def ReducedFeatureComparison(ReducedFeatureTrainingData,StateTrain):
    """

    """
    result = []
    header = []
    validation_size = 0.20
    seed = 6
    for set in ReducedFeatureTrainingData:
        FeatureTrain = ReducedFeatureTrainingData[set].values
        xtrain, xtest, ytrain, ytest = model_selection.train_test_split(FeatureTrain, StateTrain.values[:,0], test_size=validation_size, random_state=seed)

        ClassifierReduced = TrainModel(xtrain,ytrain)
        ypred,ypredstring = PredictModel(ClassifierReduced,xtest)
        result.append(classification_report(ytest,ypred))
        header.append('Feature Number ({}):'.format(set))

    return result,header

# In[52]:

def GenerateFeatureFile(string,result,header):
    """

    """
    with open(string, 'w') as writeFile:
        for i in np.arange(len(result)):
            writeFile.write("{}\n{}\n".format(header[i],result[i]))
    writeFile.close()

    return True


# encoding: utf-8
"""
@version: 1.0
@author:
@file: audio_filter
@time: 2021/11/9 10:11
"""
#  在频率域进行音频滤波
# 来源: https://blog.csdn.net/qq_39798423/article/details/86764443

import wave as we
import numpy as np
import matplotlib.pyplot as plt

dir = r'C:\\Users\\MichellJou\\Desktop\\音频识别数据集\\'

def read_wav(wavfile, plots=True, normal=False):
    f = wavfile
    params = f.getparams()
    # print(params)
    nchannels, sampwidth, framerate, nframes = params[:4]
    strData = f.readframes(nframes)  # 读取音频，字符串格式
    waveData = np.frombuffer(strData, dtype=np.int16)  # 将字符串转化为int
    # wave幅值归一化
    if normal == True:
        waveData = waveData*1.0/(max(abs(waveData)))
    # 绘图
    if plots == True:
        time = np.arange(0, nframes)*(1.0 / framerate)
        plt.figure(dpi=100)
        plt.plot(time, waveData)
        plt.xlabel("Time")
        plt.ylabel("Amplitude")
        plt.title("Single channel wavedata")
        plt.show()
        print(type(waveData))
    return (waveData, time)

def fft_wav(waveData, plots=True):
    f_array = np.fft.fft(waveData)  # 傅里叶变换，结果为复数数组
    f_abs = f_array
    axis_f = np.linspace(2500, 3000, np.int64(len(f_array)/2))  # 映射到250
    # print(type(axis_f),type(f_abs))
    # axis_f = np.linspace(0, 250, np.int(len(f_array)))  # 映射到250
    if plots == True:
        plt.figure(dpi=100)
        plt.plot(axis_f, np.abs(f_abs[0:len(axis_f)]))
        # plt.plot(axis_f, np.abs(f_abs))
        plt.xlabel("Frequency")
        plt.ylabel("Amplitude spectrum")
        plt.title("Tile map")
        plt.show()
    return f_abs

f = we.open(dir+ r'切割.wav', 'rb')
waveData, time = read_wav(f)

wavefft = fft_wav(waveData)

# 截取
step_hz = 20 / (len(waveData) / 2)
print(step_hz,type(step_hz))

tab_hz = 20
savewav = []
for i in range(int(tab_hz/step_hz)):
    savewav.append(wavefft[i])
for j in range(int(tab_hz/step_hz), (len(wavefft) - int(tab_hz/step_hz))):
    savewav.append(0)
for i in range((len(wavefft) - int(tab_hz/step_hz)), len(wavefft)):
    savewav.append(wavefft[i])
# 绘图
axis_f = np.linspace(0, 2500, np.int64(len(wavefft)/2))  # 映射到250
# axis_f = np.linspace(0, 250, np.int(len(wavefft)))  # 映射到250
plt.figure(dpi=100)
plt.plot(axis_f, np.abs(savewav[0:len(axis_f)]))
# plt.plot(axis_f, np.abs(savewav))
plt.xlabel("Frequency")
plt.ylabel("Amplitude spectrum")
plt.title("Tile map after wave filtering")
plt.show()

# 傅里叶逆变换
i_array = np.fft.ifft(savewav)
# 绘图
plt.figure(dpi=100)
plt.plot(time, i_array.real)
plt.xlabel("Time")
plt.ylabel("Amplitude")
plt.title("Single channel wavedata after wave filtering")
plt.show()

# 保存
save_wav = i_array.real.reshape((len(i_array), 1)).T.astype(np.short)
# print(save_wav.shape)
# i_array.real.tofile(dir+r'\test.bin')

f = we.open(dir+r'\提取5.wav', "wb")
# 配置声道数、量化位数和取样频率
f.setnchannels(1)
f.setsampwidth(2)
f.setframerate(16000)
# 将wav_data转换为二进制数据写入文件
f.writeframes(save_wav.tobytes())
f.close()


import requests
from bs4 import BeautifulSoup
import csv

def get_html(url):
    response = requests.get(url)
    return response

def write_csv(data):
    with open('parsing_011_Video_title.csv', 'a', newline='', encoding='utf-8') as file:
        order = ['name', 'url']
        writer = csv.DictWriter(file, delimiter='|', fieldnames=order)
        writer.writerow(data)

def get_page_data(response):
    if 'html' in response.headers['Content-Type']:
        html = response.text
    else:
        html = response.json()['content_html']

    soup = BeautifulSoup(html, 'lxml')

    items = soup.find_all('h3', class_='yt-lockup-title')

    for item in items:
        name = item.text.strip()
        url = item.find('a').get('href')

        data = {'name': name, 'url': url}
        write_csv(data)

def get_next(response):
    if 'html' in response.headers['Content-Type']:
        html = response.text
    else:
        html = response.json()['load_more_widget_html']

    soup = BeautifulSoup(html, 'lxml')

    try:
        url = 'https://youtube.com' + soup.find('button', class_='load-more-button').get('data-uix-load-more-href')
    except:
        url = ''

    return url

def main():
    # url = 'https://www.youtube.com/browse_ajax?action_continuation=1&continuation=4qmFsgI0EhhVQzZjcWF6U1I2Q25WTUNsWTBiSkkwTGcaGEVnWjJhV1JsYjNNZ0FEZ0JlZ0V5dUFFQQ%253D%253D&amp'
    url = 'https://www.youtube.com/user/TheBadComedian/videos'

    while True:
        response = get_html(url)
        get_page_data(response)

        url = get_next(response)

        if url:
            continue
        else:
            break

if __name__ == '__main__':
    main()


#coding:utf-8
#   Copyright (c) 2019  PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import time

import paddle.fluid as fluid
import numpy as np

from paddlehub.common.logger import logger
import paddlehub as hub

# Sequence label evaluation functions
def chunk_eval(np_labels, np_infers, np_lens, tag_num, dev_count=1):
    def extract_bio_chunk(seq):
        chunks = []
        cur_chunk = None
        null_index = tag_num - 1
        for index in range(len(seq)):
            tag = seq[index]
            tag_type = tag // 2
            tag_pos = tag % 2

            if tag == null_index:
                if cur_chunk is not None:
                    chunks.append(cur_chunk)
                    cur_chunk = None
                continue

            if tag_pos == 0:
                if cur_chunk is not None:
                    chunks.append(cur_chunk)
                    cur_chunk = {}
                cur_chunk = {"st": index, "en": index + 1, "type": tag_type}

            else:
                if cur_chunk is None:
                    cur_chunk = {"st": index, "en": index + 1, "type": tag_type}
                    continue

                if cur_chunk["type"] == tag_type:
                    cur_chunk["en"] = index + 1
                else:
                    chunks.append(cur_chunk)
                    cur_chunk = {"st": index, "en": index + 1, "type": tag_type}

        if cur_chunk is not None:
            chunks.append(cur_chunk)
        return chunks

    null_index = tag_num - 1
    num_label = 0
    num_infer = 0
    num_correct = 0
    labels = np_labels.reshape([-1]).astype(np.int32).tolist()
    infers = np_infers.reshape([-1]).astype(np.int32).tolist()
    all_lens = np_lens.reshape([dev_count, -1]).astype(np.int32).tolist()

    base_index = 0
    for dev_index in range(dev_count):
        lens = all_lens[dev_index]
        max_len = 0
        for l in lens:
            max_len = max(max_len, l)

        for i in range(len(lens)):
            seq_st = base_index + i * max_len + 1
            seq_en = seq_st + (lens[i] - 2)
            infer_chunks = extract_bio_chunk(infers[seq_st:seq_en])
            label_chunks = extract_bio_chunk(labels[seq_st:seq_en])
            num_infer += len(infer_chunks)
            num_label += len(label_chunks)

            infer_index = 0
            label_index = 0
            while label_index < len(label_chunks) \
                   and infer_index < len(infer_chunks):
                if infer_chunks[infer_index]["st"] \
                    < label_chunks[label_index]["st"]:
                    infer_index += 1
                elif infer_chunks[infer_index]["st"] \
                    > label_chunks[label_index]["st"]:
                    label_index += 1
                else:
                    if infer_chunks[infer_index]["en"] \
                        == label_chunks[label_index]["en"] \
                        and infer_chunks[infer_index]["type"] \
                        == label_chunks[label_index]["type"]:
                        num_correct += 1

                    infer_index += 1
                    label_index += 1

        base_index += max_len * len(lens)

    return num_label, num_infer, num_correct

def calculate_f1(num_label, num_infer, num_correct):
    if num_infer == 0:
        precision = 0.0
    else:
        precision = num_correct * 1.0 / num_infer

    if num_label == 0:
        recall = 0.0
    else:
        recall = num_correct * 1.0 / num_label

    if num_correct == 0:
        f1 = 0.0
    else:
        f1 = 2 * precision * recall / (precision + recall)
    return precision, recall, f1


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import migrations, models
import datetime
from django.utils.timezone import utc

class Migration(migrations.Migration):

    dependencies = [
        ('research', '0002_book_cover'),
    ]

    operations = [
        migrations.CreateModel(
            name='Patent',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('name', models.CharField(max_length=200, verbose_name='\u4e13\u5229\u540d\u79f0')),
                ('inventor', models.CharField(max_length=200, verbose_name='\u53d1\u660e\u8005')),
                ('serial_number', models.CharField(max_length=100, verbose_name='\u4e13\u5229\u53f7')),
                ('date', models.DateTimeField(verbose_name='\u7533\u8bf7\u65e5\u671f')),
                ('create_timestamp', models.DateTimeField(auto_now_add=True)),
                ('last_update_timestamp', models.DateTimeField(auto_now=True)),
            ],
            options={
                'verbose_name': '\u53d1\u660e\u4e13\u5229',
                'verbose_name_plural': '\u53d1\u660e\u4e13\u5229',
            },
        ),
        migrations.AddField(
            model_name='book',
            name='create_timestamp',
            field=models.DateTimeField(default=datetime.datetime(2016, 1, 22, 15, 59, 34, 353984, tzinfo=utc), auto_now_add=True),
            preserve_default=False,
        ),
        migrations.AddField(
            model_name='book',
            name='last_update_timestamp',
            field=models.DateTimeField(default=datetime.datetime(2016, 1, 22, 15, 59, 58, 844809, tzinfo=utc), auto_now=True),
            preserve_default=False,
        ),
        migrations.AlterField(
            model_name='book',
            name='cover',
            field=models.ImageField(upload_to=b'books', verbose_name='\u5c01\u9762'),
        ),
    ]


import matplotlib.pyplot as plt
import numpy as np
#define a function to calculate the iteration that is needed to find the integer 1
def func_f(n):
	if n ==1:
		return 0
	else:
		count = 0
		result = n
		while(result!=1):
			count+=1
			if result%2 == 0:
				result = result//2
			else:
				result = result*3+1
		return count

#define a function that return the dictionary L, where L[k] is the number of times one needs to apply f to turn k into 1
def func_L_k(k):
	L = {}
	for i in range(1,k+1):
		L[i] = func_f(i)
	return L

def func_T_k(k):
	dic_L = func_L_k(k)
	sum = 0
	for item in dic_L:
		sum+=dic_L[item]
	return sum

def plot_fun(k):
	X = [i for i in range(1,k+1)]
	Y = [func_T_k(item) for item in X]
	plt.xlabel("k")
	plt.ylabel("T(k)")

	#fitting with linear and quadratic
	linear_A = np.matrix([[1.0,x] for x in X]).T
	quadratic_A = np.matrix([[1.0,x,x**2] for x in X]).T
	y = np.matrix(Y).T

	linear_c = np.linalg.inv(linear_A*linear_A.T)*linear_A*y
	quadratic_c = np.linalg.inv(quadratic_A*quadratic_A.T)*quadratic_A*y

	dom = np.linspace(min(X), max(X), 1000)
	linear_ran = [linear_c[0,0] + linear_c[1,0]*x  for x in dom]
	quadratic_ran = [quadratic_c[0,0] + quadratic_c[1,0]*x + quadratic_c[2,0]*x**2 for x in dom]
	plt.plot(dom, linear_ran, 'b-', label='y=$%2.2fx+%2.2f$' %(linear_c[1,0], linear_c[0,0]))
	plt.plot(dom, quadratic_ran, 'r-', label='y=$%2.6fx^2+%2.2fx+%2.2f$' %(quadratic_c[2,0],quadratic_c[1,0], quadratic_c[0,0]))
	plt.legend()

	# plt.plot(x,y)
	plt.scatter(X,Y,alpha = 0.1)
	plt.show()

def main():
	# plot_fun(500)
	for i in range(1,11):
		print(func_f(i))

	# print the value of T(10), T(100), T(1000), T(10000), T(100000)
	# test_k = [10, 100, 1000, 10000, 100000]
	# result = [func_T_k(x) for x in test_k]
	# print("The result of T(10), T(100), T(1000), T(10000), T(100000) in a list is: ",result)

if __name__ == '__main__':
	main()


from maximo import maximo
from media import media
from moda import moda
from mediana import mediana
from desviacion_estandar import desviacion_estandar
from permutacion import permutacion
from variaciones_sin_repeticion import variaciones_sin_repeticion
from variaciones_con_repeticion import variaciones_con_repeticion
from generar_archivo_salida import generar_archivo_salida
from lista_enlazada import *

def ejecutar_funciones(nombre_del_archivo, funcion, lista_parametros):
    resultado = None

    if funcion in ["variaciones_con_repeticion", "variaciones_sin_repeticion"]:

        if len(lista_parametros) > 3:
            longitud_subconjunto = int( lista_parametros[3] )

        else:
            print("Se necesita tercer parámetro para el tamaño del subconjunto")
            exit()

    lista_archivo = ListaEnlazada()
    with open(nombre_del_archivo) as archivo_entrada:

        for linea in archivo_entrada:
            lista_archivo.insertar_ultimo( int( linea.rstrip('\n') ) )

    try:

        if funcion in ["variaciones_con_repeticion", "variaciones_sin_repeticion"]:
            resultado = obtener_funcion(funcion, True)(lista_archivo, longitud_subconjunto)

        else:
            resultado = obtener_funcion(funcion)(lista_archivo)

    except KeyError:
        print("Función no válida")
        exit()

    """if funcion in locals():
        if funcion in ["variaciones_con_repeticion", "variaciones_sin_repeticion"]:
            resultado = locals()[funcion](lista_archivo, r)
        else:
            resultado = locals()[funcion](lista_archivo)
    else:
        print("Función no válida")"""

    generar_archivo_salida(resultado, funcion)

def obtener_funcion(funcion, extra = False):
    diccionario_variaciones = {"variaciones_sin_repeticion" : variaciones_sin_repeticion,
                               "variaciones_con_repeticion" : variaciones_con_repeticion
                               }

    diccionario_otras_funciones = {"maximo" : maximo,
                                   "media" : media,
                                   "mediana" : mediana,
                                   "moda" : moda,
                                   "desviacion_estandar" : desviacion_estandar,
                                   "permutacion" : permutacion
                                   }

    if extra:
        return diccionario_variaciones[funcion]

    return diccionario_otras_funciones[funcion]


#2-7: implement a function that checks if a linked list is a palindrome
#2-5: Adding numbers in LinkedList

class Node:
    def __init__(self, data=None):
        self.data=data
        self.next=None #pointer to next node

class LinkedList:
    def __init__(self):
        self.head = Node() #head node should always be available. Doesn't contain data and shouldn't be indexiable
        #place holder
    def append(self, data):
        """
        Given a data point, append new node to linked list.
        """
        new_node = Node(data)
        current_node = self.head
        while current_node.next!=None:
            current_node = current_node.next
        current_node.next = new_node #when we are at the last node, set it's pointer to point at the new Node
    def length(self):
        """
        Returns length of linked list
        """
        current_node = self.head
        size = 0
        while current_node.next!=None:
            size += 1
            current_node = current_node.next
        return size
    def display(self):
        """
        Displays all items in the linked list
        """
        elems = [] #create a list of elements we've seen
        current_node = self.head
        while current_node.next!=None:
            current_node = current_node.next
            elems.append(current_node.data)
        print(elems)
    def get(self, index):
        """
        Returns the data point at the given index in the linked list
        """
        if index >= self.length():
            print("ERROR")
            return None
        current_index = 0
        current_node = self.head
        while True:
            current_node = current_node.next
            if current_index == index: return current_node.data
            current_index += 1
    def get_index(self, data):
        """
        Return the index at the given data point in the linked list
        """
        current_node = self.head
        current_index = 0
        while current_node.next != None:
            current_node = current_node.next
            if current_node.data == data:
                return current_index
            current_index += 1
        print("data doesn't exist")
        return None
    def erase(self, index):
        """
        Given the index, will erase the node in the linked list
        """
        if index >= self.length():
            print("ERROR")
            return None
        current_index = 0
        current_node = self.head
        while True:
            last_node = current_node
            current_node = current_node.next
            if current_index == index:
                last_node.next = current_node.next
                return
            current_index += 1
    def delete(self, data):
        """
        Given the data point, will delete the node in the linked list
        """
        current_node = self.head
        current_index = 0
        index = self.get_index(data)
        while current_node.next != None:
            last_node = current_node
            current_node = current_node.next
            if current_index == index:
                last_node.next = current_node.next
                return
            current_index += 1
    def insert_index(self, index, data):
        """
        Given the data point and index, will insert new node at the given index
        """
        new_node = Node(data)
        if index >= self.length():
            print("ERROR")
            return None
        current_index = 0
        current_node = self.head
        while True:
            last_node = current_node
            current_node = current_node.next
            if current_index == index:
                last_node.next = new_node
                new_node.next = current_node
                return
            current_index += 1

    def split(self, data):
        """
        Given a data point, will split linked list so everything less than
        will come before and everything greater than will come after
        [2, 1, 3, 4, 9, 6, 5, 7, 8] = split(4)
        """
        index = self.get_index(data)
        less_than = []
        greater_than = []
        current_node = self.head
        current_index = 0
        while current_node.next != None:
            current_node = current_node.next
            if current_node.data > data:
                greater_than.append(current_node.data)
            if current_node.data < data:
                   less_than.append(current_node.data)
            self.delete(current_node.data)
            current_index += 1
        for item in less_than:
            self.append(item)
        self.append(data)
        for item in greater_than:
            self.append(item)
    def sort(self):
        """
        Sorts all elements of list
        """
        elems = []
        current_node = self.head
        while current_node.next != None:
            current_node = current_node.next
            elems.append(current_node.data)
            self.delete(current_node.data)
        elems.sort()
        for item in elems:
            self.append(item)
    def concat_list(self, my_list=[], *args):
        result = ''
        for item in my_list:
            result += str(item)
        return result
    def addition(self):
        """
        Each node contains a single digit. Adding two different numbers and appending
        the result to the linked list
        """
        middle = int(self.length()//2)
        current_node = self.head
        current_index = 0
        less_than = []
        greater_than = []
        while current_node.next != None:
            current_node = current_node.next
            if current_index <= middle:
                less_than.append(current_node.data)
            else:
                greater_than.append(current_node.data)
            current_index += 1
        n1 = self.concat_list(less_than)
        n2 = self.concat_list(greater_than)
        self.append(int(n1)+int(n2))
        return int(n1)+int(n2)
    def reverse_addition(self):
        """
        Same as class method above, just reversed.
        """
        middle = int(self.length()//2)
        current_node = self.head
        current_index = 0
        less_than = []
        greater_than = []
        while current_node.next != None:
            current_node = current_node.next
            if current_index <= middle:
                less_than.append(current_node.data)
            else:
                greater_than.append(current_node.data)
            current_index += 1
        less_than.reverse()
        greater_than.reverse()
        n1 = self.concat_list(less_than)
        n2 = self.concat_list(greater_than)
        self.append(int(n1)+int(n2))
        return int(n1)+int(n2)

def palindrome(LinkedList):
    end = LinkedList.length()-1
    beg = 0
    while (end > beg):
        if LinkedList.get(beg) != LinkedList.get(end):
            return False
        beg += 1
        end -= 1
    return True

my_list = ["M", "O", "O", "M", ]
a = LinkedList()
for item in my_list:
    a.append(item)
a.display()
print(palindrome(a))


from django import template
from django import forms
from django.forms.forms import NON_FIELD_ERRORS
from django.forms.utils import ErrorDict
from django.utils.text import normalize_newlines
from django.template.defaultfilters import stringfilter
from django.utils.safestring import SafeData, mark_safe
from django.utils.html import escape

register = template.Library()

@register.filter(is_safe=True, needs_autoescape=True)
@stringfilter
def linebreaksxml(value, autoescape=True):
    autoescape = autoescape and not isinstance(value, SafeData)
    value = normalize_newlines(value)
    if autoescape:
        value = escape(value)
    return mark_safe(value.replace('\n', '<br />'))

@register.filter
def multiply(value, arg):
    return value * arg

@register.filter
def to_class_name(value):
    return value.__class__.__name__

@register.filter
def nice_errors(form, non_field_msg='General form errors'):
    nice_errors = ErrorDict()
    if isinstance(form, forms.BaseForm):
        for field, errors in list(form.errors.items()):
            if field == NON_FIELD_ERRORS:
                key = non_field_msg
            else:
                key = form.fields[field].label
            nice_errors[key] = errors
    return nice_errors

def paginator(context, adjacent_pages=3):
    """
    To be used in conjunction with the object_list generic view.

    Adds pagination context variables for use in displaying first, adjacent and
    last page links in addition to those created by the object_list generic
    view.

    """
    page = context['page_obj']
    paginator = context['paginator']
    startPage = max(page.number - adjacent_pages, 1)
    if startPage <= 3:
        startPage = 1
    endPage = page.number + adjacent_pages + 1
    if endPage >= paginator.num_pages - 1:
        endPage = paginator.num_pages + 1
    page_numbers = [n for n in range(startPage, endPage)
                    if n > 0 and n <= paginator.num_pages]

    dict = {
        'request': context['request'],
        'is_paginated': paginator.num_pages > 0,
        'page_obj': page,
        'paginator': paginator,
        'results': paginator.per_page,
        'page_numbers': page_numbers,
        'show_first': 1 not in page_numbers,
        'show_last': paginator.num_pages not in page_numbers,
        'first': 1,
        'last': paginator.num_pages,
        'has_next': page.has_next(),
        'has_previous': page.has_previous(),
    }

    if page.has_next():
        dict['next'] = page.next_page_number()
    if page.has_previous():
        dict['previous'] = page.previous_page_number()

    return dict

register.inclusion_tag('pagination.html', takes_context=True)(paginator)

@register.simple_tag
def url_replace(request, field, value):

    dict_ = request.GET.copy()

    dict_[field] = value

    return dict_.urlencode()

@register.simple_tag
def orderby(request, field, attr):

    dict_ = request.GET.copy()

    if dict_.__contains__(field) and dict_[field] == attr:
        if not dict_[field].startswith("-"):
            dict_[field] = "-" + attr
        else:
            dict_[field] = attr
    else:
        dict_[field] = attr

    return dict_.urlencode()


# SimpleDMRS codec
# Summary: This module implements serialization and deserialization of the
#          SimpleDMRS encoding of Distributed Minimal Recusion Semantics (DMRS). It
#          provides standard Pickle API calls of load, loads, dump, and dumps
#          for serializing and deserializing single SimpleDMRS instances. Further,
#          encode_list and decode_list are provided for lists of DMRX
#          instances, and they read and write incrementally.
#
# Author: Michael Wayne Goodman <goodmami@uw.edu>

from __future__ import print_function

from collections import OrderedDict
from io import BytesIO
import re
from delphin.mrs import (Dmrs, Node, Link, Pred, Lnk)
from delphin.mrs.components import (nodes, links)
from delphin.mrs.config import EQ_POST, CVARSORT

##############################################################################
##############################################################################
# Pickle-API methods

def load(fh, single=False):
    ms = decode(fh)
    if single:
        ms = next(ms)
    return ms

def loads(s, single=False, encoding='utf-8'):
    ms = decode(BytesIO(bytes(s, encoding=encoding)))
    if single:
        ms = next(ms)
    return ms

def dump(fh, ms, **kwargs):
    print(dumps(ms, **kwargs), file=fh)

def dumps(ms, single=False, pretty_print=False, **kwargs):
    if single:
        ms = [ms]
    return encode(ms, indent=2 if pretty_print else None)

# for convenience

load_one = lambda fh: load(fh, single=True)
loads_one = lambda s: loads(s, single=True)
dump_one = lambda fh, m, **kwargs: dump(fh, m, single=True, **kwargs)
dumps_one = lambda m, **kwargs: dumps(m, single=True, **kwargs)

##############################################################################
##############################################################################
# Decoding

tokenizer = re.compile(r'("[^"\\]*(?:\\.[^"\\]*)*"'
                       r'|[^\s:#@\[\]<>"]+'
                       r'|[:#@\[\]<>])')

def decode(fh):
    """Decode a SimpleDmrs-encoded DMRS structure."""
    # (dmrs { ... })*

def decode_dmrs(elem):
    # dmrs { NODES LINKS }
    return Dmrs(nodes=list(map(decode_node)),
                links=list(map(decode_link)),
                lnk=None,
                surface=None,
                identifier=None)

def decode_node(elem):
    return Node(pred=decode_pred(elem.find('*[1]')),
                nodeid=elem.get('nodeid'),
                sortinfo=decode_sortinfo(elem.find('sortinfo')),
                lnk=decode_lnk(elem),
                surface=elem.get('surface'),
                base=elem.get('base'),
                carg=elem.get('carg'))

def decode_pred(elem):
    if elem.tag == 'gpred':
        return Pred.grammarpred(elem.text)
    elif elem.tag == 'realpred':
        return Pred.realpred(elem.get('lemma'),
                             elem.get('pos'),
                             elem.get('sense'))

def decode_sortinfo(elem):
    return elem.attrib

def decode_link(elem):
    return Link(start=elem.get('from'),
                end=elem.get('to'),
                rargname=elem.find('rargname').text,
                post=elem.find('post').text)

def decode_lnk(elem):
    return Lnk.charspan(elem.get('cfrom', '-1'), elem.get('cto', '-1'))

##############################################################################
##############################################################################
# Encoding

_graphtype = 'dmrs'
_graph = '{graphtype} {graphid}{{{dmrsproperties}{nodes}{links}}}'
_dmrsproperties = ''
_node = '{indent}{nodeid} [{pred}{lnk}{sortinfo}];'
_sortinfo = ' {cvarsort} {properties}'
_link = '{indent}{start}:{pre}/{post} {arrow} {end};'

def encode(ms, encoding='unicode', indent=2):
    delim = '\n' if indent is not None else ' '
    return delim.join(encode_dmrs(m, indent=indent) for m in ms)

def encode_dmrs(m, indent=2):
    if indent is not None:
        delim = '\n'
        space = ' ' * indent
    else:
        delim = ''
        space = ' '

    nodes_ = [
        _node.format(
            indent=space,
            nodeid=n.nodeid,
            pred=n.pred.string,
            lnk='' if n.lnk is None else str(n.lnk),
            sortinfo=(
                '' if not n.sortinfo else
                _sortinfo.format(
                    cvarsort=n.cvarsort,
                    properties=' '.join('{}={}'.format(k, v)
                                        for k, v in n.sortinfo.items()
                                        if k != CVARSORT),
                )
            )
        )
        for n in nodes(m)
    ]

    links_ = [
        _link.format(
            indent=space,
            start=l.start,
            pre=l.rargname or '',
            post=l.post,
            arrow='->' if l.rargname or l.post != EQ_POST else '--',
            end=l.end
        )
        for l in links(m)
    ]

    return delim.join(['dmrs {'] + nodes_ + links_ + ['}'])


'''
Created on May 13, 2017

@author: scottireton
'''

marginal_tax_rate = [  # income range |  tax rate
                        ((0,     9325),   0.1),
                        ((9325,  37950),  0.15),
                        ((37950, 91900),  0.25),
                        ((91900, 191650), 0.28)
                    ]

class IncomeTax(object):

    @staticmethod
    def getTax(gross_value):
        tax = 0

        for income_range, tax_rate in marginal_tax_rate:
            if gross_value > income_range[1]:
                tax = tax + (income_range[1]-income_range[0])*tax_rate
            else: # final bracket
                tax = tax + (gross_value-income_range[0])*tax_rate

        return tax


import turtle
turtle.tracer(False)
turtle.mode('logo')
turtle.shape('turtle')

x = turtle.getcanvas().winfo_width()/2
y = turtle.getcanvas().winfo_height()/2

def square(a, b):
    turtle.pencolor("black")
    turtle.clear()
    turtle.pu()
    turtle.goto(a, b)
    turtle.pd()
    for i in range(4):
        turtle.forward(50)
        turtle.right(90)
    turtle.update()

def draw_square(event):
    square(event.x - x, y - event.y)
    # if y-event.y > 0:
    #     square(event.x-x, y-event.y)
    # if event.x-x > 0:
    #     square(event.x-x, y-event.y)
    #

cv = turtle.getcanvas()
cv.bind("<Motion>", draw_square)

turtle.done()


import os
import Ribozyme_generation
import Util_functions
import pickle
import shutil

# Load in the list of sequences to fold
struct_file = open('seq_list.pkl', 'rb')
full_list = pickle.load(struct_file)

# Get the structure of the native ribozyme for comparison
five_HHRz = 'GCUGUCACCGGAUGUGCUUUCCGGUCUGAUGAGUCCGU'
three_HHRz = 'GAGGACGAAACAGC'
[ribozyme_parts, loops] = Ribozyme_generation.RNAStructure_get_reference_structures(five_HHRz + three_HHRz,
                                                                                'ribozyme', five_HHRz)

print([ribozyme_parts, loops])

tuple_list = []
bar = Util_functions.ProgressBar(len(full_list))

for seq in full_list:

    if not os.path.exists("Test_ribozymes"):
        os.makedirs("Test_ribozymes")

    Ribozyme_generation.RNAStructure_minimal_generator(seq, 1, "Test_ribozymes/test")

    # Iterates through each tested ribozyme structure(teststruct) and finds ribozyme active and aptamer formed.
    testopen = open("Test_ribozymes/test.1.txt")
    testopen = testopen.readlines()

    # Gets the sequence of the loops for the sequence, if correctly folded.
    teststruct = ''
    if testopen != []:
        teststruct = testopen[2][:-1]
        [loops, stem_lengths] = Ribozyme_generation.get_ribozyme_loops(seq, teststruct, ribozyme_parts)

    else:
        [loops, stem_lengths] = [['', ''], [0, 0]]

    shutil.rmtree("Test_ribozymes")

    tuple_list.append((seq, [loops, stem_lengths], teststruct))

    bar.update()
    print(bar.get_bar())
    print(bar.get_time_remaining())

# Dumps list of sequences, folded structure, and loop sequences to a pickle file for storage and later analysis.
struct_file = open('Candidate_list_RNAs_min_structures.pkl', 'wb')
pickle.dump(tuple_list, struct_file)
struct_file.close()