import torch.nn as nn
import torch.optim as optim
import torch
from Part01_Custom_dataset import get_image_pd
from sklearn.model_selection import train_test_split
from torchvision import datasets, transforms
from Part01_Custom_dataset import Mydataset
from torchvision.models import resnet50, resnet101, vgg16
from torch.optim import lr_scheduler
import matplotlib.pyplot as plt
import os
from Part03_Focal_loss import FocalLoss

# 定义虚线的的Connection部分，表示通道不同
class ConvBlock(nn.Module):
    def __init__(self, in_channels, filter, s, f):
        super(ConvBlock, self).__init__()
        # 卷积->BN->RELU ->卷积->BN->RELU->卷积->BN
        F1, F2, F3 = filter
        self.stage = nn.Sequential(
            nn.Conv2d(in_channels=in_channels, out_channels=F1, kernel_size=1, stride=1, padding=0, bias=False),
            nn.BatchNorm2d(F1), nn.ReLU(True),
            nn.Conv2d(in_channels=F1, out_channels=F2, kernel_size=f, stride=s, padding=0, bias=False),
            nn.BatchNorm2d(F2), nn.ReLU(True),
            nn.Conv2d(in_channels=F2, out_channels=F3, kernel_size=1, stride=2, padding=0, bias=False),
            nn.BatchNorm2d(F3))
        self.shortcut_1 = nn.Conv2d(in_channels=in_channels, out_channels=F3, kernel_size=1, stride=s, padding=0,
                                    bias=False)
        self.batch_1 = nn.BatchNorm2d(F3)
        self.relu_1 = nn.ReLU(True)

    def forward(self, X):
        shortcut_1 = self.shortcut_1(X)
        batch_1 = self.batch_1(shortcut_1)
        X = self.stage(X)
        X = X + batch_1
        X = self.relu_1(X)
        return X

class IdentifyBlock(nn.Module):
    def __init__(self, in_channel, filters):
        super(IdentifyBlock, self).__init__()
        F1, F2, F3 = filters
        # 1*1 conv -> bn -> relu ->3*3 conv ->bn ->relu ->1*1 conv->bn
        self.stage = nn.Sequential(
            nn.Conv2d(in_channels=in_channel, out_channels=F1, kernel_size=1, stride=1, padding=0, bias=False),
            nn.BatchNorm2d(F1), nn.ReLU(True),
            nn.Conv2d(in_channels=F1, out_channels=F2, kernel_size=3, stride=1, padding=0, bias=False),
            nn.BatchNorm2d(F2), nn.ReLU(True),
            nn.Conv2d(in_channels=F2, out_channels=F3, kernel_size=1, stride=1, padding=0, bias=False),
            nn.BatchNorm2d(F3))
        self.relu = nn.ReLU(True)

    def forward(self, X):
        shortcut_1 = X
        X = self.stage(X)
        X = shortcut_1 + X
        X = self.relu(X)
        return X

class ResnetModel(nn.Module):
    def __init__(self,n_classes):
        super(ResnetModel, self).__init__()

        self.stage1 = nn.Sequential(
            nn.Conv2d(in_channels=3, out_channels=64, kernel_size=7, stride=2, padding=3, bias=False),
            nn.BatchNorm2d(64),
            nn.ReLU(True),
            nn.MaxPool2d(kernel_size=3, stride=2, padding=1))

        self.stage2 = nn.Sequential(ConvBlock(in_channels=64,filter=[64,64,256],s=1),
                                    IdentifyBlock(in_channel=256,filters=[64,64,256]),
                                    IdentifyBlock(in_channel=256,filters=[64,64,256]))
        self.stage3 = nn.Sequential(ConvBlock(in_channels=256,filter=[128,128,512],s=2),
                                    IdentifyBlock(in_channel=512,filters=[128,128,512]),
                                    IdentifyBlock(in_channel=512,filters=[128,128,512]),
                                    IdentifyBlock(in_channel=512,filters=[128,128,512]),
                                    )
        self.stage4 = nn.Sequential(ConvBlock(in_channels=512,filter=[256,256,1024],s=2),
                                    IdentifyBlock(in_channel=1024,filters=[256,256,1024]),
                                    IdentifyBlock(in_channel=1024,filters=[256,256,1024]),
                                    IdentifyBlock(in_channel=1024,filters=[256,256,1024]),
                                    IdentifyBlock(in_channel=1024,filters=[256,256,1024]),
                                    IdentifyBlock(in_channel=1024,filters=[256,256,1024]),
                                    )
        self.stage5 = nn.Sequential(ConvBlock(in_channels=1024,filter=[512,512,2048],s=2),
                                    IdentifyBlock(in_channel=2048,filters=[512,512,2048]),
                                    IdentifyBlock(in_channel=2048,filters=[512,512,2048]),
                                    IdentifyBlock(in_channel=2048,filters=[512,512,2048]))

        self.pool = nn.AdaptiveAvgPool2d((1,1))
        self.fc = nn.Linear(in_features=2048,out_features=n_classes)

    def forward(self, X):
        output = self.stage2(X)
        output = self.stage3(output)
        output = self.stage4(output)
        output = self.stage5(output)
        output = self.pool(output)
        output = torch.flatten(output, 1)
        output = self.fc(output)
        return output

def draw_loss_and_accuracy(Loss_list, Accuracy_list,name):
    plt.figure()
    x1 = range(0, len(Loss_list))
    x2 = range(0, len(Accuracy_list))
    y1 = Accuracy_list
    y2 = Loss_list
    plt.subplot(2, 1, 1)
    plt.plot(x1, y1, 'o-')
    plt.title('Test accuracy vs. step')
    plt.ylabel('Test accuracy')
    plt.subplot(2, 1, 2)
    plt.plot(x2, y2, '.-')
    plt.xlabel('Test loss vs. step')
    plt.ylabel('Test loss')
    plt.savefig(f'./{name}.jpg')

if __name__ == '__main__':
    img_root_train = './data/NEU-CLS-64'
    # 与数字标记一一对应
    defect_label = {
        'cr': '0',
        'gg': '1',
        'in': '2',
        'pa': '3',
        'ps': '4',
        'rp': '5',
        'rs': '6',
        'sc': '7',
        'sp': '8'
    }
    all_pd = get_image_pd(img_root_train,defect_label)
    train_pd, val_pd = train_test_split(all_pd, test_size=0.2, random_state=53, stratify=all_pd["label"])

    train_transform = transforms.Compose([
        transforms.Resize(size=(64, 64)),
        transforms.RandomHorizontalFlip(),
        transforms.RandomVerticalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ])

    val_transform = transforms.Compose([transforms.Resize(64),
                                        transforms.ToTensor(),
                                        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
                                        ])
    # train_pd.to_csv('./train.csv')
    # val_pd.to_csv('./val.csv')

    # 训练集加载器
    train_datsset = Mydataset(train_pd, transform=train_transform)
    val_dataset = Mydataset(val_pd, transform=val_transform)
    # 验证集加载器
    train_dataloader = torch.utils.data.DataLoader(dataset=train_datsset, batch_size=64, shuffle=True)
    val_dataloader = torch.utils.data.DataLoader(dataset=val_dataset, batch_size=64, shuffle=False)

    num_epoch = 1
    device = 'cpu'
    # # 网络模型resnet
    # model = ResnetModel(n_class=9)
    # model.to(device=device)
    # optim_sgd = optim.SGD(model.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4)

    model = resnet50(pretrained=True)
    model.avgpool = torch.nn.AdaptiveAvgPool2d(output_size=1)
    model.fc = torch.nn.Linear(model.fc.in_features, 9)
    model.to(device=device)
    optim_sgd = optim.SGD(model.fc.parameters(), lr=0.01, momentum=0.9, weight_decay=1e-4)

    # 损失函数
    criterion = nn.CrossEntropyLoss()

    acc_all = []
    loss_all = []
    val_acc = []
    val_loss = []

    for epoch in range(num_epoch):
        print("-----------epoch-------", epoch)
        model.train()
        for i, data in enumerate(train_dataloader):
            img, label = data
            img = img.float().to(device)
            label = label.long().to(device)
            # 前向传播
            output = model(img)
            # 计算损失
            loss = criterion(output, label)
            # 梯度清零
            optim_sgd.zero_grad()
            # 反向传播
            loss.backward()
            # 参数更新
            optim_sgd.step()

            _, predicted = torch.max(output.data, 1)
            # torch.max返回输出结果中，按dim=1行排列的每一行最大数据及他的索引，丢弃数据，保留索引
            total = label.size(0)
            correct = (predicted == label).sum().item()
            acc = correct / total
            print("-------pred----------", predicted)
            print("---------PRED_COUNT---------", correct)
            acc_all.append(acc)
            loss_all.append(loss)

        if epoch % 2 == 0:
            with torch.no_grad():
                model.eval()  # 使用验证集时关闭梯度计算
                for data in val_dataloader:
                    images, labels = data
                    images, labels = images.to(device), labels.to(device)
                    outputs = model(images)
                    loss = criterion(outputs, labels)
                    _, predicted = torch.max(outputs.data, 1)
                    # torch.max返回输出结果中，按dim=1行排列的每一行最大数据及他的索引，丢弃数据，保留索引
                    total = labels.size(0)
                    correct = (predicted == labels).sum().item()
                    acc = correct/total
                    val_acc.append(acc)
                    val_loss.append(loss)

            # 1、 保存整个模型
            # torch.save(model, './model_parameter/test/ckpt_best_%s.pth' % (str(epoch)))
            #
            # # 2、只保存模型参数
            # state_dict = model.state_dict()
            # torch.save(state_dict, './model_parameter/test/ckpt_best_%s.pth' % (str(epoch)))

            # 3、保存模型参数、优化器参数、epoch
            print('epoch:', epoch)
            print('learning rate:', optim_sgd.state_dict()['param_groups'][0]['lr'])
            checkpoint = {
                "net": model.state_dict(),
                'optimizer': optim_sgd.state_dict(),
                "epoch": epoch
            }
            if not os.path.isdir("./model_parameter"):
                os.mkdir("./model_parameter")
            torch.save(checkpoint, './model_parameter/ckpt_%s.pth' % (str(epoch)))

    # loss-acc 可视化
    draw_loss_and_accuracy(loss_all, acc_all,'train_acc_loss')
    draw_loss_and_accuracy(val_loss, val_acc,'val_acc_loss')


"""This module saves visual representations of binary sequences.

The sequences are done through constant addition.
"""
import matplotlib.pyplot as plt
from math import sqrt, log

def to_binary(num: int) -> str:
    """Convert int to string form of binary sequence"""
    return bin(num)[2:]

def binary_image(num: int, step=1):
    """Print a plot representing a simple sequence of binary numbers.

    Step corresponds to how much to add"""
    grid = []
    for i in range(num//step):
        grid.append([0]*int(log(num, 2)+1))
        for j, char in enumerate(to_binary(i*step)):
            grid[i][j] = float(char)
    fig1 = plt.figure(figsize=(25,100))
    plt.pcolormesh(grid[::-1], cmap='Greens')
    plt.axis('off')
    plt.show()
    plt.draw()
    fig1.savefig(f'binary_image_addition_range({num})_stepsize({step}).png', dpi=100)

# The visual pattern remains the same regardless of the stepsize you use.
# Evenly spaced sequences in binary seem to have a universal visual pattern.
binary_image(500, step=1)
binary_image(500*3, step=3)
binary_image(500*13, step=13)
binary_image(500*25, step=25)
binary_image(500*87, step=87)


import pandas as pd
import numpy as np
from scipy.stats import chi2_contingency
from itertools import product, combinations
from statsmodels.stats.multitest import multipletests
from tqdm import tqdm

np.seterr(divide='ignore', invalid='ignore')

def generate_permutations(hb, x_variables):
    """Helper function for crosstab(). Returns list containining permutations of variables in dicts"""
    var_combos = [set for r in range(1, len(x_variables)+1) for set in combinations(x_variables, r)]
    perms = []
    for combo in var_combos:
        d = {col: hb[col].unique() for col in combo}
        perms = perms + [dict(zip(d, v)) for v in product(*d.values())]
    return perms

def return_row_for_permutation(hb, perm, x_variables, y_variable, y_values, date):
    """Helper function for crosstab(). Returns row of crosstab stats for a variable permutation"""
    n_ids = hb[hb.report_date == date].scrambled_id.nunique()
    subset = control = hb[hb.report_date == date]
    row = dict(perm)
    for key, value in row.items():
        if pd.isnull(value):
            subset = subset[subset[key].isnull()]
        else:
            subset = subset[subset[key] == value]
    control = control[~control.scrambled_id.isin(subset.scrambled_id.unique())]
    sub_total_count = subset.scrambled_id.nunique()
    control_total_count = control.scrambled_id.nunique()
    assert n_ids == (sub_total_count + control_total_count), date+str(perm)
    row.update({"permutation": perm, "date": date,
                "sub_total_pop": sub_total_count,
                "control_total_pop": control_total_count})
    chi2_obs = np.empty(shape=(len(y_values), 2))
    for index, val in enumerate(y_values):
        sub_count = subset[subset[y_variable] == val].scrambled_id.nunique()
        control_count = control[control[y_variable] == val].scrambled_id.nunique()
        row.update({"sub_"+y_variable+"_"+str(val): sub_count})
        row.update({"control_"+y_variable+"_"+str(val): control_count})

        row.update({"sub_"+y_variable+"_"+str(val)+"_rate": np.float32(sub_count)/sub_total_count})
        row.update({"control_"+y_variable+"_"+str(val)+"_rate": np.float32(control_count)/control_total_count})

        chi2_obs[index, 0] = sub_count
        chi2_obs[index, 1] = control_count
    p_value = np.NaN if (~chi2_obs.any(axis=0)).any() else chi2_contingency(chi2_obs)[1]
    row.update({"p_value": p_value})
    return row

def crosstab(hb, x_columns, y_variable, dates):
    """Returns crosstab dataframe. Pass list of categorical columns as x_columns
    and list of dates to produces crosstabs for. The y_variable should be categorical.

    Example use: crosstab(hb, ['counter_dhp', 'main_claimant_gender'], 'counter_homeless', ['2012-01-01', '2013-01-01'])
    """
    y_values = hb[y_variable].unique()
    perms = generate_permutations(hb, x_columns)
    result = pd.DataFrame()
    for p in tqdm(product(perms, dates), total=len(perms)*len(dates)):
        row = return_row_for_permutation(hb, p[0], x_columns, y_variable, y_values, p[1])
        result = result.append(row, ignore_index=True)
    result['p_value'] = multipletests(result.p_value, method='bonferroni')[1]
    return result

def simplify_counter_homeless_crosstab(crosstab, min_pop=10, p_value_max=0.05):
    """Function to process crosstab result to pull out most useful sumary results."""
    crosstab = crosstab[(crosstab.control_total_pop >= min_pop) & (crosstab.sub_total_pop >= min_pop)].copy()
    crosstab = crosstab[crosstab.p_value < p_value_max]
    crosstab['sub_homeless_pc'] = round(100*crosstab.sub_counter_homeless_True_rate,2)
    crosstab['control_homeless_pc'] = round(100*crosstab.control_counter_homeless_True_rate,1)
    crosstab = crosstab[['permutation', 'date',
                         'sub_total_pop', 'sub_homeless_pc',
                         'control_total_pop', 'control_homeless_pc',
                         'p_value']]
    return crosstab.sort_values('p_value')


# uncompyle6 version 3.7.4
# Python bytecode 2.7 (62211)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: build/bdist.linux-x86_64/egg/nodeconductor_saltstack/exchange/migrations/0007_user_additional_fields.py
# Compiled at: 2016-09-28 02:05:53
from __future__ import unicode_literals
from django.db import models, migrations
import nodeconductor_saltstack.exchange.validators

class Migration(migrations.Migration):
    dependencies = [
     ('exchange', '0006_group_manager_foreign_key')]
    operations = [
     migrations.AddField(model_name=b'user', name=b'company', field=models.CharField(max_length=255, blank=True), preserve_default=True),
     migrations.AddField(model_name=b'user', name=b'department', field=models.CharField(max_length=255, blank=True), preserve_default=True),
     migrations.AddField(model_name=b'user', name=b'manager', field=models.ForeignKey(blank=True, to=b'exchange.User', null=True), preserve_default=True),
     migrations.AddField(model_name=b'user', name=b'office', field=models.CharField(max_length=255, blank=True), preserve_default=True),
     migrations.AddField(model_name=b'user', name=b'phone', field=models.CharField(max_length=255, blank=True), preserve_default=True),
     migrations.AddField(model_name=b'user', name=b'title', field=models.CharField(max_length=255, blank=True), preserve_default=True),
     migrations.AlterField(model_name=b'exchangetenant', name=b'domain', field=models.CharField(max_length=255, validators=[nodeconductor_saltstack.exchange.validators.domain_validator]), preserve_default=True),
     migrations.AlterField(model_name=b'user', name=b'username', field=models.CharField(unique=True, max_length=255), preserve_default=True)]


import csv
import http.client
import logging
import os
import redis
from zipfile import ZipFile
import schedule
import time

def get_bhavcopy():
    conn = http.client.HTTPSConnection("www.bseindia.com")
    payload = ''
    headers = {}
    try:
        conn.request("GET", "/download/BhavCopy/Equity/EQ290421_CSV.ZIP",
                     payload,
                     headers)
        res = conn.getresponse()
        data = res.read()
        with open('bhavcopy.zip', 'wb') as fd:
            fd.write(data)
        zf = ZipFile("./bhavcopy.zip")
        zf.extractall(path='extraction_bhavcopy')
        zf.close()
    except:
        return False
    return True

def poll():
    print('polling started')
    if get_bhavcopy():
        conn = redis.Redis('localhost')
        try:
            for (dirpath, dirnames, filenames) in os.walk(
                    'extraction_bhavcopy'):

                if filenames:
                    with open("/".join([dirpath, filenames[0]]),
                              mode='r') as infile:

                        for line in csv.DictReader(infile):
                            conn.hset("zerodha" + dict(line)['SC_NAME'].strip(),

                                      mapping=dict(line))

        except Exception as e:
            logging.exception(e)

if __name__ == '__main__':

    schedule.every().day.at("17:30").do(poll, 'It is 17:30')

    while True:
        schedule.run_pending()
        time.sleep(60*60)  # wait one minute


# -*- coding: utf-8 -*-
"""
Created on Tue Feb 25 14:08:25 2020

@author: xinyu
"""

# -*- coding: utf-8 -*-
"""
Created on Tue Feb 25 11:24:46 2020

@author: xinyu
"""

#import ruptures as rpt
from scipy import signal, stats, fftpack, arange
from scipy.signal import butter, lfilter
import matplotlib.pyplot as plt
from pylab import plot, show, title, xlabel, ylabel, subplot, bar
from sklearn.model_selection import train_test_split, ShuffleSplit
import tensorflow as tf
import random
import numpy as np
import matplotlib.ticker as ticker
import os

class dataClass:
    def __init__(self, dataIn, idNum, measNum):
        dataIn = dataIn[(dataIn[:,0]==idNum) & (dataIn[:,6]==measNum)]
        self.t = dataIn[:,1]
        self.s = dataIn[:,2]
        self.p = dataIn[:,3]
        self.F = dataIn[:,4]
        self.A = dataIn[:,5]

class dataSet:
    def __init__(self, varIn):
        self.dat = varIn

class resClass:
    def __init__(self, resIn, idNum, measNum):
        res = resIn[(resIn[:,0]==idNum) & (resIn[:,1]==measNum)].T
        self.to = res[2]
        self.Fo = res[3]
        self.so = res[4]
        self.tc = res[5]
        self.Fc = res[6]
        self.sc = res[7]

class infoClass:
    def __init__(self, infoIn, idNum):
        self.info = infoIn[(infoIn[:,0]==idNum)].T
        self.p_set = self.info[1]
        self.F_tol = self.info[2]
        self.s_tol = self.info[3]
        self.p_sys = self.info[4]
        #self.DN = self.info[5]
        self.PN = self.info[5]
        self.d0 = self.info[6]
        self.kalt_heiss = self.info[7]

class timeSeries:
    def __init__(self, db, idNum, measNum):
        data = db[0]
        result = db[1]
        info = db[2]
        self.idNum = idNum
        self.measNum = measNum
        self.data = dataClass(data, idNum, measNum)
        self.points = resClass(result, idNum, measNum)
        self.info = infoClass(info,idNum)

###############################################################################

def loop_from_to(IDs, id_from, steps):
    if id_from == 0:
        id_from = 1
        id_to = IDs + 1
    else:
        id_to = id_from + steps + 1
    return id_from, id_to

def safe_div(x,y):
    if y == 0:
        return 0
    return x / y

def delta(var):
    if var.size > 0:
        dvar = var[1:]-var[:-1]
        dvar = np.append(dvar,0)
        dvar[-1] = dvar[-2]
        return dvar
    else:
        return var

def grad(var, idNum):
    if var.size > 0:
        if idNum < 775:
            grad = np.gradient(var, 0.1)
        else:
            grad = np.gradient(var, 0.05)
        return grad
    else:
        return var

def diff(var):
    dvar = np.diff(var)
    dvar = np.append(dvar,dvar[-1])
    return dvar

def diff2(var):
    dvar = np.diff(var,n=2)
    dvar = np.append(dvar,dvar[-1])
    return dvar

def grad_fwd(var,idNum,win_len):
    if var.size > 0:
        if idNum < 775:
            sampRate = 0.1
        else:
            sampRate = 0.05

        grad_fwd = np.zeros(len(var))

        for l in range(0,len(var)-win_len):
            grad_fwd[l] = (var[l+win_len] - var[l])/(win_len*sampRate)
        for l in range(len(var)-win_len, len(var)-1):
            w = (len(var)-1)-l
            grad_fwd[l] = (var[l+w] - var[l])/(w*sampRate)
        grad_fwd[len(var)-1] = (var[-1] - var[-2])/sampRate
        return grad_fwd
    else:
        return var

def grad_bwd(var,idNum,win_len):
    if var.size > 0:
        if idNum < 775:
            sampRate = 0.1
        else:
            sampRate = 0.05

        grad_bwd = np.zeros(len(var))
        grad_bwd[0] = (var[1] - var[0])/sampRate
        for l in range(1,win_len):
            grad_bwd[l] = (var[l] - var[0])/(win_len*sampRate)
        for l in range(win_len, len(var)):
            grad_bwd[l] = (var[l] - var[l-win_len])/(win_len*sampRate)
        return grad_bwd
    else:
        return var

def grad_win(var,idNum,win_len):
    if var.size > 0:
        if idNum < 775:
            sampRate = 0.1
        else:
            sampRate = 0.05

        grad_win = np.zeros(len(var))

        grad_win[0] = (var[1] - var[0])/sampRate
        for l in range(1,win_len):
            grad_win[l] = (var[l+l] - var[l-l])/(2*l*sampRate)
        for l in range(win_len,len(var)-win_len):
            grad_win[l] = (var[l+win_len] - var[l-win_len])/(2*win_len*sampRate)
        for l in range(len(var)-win_len, len(var)):
            grad_win[l] = (var[-1] - var[l -((len(var)-1)-l)])/(2*((len(var)-1)-l)*sampRate)
        grad_win[-1] = (var[-1] - var[-2])/sampRate
        return grad_win
    else:
        return var

def smooth(var,win_len,polyorder):
    try:
        var = signal.savgol_filter(var, window_length=win_len, polyorder=polyorder, mode='nearest')
        return var
    except ValueError:
        return var

def norm_std(var):
    var = (var-var.mean(0))/var.std(0)
    return var
def norm(var):
    if max(var)-min(var) != 0:
        var = (var-min(var))/(max(var)-min(var))
        return var
    else:
        #print("Input could not be normalized")
        return var

def norm2(var):
    if max(abs(var)) != 0:
        var = var/(max(abs(var)))
        return var
    else:
        return var

def norm3(var):
    for i in range(0,len(var)):
        if var[i] < 0:
            var[i]=0
    try:
        var = var/(max(abs(var)))
        return var
    except ValueError:
        return var

def detectOpt(signalY, signalX):

    signal = np.column_stack((signalY,signalX))
    n_bkps = 4
    #c = rpt.costs.CostLinear().fit(signal)
    # change point detection
    model = "linear" # "l1", "l2", "rbf", "normal", "linear"
    algo = rpt.Dynp(model=model, min_size=1, jump=1).fit(signal)
    bkps = algo.predict(n_bkps)
    return bkps

def butter_lowpass(lowcut, fs, order=2):
    nyq = 0.5 * fs
    low = lowcut / nyq
    b, a = butter(order, low, btype='lowpass')
    return b, a

def butter_lowpass_filter(data, lowcut, fs, order=5):
    b, a = butter_lowpass(lowcut, fs, order=order)
    y = lfilter(b, a, data)
    return y

def butter_zerophase_filter(data,lowcut,fs,order=2):
    b, a = butter_lowpass(lowcut, fs, order=order)
    y = signal.filtfilt(b, a, data)
    return y

def butter_bandpass(lowcut, highcut, fs, order=5):
    nyq = 0.5 * fs
    low = lowcut / nyq
    high = highcut / nyq
    b, a = butter(order, [low, high], btype='band')
    return b, a

def butter_bandpass_filter(data, lowcut, highcut, fs, order=5):
    b, a = butter_bandpass(lowcut, highcut, fs, order=order)
    y = lfilter(b, a, data)
    return y

def round_arr(data):
    data[:,2] = np.round(data[:,2],3)
    data[:,3] = np.round(data[:,3],7)
    return data

def create_y(data,idxo):
    y = np.zeros(len(data))
    y[idxo[0]:] = 1
    return y

def frequency_sepectrum(x, sf):
    x = x - np.average(x)  # zero-centering
    n = len(x)
    k = np.arange(n)
    tarr = n / float(sf)
    frqarr = k / float(tarr)  # two sides frequency range

    frqarr = frqarr[range(n // 2)]  # one side frequency range

    x = fftpack.fft(x) / n   # fft computing and normalization
    x = x[range(n // 2)]
    return frqarr, abs(x)

def interpo_point(x, p): #makes sure that Fo exists in resampled data
    n = 0
    for i in x:
        if i == p:
            break
        elif i > p:
            if (i-p)>(x[n-1]-p):
                x[n-1] = p
                n -= 1
            else:
                x[n] = p
            break
        n += 1
    return x, n

def generate_train(data,Po,l):
    data = signal.resample(data,l)
    train_x, n = interpo_point(data,Po)
    if n >= l:
        return np.zeros(l),0,0
    train_y = np.zeros(l)
    #train_y[n] = 2
    train_y[n:] = 1
    return train_x, train_y, n
def padding_len(xseq):
    Len = []
    for i in range(len(xseq)):
        Len.append(len(xseq[i]))
    pad_len = np.array(Len).max()
    return pad_len
def data_padding(Xseq,Yseq):
    padlen = padding_len(Xseq)
    padded_xseq = np.zeros((len(Xseq),padlen,Xseq[0].shape[-1]))
    padded_yseq = np.zeros((len(Yseq),padlen,1))
    for i in range(len(Xseq)):
        padded_xseq[i,...] = np.pad(Xseq[i],((0,padlen-len(Xseq[i])),(0,0)),'constant',constant_values = 0.)
        padded_yseq[i,...] = np.pad(Yseq[i].reshape(-1,1),((0,padlen-len(Yseq[i])),(0,0)),'constant',constant_values = 0.)
    return padded_xseq, padded_yseq

def gate(y_pred):
    y_bool = y_pred!=y_pred[-1]
    y =y_pred[y_bool==True]
    idx = np.where(y_pred == y[-1])[0][-1]
    y_ = y_pred[:idx+1]
    return y_
def evaluate_plot(*ypre,to):
    To = np.zeros(to.shape)
    for y_pre in ypre:
        for i in range(y_pre.shape[0]):
            y_ = gate(y_pre[i])
            To[i] = np.argmax(y_)
        diff_to = np.array(To)-np.array(to)
        diff_to = np.where(np.abs(diff_to)>=3,3,diff_to)
        unique, count = np.unique(diff_to, return_counts=True)
        fig = plt.figure(1)
        ax = fig.add_subplot(1,1,1)
        ax.bar(unique, 100*count/len(to))

        x_ticks_labels = ['-2','-1','0','1','2','>3']
        fmt = '%.0f%%' # Format you want the ticks, e.g. '40%'
        yticks = ticker.FormatStrFormatter(fmt)
        ax.yaxis.set_major_formatter(yticks)
        plt.xticks(np.arange(6)-2, x_ticks_labels)
        #ax.set_xticklabels(x_ticks_labels)

        xlabel('Error in timesteps')
        ylabel('Distribution')
        plt.show()
    return unique, count, diff_to

def FoErrEval(y_,F,Fos):
    Fo_pred = [F[i][np.argmax(gate(y_[i]))] for i in range(y_.shape[0])]
    FoErr = (np.array(Fos)-np.array(Fo_pred))/np.array(Fos)
    FoErr = np.array([round(x,2) for x in FoErr])
    FoErr = np.where(FoErr<-1,-1,FoErr)
    for i in np.linspace(0,0.4,5):
        low = np.where(FoErr>i)[0]
        up = np.where(FoErr<=i+0.1)[0]
        FoErr[np.intersect1d(low, up)]=i+0.1
        low = np.where(FoErr>=-i-0.1)[0]
        up = np.where(FoErr<-i)[0]
        FoErr[np.intersect1d(low,up)]=-i-0.1
    FoErr = np.where(FoErr>0.5,1,FoErr)
    FoErr = np.where(FoErr<-0.5,-1,FoErr)
    unique, count = np.unique(FoErr, return_counts=True)
    fig = plt.figure(1)
    ax = fig.add_subplot(1,1,1)
    ax.bar(100*unique, 100*count/len(FoErr),width=8)
    fmt = '%.0f%%' # Format you want the ticks, e.g. '40%'
    yticks = ticker.FormatStrFormatter(fmt)
    ax.yaxis.set_major_formatter(yticks)
    fmt = '%.2f%%'
    xticks = ticker.FormatStrFormatter(fmt)
    ax.xaxis.set_major_formatter(xticks)
    xlabel('Fo_err/Fo [%]')
    ylabel('Distribution')
    plt.show()
    return Fo_pred,unique,count

def evaluate_dense(y_pred,Yseq):
    xfrom=0
    xto=0
    idxo_pred = []
    idxo_true = []
    for i in Yseq:
        xto = xto + len(i)
        idxo_pred.append(np.argmax(y_pred[xfrom:xto]))
        idxo_true.append(np.argmax(i))
        xfrom = xto

    Err = np.array([idxo_pred[i] - idxo_true[i] for i in range(len(idxo_pred))])

    err = np.where(Err>=3,3,Err)
    err = np.where(err<=-3,-3,err)
    unique, count = np.unique(err, return_counts=True)
    fig = plt.figure(1)
    ax = fig.add_subplot(1,1,1)
    ax.bar(unique, 100*count/len(err))

    x_ticks_labels = ['<-3','-2','-1','0','1','2','>3']
    fmt = '%.0f%%' # Format you want the ticks, e.g. '40%'
    yticks = ticker.FormatStrFormatter(fmt)
    ax.yaxis.set_major_formatter(yticks)
    plt.xticks(np.arange(7)-3, x_ticks_labels)
    ax.set_xticklabels(x_ticks_labels)

    xlabel('Error in timesteps')
    ylabel('Distribution')
    plt.show()
    return Err

def FoErrEvalDense(y_,F,Fos):
    xfrom=0
    xto=0
    Fo_pred = []
    j=0
    for i in Yseq:
        xto = xto + len(i)
        Fo_pred.append(F[j][np.argmax(y_[xfrom:xto])])
        xfrom = xto
        j+=1
    FoErr = (np.array(Fos)-np.array(Fo_pred))/np.array(Fos)
    FoErr = np.array([round(x,2) for x in FoErr])
    FoErr = np.where(FoErr<-1,-1,FoErr)
    for i in np.linspace(0,0.4,5):
        low = np.where(FoErr>i)[0]
        up = np.where(FoErr<=i+0.1)[0]
        FoErr[np.intersect1d(low, up)]=i+0.1
        low = np.where(FoErr>=-i-0.1)[0]
        up = np.where(FoErr<-i)[0]
        FoErr[np.intersect1d(low,up)]=-i-0.1
    FoErr = np.where(FoErr>0.5,1,FoErr)
    FoErr = np.where(FoErr<-0.5,-1,FoErr)
    unique, count = np.unique(FoErr, return_counts=True)
    fig = plt.figure(1)
    ax = fig.add_subplot(1,1,1)
    ax.bar(100*unique, 100*count/len(FoErr),width=8)
    fmt = '%.0f%%' # Format you want the ticks, e.g. '40%'
    yticks = ticker.FormatStrFormatter(fmt)
    ax.yaxis.set_major_formatter(yticks)
    fmt = '%.2f%%'
    xticks = ticker.FormatStrFormatter(fmt)
    ax.xaxis.set_major_formatter(xticks)
    xlabel('Fo_err/Fo [%]')
    ylabel('Distribution')
    plt.show()
    return Fo_pred,unique,count

###############################################################################

'''load measurement data and result data from respective txt-files, skip header rows'''
dataArr = np.loadtxt("U:/Data/OUTPUT_diagram_ll_clean.txt", skiprows=1) #contains raw measurement data
resArr = np.loadtxt("U:/Data/OUTPUT_tocke_ll_clean_2020.txt", skiprows=1) #contains determined opening point for each measurement
infoArr = np.loadtxt("U:/Data/OUTPUT_meritve_ll.txt", skiprows=1) #additional information
IDs = int(dataArr[-1,0]) #Anzahl Messdurchläufe in der Datenbank mit jeweils bis zu 3 Messungen

DB = [dataArr,resArr,infoArr]

initialize = 1
trim = 1

loopThroughData = 1
sequence = 1

ruleBased = 0
findBySort = 0
findBySum = 1

optSegment = 0
segFlinear = 1
segFslinear = 0
segPlt = 0
savefigure = 0

NNseq = 1

showPlt = 0
evaluation = 1

gradWin = 0
gradFwd = 0
gradBwd = 1
loadmodel = 1

if initialize != 0:
    dataList = []
    featureList = []
    X = np.zeros(shape = (1,12))
    Y = np.zeros(shape=1)
    F_origin = np.zeros(shape=1)
    raw_F = []
    FSA = []
    FSAtrim = []
    tos = np.zeros(1+IDs*3)
    Fos = np.zeros(1+IDs*3)
    idxos = np.zeros(1+IDs*3)
    Fos_pred = np.zeros(1+IDs*3)
    idxos_pred = np.zeros(1+IDs*3)
    Xseq = []
    Yseq = []
    Frng = np.zeros(1+IDs*3)
    label=[]
    measLen = np.zeros(1+IDs*3)

    #errArr = np.zeros((IDs,6))
if loopThroughData !=0:
    #r = random.randrange(1,935)
    id_from, id_to = loop_from_to(IDs, id_from=0, steps=0) #id_from=0 loops through all
    loopNo = (id_from-1)*3 #adjust initial value according to id_from
    for i in range(id_from,id_to):
        for j in range(1,4):
            print(str(i) + ", " + str(j))
            loopNo = loopNo + 1

            dat = timeSeries(DB,i,j).data #data
            res = timeSeries(DB,i,j).points #data labels
            info = timeSeries(DB,i,j).info #additional labels

            dataList.append(dat)

            t = dat.t
            F = dat.F
            s = dat.s
            p = dat.p
            A = dat.A

            to = res.to #opening time
            Fo = res.Fo #opening force

            if F.size > 0:
                Frng[loopNo] = max(F)-min(F)
            if to.size > 0:
                tos[loopNo] = to

            if F.size > 0 and Fo.size > 0 and to.size > 0 and to > 0.1:
                to = round(to[0],2)
                idxo = np.searchsorted(t,to,'left')
                idxos[loopNo] = idxo
                Frng[loopNo] = max(F)-min(F)
                raw_F.append(F)
                y = np.zeros_like(F)
                y[idxo] = 1
                Fos[loopNo] = Fo
            else:
                 continue

            if trim != 0:
                trim_to = np.argmax(F) + 5
                measLen[loopNo] = trim_to
                label.append([i,j])
                F = F[:trim_to]
                s = s[:trim_to]
                A = A[:trim_to]
                t = t[:trim_to]
                y = y[:trim_to]

            if gradFwd != 0:
                gradF = grad_fwd(F,i,1)
                grad2F = grad_fwd(gradF,i,1)
                grad3F = grad_fwd(grad2F,i,1)
                grads = grad_fwd(s,i,3)
                grad2s = grad_fwd(grads,i,3)
                grad3s = grad_fwd(grad2s,i,3)
                #gradA = grad_fwd(A,i,5)
                #grad2A = grad_fwd(gradA,i,5)
                #grad3A = grad_fwd(grad2A,i,5)
            elif gradBwd !=0:
                gradF = grad_bwd(F,i,1)
                grad2F = grad_fwd(gradF,i,1)
                grad3F = grad_bwd(grad2F,i,1)
                grads = grad_bwd(s,i,1)
                grad2s = grad_fwd(grads,i,1)
                grad3s = grad_bwd(grad2s,i,1)
                gradA = grad_bwd(A,i,1)
                grad2A = grad_fwd(gradA,i,1)
                grad3A = grad_bwd(grad2A,i,1)
            else:
                gradF = grad_win(F,i,1)
                grad2F = grad_win(gradF,i,1)
                grad3F = grad_win(grad2F,i,1)
                grads = grad_win(s,i,3)
                grad2s = grad_win(grads,i,3)
                grad3s = grad_win(grad2s,i,3)
                #gradA = grad_win(A,i,1)
                #grad2A = grad_win(gradA,i,1)
                #grad3A = grad_win(grad2A,i,1)

            Fn = norm2(F)
            gradFn = norm2(gradF)
            grad2Fn = norm2(grad2F)
            grad3Fn = norm2(grad3F)

            sn = norm2(s)
            gradsn = norm2(grads)
            grad2sn = norm2(grad2s)
            grad3sn = norm2(grad3s)

            An = norm2(A)
            gradAn = norm2(gradA)
            grad2An = norm2(grad2A)
            grad3An = norm2(grad3A)
            if NNseq != 0:
                x = np.column_stack((Fn, gradFn, grad2Fn, grad3Fn,
                                     sn, gradsn, grad2sn, grad3sn,
                                     An, gradAn, grad2An, grad3An))

                if X.size != 0:
                    X = np.concatenate((X, x), axis=0)
                    Y = np.concatenate((Y, y), axis=0)
                    F_origin = np.concatenate((F_origin, F), axis=0)
                else:
                    X = x
                    Y = y
                    F_origin = F

###########-Form data in sequential
            if sequence > 0:
                Xseq.append(np.array(x))
                Yseq.append(np.array(y))

#####################################################################################
# Datasplit

n = np.arange(len(Xseq))
label = np.array(label)
test_size = 0.15
trial,uniidx = np.unique(label[:,0],return_index=True)
test_vol = test_size*len(Xseq)/len(trial)
ss = ShuffleSplit(n_splits=1,test_size=test_vol,random_state=10)
for trainidx, testidx in ss.split(trial):
    test = uniidx[testidx]

train = n[np.isin(n, test,invert=True)]

entire = n

#Data for Feedforward Dense NN
Trainx = [Xseq[i] for i in train]
Testx = [Xseq[i] for i in test]
Trainy = [Yseq[i] for i in train]
Testy = [Yseq[i] for i in test]
x_train = np.array([y for x in Trainx for y in x]  )
x_test = np.array([y for x in Testx for y in x] )
y_train = np.array([y for x in Trainy for y in x])
y_test= np.array([y for x in Testy for y in x] )
XX = np.array([y for x in Xseq for y in x]  )
YY = np.array([y for x in Yseq for y in x]  )

#Data for Recurrent layer
sequential_x, sequential_y = data_padding(Xseq,Yseq)
idxo_masked = np.array([np.argmax(y) for y in sequential_y])
recurr_xtrain = np.array([sequential_x[i] for i in train])
recurr_xtest = np.array([sequential_x[i] for i in test])
recurr_ytrain = np.array([sequential_y[i] for i in train])
recurr_ytest = np.array([sequential_y[i] for i in test])

########################################################################
if loadmodel:
    model_dense = tf.keras.models.load_model('dense_withA.h5')
    model_lstm = tf.keras.models.load_model('lstm_withA.h5')
    model_bilstm = tf.keras.models.load_model('bilstm_withA.h5')

else:
    # Feedforward Dense NN
    model_dense = tf.keras.models.Sequential()
    model_dense.add(tf.keras.layers.Dense(80, input_dim=12, activation='relu'))
    model_dense.add(tf.keras.layers.Dense(50, activation='relu'))
    model_dense.add(tf.keras.layers.Dense(30, activation='relu'))
    model_dense.add(tf.keras.layers.Dense(1, activation='sigmoid'))
    model_dense.summary()
    model_dense.compile(optimizer='adam', loss='binary_crossentropy', metrics=[tf.keras.metrics.Precision(),tf.keras.metrics.Recall()])
    history = model_dense.fit(x_train, y_train,validation_split=0.15, shuffle = True,
                  epochs=20)

    # LSTM NN
    def create_lstm(input_shape):
        inputs = tf.keras.Input(shape = input_shape)
        x = tf.keras.layers.Masking(mask_value=0.)(inputs)
        x1 = tf.keras.layers.LSTM(20, activation='relu',return_sequences = True,name='lstm')(x)
        x1 = tf.keras.layers.BatchNormalization()(x1)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(100, activation='relu'))(x1)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(20, activation='relu'))(x6)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(20, activation='relu'))(x6)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        y_ = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(1, activation='sigmoid'))(x6)
        return tf.keras.Model(inputs=inputs,outputs=y_)

    model_lstm = create_lstm(recurr_xtrain.shape[1:])

    loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
    model_lstm.compile(loss=loss,metrics=[tf.keras.metrics.Precision(),tf.keras.metrics.Recall()],optimizer='adam')
    model_lstm.summary()
    model_lstm.fit(recurr_xtrain, recurr_ytrain,
              validation_split=0.15,
              shuffle = True,
              epochs=50,
              verbose=1)

    # Bidirectional LSTM
    def create_bilstm(input_shape):
        inputs = tf.keras.Input(shape = input_shape)
        x = tf.keras.layers.Masking(mask_value=0.)(inputs)
        x1 = tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(20, activation='relu',return_sequences = True,name='lstm'))(x)
        x1 = tf.keras.layers.BatchNormalization()(x1)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(100, activation='relu'))(x1)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(20, activation='relu'))(x6)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        x6 = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(20, activation='relu'))(x6)
        x6 = tf.keras.layers.BatchNormalization()(x6)
        y_ = tf.keras.layers.TimeDistributed(tf.keras.layers.Dense(1, activation='sigmoid'))(x6)
        return tf.keras.Model(inputs=inputs,outputs=y_)

    model_bilstm = create_bilstm(recurr_xtrain.shape[1:])

    loss = tf.keras.losses.BinaryCrossentropy(from_logits=False)
    model_bilstm.compile(loss=loss,metrics=[tf.keras.metrics.Precision(),tf.keras.metrics.Recall()],optimizer='adam')
    model_bilstm.summary()
    model_bilstm.fit(recurr_xtrain, recurr_ytrain,
              validation_split=0.15,
              shuffle = True,
              epochs=50,
              verbose=1)

if evaluation:

    dataset = 'test' #  test or entire

    if dataset == 'entire':
        dataset = n
        data = XX
        recurr_data = sequential_x
    elif dataset == 'test':
        dataset = test
        data = x_test
        recurr_data = recurr_xtest

    Force = [raw_F[i] for i in dataset]
    Fo = Fos[np.where(Fos!=0)[0]]
    Force_open = np.array([Fo[i] for i in dataset])

    y_pred_dense = model_dense.predict(data)
    Err_dense = evaluate_dense(y_pred_dense,np.array(Yseq)[dataset])
    #FoPre, FoErr, count =  FoErrEvalDense(y_pred_dense,Force,Force_open)
    print('Error (Dense) <= 2 steps: %.4f'%(100*len(np.where(abs(Err_dense)<=2)[0])/len(Err_dense)))

    y_pred_lstm = model_lstm.predict(sequential_x[dataset])
    Err_lstm, lstm_err = evaluate_plot(y_pred_lstm,to=idxo_masked[dataset])
    #FoPre, FoErr, count =  FoErrEval(y_pred_lstm,Force,Force_open)
    print('Error (LSTM) <= 2  steps: %.4f'%(100*np.sum(lstm_err[np.where(Err_lstm<=2)[0]])/np.sum(lstm_err)))

    y_pred_bilstm = model_bilstm.predict(sequential_x[dataset])
    Err_bilstm, bilstm_err = evaluate_plot(y_pred_bilstm,to=idxo_masked[dataset])
    #FoPre, FoErr, count =  FoErrEval(y_pred_bilstm,Force,Force_open)
    print('Error (BiLSTM) <= 2  steps: %.4f '%(100*np.sum(bilstm_err[np.where(Err_bilstm<=2)[0]])/np.sum(bilstm_err)))


import yaml
import io

import os, django

# from django.contrib.gis.geoip2.resources import Country

os.environ['DJANGO_SETTINGS_MODULE'] = 'portfolio_ecom.settings'
django.setup()
from core.models import Category, Country

# data = {
# 	'a list': [
# 		1,
# 		42,
# 		3.141,
# 		1337,
# 		'help',
# 		u'€'
# 	],
# 	'a string': 'bla',
# 	'another dict': {
# 		'foo': 'bar',
# 		'key': 'value',
# 		'the answer': 42
# 	}
# }

def write_data(data):
	"""
	:type data: dict
	"""
	with io.open('data.yaml', 'w', encoding='utf8') as outfile:
		try:
			yaml.dump(data, outfile, default_flow_style=False, allow_unicode=True)
		except yaml.YAMLError as exc:
			print(exc)

# return dict
def get_file_data() -> dict:
	with open('data.yaml', 'r') as stream:
		try:
			return yaml.safe_load(stream)
		except yaml.YAMLError as exc:
			print(exc)

class CategoryData:
	name = 'core_category'

	@classmethod
	def tofile(cls, data: dict = None):
		arr = []
		for r in Category.objects.all():
			arr.append({
				'id': r.id,
				'name': r.name,
				'parent': r.parent,
				'is_selectable': r.is_selectable
			})
		write_data({cls.name: arr})

	@classmethod
	def todb(cls, data: dict = None):
		for i in get_file_data()[cls.name]:
			Category(**i).save()

class CountryData:
	name = 'core_country'

	@classmethod
	def tofile(cls, data: dict = None, override=False):
		if data is None:
			arr = []
			for r in Category.objects.all():
				arr.append({
					'id': r.id,
					'name': r.name,
				})
			write_data({cls.name: arr})
		else:
			yaml_data = get_file_data()
			if not override:  # if append
				if not cls.name in yaml_data:
					yaml_data[cls.name] = []
					for index, item in enumerate(data):
						item['id'] = index
						yaml_data[cls.name].append(item)
				else:
					max2 = 0
					if len(yaml_data[cls.name]) > 0:
						arr = []
						for item in yaml_data[cls.name]:
							arr.append(float(item['id']))
						max2 = max(arr) + 1

					for index, item in enumerate(data):
						item['id'] = index + max2
						yaml_data[cls.name].append(item)
			else:
				yaml_data[cls.name] = []
				for index, item in enumerate(data):
					item['id'] = index
					yaml_data[cls.name].append(item)

	@classmethod
	def todb(cls, data: list = None, override=False):
		if data is None:
			for i in get_file_data()[cls.name]:
				Country(**i).save()
		else:
			if not override:  # if append
				for i in data:
					if 'id' in i:
						raise Exception('can\'t override with id, remove the id')
					Country(**i).save()
			else:
				for i in data:
					if 'id' not in i:
						raise Exception('if you want to override you need id')
					el = Country.objects.get(id=i['id'])
					el.name = i['name']

# CategoryData.tofile()
# CategoryData.todb()

# test if exist replace else create yaml
# data = get_file_data()
# data['sambal'][2] = {
# 	'wow': '555555555555555',
# 	'wow111111111111': 'dam',
# 	'wow1111111112': 'dam',
# 	'wodddddddw3': 'dam',
# }
#
# write_data(data)

CountryData.todb([
	{'name': 'Sri Lanka', },
	{'name': 'India', },
	{'name': 'USA', },
	{'name': 'Germany', },
])


# from typing import List, Optional, Union

import cv2
import imutils
import numpy as np

from Object import Bucket, Cube
#from dominant_color import DominantColorDetector

class ObjectsDetector:

    def stupid_detection(self, frame):
        hsv, thresh = self.__prepare_frame(frame, height=self._height, width=self._width)
        cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                                cv2.CHAIN_APPROX_SIMPLE)
        cnts = imutils.grab_contours(cnts)
        for c in cnts:
            if cv2.contourArea(c) < self._min_area:
                continue
            (x, y, w, h) = cv2.boundingRect(c)

            return x + w / 2, y + h / 2

    """
    Constants
    #
    # :param width
    # :param height
    #
    # :param circles_pool_length length of pool with previous circles to prevent fluctuations of contours
    # :param min_area_to_detect min area to detect contours
    #
    # :param circle_factor coefficient to prevent fluctuations of contours with circles
    # :param debug_mode if true call cv2.imshow() to take a look at result
    # :param rotation_factor coefficient for is_rotated. The smaller it is, the more precise would be algorithm.
    # :param daytime preset for day or night light. Should be customize for your environment.
    """

    def __init__(self,
                 # TODO determine this factor
                 rotation_factor=22,
                 width=640,
                 height=360,
                 circles_pool_length=1,
                 min_area_to_detect=400,
                 circle_factor=1.25,
                 debug_mode=False,
                 # TODO change min are
                 min_area_to_compute_mean_colors=100000,
                 daytime="DAY"):

        self._width = width
        self._height = height
        self._pool_size = circles_pool_length
        self._min_area = min_area_to_detect
        self._circle_factor = circle_factor
        self._circles_coords_pool = list()
        self._debug_mode = debug_mode
        self._min_area_for_dominant = min_area_to_compute_mean_colors
        self._daytime = 100 if daytime == "DAY" else 125
        self._rotation_factor = rotation_factor
        #self._dominant_detector = DominantColorDetector(n_clusters=7)

    # crop image
    def _get_subimage_by_pxs(self, image, start, shift):
        cropped_image = []

        for i in range(shift[1]):
            cropped_image.append([])
            for j in range(shift[0]):
                cropped_image[i].append([])
                cropped_image[i][j] = image[start[1] + i][start[0] + j]

        return np.array(cropped_image)

    # Can detect that cube is rotated relatively to the camera
    def is_rotated(self, frame):
        hsv, thresh = self.__prepare_frame(frame, height=self._height, width=self._width)

        cv2.imshow("thresh", thresh)
        # find cube
        cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                                cv2.CHAIN_APPROX_SIMPLE)
        cnts = imutils.grab_contours(cnts)

        for c in cnts:
            if cv2.contourArea(c) < self._min_area:
                continue
            (x, y, w, h) = cv2.boundingRect(c)
            # take a small image of it
            area = self._get_subimage_by_pxs(thresh, start=(x, y), shift=(w, h))
            print(area.shape)
            # cv2.imshow('result', area)
            # compute a factor
            factor = self.__get_difference(area)

            if cv2.waitKey(25) & 0xFF == ord('q'):
                return None

            if factor >= self._rotation_factor:
                return True
            else:
                return False

    # using absdiff to get all pixels that deviate from reference
    def __get_difference(self, frame_thresh):
        reference = cv2.imread('./src/reference.png')
        shape = min(min((frame_thresh.shape, reference.shape[:-1])))
        frame = cv2.resize(reference, (shape, shape))
        hsv, reference_thresh = self.__prepare_frame(reference, width=shape, height=shape)
        frame_thresh = cv2.resize(frame_thresh, (shape, shape))

        frame_delta = cv2.absdiff(frame_thresh, reference_thresh)
        frame_delta = cv2.dilate(frame_delta, None, iterations=2)

        # count pixels
        counter = 0
        for row in frame_delta:
            for pixel in row:
                if pixel == 255:
                    counter += 1
        # devide by shape to get relation with size of the cropped image
        return counter / shape

    # function that resize and make hsv and thresh copy of a frame
    def __prepare_frame(self, frame, width, height):

        hsv = cv2.cvtColor(frame, cv2.COLOR_RGB2HSV)
        blur = cv2.GaussianBlur(hsv, (19, 19), 0)

        gray = cv2.cvtColor(blur, cv2.COLOR_BGR2GRAY)
        ret, thresh = cv2.threshold(gray, self._daytime, 255, 1)
        thresh = cv2.bitwise_not(thresh)
        return hsv, thresh

    # Check if this object is circle and we shouldn't detect it
    # iterate though pool with circles and check a distance between coordinates and circles
    def _circle_check(self, pool, coords):
        if pool:
            for circles in pool:
                for circle in circles:
                    if max(abs(np.array(circle.get_position()) - np.array(coords))) < \
                            self._circle_factor * circle.get_radius():
                        return False

        return True

    """
    # :brief Color detection by hue, check HSV wiki for details. Take a 20x20 px square and count mean H component
    # then translate it to 360 points scale. Then just compare it with table (wiki)
    # :note if a color cannot be detected return 'no_color'
    # :param image just image where needed to detect color

    # :param x coordinate of a dot where to detect color
    # :param y

    """

    def _get_color(self, image, x, y):
        # if contour nearby edge use square with smaller side
        if x == self._width:
            x -= 1
        if y == self._height:
            y -= 1

        area = 0
        for i in range(10):
            if x - i < 0 or x + i >= self._width:
                break
            elif y - i < 0 or y + i >= self._height:
                break
            else:
                area = i

        avg_color = 0
        # Count mean
        for i in range(x - area, x + area):
            for j in range(y - area, y + area):
                avg_color += image[j, i, 0]

        # translate result
        avg_color = avg_color / area ** 2 if area != 0 else image[y, x, 0]
        hsv_color = 255 * avg_color / 360
        # compare
        if 0 < hsv_color <= 13 or 330 <= hsv_color:
            return 'RED'
        elif 13 <= hsv_color < 35:
            return 'BLUE'
        elif 35 <= hsv_color < 70:
            return 'ORANGE'
        elif 70 <= hsv_color < 180:
            return 'GREEN'
        elif 180 <= hsv_color < 300:
            return 'YELLOW'
        else:
            return 'no_color'

    """
    # :brief Detect objects(bucket or cube) in image
    # :param frame take an image (RGB) were need to detect objects(bucket or cube).
    """

    def get_objects(self, frame):
        if frame is None:
            raise ValueError('Frame is none')

        frame = cv2.resize(frame, (self._width, self._height))

        # Font
        font = cv2.FONT_HERSHEY_SIMPLEX
        fontScale = 0.5
        fontColor = (255, 255, 255)

        hsv, thresh = self.__prepare_frame(frame, height=self._height, width=self._width)

        # Store here all objects
        objects_in_frame = []

        """
        # First detect only buckets. It's more simple to detect circle in frame then
        # we iterate each contour and check if it is circle or not.
        # We conclude a contour a circle if it has the center nearby a center of already detected circles.
        # `get_color` - function that make a verification for contour
        """

        #  Pool must not be bigger than max size
        if self._circles_coords_pool is not None and len(self._circles_coords_pool) >= self._pool_size:
            self._circles_coords_pool.pop()

        # Get circles
        circles = cv2.HoughCircles(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY), cv2.HOUGH_GRADIENT, dp=1.5, minDist=1000,
                                   param1=65, param2=90)
        if circles is not None:
            # convert the (x, y) coordinates and radius of the circles to integers
            circles = np.round(circles[0, :]).astype("int")
            # loop over the (x, y) coordinates and radius of the circles
            for (x, y, r) in circles:
                # draw the circle in the output image, then draw a rectangle
                # corresponding to the center of the circle
                cv2.circle(frame, (x, y), r, (0, 255, 0), 4)
                circles = list()
                bucket = Bucket(position=(x, y), radius=r, color=self._get_color(hsv, x, y))
                cv2.putText(img=frame, text=str(bucket),
                            org=(x, y),
                            fontFace=font,
                            fontScale=fontScale,
                            color=fontColor)
                circles.append(bucket)
                objects_in_frame.append(bucket)
            self._circles_coords_pool.append(circles)

        cv2.imshow('thresh', thresh)
        # Check all contours
        cnts = cv2.findContours(thresh.copy(), cv2.RETR_EXTERNAL,
                                cv2.CHAIN_APPROX_SIMPLE)
        cnts = imutils.grab_contours(cnts)

        for c in cnts:
            # if the contour is too small, ignore it
            (x, y, w, h) = cv2.boundingRect(c)
            if cv2.contourArea(c) < self._min_area:
                continue
            elif cv2.contourArea(c) > self._min_area_for_dominant:
                # TODO find dominant color
                #self._reveal_clusters(frame, position=(x, y), shift=(w, h))
                pass
                # area = self._get_image_by_px(frame, [x, y], [x + w, y + h])
                # cv2.imshow("cropped", area)

            # area = self._get_image_by_px(frame, [x, y], [x + w, y + h])
            # Validation with circle_check
            if self._circle_check(self._circles_coords_pool, [x, y]):
                cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
                center_x = int(x + w / 2)
                center_y = int(y + h / 2)
                cube = Cube(position=(center_x, center_y), color=self._get_color(hsv, x=center_x, y=center_y))
                cv2.putText(img=frame, text=str(cube),
                            org=(x, y),
                            fontFace=font,
                            fontScale=fontScale,
                            color=fontColor)
                objects_in_frame.append(cube)

        cv2.imshow('result', frame)

        if cv2.waitKey(25) & 0xFF == ord('q'):
            return None

        return objects_in_frame, frame


from django import template
from django.db import connection
from apps.products.dashboard import dictfetchall
from django.template import loader, Context, Template

register = template.Library()

@register.simple_tag
def lastNews():
    html = ''

    cursor = connection.cursor()
    cursor.execute('''SELECT `id`, `title`, `content`, `timestamp`, `image` FROM `news` WHERE `active` = 1 ORDER BY `timestamp` DESC LIMIT 9''')
    news = dictfetchall(cursor)

    if news:
        template = loader.get_template('_newsTpl.html')
        for item in news:
            html += template.render(Context(item))
    return html

@register.simple_tag
def tags():
    html = ''
    tag_list = list()

    cursor = connection.cursor()
    cursor.execute('''SELECT `tags` FROM `news` WHERE (`active` = 1 AND (`tags` != "" OR `tags` IS NOT NULL))''')
    tags = dictfetchall(cursor)

    if tags:
        for tag in tags:
            [tag_list.append(x.strip()) for x in tag['tags'].split(",") if x.strip() not in tag_list]

    if tag_list:
        tpl = Template('{{tag}}')
        html = '
'
        for t in tag_list:
            if t:
                html += tpl.render(Context({"tag": t}))
        html += '
'
    return html


# -*- coding: utf-8 -*-
"""
This programs runs various classification models
on the moons data set. Each run results in a set
of PNG files displaying the result from the model
and parameters that were used. The PNG files are
saved in the system temp folder.

@author: Brian Munksgaard.
"""
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.datasets import make_moons
import numpy as np
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn import svm
from sklearn import linear_model
from sklearn.cluster import KMeans
from sklearn.cluster import DBSCAN

#
# Returns a string with parameters of the
# specified MLP model. Each parameter is
# separated by a newline character.
#
def get_mlp_legend(p_mlp_model):
    legend = "hl: " + str(p_mlp_model.hidden_layer_sizes)
    legend = legend + "\nmax_iter: " + str(p_mlp_model.max_iter)
    legend = legend + "\nbatch_size: " + str(p_mlp_model.batch_size)
    legend = legend + "\nrandom_state: " + str(p_mlp_model.random_state)
    legend = legend + "\nactivation: " + p_mlp_model.activation
    legend = legend + "\nalpha: " + str(p_mlp_model.alpha)
    return legend

#
# Returns a string with parameters of the
# specified SVC model. Each parameter is
# separated by a newline character.
#
def get_svc_legend(p_svc_model):
    legend = "kernel: " + p_svc_model.kernel
    legend = legend + "\ngamma: " + str(p_svc_model.gamma)
    legend = legend + "\nC: " + str(p_svc_model.C)
    legend = legend + "\ndegree: " + str(p_svc_model.degree)
    legend = legend + "\nrandom_state: " + str(p_svc_model.random_state)
    return legend

#
# Returns a string with parameters of the
# specified logistic model. Each parameter is
# separated by a newline character.
#
def get_log_legend(p_log_model):
    legend = "C: " + str(p_log_model.C)
    legend = legend + "\nmax_iter: " + str(p_log_model.max_iter)
    legend = legend + "\nrandom_state: " + str(p_log_model.random_state)
    return legend

#
# Returns a string with parameters of the
# specified KMEANS model. Each parameter is
# separated by a newline character.
#
def get_kmeans_legend(p_k_model):
    legend = "clusters: " + str(p_k_model.n_clusters)
    legend = legend + "\nmax_iter: " + str(p_k_model.max_iter)
    legend = legend + "\nrandom_state: " + str(p_k_model.random_state)
    return legend

#
# Split data into training & test and perform feature scaling.
#
def split_and_scale(x_data, y_data):
    # Split in train and test set
    x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=0.20)

    # Perform feature scaling.
    scaler = StandardScaler()
    scaler.fit(x_train)
    x_train = scaler.transform(x_train)
    x_test = scaler.transform(x_test)

    return x_train, x_test, y_train, y_test

#
# Displays a mesh plot with test data and predictions.
# Parameters:
#   p_title: The title of the plot.
#   p_legend: A text with the parameters used in the model.
#   p_x_test: All the test values for x.
#   p_y_test: All the test values for y.
#   p_model: The model itself.
#
def display_mesh(p_title, p_legend, p_x_test, p_y_test, p_model):
    h = .02  # Step size in th mesh.

    x_min, x_max = p_x_test[:, 0].min() - 1, p_x_test[:, 0].max() + 1
    y_min, y_max = p_x_test[:, 1].min() - 1, p_x_test[:, 1].max() + 1

    xx, yy = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))
    plt.subplot(1, 1, 1)
    plt.subplots_adjust(wspace=0.4, hspace=0.4)
    z = p_model.predict(np.c_[xx.ravel(), yy.ravel()])

    # Put the result into a color plot
    z = z.reshape(xx.shape)
    plt.contourf(xx, yy, z, cmap=plt.cm.coolwarm, alpha=0.8)

    # Plot the training points
    plt.scatter(p_x_test[:, 0], p_x_test[:, 1], c=p_y_test)

    # Title and model parameters.
    plt.title(p_title)
    x_pos = x_max / 2
    y_pos = y_min
    plt.text(x_pos, y_pos, p_legend, bbox=dict(boxstyle="square", facecolor="white"))

    # Save the figure in the system temp folder.
    filename = "/tmp/" + p_title + "-" + p_legend.replace("\n", ", ").replace(":", "->") + ".png"
    print("Filename: " + filename)
    plt.savefig(filename)

    # Execute display and save.
    plt.show()

#
# This procedure is used to train and display
# the results of an MLP model.
#
def train_and_display_mlp(p_mlp_model, x_data, y_data):
    x_train, x_test, y_train, y_test = split_and_scale(x_data, y_data)
    p_mlp_model.fit(x_train, y_train)
    display_mesh("Model type - MLP", get_mlp_legend(p_mlp_model), x_test, y_test, p_mlp_model)

#
# This procedure is used to train and display
# the results of a SVC model.
#
def train_and_display_svc(p_svc_model, x_data, y_data):
    x_train, x_test, y_train, y_test = split_and_scale(x_data, y_data)
    p_svc_model.fit(x_train, y_train)
    display_mesh("Model type - SVC", get_svc_legend(p_svc_model), x_test, y_test, p_svc_model)

#
# This procedure is used to train and display
# the results of a logistic regression model.
#
def train_and_display_log(p_log_model, x_data, y_data):
    x_train, x_test, y_train, y_test = split_and_scale(x_data, y_data)
    p_log_model.fit(x_train, y_train)
    display_mesh("Model type - LOG", get_log_legend(p_log_model), x_test, y_test, p_log_model)

#
# This procedure is used to train and display
# the results of a KMEANS model.
#
def train_and_display_kmeans(p_k_model, x_data, y_data):
    x_train, x_test, y_train, y_test = split_and_scale(x_data, y_data)
    p_k_model.fit(x_train)
    display_mesh("Model type - KMEANS", get_kmeans_legend(p_k_model), x_test, y_test, p_k_model)

# -------------------------------------------
# Generate moons 2d classification data set
# -------------------------------------------
X, y = make_moons(n_samples=1000, noise=0.1)

# Visualize the data set.
plt.figure()
colormap = ListedColormap(['#FF0000', '#00FF00', '#0000FF'])
plt.scatter(X[:, 0], X[:, 1], c=y, cmap=colormap,edgecolor='black', s=20)

filename = "/tmp/" + "moons_data.png"
print("Filename: " + filename)
plt.savefig(filename)
plt.show()

#
# Now run the various models.
#

# ----------------------
# MLP models.
# ----------------------
#mlp = MLPClassifier(hidden_layer_sizes=(4, 4), max_iter=1000)
#train_and_display_mlp(mlp, X, y)

#mlp = MLPClassifier(hidden_layer_sizes=(8, 8), max_iter=1000)
#train_and_display_mlp(mlp, X, y)

#mlp = MLPClassifier(hidden_layer_sizes=(4, 4, 4, 4), max_iter=1000)
#train_and_display_mlp(mlp, X, y)

#mlp = MLPClassifier(hidden_layer_sizes=(8, 8, 8), max_iter=1000)
#train_and_display_mlp(mlp, X, y)

#mlp = MLPClassifier(hidden_layer_sizes=(8, 8, 8, 8), max_iter=1000)
#train_and_display_mlp(mlp, X, y)

#mlp = MLPClassifier(hidden_layer_sizes=(10, 10, 10, 10), max_iter=1000, alpha=0.01)
#train_and_display_mlp(mlp, X, y)

#mlp = MLPClassifier(hidden_layer_sizes=(10, 10, 10, 10), max_iter=1000, alpha=1.0)
#train_and_display_mlp(mlp, X, y)

#mlp = MLPClassifier(hidden_layer_sizes=(10, 10, 10, 10), max_iter=1000, alpha=1.0, activation='tanh')
#train_and_display_mlp(mlp, X, y)

# ----------------------------
# Logistic regression models.
# ----------------------------
#logreg = linear_model.LogisticRegression(C=1000)
#train_and_display_log(logreg, X, y)

# ----------------------------
# KMeans clustering models.
# ----------------------------
#kmeans = KMeans(n_clusters=4, random_state=0)
#train_and_display_kmeans(kmeans, X, y)

# ----------------------------
# SVC clustering models.
# ----------------------------
#svc = svm.SVC(kernel='linear', gamma=0.7, C=1)
#train_and_display_svc(svc, X, y)

svc = svm.SVC(kernel='poly', gamma=3.7, C=2, degree=5)
train_and_display_svc(svc, X, y)

# RBF.
# Vary C.
svc = svm.SVC(kernel='rbf', gamma=0.7, C=1)
train_and_display_svc(svc, X, y)

svc = svm.SVC(kernel='rbf', gamma=0.7, C=5)
train_and_display_svc(svc, X, y)

svc = svm.SVC(kernel='rbf', gamma=0.7, C=0.2)
train_and_display_svc(svc, X, y)

# Vary gamma.
svc = svm.SVC(kernel='rbf', gamma=0.3, C=1)
train_and_display_svc(svc, X, y)

svc = svm.SVC(kernel='rbf', gamma=2, C=1)
train_and_display_svc(svc, X, y)


import os
import sys
import datetime
import traceback
import pandas as pd
import numpy as np
import utils
sys.path.append('/bearbin-data/home/sdou/gdrive/segpy/')
from segpy.reader import create_reader

UTC = datetime.timezone.utc

class PeekSegy:
    def __init__(self, file_segy):
        self.file_segy = file_segy
        with open(file_segy, 'rb') as f:
            try:
                self.segy_reader = create_reader(f)
            except UnicodeDecodeError as ue:
                try:
                    self.segy_reader = create_reader(f, encoding='cp037')
                except UnicodeDecodeError('encoding not deciphered: neither ascii nor cp037') as ue:
                    traceback.print_tb(ue.__traceback__)
                    sys.exit(1)

    @staticmethod
    def get_timestamp(file_segy):
        """
        Get time stamp from segy file name
        :param file_segy: full path of input segy file
        :return: time stamp as datetime object
        :how to use:
         PeekSegy.get_timestamp(file_segy)
        """
        filename = os.path.splitext(os.path.basename(file_segy))[0]
        # Select the longest string
        timestamp_str = max(filename.split('_'), key=len)
        dateformat = {12: '%y%m%d%H%M%S', 14:'%Y%m%d%H%M%S'}
        return datetime.datetime.strptime(
                    timestamp_str, dateformat[len(timestamp_str)]
                ).replace(tzinfo=UTC)

    @property
    def filesize(self):
        return utils.filesize(self.file_segy)

    @property
    def dt(self):
        file_header = self.segy_reader.binary_reel_header
        return 1.0e-6 * getattr(file_header, 'sample_interval')

    @property
    def shape(self):
        n_samples = self.segy_reader.max_num_trace_samples()
        n_traces = self.segy_reader.num_traces()
        return (n_samples, n_traces)

    @property
    def duration(self):
        return self.dt*(self.shape[0] - 1)

    @property
    def tb(self):
        trace_header = self.segy_reader.trace_header(0)
        th_list = ['year_recorded', 'day_of_year', 'hour_of_day',
                   'minute_of_hour', 'second_of_minute']
        year, doy, hour, min, sec = (
            getattr(trace_header, th)  for th in th_list
        )
        return (
            datetime.datetime(year=year, month=1, day=1, tzinfo=UTC) +
            datetime.timedelta(
                days=doy - 1, hours=hour, minutes=min, seconds=sec
            )
        )

    @property
    def te(self):
        return self.tb + datetime.timedelta(seconds=self.duration)

class SegyToSeis:
    def __init__(self, file_segy, tmin=0, tmax=None, trace_indexes=None):
        try:
            self.param = dict(tmin=tmin, tmax=tmax, trace_indexes=trace_indexes)
            self.segy = PeekSegy(file_segy)
            if tmin > self.segy.duration:
                raise ValueError('tmin larger than duration time')
            elif tmax and tmin >= tmax:
                raise ValueError('tmax must be larger than tmin')
        except ValueError as ve:
            traceback.print_tb(ve.__traceback__)
            sys.exit(1)

    def _t_index(self):
        i_tmin = 0
        i_tmax = self.segy.shape[0]
        t = self.segy.dt * np.arange(self.segy.shape[0])
        if self.param['tmin'] is not None:
            i_tmin = utils.nearest_index(t, self.param['tmin'])
        if self.param['tmax'] is not None:
            i_tmax = utils.nearest_index(t, self.param['tmax'])
        return (i_tmin, i_tmax)

    @property
    def trace_indexes(self):
        if self.param['trace_indexes'] is not None:
            return self.param['trace_indexes']
        else:
            return list(self.segy_reader.trace_indexes())

    @property
    def t(self):
        (i_min, i_max) = self._t_index()
        return self.segy.dt * np.arange(i_min, i_max)

    def _tb(self):
        tmin = self.param['tmin']
        if (tmin is not None) and (tmin > 0):
            return self.segy.tb + datetime.timedelta(seconds=tmin)
        else:
            return self.segy.tb

    def _te(self):
        tmax = self.param['tmax']
        if (tmax is not None) and (tmax < self.segy.duration):
            return self.segy.te - datetime.timedelta(
                seconds=(self.segy.duration - tmax)
            )
        else:
            return self.segy.te

    def _duration(self):
        return (self._te() - self._tb()).total_seconds()

    def _shape(self):
        return (len(self.t), len(self.trace_indexes))

    @property
    def dat(self):
        segy_reader = self.segy.segy_reader
        (i_tmin, i_tmax) = self._t_index()
        self.__dat = np.zeros(self._shape(), dtype=float)
        for i, trace_index in enumerate(self.trace_indexes):
            self.__dat[:, i] = segy_reader.trace_samples(
                trace_index, start=i_tmin, stop=i_tmax
            )
        return self.__dat

    @property
    def fh(self):
        return pd.Series(
            dict(tb=self._tb(), te=self._te(), dt=self.segy.dt,
                 duration=self._duration(),
                 n_samples=self._shape()[0], n_traces=self._shape()[1])
        )

    def to_Series(self):
        """
        Output bare-bone data as pandas Series to memory
        :return: pandas Series; fields = [fh, t, trace_indexes, dat]
        """
        return pd.Series(
            dict(fh=self.fh, t=self.t, trace_indexes=self.trace_indexes,
                 dat=self.dat)
        )

    def to_pickle(self, path):
        """
        Save bare-bone data as pickle
        :param path: full path of output pickle file
        :return: None
        """
        self.to_Series().to_pickle(path)
        return None


"""Test model generation."""
import io
from textwrap import dedent

import numpy as np
import pandas as pd
import pytest

from respy.config import EXAMPLE_MODELS
from respy.likelihood import get_crit_func
from respy.pre_processing.model_checking import validate_options
from respy.pre_processing.model_processing import _convert_labels_in_formulas_to_codes
from respy.pre_processing.model_processing import _identify_relevant_covariates
from respy.pre_processing.model_processing import _parse_initial_and_max_experience
from respy.pre_processing.model_processing import process_params_and_options
from respy.tests.random_model import generate_random_model
from respy.tests.random_model import simulate_truncated_data
from respy.tests.utils import process_model_or_seed

@pytest.mark.parametrize("seed", range(5))
def test_generate_random_model(seed):
    """Test if random model specifications can be simulated and processed."""
    np.random.seed(seed)

    params, options = generate_random_model()

    df = simulate_truncated_data(params, options)

    crit_func = get_crit_func(params, options, df)

    crit_val = crit_func(params)

    assert isinstance(crit_val, float)

@pytest.mark.parametrize("model_or_seed", EXAMPLE_MODELS + list(range(10)))
def test_model_options(model_or_seed):
    _, options = process_model_or_seed(model_or_seed)

    _, options = process_params_and_options(_, options)

    validate_options(options)

@pytest.mark.parametrize("model_or_seed", EXAMPLE_MODELS + list(range(10)))
def test_sorting_of_type_probability_parameters(model_or_seed):
    # Set configuration for random models.
    n_types = np.random.randint(2, 5)
    n_type_covariates = np.random.randint(2, 4)

    params, options = process_model_or_seed(
        model_or_seed, n_types=n_types, n_type_covariates=n_type_covariates
    )

    optim_paras, options = process_params_and_options(params, options)

    # Update variables if not a random model, but an example model is tested.
    if isinstance(model_or_seed, str):
        n_types = optim_paras["n_types"]
        n_type_covariates = (
            None if n_types == 1 else len(optim_paras["type_covariates"])
        )

    if optim_paras["n_types"] > 1:
        # Resort type probability parameters.
        types = [f"type_{i}" for i in range(2, optim_paras["n_types"] + 1)]
        params.loc[types] = params.sort_index(ascending=False).loc[types]

        optim_paras_, _ = process_params_and_options(params, options)

        assert (optim_paras["type_prob"] == optim_paras_["type_prob"]).all()

def test_parse_initial_and_max_experience():
    """Test ensures that probabilities are transformed with logs and rest passes."""
    choices = ["a", "b"]

    options = {"n_periods": 10}
    optim_paras = {"choices_w_exp": choices, "choices": {"a": {}, "b": {}}}
    params = pd.DataFrame(
        {
            "category": [
                "initial_exp_a_0",
                "initial_exp_a_5",
                "initial_exp_b_0",
                "initial_exp_b_5",
                "maximum_exp",
            ],
            "name": ["probability"] * 2 + ["constant"] * 2 + ["b"],
            "value": [2, 2, np.log(2), np.log(2), 5],
        }
    ).set_index(["category", "name"])["value"]

    optim_paras = _parse_initial_and_max_experience(optim_paras, params, options)

    assert (
        optim_paras["choices"]["a"]["start"][0]
        == optim_paras["choices"]["a"]["start"][5]
    ).all()
    assert (
        optim_paras["choices"]["b"]["start"][0]
        == optim_paras["choices"]["b"]["start"][5]
    ).all()
    assert optim_paras["choices"]["a"]["max"] == options["n_periods"] - 1 + max(
        optim_paras["choices"]["a"]["start"]
    )
    assert optim_paras["choices"]["b"]["max"] == 5

@pytest.mark.parametrize("seed", list(range(10)))
def test_normalize_probabilities(seed):
    np.random.seed(seed)
    constraints = {"observables": [3]}
    params, options = generate_random_model(point_constr=constraints)
    optim_paras_1, _ = process_params_and_options(params, options)

    for group in ["initial_exp_edu", "observable_"]:
        mask = params.index.get_level_values(0).str.contains(group)
        params.loc[mask, "value"] = params.loc[mask, "value"].to_numpy() / 2

    optim_paras_2, _ = process_params_and_options(params, options)

    for key in optim_paras_1["choices"]["edu"]["start"]:
        np.testing.assert_array_almost_equal(
            optim_paras_1["choices"]["edu"]["start"][key],
            optim_paras_2["choices"]["edu"]["start"][key],
        )
    for level in optim_paras_1["observables"]["observable_0"]:
        np.testing.assert_array_almost_equal(
            optim_paras_1["observables"]["observable_0"][level],
            optim_paras_2["observables"]["observable_0"][level],
        )

def test_identify_relevant_covariates():
    params = pd.read_csv(
        io.StringIO(
            dedent(
                """
                category,name,value
                wage_a,constant,1
                nonpec_b,upper_upper,1
                wage_c,upper_upper_with_spacing_problem,1
                """
            )
        ),
        index_col=["category", "name"],
    )

    options = {
        "covariates": {
            "constant": "1",
            "nested_covariate": "2",
            "upper": "nested_covariate > 2",
            "upper_upper": "upper == 5",
            "unrelated_covariate": "2",
            "unrelated_covariate_upper": "unrelated_covariate",
            "upper_upper_with_spacing_problem": "upper>2",
        }
    }

    relevant_covariates = _identify_relevant_covariates(options, params)

    expected = {
        "covariates": {
            "constant": "1",
            "nested_covariate": "2",
            "upper": "nested_covariate > 2",
            "upper_upper": "upper == 5",
            "upper_upper_with_spacing_problem": "upper>2",
        }
    }

    assert expected == relevant_covariates

def test_convert_labels_in_covariates_to_codes():
    optim_paras = {
        "choices": ["fishing", "hammock"],
        "observables": {"fishing_grounds": ["poor", "rich"]},
        "choices_w_exp": ["fishing"],
    }

    options = {
        "covariates": {
            "rich_fishing_grounds": "fishing_grounds == 'rich'",
            "do_fishing": "choice == 'fishing'",
            "do_hammock": 'choice == "hammock"',
        },
        "core_state_space_filters": [],
        "inadmissible_states": {},
    }

    options = _convert_labels_in_formulas_to_codes(options, optim_paras)

    expected = {
        "rich_fishing_grounds": "fishing_grounds == 1",
        "do_fishing": "choice == 0",
        "do_hammock": "choice == 1",
    }

    assert options["covariates"] == expected


from constraint import Constraint

class BaseConstraint(Constraint):
	def __init__(self, order, machine):
		Constraint.__init__(self)
		self.order = order
		self.machine = machine

class MachineBreaksConstraint(BaseConstraint):
	def __call__(self, variables, domains, assignments, forwardcheck = False):
		enterTime = assignments[variables[0]]
		orderRange = range(enterTime,
						   enterTime + self.order.recipe[self.machine.name])

		for t in self.machine.setOfBreaks():
			for t2 in orderRange:
				if t == t2:
					return False
		return True


from flask import Flask
'''
当访问路径是以下任何一个的时候：
    127.0.0.1:5000/
    127.0.0.1:5000/index
    127.0.0.1:5000/数字
    127.0.0.1:5000/index/数字
将程序交给 index()视图处理
根据数字的传递情况给出响应
    如果传递数字到路由中：
    响应：您当前看到的页数为：xxx
    如果没有传递数字到路由中：
        响应：您当前看的页数为：1
'''
app = Flask(__name__)

@app.route('/')
@app.route('/index')
@app.route('/<int:number>')
@app.route('/index/<int:number>')
def show(number=1):
    return "您当前看到的页数为%d"%number

if __name__ == '__main__':
    app.run(debug=True)


from shutil import copyfile
import numpy as np
import os
from scipy.optimize import curve_fit
from scipy.optimize import leastsq
import pickle
import matplotlib.pyplot as plt
from matplotlib import rcParams
import sys

def get_populations(ics, datadir, state_from, state_to):
    ''' Get population data for transfer events from state_from to state_to. '''

    print('Loading excited state trajectories and extracting population information.')
    absolute_transfer = {}
    relative_transfer = {}
    energy_gaps = {}

    for ic in ics:
        data = pickle.load(open(datadir+('/%04d.pickle' %ic), 'rb'))
        total_ic_transfer = 0
        for tbf_key in data.keys():
            tbf = data[tbf_key]
            tbf_state = tbf['spawn_info']['tbf_state']
            if tbf_state==state_to:
                parent_state = tbf['spawn_info']['parent_state']
                if parent_state==state_from:

                    # Get the population of the parent TBF at the time of spawning
                    parent_key = '%04d-%04d' %(ic, tbf['spawn_info']['parent_id'])
                    parent_tbf = data[parent_key]
                    spawn_time = tbf['spawn_info']['spawn_time']
                    parent_tbf_times = parent_tbf['time_steps']
                    spawn_ind_parent = np.argmin(np.abs(parent_tbf_times - spawn_time))
                    parent_pop = parent_tbf['populations'][spawn_ind_parent]

                    # Get the population of the current TBF after population exchange has stabilized
                    tbf_pop = tbf['populations']
                    if len(tbf_pop) > 20:
                        pop_transferred = tbf_pop[20]
                        upper = tbf['energies']['s%d' %state_from][:20]
                        lower = tbf['energies']['s%d' %state_to][:20]
                        min_gap = np.min(np.abs(upper-lower)) * 27.21138602
                    else:
                        pop_transferred = tbf_pop[-1]
                        upper = tbf['energies']['s%d' %state_from][:-1]
                        lower = tbf['energies']['s%d' %state_to][:-1]
                        min_gap = np.min(np.abs(upper-lower)) * 27.21138602

                    if parent_pop > pop_transferred:
                        transfer_efficiency = pop_transferred / parent_pop
                        total_ic_transfer += pop_transferred
                        absolute_transfer[tbf_key] = pop_transferred
                        relative_transfer[tbf_key] = transfer_efficiency
                        energy_gaps[tbf_key] = min_gap

        print('Total transferred: ', total_ic_transfer)
        print()

    return absolute_transfer, relative_transfer, energy_gaps

def plot_transfer(absolute_transfer, relative_transfer, energy_gaps, bin_spacing=0.05):

    tbf_keys = [ x for x in absolute_transfer.keys() ]
    all_populations = [ absolute_transfer[key] for key in tbf_keys ]
    all_efficiencies = [ relative_transfer[key] for key in tbf_keys ]
    all_energy_gaps = [ energy_gaps[key] for key in tbf_keys ]

    nbins = int( np.ceil(np.max(all_energy_gaps) / bin_spacing) )

    # Compute fractional population transferred from absolute transfer
    abs_bins = np.zeros(nbins)
    total_population_transfer = np.sum(all_populations)
    for pop, gap in zip(all_populations, all_energy_gaps):
        abs_bins[ int(np.floor(gap/bin_spacing)) ] += pop
    abs_bins = abs_bins / total_population_transfer

    # Compute average efficiency for each gap
    rel_bins = {}
    for b in range(nbins):
        rel_bins['%d' %b] = []
    for pop, gap in zip(all_efficiencies, all_energy_gaps):
        rel_bins['%d' %(int(np.floor(gap/bin_spacing)))].append(pop)
    rel_bins2 = np.zeros(nbins)
    for b in range(nbins):
        nevents = len(rel_bins['%d' %b])
        rel_bins2[b] = np.sum(rel_bins['%d' %b]) / nevents
    # If fractional transfer is small in a bin, its efficiency is more or less an artifact,
    # so zero it out. Threshold set at 0.01.
    rel_bins = [ rel_bins2[b] if abs_bins[b]>0.01 else 0 for b in range(nbins) ]

    rcParams.update({'figure.autolayout': True})
    fig, ax1 = plt.subplots(figsize=(6,5))
    labelsize = 14
    ticksize = 12
    plt.rc('xtick',labelsize=ticksize)
    plt.rc('ytick',labelsize=ticksize)

    width = 0.3 * bin_spacing
    x_labels = np.arange(nbins)*bin_spacing
    abs_bars = ax1.bar(x_labels - width/2, abs_bins, width, label='Absolute Population Transfer', color='darkslateblue')
    ax1.set_ylabel('Fractional Population Transferred', fontsize=labelsize, color='darkslateblue')
    ax1.set_xlabel('Minimum Energy Gap [eV]', fontsize=labelsize, color='black')
    ax1.set_xticks([ bin_spacing*x for x in range(0, nbins) ])
    ax1.tick_params('y', colors='darkslateblue', labelsize=ticksize)
    ax1.tick_params('x', colors='black', labelsize=ticksize)

    ax2 = ax1.twinx()
    rel_bars = ax2.bar(x_labels + width/2, rel_bins, width, label='Population Transfer Efficiency', color='mediumvioletred')
    ax2.set_ylabel('Relative Population Transferred', fontsize=labelsize, color='mediumvioletred', rotation=270, labelpad=20)
    ax2.tick_params('y', colors='mediumvioletred', labelsize=ticksize)

    if not os.path.isdir('./figures/'):
        os.mkdir('./figures/')
    plt.savefig('./figures/population_transfer.pdf', dpi=300)
    plt.close()

'''
Specify the time grid and ICs to use.
Can use a coarser time grid than is used here and it shouldn't change the result.
'''
datadir = '../../1_collect_data/data/'
fmsinfo = pickle.load(open(datadir+'/fmsinfo.pickle', 'rb'))
ics = fmsinfo['ics']
nstates = fmsinfo['nstates']
absolute_transfer, relative_transfer, energy_gaps = get_populations(ics, datadir, state_from=1, state_to=0)
plot_transfer(absolute_transfer, relative_transfer, energy_gaps)


import numpy as np
import healpy as hp
import h5py
import re
import os
import glob

if __name__ == "__main__":
    LOCAL = False

    # specify parameters for RHT data
    nchunks = 32
    startvel = 73
    stopvel  = 111

    # need to step through all R(n, v, theta) chunks
    nside = 1024
    npix = 12*nside**2
    stepsize = np.int(npix/nchunks)
    all_startpix = np.arange(0, npix, stepsize, dtype=np.int_)
    all_stoppix = all_startpix + stepsize

    # final data will be NHI(n, theta) with dimensions (npix, ntheta)
    ntheta = 165
    NHI_n_theta = np.zeros((npix, ntheta), np.float_)

    # need sum of I*R over all v and theta for denominator
    sum_I_R_n = np.zeros(npix)

    # step through each chunk of RHT data
    for _i, (startpix, stoppix) in enumerate(zip(all_startpix, all_stoppix)):
        print("beginning chunk {}: from {} to {}".format(_i, startpix, stoppix))

        # load RHT data
        rht_root = '../data/'
        rht_fn = rht_root + 'Rtheta_{}_{}_1_hpix{}_{}.h5'.format(startvel, stopvel, startpix, stoppix)

        with h5py.File(rht_fn, 'r') as rht_file:
            RHT_n_v_theta = rht_file['hthets_arr'][:] # needs colon indexer or will just be pointer #[startpix:stoppix]
            v_index_list = rht_file['hthets_arr'].attrs['velocities']
            nvels = len(v_index_list)

        # load HI intensity data I(n, v)
        if LOCAL:
            hi4pi_root = '/Users/Dropbox/HI4PI/'
        else:
            hi4pi_root = '/data/seclark/HI4PI/full_data/'
        HI_n_v = np.zeros((stoppix-startpix, nvels), np.float_)
        hi4pi_fn = hi4pi_root + 'HI4PI_120kms.h5'
        with h5py.File(hi4pi_fn, 'r') as f:
            for _i, _vel in enumerate(v_index_list):
                HI_n_v[:, _i] = f['survey'][startpix:stoppix, _vel]

        # Intensity-weighted RHT data
        I_weighted_RHT = RHT_n_v_theta * HI_n_v[:, :, np.newaxis]

        # sum over velocity axis
        R_I_n_theta = np.nansum(I_weighted_RHT, axis=1)
        NHI_n_theta[startpix:stoppix, :] = R_I_n_theta

        # sum over velocity and theta axes
        sum_I_R_n[startpix:stoppix] = np.nansum(R_I_n_theta, axis=1)

    # load NHI(n)
    if LOCAL:
        hi4pi_root = '/Users/susanclark/Dropbox/HI4PI/'
    else:
        hi4pi_root = '/data/seclark/HI4PI/full_data/'
    NHI = hp.fitsfunc.read_map(hi4pi_root+'HI4PI_NHI.fits')

    # compute Eq. 1
    NHI_over_denominator = NHI / sum_I_R_n
    final_NHI_n_theta = (NHI_over_denominator[:, np.newaxis] * NHI_n_theta)

    print("final_NHI_n_theta shape: {}".format(final_NHI_n_theta.shape))

    compress = False
    out_root = '../data/'
    NHI_n_theta_fn = out_root + 'NHI_n_theta_{}_{}_1.h5'.format(startvel, stopvel)
    with h5py.File(NHI_n_theta_fn, 'w') as f:
        if compress:
            dset = f.create_dataset(name='NHI_n_theta', data=final_NHI_n_theta, compression="gzip")
        else:
            dset = f.create_dataset(name='NHI_n_theta', data=final_NHI_n_theta)
        dset.attrs['velocities'] = v_index_list

    # sanity check: the sum along theta of this map should be NHI
    NHI_n_theta_thetsum = np.nansum(final_NHI_n_theta, axis=1)
    print("sum of difference: {}".format(np.nansum(NHI_n_theta_thetsum - NHI)))
    print("sum of NHI n theta: {}".format(np.nansum(NHI_n_theta_thetsum)))
    print("sum of NHI: {}".format(np.nansum(NHI)))


import pandas
import sys
from google.cloud import firestore
import math

pre_column_names = [
    "Form inserted",
    "* Are you?",
    "* How old are you?",
    "* What Province are you contacting us from?",
    "On a scale of 0 to 7, how upset are you right now right now?",
    "siteLanguage",
    "chat id"
]

post_column_names = [
    "chatid",
    "2. On a scale of 0 to 7, how upset are you right now?"
]

if __name__ == '__main__':
    pre_file_loc = sys.argv[1]
    pre_sheet_num = sys.argv[2]
    post_file_loc = sys.argv[3]
    post_sheet_num = sys.argv[4]
    pre_sheet = pandas.read_excel(pre_file_loc, int(pre_sheet_num))
    post_sheet = pandas.read_excel(post_file_loc, int(post_sheet_num))
    db = firestore.Client('kids-help-phone-dash')
    the_datas = []
    nums = {}
    for pre_name in pre_column_names:
        the_datas.append(pre_sheet[pre_name])
    post_datas = []
    for post_name in post_column_names:
        post_datas.append(post_sheet[post_name])

    tot_lt_5 = 0
    tot_6_to_10 = 0
    tot_11_to_15 = 0
    tot_16_to_20 = 0
    tot_21_plus = 0
    for i in range(len(the_datas[0])):
        call_id = the_datas[6][i]
        gender = the_datas[1][i]
        if gender == 'I identify differently than the above':
            gender = 'Other'

        age = str(the_datas[2][i]).split(' ')[0]
        if age == 'Over':
            age = '21'
        age = int(age)
        ind = post_datas[0][post_datas[0] == call_id]
        if ind.size == 0:
            print('No after call log for: ' + str(call_id))
            continue
        print(ind)
        ind = ind.index[0]
        print(ind)
        if age in nums.keys():
            nums[age] = nums[age]+1
        else:
            nums[age] = 1
        data = {
            'date': the_datas[0][i],
            'gender': gender,
            'age': age,
            'province': the_datas[3][i],
            'language': the_datas[5][i],
            'before_rating': the_datas[4][i],
            'after_rating': post_datas[1][ind]
        }
        db.collection(u'call-stats').document(str(call_id)).set(data)
    print(nums)
    total = 0
    for key,val in nums.items():
        total += val
    print(total)


#!/usr/bin/env python

#
# This work is licensed under the terms of the MIT license.
# For a copy, see <https://opensource.org/licenses/MIT>.

#
# This file contains the basic scenario and scenario manager classes
# These must not be modified and are for reference only!

"""
This module provides the Scenario and ScenarioManager implementations.
"""

import time
import py_trees

class Scenario(object):

    """
    Basic scenario class. This class holds the behavior_tree describing the
    scenario and the test criteria.

    The user must not modify this function.

    Important parameters:
    - behavior_tree: User defined scenario with py_tree
    - criteria_list: List of user defined test criteria with py_tree
    - timeout (default = 60s): Timeout of the scenario in seconds
    - terminate_on_failure: Terminate scenario on first failure
    """

    def __init__(self, behavior_tree, timeout=60, terminate_on_failure=False):
        self.behavior = behavior_tree

        # Create node for timeout
        self.timeout_node = py_trees.timers.Timer(
            name="TimeOut", duration=timeout)

        # Create overall py_tree
        self.scenario = py_trees.composites.Parallel(
            name="Scenario with TimeOut and TestCriteria",
            policy=py_trees.common.ParallelPolicy.SUCCESS_ON_ONE)
        self.scenario.add_child(self.behavior)
        self.scenario.add_child(self.timeout_node)
        self.scenario.setup(timeout=1)

class ScenarioManager(object):

    """
    Basic scenario manager class. This class holds all functionality
    required to start, and analyze a scenario.

    The user must not modify this function.
    """

    def __init__(self, scenario):
        """
        Init requires scenario as input
        """
        self.scenario = scenario
        self.scenario_tree = scenario.scenario
        self.scenario_duration = 0

        # To print the scenario tree uncomment the next line
        # py_trees.display.render_dot_tree(self.scenario_tree)

    def run_scenario(self):
        """
        Execute scenario
        """
        start = time.time()
        while self.scenario_tree.status != py_trees.common.Status.SUCCESS:
            # print("\n--------- Tick ---------\n")
            self.scenario_tree.tick_once()
            # print("\n")
            # py_trees.display.print_ascii_tree(
            #    self.scenario_tree, show_status=True)

            if self.scenario_tree.status == py_trees.common.Status.FAILURE:
                print("Terminated due to failure")
                break

            # Sleep for a small time to avoid high cpu load
            time.sleep(0.1)

        end = time.time()
        self.scenario_duration = end - start

    def run_scenario_per_frame(self):
        """
        Execute scenario
        """
        if self.scenario_tree.status != py_trees.common.Status.SUCCESS:
            # print("\n--------- Tick ---------\n")
            self.scenario_tree.tick_once()
            # print("\n")
            # py_trees.display.print_ascii_tree(
            #    self.scenario_tree, show_status=True)

            if self.scenario_tree.status == py_trees.common.Status.FAILURE:
                print("Terminated due to failure")