import sys

from awswrangler.exceptions import UnsupportedType
from awswrangler.glue.utils import (
    delete_table_if_exists,
    create_table,
    add_partitions,
    table_exists,
)

if sys.version_info.major > 2:
    string_types = str  # noqa
else:
    # noinspection PyUnresolvedReferences
    string_types = basestring  # noqa

def _type_pandas2athena(dtype):
    dtype = dtype.lower()
    if dtype == "int32":
        return "int"
    elif dtype == "int64":
        return "bigint"
    elif dtype == "float32":
        return "float"
    elif dtype == "float64":
        return "double"
    elif dtype == "bool":
        return "boolean"
    elif dtype == "object" and isinstance(dtype, string_types):
        return "string"
    elif dtype[:10] == "datetime64":
        return "string"
    else:
        raise UnsupportedType("Unsupported Pandas type: " + dtype)

def _build_schema(df, partition_cols, preserve_index):
    schema_built = []
    if preserve_index:
        name = str(df.index.name) if df.index.name else "index"
        df.index.name = "index"
        dtype = str(df.index.dtype)
        if name not in partition_cols:
            athena_type = _type_pandas2athena(dtype)
            schema_built.append((name, athena_type))
    for col in df.columns:
        name = str(col)
        dtype = str(df[name].dtype)
        if name not in partition_cols:
            athena_type = _type_pandas2athena(dtype)
            schema_built.append((name, athena_type))
    return schema_built

def _get_table_name(path):
    if path[-1] == "/":
        path = path[:-1]
    return path.rpartition("/")[2]

def write_metadata(
    df,
    path,
    session_primitives,
    partition_paths,
    database=None,
    table=None,
    partition_cols=None,
    preserve_index=True,
    file_format="parquet",
    mode="append",
):
    schema = _build_schema(
        df=df, partition_cols=partition_cols, preserve_index=preserve_index
    )
    table = table if table else _get_table_name(path)
    if mode == "overwrite":
        delete_table_if_exists(
            database=database, table=table, session_primitives=session_primitives
        )
    exists = table_exists(
        database=database, table=table, session_primitives=session_primitives
    )
    if not exists:
        create_table(
            database=database,
            table=table,
            schema=schema,
            partition_cols=partition_cols,
            path=path,
            file_format=file_format,
            session_primitives=session_primitives,
        )
    add_partitions(
        database=database,
        table=table,
        partition_paths=partition_paths,
        file_format=file_format,
        session_primitives=session_primitives,
    )


# -*- coding: utf-8 -*-
#输入为每个点的坐标信息
#输出为每个点对应的13feature的value

import math
import os
import sys
import numpy
from osgeo import gdal
from osgeo.gdalconst import *
import csv
import struct
import pdb
from os.path import join
import pandas as pd
import better_exceptions
import glob

# extract points one by one band
def pt2fmt(pt):
    fmttypes = {GDT_Byte: 'B',GDT_Int16: 'h',GDT_UInt16: 'H',GDT_Int32: 'i',GDT_UInt32: 'I',GDT_Float32: 'f',GDT_Float64: 'f'}
    return fmttypes.get(pt, 'x')
fire_dir = "/home/lan/data_pool/orange/sample_point.csv"
immutable_dir = "/home/lan/data_pool/orange/immutable"
variable_dir = "/home/lan/data_pool/orange/variable"
wetness_dir = "/home/lan/data_pool/orange/vegetation_wetness"

col = 1018
row = 844
immutable_file = glob.glob(join(immutable_dir,"*","*.tif"))
immutable_file.sort()
# print(immutable_file)
dictpoints = {}
for filee in immutable_file:
    print(filee)
    bands = gdal.Open(filee)
    bandsdata = bands.GetRasterBand(1)
    with open(fire_dir) as f:
        next(f)
        for line in f:
            seg = line.strip().split(',')
            px = int(seg[1])
            py = int(seg[2])
            dt = seg[4]
            key1 = str(px) + "-" + str(py) +"-" + dt
            if px > col or py > row:
                continue
            result = bandsdata.ReadRaster(px-1, py-1, 1, 1,buf_type=bandsdata.DataType)
            fmt = pt2fmt(bandsdata.DataType)
            intval = struct.unpack(fmt, result)
            value = intval[0]
            dictpoints.setdefault(key1, []).append(value)

for v_folder in os.listdir(variable_dir):
    print(v_folder)
    v_ = join(variable_dir,v_folder)
    for filee in os.listdir(v_):
        bands = gdal.Open(join(v_,filee))
        bandsdata = bands.GetRasterBand(1)
        with open(fire_dir) as f:
            next(f)
            for line in f:
                seg = line.strip().split(',')
                if filee.split(".")[0] != seg[4]:
                    continue
                else:
                    px = int(seg[1])
                    py = int(seg[2])
                    dt = seg[4]
                    key1 = str(px) + "-" + str(py) +"-" + dt
                    if px > col or py > row:
                        continue
                    result = bandsdata.ReadRaster(px-1, py-1, 1, 1,buf_type=bandsdata.DataType)
                    fmt = pt2fmt(bandsdata.DataType)
                    intval = struct.unpack(fmt, result)
                    value = intval[0]
                    dictpoints.setdefault(key1, []).append(value)
for wetness_file in os.listdir(wetness_dir):
    print(" process vegetation_wetness")
    w_ = join(wetness_dir,wetness_file)
    bands = gdal.Open(w_)
    bandsdata = bands.GetRasterBand(1)
    with open(fire_dir) as f:
        next(f)
        for line in f:
            seg = line.strip().split(',')
            if ((int(wetness_file.split(".")[0])+ 8) > int(seg[4]))and ((int(wetness_file.split(".")[0])) <= int(seg[4])):
                px = int(seg[1])
                py = int(seg[2])
                dt = seg[4]
                key1 = str(px) + "-" + str(py) +"-" + dt
                if px > col or py > row:
                    continue
                result = bandsdata.ReadRaster(px-1, py-1, 1, 1,buf_type=bandsdata.DataType)
                fmt = pt2fmt(bandsdata.DataType)
                intval = struct.unpack(fmt, result)
                value = intval[0]
                dictpoints.setdefault(key1, []).append(value)
                if len(dictpoints[key1]) > 12:
                    dictpoints.setdefault(key1, []).append(int(seg[3]))

dataset = pd.DataFrame.from_dict(data = dictpoints,orient = 'index')
dataset .to_csv(join("/home/lan/data_pool/orange",'sample.csv'),header = False)
print ("process is over")


#!/usr/bin/python

#    Main script to execute. Creates Predictions.csv file.
#
#    Copyright Orange (C) 2016. All rights reserved.
#    G. Antipov, M. Baccouche and S. A. Berrani.
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU General Public License for more details.
#
#    You should have received a copy of the GNU General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.

import os
import glob

os.chdir('General_age_estimation')

# BIG ONE:

allpointerfiles = glob.glob("../img_pointers_*")

print("Found {} pointers.".format(len(allpointerfiles)))
#programPause = raw_input("Press the <ENTER> key to continue...")

for i in range(len(allpointerfiles)):
    os.system('./PREDICT_AGE_FINAL.py -pf ../img_pointers_{} -in ../img_names_{} -w list_of_models.txt -rf result_general.txt -lf log_general.txt'.format(i+1,i+1))

# Uncomment to include the kids:
#os.chdir('../Kids_age_estimation')
#os.system('./PREDICT_AGE_FINAL.py -pf ../test_image_pointers.txt -in ../image_list.txt -w list_of_models.txt -rf result_kids.txt -lf log_kids.txt')
#os.chdir('..')
#os.system('./combine_general_and_kids_estimations.py -op General_age_estimation/result_general.txt -kp Kids_age_estimation/result_kids.txt -rp Predictions.csv')


'''
@author: Xuankang
@description: Handling the search requests
'''

import urllib
import webapp2

import utils
import auth
from books import TagHelper

class SearchHandler(webapp2.RequestHandler):
    """ Handling the search request from nav bar. """

    def get(self):
        email = auth.get_email_from_cookies(self.request.cookies)
        user = auth.user.User.get_by_email(email)
        if not user:
            self.redirect('/login')
            return

        keywords = self.request.get('q')
        if self._is_isbn(keywords):
            # show that book
            self.redirect('/book/' + keywords)
            return

        if self._is_tag(keywords, user):
            # show the books of that tag
            self.redirect('/tags?t=' + keywords)
            return

        msg = "Invalid search words: " + keywords
        params = {'msg': msg}
        self.redirect('/error?' + urllib.urlencode(params))
        return

    def _is_isbn(self, text):
        """ Determine whether the user's input is a valid isbn number. """
        try:
            utils.validate_isbn(text)
        except Exception:
            # not a isbn
            return False
        else:
            return True

    def _is_tag(self, text, user):
        """ Determine whether the user's input is a valid isbn number. """
        if not text:
            # there must be something
            return False

        if ' ' in text:
            # can not be multiple tags
            return False

        helper = TagHelper(user)
        if helper.isbns(text):
            return True
        else:
            return False


import numpy as np
import matplotlib
from datetime import datetime

#import pywt
#from scipy.integrate import quad

#from sklearn.neighbors.kde import KernelDensity

def varlist(data):
#Calculate the mean of the magnetic field components and return the variance matrix as a (3,3) array
    Bxm=data[0].mean()
    Bym=data[1].mean()
    Bzm=data[2].mean()
    Bxsqm=(data[0]**2).mean()
    Bysqm=(data[1]**2).mean()
    Bzsqm=(data[2]**2).mean()
    Bxym=(data[0]*data[1]).mean()
    Bxzm=(data[0]*data[2]).mean()
    Byzm=(data[1]*data[2]).mean()
    Varx= Bxsqm-Bxm**2
    Varxy=Bxym-Bxm*Bym
    Varxz=Bxzm-Bxm*Bzm
    Vary=Bysqm-Bym**2
    Varyz=Byzm-Bym*Bzm
    Varz=Bzsqm-Bzm**2
    var=[Varx,Varxy, Varxz,Varxy,Vary,Varyz,Varxz,Varyz,Varz]
    var=np.array(var)
    var=var.reshape((3,3))
    return var

def xyz2polar(B_x,B_y,B_z):
    B_mag = []
    theta = []
    phi = []
    for i in range(0,len(B_x)):
        B_mag.append( np.sqrt((B_x[i])**2 + (B_y[i])**2 + (B_z[i])**2 ) )
        theta.append( (np.arctan2( [np.sqrt((B_x[i])**2 + (B_y[i])**2 )],B_z[i] ))[0])
        phi.append( (np.arctan2( [B_y[i]],[B_x[i] ]) )[0] )
        #theta.append(np.arccos( np.dot(B_vec[i],[0,0,1]) ))
        #phi.append(np.arccos( np.dot( ([B_x[i],B_y[i],0]/B_xy[i]),[1,0,0]) ))
    return(np.array([B_mag,theta,phi]))

def data_in_interval(data,se_times):
    t = data[0]
    B_x = data[1]
    B_y = data[2]
    B_z = data[3]

    data_start_time = se_times[0]
    data_end_time = se_times[1]

    t_datetime = []
    for i in range(0,len(t)):
        strpdtime=datetime.strptime(t[i],'%Y-%m-%dT%H:%M:%S.%fZ')
        t_datetime.append(strpdtime)
    t_days = matplotlib.dates.date2num(t_datetime)
    t_secs= t_days*24*3600

    if data_start_time > t_days[0]:
        data_start_index = np.argmax(t_days>data_start_time)
    else:
        data_start_index = 0
    if data_end_time < t_days[-1]:
        data_end_index = np.argmax(t_days>data_end_time)
    else:
        data_end_index = len(t_days)

    t = t[data_start_index:data_end_index]
    B_x = B_x[data_start_index:data_end_index]
    B_y = B_y[data_start_index:data_end_index]
    B_z = B_z[data_start_index:data_end_index]
    #B_mag = B_mag[data_start_index:data_end_index]
    B_x.astype(float)
    B_y.astype(float)
    B_z.astype(float)
    B_mag = (B_x**2+B_y**2+B_z**2)**(0.5)

    t_days = t_days[data_start_index:data_end_index]
    t_secs = t_secs[data_start_index:data_end_index]

    return([[t,t_days,t_secs],[B_x,B_y,B_z],B_mag])

def sitvfy(input_data,settings_params):
    t_secs = input_data[0]
    B_x = input_data[1][0]
    B_y = input_data[1][1]
    B_z = input_data[1][2]
    B_mag = input_data[2]
    rangelist = input_data[3]

    t_int = settings_params[0]
    shift = settings_params[1]

    B_xy = []
    for i in range(0,len(t_secs)):
        B_xy.append (np.sqrt((B_x[i])**2 + (B_y[i])**2 ) )
    B_xy = np.array(B_xy)

    subintervals = np.empty(( int((t_secs[-1]-t_secs[0]-t_int+shift)/shift) ,2))
    for i in range(0,int((t_secs[-1]-t_secs[0]-t_int+shift)/shift)):
        subintervals[i][0] = t_secs[0] + i*shift
        subintervals[i][1] = t_secs[0] + i*shift + t_int

    Bx_sitv = []
    By_sitv = []
    Bz_sitv = []
    Bmag_sitv=[]
    Bxy_sitv = []
    range_sitv = []

    Bx_sitv_mean = []
    By_sitv_mean = []
    Bz_sitv_mean = []
    Bmag_sitv_mean = []

    Bxy_sitv_min = []
    Bxy_sitv_max = []
    Bxy_sitv_mean = []

    gapadj_subintervals = []
    for i in range(0,int((t_secs[-1]-t_secs[0]-t_int+shift)/shift)):
        si_start = np.argmax(t_secs>subintervals[i][0])
        si_end = np.argmax(t_secs>subintervals[i][1])

        if len(B_x[si_start:si_end])!=0:
            gapadj_subintervals.append(subintervals[i])

            Bx_sitv.append(B_x[si_start:si_end])
            By_sitv.append(B_y[si_start:si_end])
            Bz_sitv.append(B_z[si_start:si_end])
            Bmag_sitv.append(B_mag[si_start:si_end])
            Bxy_sitv.append(B_xy[si_start:si_end])

            Bx_sitv_mean.append(np.mean(B_x[si_start:si_end]))
            By_sitv_mean.append(np.mean(B_y[si_start:si_end]))
            Bz_sitv_mean.append(np.mean(B_z[si_start:si_end]))
            Bmag_sitv_mean.append(np.mean(B_mag[si_start:si_end]))

            Bxy_sitv_min.append(min(Bxy_sitv[-1]))
            Bxy_sitv_max.append(max(Bxy_sitv[-1]))
            Bxy_sitv_mean.append(np.mean(Bxy_sitv[-1]))

            local_rangelist = rangelist[si_start:si_end]
            contains_2 = False
            contains_3 = False
            for i in range(0,len(local_rangelist)):
                if local_rangelist[i]==2:
                    contains_2=True
                elif local_rangelist[i]==3:
                    contains_3=True
            if contains_2:
                range_sitv_val = 2
                if contains_3:
                    range_sitv_val = 23
            else:
                if contains_3:
                    range_sitv_val = 3
                else:
                    range_sitv_val = -1

            range_sitv.append(range_sitv_val)

    Bx_sitv_mean = np.array(Bx_sitv_mean)
    By_sitv_mean = np.array(By_sitv_mean)
    Bz_sitv_mean = np.array(Bz_sitv_mean)

    range_sitv = np.array(range_sitv)

    subintervals = np.array(gapadj_subintervals.copy())
    sitv_midpoints_secs = (subintervals[:,0]+subintervals[:,1])/2

    return([sitv_midpoints_secs,[Bx_sitv,By_sitv,Bz_sitv],Bmag_sitv_mean,[Bxy_sitv_min,Bxy_sitv_max,Bxy_sitv_mean],range_sitv])

def MVA(input_data):

    sitv_midpoints_secs = input_data[0]
    Bx_sitv = input_data[1][0]
    By_sitv = input_data[1][1]
    Bz_sitv = input_data[1][2]
    Bxy_sitv_min = input_data[2][0]
    Bxy_sitv_max = input_data[2][1]
    Bxy_sitv_mean = input_data[2][2]
    Bx_sitv_mean = [np.mean(Bx_sitv[i]) for i in range(0,len(sitv_midpoints_secs))]
    By_sitv_mean = [np.mean(By_sitv[i]) for i in range(0,len(sitv_midpoints_secs))]
    Bz_sitv_mean = [np.mean(Bz_sitv[i]) for i in range(0,len(sitv_midpoints_secs))]

    Bxy_fluct_PN16 = (np.array(Bxy_sitv_max)-np.array(Bxy_sitv_min))/np.array(Bxy_sitv_mean)
    phi_PN16 = []
    theta_B_PN16 = []
    theta_D_PN16 = []

    B_x1_angle = []
    theta_D = []
    phi_D = []
    lam1_lam2 = []
    lam3_lam2 = []

    O_z_unfiltered = []

    for i in range(0,len(sitv_midpoints_secs)):
        #if i%int(len(sitv_midpoints_secs)/50)==0:
            #print(matplotlib.dates.num2date(sitv_midpoints_secs[i]/3600/24))
        data = np.array([Bx_sitv[i],By_sitv[i],Bz_sitv[i]])
        eigen = np.linalg.eig(varlist(data))

        lam1_index = np.argmax(eigen[0])
        lam3_index = np.argmin(eigen[0])
        for eigenindex in [0,1,2]:
            if eigenindex!=lam1_index and eigenindex!=lam3_index:
                lam2_index=eigenindex
                break

        lam1 = eigen[0][lam1_index]
        lam2 = eigen[0][lam2_index]
        lam3 = eigen[0][lam3_index]

        lam1_lam2.append(lam1/lam2)
        lam3_lam2.append(lam3/lam2)

        x1 = eigen[1][:,lam1_index]
        x1_xy = np.sqrt(x1[0]**2+x1[1]**2)

        B_dir = np.array([Bx_sitv_mean[i],By_sitv_mean[i],Bz_sitv_mean[i]])
        B_dir /= np.sqrt(Bx_sitv_mean[i]**2+By_sitv_mean[i]**2+Bz_sitv_mean[i]**2)
        B_dir_xy = np.sqrt(B_dir[0]**2+B_dir[1]**2)

        if np.dot(x1,B_dir) < 0:
            x1=-x1

        theta_D.append( (np.arctan2( [x1_xy],[x1[2]] ))[0])
        phi_D.append( (np.arctan2( [x1[1]],[x1[0] ]) )[0] )

        B_x1_angle.append( np.arccos(np.dot(x1,B_dir)))
        phi_PN16.append( np.arccos(np.dot([x1[0],x1[1]],[B_dir[0],B_dir[1]]) \
                                         /np.sqrt(x1[0]**2+x1[1]**2)/np.sqrt(B_dir[0]**2+B_dir[1]**2) ))
        """
        else:
            B_x1_angle.append( np.arccos(np.dot(x1,B_dir))  )
            phi_PN16.append( np.arccos(np.dot([x1[0],x1[1]],[B_dir[0],B_dir[1]]) \
                                             /np.sqrt(x1[0]**2+x1[1]**2)/np.sqrt(B_dir[0]**2+B_dir[1]**2) ) )
        """

        theta_B_PN16.append( np.arctan2(B_dir[2],B_dir_xy) )
        theta_D_PN16.append( np.arctan2(x1[2],x1_xy) )

        O_z_unfiltered.append( Bz_sitv_mean[i] - x1[2]/x1_xy*Bxy_sitv_mean[i] )

    return([[phi_PN16,theta_D_PN16,theta_B_PN16,Bxy_fluct_PN16]\
                ,[B_x1_angle,theta_D,phi_D,lam1_lam2,lam3_lam2]\
                ,O_z_unfiltered] )

def MVA_offsetref(input_data):

    sitv_midpoints_secs = input_data[0]
    Bx_sitv = input_data[1][0]
    By_sitv = input_data[1][1]
    Bz_sitv = input_data[1][2]
    Bxy_sitv_min = input_data[2][0]
    Bxy_sitv_max = input_data[2][1]
    Bxy_sitv_mean = input_data[2][2]
    Bx_sitv_mean = [np.mean(Bx_sitv[i]) for i in range(0,len(sitv_midpoints_secs))]
    By_sitv_mean = [np.mean(By_sitv[i]) for i in range(0,len(sitv_midpoints_secs))]
    Bz_sitv_mean = [np.mean(Bz_sitv[i]) for i in range(0,len(sitv_midpoints_secs))]

    Bxy_fluct_PN16 = (np.array(Bxy_sitv_max)-np.array(Bxy_sitv_min))/np.array(Bxy_sitv_mean)
    phi_PN16 = []
    theta_B_PN16 = []
    theta_D_PN16 = []

    B_x1_angle = []
    theta_D = []
    phi_D = []
    lam1_lam2 = []
    lam3_lam2 = []

    O_z_unfiltered = []
    dO=[]

    d_g=10**(-4)
    d_bn=10**(-12)
    d_thetaD=[]
    d_b=[]
    bxymirror=[]
    bzmirror=[]
    d_thetaB=[]
    theta_mb=[]
    theta_md=[]
    test=[]
    oz1=[]

    for i in range(0,len(sitv_midpoints_secs)):
        data = np.array([Bx_sitv[i],By_sitv[i],Bz_sitv[i]])
        eigen = np.linalg.eig(varlist(data))

        lam1_index = np.argmax(eigen[0])
        lam3_index = np.argmin(eigen[0])
        for eigenindex in [0,1,2]:
            if eigenindex!=lam1_index and eigenindex!=lam3_index:
                lam2_index=eigenindex
                break

        lam1 = eigen[0][lam1_index]
        lam2 = eigen[0][lam2_index]
        lam3 = eigen[0][lam3_index]

        lam1_lam2.append(lam1/lam2)
        lam3_lam2.append(lam3/lam2)

        x1 = eigen[1][:,lam1_index]
        x1_xy = np.sqrt(x1[0]**2+x1[1]**2)

        B_dir = np.array([Bx_sitv_mean[i],By_sitv_mean[i],Bz_sitv_mean[i]])
        B_dir /= np.sqrt(Bx_sitv_mean[i]**2+By_sitv_mean[i]**2+Bz_sitv_mean[i]**2)
        B_dir_xy = np.sqrt(B_dir[0]**2+B_dir[1]**2)

        if np.dot(x1,B_dir) < 0:
            x1=-x1

        theta_D.append( (np.arctan2( [x1_xy],[x1[2]] ))[0])
        phi_D.append( (np.arctan2( [x1[1]],[x1[0] ]) )[0] )
        B_magn=np.sqrt(Bx_sitv_mean[i]**2+By_sitv_mean[i]**2+Bz_sitv_mean[i]**2)

        B_x1_angle.append( np.arccos(np.dot(x1,B_dir)))
        phi_PN16.append( np.arccos(np.dot([x1[0],x1[1]],[B_dir[0],B_dir[1]]) \
                                         /np.sqrt(x1[0]**2+x1[1]**2)/np.sqrt(B_dir[0]**2+B_dir[1]**2) ))
        """
        else:
            B_x1_angle.append( np.arccos(np.dot(x1,B_dir))  )
            phi_PN16.append( np.arccos(np.dot([x1[0],x1[1]],[B_dir[0],B_dir[1]]) \
                                             /np.sqrt(x1[0]**2+x1[1]**2)/np.sqrt(B_dir[0]**2+B_dir[1]**2) ) )
        """

        theta_B_PN16.append( np.arctan2(B_dir[2],B_dir_xy) )
        theta_D_PN16.append( np.arctan2(x1[2],x1_xy) )

        O_z_unfiltered.append( Bz_sitv_mean[i] - x1[2]/x1_xy*Bxy_sitv_mean[i] )

    return([[phi_PN16,theta_D_PN16,theta_B_PN16,Bxy_fluct_PN16]\
                ,[B_x1_angle,theta_D,phi_D,lam1_lam2,lam3_lam2]\
                ,O_z_unfiltered] )

def get_Mmode_times(sitv_midpoints_days,Mmode_indices_PN16_only\
                                     ,Mmode_indices_SLD08_only\
                                     ,Mmode_indices_overlapping):

    #Mmode_sitv_times_PN16 = []
    #Mmode_sitv_times_SLD08 = []
    Mmode_sitv_times_PN16_only = []
    #Mmode_sitv_times_PN16_days = []
    Mmode_sitv_times_SLD08_only = []
    #Mmode_sitv_times_SLD08_days = []
    Mmode_sitv_times_overlapping = []
    #Mmode_sitv_times_overlapping_days = []

    #for i in range(0,len(Mmode_indices_PN16)):
        #Mmode_sitv_times_PN16.append(matplotlib.dates.num2date(sitv_midpoints_days[Mmode_indices_PN16[i]]))
        #Mmode_sitv_times_PN16_days.append(sitv_midpoints_days[Mmode_indices_PN16[i]])
    #for i in range(0,len(Mmode_indices_SLD08)):
        #Mmode_sitv_times_SLD08.append(matplotlib.dates.num2date(sitv_midpoints_days[Mmode_indices_SLD08[i]]))
        #Mmode_sitv_times_SLD08_days.append(sitv_midpoints_days[Mmode_indices_SLD08[i]])
    for i in range(0,len(Mmode_indices_PN16_only)):
        Mmode_sitv_times_PN16_only.append(matplotlib.dates.num2date(sitv_midpoints_days[Mmode_indices_PN16_only[i]]))
    for i in range(0,len(Mmode_indices_SLD08_only)):
        Mmode_sitv_times_SLD08_only.append(matplotlib.dates.num2date(sitv_midpoints_days[Mmode_indices_SLD08_only[i]]))
    for i in range(0,len(Mmode_indices_overlapping)):
        Mmode_sitv_times_overlapping.append(matplotlib.dates.num2date(sitv_midpoints_days[Mmode_indices_overlapping[i]]))
        #Mmode_sitv_times_overlapping_days.append(sitv_midpoints_days[Mmode_indices_overlapping[i]])

    return([Mmode_sitv_times_PN16_only,Mmode_sitv_times_SLD08_only,Mmode_sitv_times_overlapping])

def sitv_in_sheath_frac(sitv_midpoints_days,indices_arrs,sheath_times):

    Mmode_indices_PN16 = indices_arrs[0]
    Mmode_indices_SLD08 = indices_arrs[1]
    Mmode_indices_overlapping = indices_arrs[2]

    sheath_start_time = sheath_times[0]
    sheath_end_time = sheath_times[1]

    Mmode_sitv_times_PN16_days = [(sitv_midpoints_days[Mmode_indices_PN16[i]]) for i in range(0,len(Mmode_indices_PN16))]
    Mmode_sitv_times_SLD08_days = [(sitv_midpoints_days[Mmode_indices_SLD08[i]]) for i in range(0,len(Mmode_indices_SLD08))]
    Mmode_sitv_times_overlapping_days =[(sitv_midpoints_days[Mmode_indices_overlapping[i]]) for i in range(0,len(Mmode_indices_overlapping))]

    PN16_frac_in_sheath = (np.argmax(np.array(Mmode_sitv_times_PN16_days)>sheath_end_time) - np.argmax(np.array(Mmode_sitv_times_PN16_days)>sheath_start_time) )/len(Mmode_indices_PN16)
    SLD08_frac_in_sheath = (np.argmax(np.array(Mmode_sitv_times_SLD08_days)>sheath_end_time) - np.argmax(np.array(Mmode_sitv_times_SLD08_days)>sheath_start_time) )/len(Mmode_indices_SLD08)
    overlapping_frac_in_sheath = (np.argmax(np.array(Mmode_sitv_times_overlapping_days)>sheath_end_time) - np.argmax(np.array(Mmode_sitv_times_overlapping_days)>sheath_start_time) )/len(Mmode_indices_overlapping)

    print(PN16_frac_in_sheath)
    print(SLD08_frac_in_sheath)
    print(overlapping_frac_in_sheath)

    return()

def dailyOff(O_z,starttim):

    #takes in a list of O_z and separate them into days, returns in list form. Together
    #with the associated error
    O_z = np.array(O_z)
    O_zd=[]
    errord=[]
    for i in range(len(np.unique(starttim))):
        O_zd.append([])
        errord.append([])

    count=0
    for i in range(len(O_z)):
            if i==0 and starttim[0]==starttim[1]:
                O_zd[count].append(O_z[i])
                errord[count].append(error[i])
            elif starttim[i-1]==starttim[i]:
                O_zd[count].append(O_z[i])
                errord[count].append(error[i])
            elif count<len(O_zd)-1:
                count+=1

    for i in errord:
        i=np.array(i)
    for i in O_zd:
        i=np.array(i)

    O_ztd=[]

    for i in range(len(O_zd)):
        O_ztd.append(np.transpose(np.array(np.array(O_zd[i]))))

    return O_ztd,errord

def kde(mean,sigma,weight):
    x=np.linspace(-20,20,100000)
    gaussian=[]
    ker=0
    for i in range(len(mean)):
        mu, width=mean, sigma
        y=1/(width[i] * np.sqrt(2 * np.pi)*weight[i])*np.exp( - (x - mu[i])**2 / (2 * width[i]**2))
        gaussian.append(y)
        ker+=y
    ker=ker/len(mean)
    plt.plot(x,ker,color='blue',label='weighted kde h=1')
    plt.legend()
    return ker,x[np.where(ker==np.max(ker))[0][0]]

def adapkde(mean):
    h=bandwidth.sskernel(mean)[2]
    x=np.linspace(-20,20,100000)
    ker=0
    for i in range(len(mean)):
        mu, width=mean, sigma
        y=1/(h * np.sqrt(2 * np.pi))*np.exp( - (x - mu[i])**2 / (2 * h**2))
        ker+=y
    ker=ker/len(mean)
    plt.plot(x,ker,color='green',label='adaptive kde no weighting')
    plt.legend()
    plt.title("Types of KDE")
    plt.xlabel("$O_z/nT$")
    plt.ylabel("P/$nT^{-1}$")
    plt.show()
    return x[np.where(ker==np.max(ker))[0][0]]

def wadapkde(mean,weight):
    h=bandwidth.sskernel(mean)[2]
    x=np.linspace(-20,20,100000)
    ker=0
    for i in range(len(mean)):
        mu, width=mean, sigma
        y=1/(h * np.sqrt(2 * np.pi)*weight[i])*np.exp( - (x - mu[i])**2 / (2 * h**2))
        ker+=y
    ker=ker/len(mean)

    plt.plot(x,ker,color='red',label='adaptive kde with weighting')
    plt.legend()
    plt.show()

    return ker,x[np.where(ker==np.max(ker))[0][0]]

def fwhm(arr):
    x=np.linspace(-20,20,100000)
    half_max = max(arr) / 2.
    d =np.sign(half_max - arr[0:-1]) - np.sign(half_max -(arr[1:]))
    for i in range(len(d)):
        if d[i]>0:
            leftid= i
            print(leftid)
            print(x[leftid])
        elif d[i]<0:
            rightid=i
    rightu= x[rightid]-x[np.where(arr==np.max(arr))[0][0]]
    leftu= x[np.where(arr==np.max(arr))[0][0]]-x[leftid]
    return [leftu,rightu, x[rightid]-x[leftid]]


from vital import adjustment

if __name__ == '__main__':
    """
    ----------------------------------------用户输入部分-------------------------------------------------
    """
    analyst_info = ('胡冠宇', 2019, 8, 27)

    """代码所在路径"""
    project_code_path = 'C:\\Users\\huguanyu\\Desktop\\数据测试\\AutoTestReport'

    """数据文件夹所在路径"""
    test_project_folder_path = r'C:\Users\huguanyu\Downloads\苏宁2w\新建文件夹'

    """输出的报告文档名"""
    report_doc_name = '数据测试报告'

    """规则导出为银行规则或非银规则：bank==True/nbank==False （文档和附件中都有规则）"""
    is_bank = False

    """需要删除的排序文件表头, 不需要删除就置空，注意输入list"""
    replace_str = ['测试数据20190828']

    """导出报告类型：(Rule,Report,Both,N)"""
    select_type = 'Report'

    """仅导出字典("Y","N","ONLY")"""
    select_dictionary = "Y"

    """仅导出附件("Y","N","ONLY")"""
    select_attachment = "Y"

    """仅排序("Y","N","ONLY")"""
    select_sort = "Y"

    """
    ----------------------------------------用户输入部分-------------------------------------------------
    """
    adjustment.main(analyst_info, project_code_path, test_project_folder_path, report_doc_name, is_bank, replace_str,
                    select_type, select_dictionary, select_attachment, select_sort)


# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
from . import transforms as T

def build_transforms(cfg, is_train=True):
    if is_train:
        min_size = cfg.INPUT.MIN_SIZE_TRAIN
        max_size = cfg.INPUT.MAX_SIZE_TRAIN
        flip_prob = 0.5  # cfg.INPUT.FLIP_PROB_TRAIN
        brightness = cfg.INPUT.BRIGHTNESS
        contrast = cfg.INPUT.CONTRAST
        saturation = cfg.INPUT.SATURATION
        hue = cfg.INPUT.HUE
        adaptive = cfg.INPUT.COLORJITTER_ADAPTIVE
    else:
        min_size = cfg.INPUT.MIN_SIZE_TEST
        max_size = cfg.INPUT.MAX_SIZE_TEST
        flip_prob = 0
        brightness = 0.0
        contrast = 0.0
        saturation = 0.0
        hue = 0.0
        adaptive = None

    to_bgr255 = cfg.INPUT.TO_BGR255
    from qd.qd_pytorch import (DictTransformMaskNormalize,
            DictTransformMaskColorJitter,
            DictTransformMaskToTensor,
            DictTransformMaskRandomHorizontalFlip,
            )
    normalize_transform = DictTransformMaskNormalize(
            mean=cfg.INPUT.PIXEL_MEAN, std=cfg.INPUT.PIXEL_STD, to_bgr255=to_bgr255
            )
    color_jitter = DictTransformMaskColorJitter(
            brightness=brightness,
            contrast=contrast,
            saturation=saturation,
            hue=hue,
            adaptive=adaptive,
            )
    to_tensor = DictTransformMaskToTensor()
    flipper = DictTransformMaskRandomHorizontalFlip(flip_prob)

    from qd.qd_pytorch import DictTransformMaskResize
    if is_train:
        if cfg.INPUT.TRAIN_RESIZER != '':
            from qd.qd_common import load_from_yaml_str
            from qd.qd_common import execute_func
            resizer = execute_func(load_from_yaml_str(cfg.INPUT.TRAIN_RESIZER))
        elif not cfg.INPUT.USE_FIXED_SIZE_AUGMENTATION:
            resizer = DictTransformMaskResize(min_size, max_size,
                                              cfg.INPUT.MIN_SIZE_ON_ITER,
                                              cfg.INPUT.TREAT_MIN_AS_MAX,
                                              )
        else:
            from qd.qd_yolov2pt import DictResizeAndPlaceForMaskRCNN
            resizer = DictResizeAndPlaceForMaskRCNN(cfg)
    else:
        resizer = DictTransformMaskResize(min_size, max_size,
                cfg.INPUT.MIN_SIZE_ON_ITER)

    from qd.qd_pytorch import DictTransformCompose
    transform = DictTransformCompose(
            [
                color_jitter,
                resizer,
                flipper,
                to_tensor,
                normalize_transform,
            ]
        )
    return transform


"""
This example demonstrates how to make a graphical interface to preform
IV characteristic measurements. There are a two items that need to be
changed for your system:

1) Correct the GPIB addresses in IVProcedure.startup for your instruments
2) Correct the directory to save files in MainWindow.queue

Run the program by changing to the directory containing this file and calling:

python iv_keithley.py

"""

import logging
import datetime

import numpy as np
import pandas as pd
# import pprint
from time import sleep

import os
import sys, time, msvcrt

askuser_timeout = 3

# sys.path.append('D:/GitHub/pymeasure')

from pymeasure.instruments.agilent import Agilent34465A, Agilent34970A
# from pymeasure.virtual.thermometer import Thermometer
from pymeasure.virtual.motor import HSM_BMWi3, ASM_EZW1, SSM_ZOE
# from pymeasure.instruments.chroma import Chroma62024p6008
from pymeasure.instruments.regatron import Regatron
from pymeasure.instruments.lecroy import WaveRunner606Zi

eps = np.spacing(1)

def close_all_visa_resources():
    import visa
    rm = visa.ResourceManager()
    rm.close()
    rm.list_resources()

def calc_power(current: float, voltage: float):
    power = voltage*current
    return power

def calc_loss_power(input_power: float, ouput_power: float):
    loss_power = input_power-ouput_power
    return loss_power

def derating_current_point(Vcurr: float, Pcurr: float, Vmax: float, Imax: float, Pmax: float):
    # applying current and voltage derating to power
    Inew = np.minimum(Imax, Pcurr / (Vcurr+eps))
    Pnew = Vcurr * Inew
    return Inew, Pnew

def measurements_pts():
    pass

def timeStamped(fname: str, fmt='%Y%m%d%H%M%S{fname}'):
    return datetime.datetime.now().strftime(fmt).format(fname=fname)

def getFilenamePrintScreen(set_input_voltage, set_output_power, set_duty, set_fsw):
    filename = ('%sVin%d_Po%.2e_D%.2f_fsw%.1fk' % (timeStamped('_'),
                set_input_voltage,
                set_output_power,
                set_duty,
                set_fsw/1000))
    return filename

def ask_user():
    print('================================')
    check = str(input("Continue ? ([y]/n): ")).lower().strip()
    try:
        if check[:1] == 'y':
            return True
        elif check[:1] == 'n':
            return False
        elif check[:1] == '':
            return True
        else:
            print('Invalid Input')
            return ask_user()
    except Exception as error:
        print("Please enter valid inputs")
        print(error)
        return ask_user()

def ask_user_timeout():
    startTime = time.time()
    inp = None
    print('\n================================')
    print("Press any key to stop or wait x seconds...")
    while True:
        if msvcrt.kbhit():
            inp = msvcrt.getch()
            break
        elif time.time() - startTime > askuser_timeout:
            break
    if inp:
        print("Stopping...")
        return False
    else:
        print("Timed out and go on...")
        return True

def print_preset_value(psu:Regatron):
    print('VQ1[V] = %f' % psu.getVoltagePreset())
    print('IQ1[A] = %f' % psu.getCurrentPreset())
    print('IQ4[A]  = %f' % psu.getCurrentLimitQ4())
    print('PQ1[kW] = %f' % psu.getPowerPreset())
    print('PQ4[kW]  = %f' % psu.getPowerLimitQ4())

def run(input_data: dict):
    min_input_voltage = input_data.min_output_voltage * input_data.D_set
    step_input_voltage = input_data.output_voltage_step * input_data.D_set

    #max_output_current = max_output_power/min_output_voltage

    max_input_voltage = input_data.max_output_voltage * input_data.D_set
    max_input_power = input_data.max_output_power + input_data.worstcase_losses
    max_input_current = max_input_power / min_input_voltage
    max_input_power = input_data.max_output_power + input_data.worstcase_losses

    if max_input_voltage > 470:
        max_input_voltage = 470

    if max_input_current > 80:
        max_input_current = 80

    if max_input_power > 32e3:
        max_input_power = 32e3

#    delay = .1
#
#    DATA_COLUMNS = ['DateTime',
#                    'Modulation',
#                    'fs_set(Hz)',
#                    'D_set(-)',
#                    'Vin_set',
#                    'Pout_set',
#                    # INPUT MEASUREMENTS
#                    'SourceInputVoltage(V)',
#                    'DMMInputVoltage(V)',
#                    'SourceInputCurrent(A)',
#                    'SourceInputPower(A)',
#                    # OUTPUT MEASUREMENTS
#                    'SinkOutputVoltage(V)',
#                    'DMMOutputVoltage(V)',
#                    'SinkOutputCurrent(A)',
#                    'SinkOutputPower(A)',
#                    # OSCILLOSCOPE MEASUREMENTS
#                    'fs_meas(Hz)',
#                    'Duty100(-)']
#    data = pd.DataFrame(columns=DATA_COLUMNS)

    data =  pd.DataFrame()

    logging.basicConfig(filename='example.log', level=logging.DEBUG)
    print('================================')
    print("Setting up instruments")
    logging.info("Setting up instruments")

    com_port_source = 'COM13'
    com_port_sink = 'COM7'

    source = Regatron(com_port_source)
    source.open()
    print(source.getSerialNumber())
    sink = Regatron(com_port_sink)
    sink.open()
    print(sink.getSerialNumber())

    # # DSO 10GHz BW
    # scope = WaveRunner606Zi('146.136.35.172')
    # # Waverunner 3024
    # scope = WaveRunner606Zi('146.136.35.170')
    # # Waverunner 8104-MS
    # scope = WaveRunner606Zi('146.136.35.134')
    # Waverunner 625Zi
    scope = WaveRunner606Zi('146.136.35.172')
    print(scope.id)
    dmm_vin = Agilent34465A('TCPIP0::146.136.35.174::inst0::INSTR')
    print(dmm_vin.id)
    dmm_vout = Agilent34465A('TCPIP0::146.136.35.176::inst0::INSTR')
    print(dmm_vout.id)
    # thermo = Thermometer(Agilent34970A('ASRL1::INSTR'))
    # print(thermo.id())

    source.powerOff()
    source.setVoltage(voltage=0)
    source.setCurrent(neg=0, pos=max_input_current)
    source.setPower(neg=0, pos=max_input_power)
    print('==SOURCE INIT==============================')
    print_preset_value(source)

    sink.powerOff()
    sink.setVoltage(voltage=20)
    sink.setCurrent(neg=-0, pos=0)
    sink.setPower(neg=-max_input_power, pos=0)
    print('==SINK INIT==============================')
    print_preset_value(sink)

    input_voltages = np.linspace(min_input_voltage, max_input_voltage,
                                 np.ceil((max_input_voltage - min_input_voltage) / step_input_voltage))
    output_powers = np.arange(input_data.min_output_power, input_data.max_output_power * 1.05,
                              input_data.step_output_power)
    input_voltages_rev = np.flip(input_voltages, axis=0)
    output_powers_rev = np.flip(output_powers, axis=0)

    loop_v = np.zeros((1, 2))

    # build startup voltage ramp
    startup_input_voltages = np.arange(step_input_voltage, max_input_voltage, step_input_voltage)
    for ii, voltage in enumerate(startup_input_voltages):
        loop_v = np.vstack((loop_v, [voltage, 0]))

    # build loops voltage/power
    direction = 1
    for ii, voltage in enumerate(input_voltages_rev):
        direction = (direction + 1) % 2
        for jj, power in enumerate(output_powers):
            if direction == 0:
                loop_v = np.vstack((loop_v, [voltage, output_powers[jj]]))
            elif direction == 1:
                loop_v = np.vstack((loop_v, [voltage, output_powers_rev[jj]]))

    input_powers = loop_v[:, 1] + input_data.worstcase_losses
    input_currents = input_powers / (loop_v[:, 0] + eps)
    output_voltages = loop_v[:, 0] / input_data.D_set

    print('==LOOPS OVERVIEW==============================')
    print('min input voltage: %f' % min_input_voltage)
    print('max input voltage: %f' % max_input_voltage)
    print('loop iterations:')
    print('¦¦ Input Voltage ¦¦ Output power ¦¦')
    print(loop_v)

    if ask_user():

        logging.info("Starting to sweep through voltages and powers")
        source_input_voltage = 0
        source_input_current = 0

        sink_output_voltage = 0
        sink_output_current = 0

        for row in loop_v:
            set_output_power = row[1]
            set_input_voltage = row[0]

            print('\n==NEXT STEP===============================')
            # logging.info('Next load power: %f' % set_output_power)
            print('Next input voltage: %f' % set_input_voltage)
            print('Next output voltage: %f' % (set_input_voltage / input_data.D_set))
            print('Next load power: %f' % set_output_power)
            set_output_current = -float(set_output_power / (set_input_voltage + eps) * input_data.D_set)
            print('Next load current: %f' % set_output_current)

            # logging.info('Next voltage step: %f' % set_input_voltage)

            if ask_user_timeout():
                source.powerOn()
                sink.powerOn()
                try:
                    source.setVoltage(voltage=set_input_voltage)
                    sleep(2)
                    #set_output_current = -1
                    sink.setCurrent(neg=set_output_current, pos=0)
                    sleep(2)
                    print('\n==SOURCE READOUT==============================')
                    print_preset_value(source)

                    print('\n==SINK READOUT==============================')
                    print_preset_value(sink)

                except:
                    source.powerOff()
                    sink.powerOff()

                try:
                    source_input_voltage = source.getTerminalVoltage()
                    source_input_current = source.getTerminalCurrent()
                    source_input_power = source.getTerminalPower() * 1000

                    sink_output_voltage = sink.getTerminalVoltage()
                    sink_output_current = sink.getTerminalCurrent()
                    sink_output_power = sink.getTerminalPower() * 1000

                    meas_input_voltage = dmm_vin.voltage_dc
                    meas_output_voltage = dmm_vout.voltage_dc
                except:
                    print('error occurred at Vin=%f, Pout=%f' % (set_input_voltage, set_output_power))
                    source.powerOff()
                    sink.powerOff()

                try:
                    # if waverunner the frequency and duty are measur. Nr1-2
                    # input_current_ph = np.nan #float(scope.get_measurement_Px(3))
                    # switching_freq = float(scope.get_measurement_Px(2))
                    # duty_boost = float(scope.get_measurement_Px(1))
                    # if DSO the frequency and duty are measur. Nr7-8

                    switching_freq = float(scope.get_measurement_Px(8))
                    duty_boost = float(scope.get_measurement_Px(7))
                except:
                    switching_freq = None
                    duty_boost = None

                scope.save_screen_to_file(filename=getFilenamePrintScreen(set_input_voltage,
                                                                            set_output_power,
                                                                            input_data.D_set,
                                                                            input_data.fsw_set))
            else:
                print('\n================================')
                print('Finishing')
                scope.disconnect()

                source.setVoltage(voltage=0)
                sleep(1)
                source.powerOff()
                source.close()
                sink.setCurrent(neg=0, pos=0)
                sleep(1)
                sink.powerOff()
                sink.close()

                data.to_csv(timeStamped('_') + '.csv')

                return 0

            # sleep(2)

            if abs(source_input_current) <= 1e-10:
                input_resistance = np.nan
            else:
                input_resistance = meas_input_voltage / source_input_current

            print('\n== ACTUAL STEP ===============================')
            print('Vin_meas = %f' % meas_input_voltage)
            print('Iin_meas = %f' % source_input_current)
            print('Vout_meas = %f' % meas_output_voltage)
            print('Iout_meas = %f' % sink_output_current)
            # print('Rin = %f' %input_resistance)
            Pin = (source_input_voltage * source_input_current)
            # print('Pin_calc = %f' % Pin)
            print('Pin_meas = %f' % source_input_power)
            Pout = (sink_output_voltage * sink_output_current)
            # print('Pout_calc = %f' % Pout)
            print('Pout_meas = %f' % sink_output_power)
            Ploss = Pin + Pout
            print('Ploss = %f' % Ploss)
            # Rloss=source_input_voltage**2/(Ploss+eps)
            # print('Rloss = %f' % Rloss)

            data_dict = {
                'DateTime': timeStamped('_'),
                'Modulation': input_data.modulation,
                'fs_set(Hz)': input_data.fsw_set,
                'D_set(-)': input_data.D_set,
                'Vin_set': set_input_voltage,
                'Pout_set': set_output_power,

                'SourceInputVoltage(V)': source_input_voltage,
                'DMMInputVoltage(V)': meas_input_voltage,
                'SourceInputCurrent(A)': source_input_current,
                'SourceInputPower(A)': source_input_power,

                'SinkOutputVoltage(V)': sink_output_voltage,
                'DMMOutputVoltage(V)': meas_output_voltage,
                'SinkOutputCurrent(A)': sink_output_current,
                'SinkOutputPower(A)': sink_output_power,

                'fs_meas(Hz)': switching_freq,
                'Duty100(-)': duty_boost}

            # pprint.pprint(data_dict, depth=1, width=60)
            # temp_lst=thermo.get_all_temperatures()
            # print(dict(temp_lst))

            df2 = pd.DataFrame([data_dict], columns=data_dict.keys())
            # df_temp = pd.DataFrame([dict(temp_lst)])
            # foo=pd.concat([df2, df_temp], axis=1, ignore_index=False)
            foo=pd.concat([df2], axis=1, ignore_index=False)

            data = pd.concat([data, foo], axis=0, ignore_index=True)
            data.to_csv('temp.csv')
        print('================================')
        print('Finishing')

        scope.disconnect()

        source.setVoltage(voltage=0)
        sleep(1)
        source.powerOff()
        source.close()
        sink.setCurrent(neg=0, pos=0)
        sleep(1)
        sink.powerOff()
        sink.close()
        # thermo.close()

        data.to_csv(timeStamped('_') + '.csv')

    return -1

if __name__ == "__main__":

    input_data=SSM_ZOE()
    # Set the input parameters
    input_data.modulation = 'CCM_Y2VW'  # YUVW configuration 1=enable
    # os.system("python .\hwrcommunication.py setFrequency 15")
    input_data.D_set = .5
    input_data.fsw_set = 25e3

    # dsp = Microcontroller()
    # dsp.set_duty(duty=input_data.D_set)
    # dsp.set_switching_frequency(input_data.fsw_set)
    # dsp.set_UV2W()

    input_data.worstcase_losses = 8000
    input_data.max_output_voltage = 200
    input_data.output_voltage_step = 50
    # max_output_voltage_step = 50

    input_data.max_output_power = 0
    input_data.min_output_power = 0
    input_data.step_output_power = 500

    run(input_data)


#!/usr/bin/python

#import spidev
import os
import random
import subprocess
import time
import statistics
import sqlite3

# delay between measurements in session
delay = 0.2
# delay between measurements posted into database
# sessionDelay = 60 #default: 600 (10 minutes)

# values under minThreshold are not representative and therefore filtered
minThreshold = 273 #100% humidity
maxThreshold = 1023 #0% humidity

# spidev
spi = spidev.SpiDev()
spi.open(0,0)
spi.max_speed_hz=1000000

def readChannel(channel):
  val = spi.xfer2([1,(8+channel)<<4,0])
  data = ((val[1]&3) << 8) + val[2]
  return data

def convertToPercentage(value):
  percentage = abs((value - maxThreshold) / (minThreshold - maxThreshold)) * 100
  return int(round(percentage, 0))

def insertIntoDb(humidity):
  connection = sqlite3.connect("database.db")
  cursor = connection.cursor()

  sql = "INSERT INTO sensorLog(value) VALUES(" + str(humidity) + ")"
  cursor.execute(sql)
  connection.commit()

  connection.close()

if __name__ == "__main__":

  try:

      ## sensor data array
      measurements = []
      print("★ Starting humidity measurement")

      while (len(measurements) <= 10):
        # get sensor data
        val = readChannel(0)

        # filter invalid values
        if (val > minThreshold):
          # if data is valid, append
          measurements.append(val)
          print("★ Sensor measurement " + str(len(measurements)) + ": " + str(val))

        # wait time between next measurement
        time.sleep(delay)

      # calculate median
      median = statistics.median(measurements)
      print("★ Median of measurements: " + str(median))

      # convert to percent
      humidityPercentage = convertToPercentage(median)
      print("★ Humidity: " + str(humidityPercentage) + "%")

      # save to db
      insertIntoDb(humidityPercentage)
      print("★ Humidity saved to DB")

      # print("★ Measurement session complete. Next measurement in " + str(sessionDelay/60) + " minutes")
      # time.sleep(sessionDelay)

  except KeyboardInterrupt:
    print ("★ sensor.py closed")


import random

import Preprocessor

spam_word_count = {}
ham_word_count = {}

words_in_spam = 0
words_in_ham = 0

raw_spam_prob = {}
raw_ham_prob = {}

def customized_train(samples_proportion=0.8):
    global words_in_ham, ham_word_count, words_in_spam, spam_word_count, raw_ham_prob, raw_spam_prob

    ham, spam = read_spam_ham()

    spam_size = len(spam)
    ham_size = len(ham)

    all_emails = append_ham_and_spam(ham, spam)

    random.shuffle(all_emails)

    print('Corpus size = ' + str(len(all_emails)) + ' emails')

    features = [(Preprocessor.get_features(email, 'bow'), label) for (email, label) in all_emails]

    print('Collected ' + str(len(features)) + ' feature sets')

    print ("Spam size: " + str(len(spam)) + " Ham size: " + str(len(ham)))

    # define Support value in %
    print('Enter support value to calculate frequent words')
    support = input()

    spam_support_count = spam_size * support / 100
    ham_support_count = ham_size * support / 100

    print('Spam support count:' + str(spam_support_count))
    print('Ham support count:' + str(ham_support_count))

    # get the spam frequent itemset and ham frequent itemset

    spam_frequent, ham_frequent = Preprocessor.get_frequent(features, spam_support_count, ham_support_count)

    # train the our own naivebayes classifier and collect dictionary of raw probabilities of words

    train_size = int(len(features) * samples_proportion)

    train_set, test_set = features[:train_size], features[train_size:]

    ham_mail_count, spam_mail_count = mails_in_ham_spam(train_set)

    spam_prior = 1.0 * spam_mail_count / len(train_set)
    ham_prior = 1.0 * ham_mail_count / len(train_set)

    words_in_ham, words_in_spam = frequency_in_ham_spam(train_set)

    spam_vocab = len(spam_word_count)
    ham_vocab = len(ham_word_count)

    t = get_probabilities_in_each_class(ham_prior, words_in_ham, ham_vocab, ham_word_count, raw_ham_prob, raw_spam_prob,
                                        spam_prior, words_in_spam, spam_vocab, spam_word_count, test_set, train_set)

    ham_prior, words_in_ham, ham_vocab, raw_ham_prob, raw_spam_prob, spam_prior, words_in_spam, spam_vocab, test_set, train_set = get_parameters(
        t)
    print("Train Size:" + str(len(train_set)) + str(' Test size:') + str(len(test_set)))

    get_tweaked_probabilies(spam_frequent, ham_frequent, spam_size, ham_size, spam_vocab, ham_vocab)

    evaluate(train_set, test_set, raw_spam_prob, raw_ham_prob, words_in_spam, words_in_ham, spam_vocab, ham_vocab,
             spam_prior,
             ham_prior)

def get_probabilities_in_each_class(ham_prior, ham_total, ham_vocab, ham_word_count, raw_ham_prob, raw_spam_prob,
                                    spam_prior, spam_total, spam_vocab, spam_word_count, test_set, train_set):
    for (words, label) in train_set:
        if label == 'spam':
            for word in words:
                raw_spam_prob[word] = (float)((spam_word_count.setdefault(word, 0)) / (spam_total))
        else:
            for word in words:
                raw_ham_prob[word] = (float)((ham_word_count.setdefault(word, 0)) / (ham_total))
    return [train_set, test_set, raw_spam_prob, raw_ham_prob, spam_total, ham_total, spam_vocab, ham_vocab, spam_prior,
            ham_prior]

def get_tweaked_probabilies(spam_frequent, ham_frequent, spam_total, ham_total, spam_vocab, ham_vocab):
    for spam_frequent_word, count in spam_frequent.iteritems():
        raw_spam_prob[spam_frequent_word] = count/(spam_total + spam_vocab)

    for ham_frequent_word, count in ham_frequent.iteritems():
        raw_ham_prob[ham_frequent_word] = count/(ham_total + ham_vocab)
    return raw_spam_prob, raw_ham_prob

def frequency_in_ham_spam(train_set):
    global spam_word_count, ham_word_count

    spam_total = 0
    ham_total = 0
    for (frequency, label) in train_set:
        if label == 'spam':
            for key in frequency:
                spam_word_count.setdefault(key, 0.0)
                spam_word_count[key] += frequency[key]
                spam_total += 1
        else:
            for key in frequency:
                ham_word_count.setdefault(key, 0.0)
                ham_word_count[key] += frequency[key]
                ham_total += 1
    return ham_total, spam_total

def mails_in_ham_spam(train_set):
    spam_count = 0
    ham_count = 0
    for (words, label) in train_set:
        if (label == 'spam'):
            spam_count += 1
        else:
            ham_count += 1
    return ham_count, spam_count

def classify(data_set, raw_spam_prob, raw_ham_prob, spam_total, ham_total, spam_vocab, ham_vocab, spam_prior,
             ham_prior):
    total_mail = 0
    correct_count = 0.0000000
    for (features, label) in data_set:
        spam_prob = 1.000000
        ham_prob = 1.000000
        is_spam = check_spam(features, ham_prior, ham_prob, ham_total, ham_vocab, raw_ham_prob, raw_spam_prob,
                             spam_prior, spam_prob, spam_total, spam_vocab)

        if (label == 'spam') and is_spam:
            correct_count += 1

        if (label == 'ham') and not (is_spam):
            correct_count += 1
        total_mail += 1

    print('correct count' + str(correct_count))
    return correct_count / total_mail

def check_spam(features, ham_prior, ham_prob, ham_total, ham_vocab, raw_ham_prob, raw_spam_prob, spam_prior, spam_prob,
               spam_total, spam_vocab):
    for word in features:
        # Handling probability of non occuring words with laplaces

        try:
            spam_prob = raw_spam_prob[word] * spam_prob
        except KeyError:
            raw_spam_prob[word] = (1 / (spam_total + spam_vocab + 1))
            spam_prob = raw_spam_prob[word] * spam_prob

        try:
            ham_prob = raw_ham_prob[word] * ham_prob
        except KeyError:
            raw_ham_prob[word] = (1 / (ham_total + ham_vocab + 1))
            ham_prob = raw_ham_prob[word] * ham_prob

    spam_prob *= spam_prior
    ham_prob *= ham_prior
    if spam_prob > ham_prob:
        is_spam = True
    else:
        is_spam = False
    return is_spam

def evaluate(train_set, test_set, raw_spam_prob, raw_ham_prob, spam_total, ham_total, spam_vocab, ham_vocab, spam_prior,
             ham_prior):

    train_accuracy = classify(train_set, raw_spam_prob, raw_ham_prob, spam_total, ham_total, spam_vocab, ham_vocab,
                              spam_prior, ham_prior)

    test_accuracy = classify(test_set, raw_spam_prob, raw_ham_prob, spam_total, ham_total, spam_vocab, ham_vocab,
                             spam_prior, ham_prior)
    print('Accuracy on the training set = ' + str(train_accuracy))
    print('Accuracy of the test set = ' + str(test_accuracy))

    #TODO: check which words are most informative for the classifier

def read_spam_ham():
    spam = Preprocessor.init_lists('enron1 (copy)/spam/')
    ham = Preprocessor.init_lists('enron1 (copy)/ham/')
    return ham, spam

def get_parameters(t):
    train_set = t[0]
    test_set = t[1]
    raw_spam_prob = t[2]
    raw_ham_prob = t[3]
    spam_total = t[4]
    ham_total = t[5]
    spam_vocab = t[6]
    ham_vocab = t[7]
    spam_prior = t[8]
    ham_prior = t[9]
    return ham_prior, ham_total, ham_vocab, raw_ham_prob, raw_spam_prob, spam_prior, spam_total, spam_vocab, test_set, train_set

def append_ham_and_spam(ham, spam):
    all_emails = [(email, 'spam') for email in spam]
    all_emails += [(email, 'ham') for email in ham]
    return all_emails

def get_spam_ham_features(ham_emails, spam_emails):
    spam_features = [(Preprocessor.get_features(email, 'bow'), label) for (email, label) in spam_emails]
    ham_features = [(Preprocessor.get_features(email, 'bow'), label) for (email, label) in ham_emails]
    return ham_features, spam_features


# -*- coding:utf-8 -*-

#collections模块提供有用的集合类
from collections import namedtuple,deque,defaultdict,OrderedDict,ChainMap,Counter

#namedtuple是一个函数，它用来创建一个自定义的tuple对象，并且规定了tuple元素的个数，并可以用属性而不是索引来引用tuple的某个元素。
Point = namedtuple('Point', ['x', 'y'])
p = Point(1,2)
print(p.x)
print(p.y)

#deque,可实现高效插入和删除的双向列表
q = deque(['a', 'b', 'c'])
q.append('x')
q.append('y')
print(q)
q.appendleft(1)
q.appendleft(2)
print(q)

q.pop()
print(q)
q.popleft()
print(q)

#defaultdict,字典的key不存在的时候传回默认值
dd = defaultdict(lambda:'N/A')
dd['key1'] = 'abc'
print(dd['key1'])
print(dd['key2'])

#OrderedDict,创建字典按照插入顺序排队
od1 = OrderedDict([('a', 1),('b', 2),('c', 3)])
print(od1)
od2 = OrderedDict()
od2['z'] = 3
od2['x'] = 1
od2['y'] = 2
print(od2)
print(list(od2.keys()))

#使用OrderedDict创建FIFO（先进先出）的字典
from collections import OrderedDict

class LastUpdateOrderedDict(OrderedDict):
    def __init__(self, capacity):
        super(LastUpdateOrderedDict, self).__init__()
        self._capacity = capacity

    def __setitem__(self, key, value):
        containsKey = 1 if key in self else 0
        if len(self) - containsKey >= self._capacity:
            last = self.popitem(last=False)
            print('remove:', last)
        if containsKey:
            del self[key]
            print('set:', (key, value))
        else:
            print('add:', (key, value))
        OrderedDict.__setitem__(self, key, value)

od3 = LastUpdateOrderedDict(2)
od3['a'] = 1
od3['b'] = 2
print('od3:', od3)

#ChainMap可以把一组dict串起来组成逻辑上的dict

#Counter计数器,dict的一个子类
from collections import Counter
c = Counter()
for ch in 'programming':
    c[ch] = c[ch] + 1

print(c)


# -*- coding:utf8 -*-
import numpy as np
from random import randint, uniform
import csv
import nibabel as nib
from skimage import transform
from sklearn.model_selection import StratifiedKFold
from ResNetBuild import Resnet3DBuilder
from keras.callbacks import (
    ModelCheckpoint,
    EarlyStopping,
    TensorBoard,
    LearningRateScheduler,
    ReduceLROnPlateau
)
from keras.optimizers import SGD
from keras import backend as K

# Set global var
def set_globals():
    global RAW_CSV
    global TV_CSV
    global TEST_CSV
    global CSV_PATH
    global TRAIN_DATA

    RAW_CSV = "/home/lb/ADNI/data/CSV/raw.csv"
    TV_CSV = "/home/lb/ADNI/data/CSV/train_val.csv"
    TEST_CSV = "/home/lb/ADNI/data/CSV/test.csv"
    CSV_PATH = "/home/lb/ADNI/data/CSV/"
    TRAIN_DATA = "/home/lb/ADNI/data/TRAIN/"

# Read csv
def read_csv(cfile):
    csv_file = open(cfile)
    csv_reader_lines = csv.reader(csv_file)
    raw_data = []
    for one_line in csv_reader_lines:
        raw_data.append(one_line)
    np_data = np.array(raw_data)
    X = np_data[:, 0]
    Y = np_data[:, 1]
    Y = Y.astype(int)
    csv_file.close()
    return X, Y

# Get data from the path to a matrix data
def read_data(path):
    image_data = nib.load(path).get_data()
    return image_data

# Split test datas, type 1 = AD/MCI, 2 = AD/CN, 3 = MCI/NC, 4 = AD/MCI/CN
def split_test(type = 4):
    test_sfolder = StratifiedKFold(n_splits=6, shuffle=True, random_state=0)
    Xraw, Yraw = read_csv(RAW_CSV)

    if type == 4:
        pass
    else:
        Xtmp = []
        Ytmp = []
        t1 = 0
        t2 = 0
        if type == 3:
            t1 = 1
            t2 = 0
        elif type == 2:
            t1 = 2
            t2 = 0
        elif type == 1:
            t1 = 2
            t2 = 1

        for i in range(len(Xraw)):
            if Yraw[i] == t1 or Yraw[i] == t2:
                Xtmp.append(Xraw[i])
                Ytmp.append(Yraw[i])
        Xraw = np.array(Xtmp)
        Yraw = np.array(Ytmp)

    for tv, test in test_sfolder.split(Xraw, Yraw):
        write_csv(TV_CSV, TEST_CSV, tv, test, Xraw, Yraw)
        break

# Write csv files
def write_csv(a_csv, b_csv, a, b, araw, braw):
    a_out = open(a_csv, 'a', newline='')
    b_out = open(b_csv, 'a', newline='')
    a_writer = csv.writer(a_out)
    b_writer = csv.writer(b_out)

    for i in range(len(a)):
        a_writer.writerow([araw[a][i], braw[a][i]])
    for j in range(len(b)):
        b_writer.writerow([araw[b][j], braw[b][j]])

    a_out.close()
    b_out.close()

def random_flip(single_data, dim=3):
    if dim == 3:
        fliptype = randint(0, 7)
        if fliptype == 0:
            single_data = np.flip(single_data, 0)
        elif fliptype == 1:
            single_data = np.flip(single_data, 1)
        elif fliptype == 2:
            single_data = np.flip(single_data, 2)
        elif fliptype == 3:
            single_data = np.flip(single_data, 0)
            single_data = np.flip(single_data, 1)
        elif fliptype == 4:
            single_data = np.flip(single_data, 0)
            single_data = np.flip(single_data, 2)
        elif fliptype == 5:
            single_data = np.flip(single_data, 1)
            single_data = np.flip(single_data, 2)
        elif fliptype == 6:
            single_data = np.flip(single_data, 0)
            single_data = np.flip(single_data, 1)
            single_data = np.flip(single_data, 2)
        else:
            pass
    elif dim == 2:
        fliptype = randint(0, 3)
        if fliptype == 0:
            single_data = np.flip(single_data, 0)
        elif fliptype == 1:
            single_data = np.flip(single_data, 1)
        elif fliptype == 2:
            single_data = np.flip(single_data, 0)
            single_data = np.flip(single_data, 1)
        else:
            pass
    else:
        pass
    return single_data

def random_crop(single_data, dim=3):
    if dim == 3:
        d = single_data.shape
        is_crop = randint(0, 1)
        if is_crop == 0:
            pass
        else:
            crop_rate = uniform(0.8, 1)
            start_rate = uniform(0, 0.2)
            crop_len = [int(crop_rate * d[0]), int(crop_rate * d[1]), int(crop_rate * d[2])]
            start_point = [int(start_rate * d[0]), int(start_rate * d[1]), int(start_rate * d[2])]
            single_data_crop = single_data[start_point[0]:(start_point[0] + crop_len[0]),
                               start_point[1]:(start_point[1] + crop_len[1]),
                               start_point[2]:(start_point[2] + crop_len[2])]
            single_data = transform.resize(single_data_crop, (d[0], d[1], d[2]))
    elif dim == 2:
        d = single_data.shape
        is_crop = randint(0, 1)
        if is_crop == 0:
            pass
        else:
            crop_rate = uniform(0.8, 1)
            start_rate = uniform(0, 0.2)
            crop_len = [int(crop_rate * d[0]), int(crop_rate * d[1])]
            start_point = [int(start_rate * d[0]), int(start_rate * d[1])]
            single_data_crop = single_data[start_point[0]:(start_point[0] + crop_len[0]),
                               start_point[1]:(start_point[1] + crop_len[1]), :]
            single_data = transform.resize(single_data_crop, (d[0], d[1], d[2]))
    else:
        pass
    return single_data

# Generate batch data to same memory
def generate_batch_data_random(cc, batch_size, dpath, class_num, is_val):
    ylen = len(cc)
    loopcount = ylen // batch_size
    np.random.shuffle(cc)
    x = np.array([])
    y = np.array([])
    for i in range(0, ylen):
        x = np.append(x, cc[i]['x'])
        y = np.append(y, cc[i]['y'])
    while (True):
        seed = randint(0, loopcount - 1)
        train_x_url = x[seed * batch_size:(seed + 1) * batch_size]
        train_x = np.array([])
        dim = None
        for r in train_x_url:
            single_path = dpath + r
            single_data = read_data(single_path)
            if not is_val:
                single_data = random_flip(single_data)
                single_data = random_crop(single_data)
            dim = single_data.shape
            train_x = np.append(train_x, single_data)
        train_x = train_x.reshape(-1, dim[0], dim[1], dim[2], 1)
        train_y = np.array([])
        for b in range(seed * batch_size, (seed + 1) * batch_size):
            tmp = [0 for i in range(class_num)]
            if class_num > 2:
                tmp[int(y[b])] = 1
            else:
                tmp = 0
                if int(y[b]) == int(max(y)):
                    tmp = 1
            train_y = np.append(train_y, tmp)
        if class_num > 2:
            train_y = train_y.reshape(-1, class_num)
        else:
            pass
        yield (train_x, train_y)

# Cross_val train models
def train_models(batches = 4, is_cross_val = False, class_num = 3):
    tv_sfolder = StratifiedKFold(n_splits=5, shuffle=True, random_state=0)
    Xtv, Ytv = read_csv(TV_CSV)
    cross_val = 0

    for train, val in tv_sfolder.split(Xtv, Ytv):
        tpath = CSV_PATH + 'train' + str(cross_val) + '.csv'
        vpath = CSV_PATH + 'val' + str(cross_val) + '.csv'
        write_csv(tpath, vpath, train, val, Xtv, Ytv)

        Xt = Xtv[train]
        Yt = Ytv[train]
        Xv = Xtv[val]
        Yv = Ytv[val]

        aa = np.array([])
        for a in range(0, Xt.size):
            aa = np.append(aa, {'x': Xt[a], 'y': Yt[a]})
        bb = np.array([])
        for b in range(0, Xv.size):
            bb = np.append(bb, {'x': Xv[b], 'y': Yv[b]})

        n = class_num
        if class_num == 2:
            n = 1

        model = Resnet3DBuilder.build_resnet_101((128, 128, 128, 1), n, 1e-3)
        sgd = SGD(lr=0.001, momentum=0.9, decay=0.0, nesterov=False)
        if class_num == 2:
            model.compile(loss="binary_crossentropy", metrics=['acc'], optimizer=sgd)
        else:
            model.compile(loss="categorical_crossentropy", metrics=['acc'], optimizer=sgd)
        checkpointer = ModelCheckpoint(filepath="./models/weights" + str(cross_val) + ".{epoch:02d}.hdf5", save_best_only=True, period=5)

        # Scheduler reduce learn rate
        # def scheduler(epoch):
        #     if epoch % 15 == 0 and epoch != 0:
        #         lr = K.get_value(model.optimizer.lr)
        #         K.set_value(model.optimizer.lr, lr * 0.1)
        #         print("lr changed to {}".format(lr * 0.1))
        #     return K.get_value(model.optimizer.lr)

        earlystoper = EarlyStopping(monitor='val_loss', patience=20)
        tensorboard = TensorBoard(log_dir='./logs', batch_size=batches)
        reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=6, verbose=1, min_lr=1e-8)
        # reduce_lr = LearningRateScheduler(scheduler)
        model.fit_generator(generate_batch_data_random(aa, batches, TRAIN_DATA, class_num, False),
                            steps_per_epoch=(Xt.size // batches), epochs=500, verbose=1,
                            validation_data=generate_batch_data_random(bb, batches, TRAIN_DATA, class_num, True),
                            validation_steps=(Xv.size // batches),
                            callbacks=[checkpointer, earlystoper, tensorboard, reduce_lr])
        model.save("./models/weights" + str(cross_val) + ".h5")
        cross_val = cross_val + 1
        if not is_cross_val:
            break

if __name__ == '__main__':
    set_globals()
    split_test(type = 3)
    train_models(batches = 4, class_num = 2)


import numpy as np
import pandas as pd
import os

import matplotlib.pyplot as plt
### matplotlib inline
from tqdm import tqdm_notebook
from sklearn.preprocessing import StandardScaler
from sklearn.svm import NuSVR, SVR
from sklearn.metrics import mean_absolute_error
pd.options.display.precision = 15
from sklearn.model_selection import train_test_split, KFold, cross_val_score, StratifiedKFold, GridSearchCV
from sklearn import svm
import lightgbm as lgb
import xgboost as xgb
import time
import datetime
from catboost import CatBoostRegressor
from sklearn.preprocessing import LabelEncoder
from sklearn import neighbors
from sklearn.metrics import mean_absolute_error
from sklearn import linear_model
import gc
import seaborn as sns
import warnings
warnings.filterwarnings("ignore")
from sklearn.ensemble import ExtraTreesRegressor, AdaBoostRegressor

from scipy.signal import hilbert
from scipy.signal import hann
from scipy.signal import convolve
from scipy import stats
from sklearn.kernel_ridge import KernelRidge
from sklearn.neighbors import NearestNeighbors
import librosa, librosa.display
import builtins
from sklearn.ensemble import RandomForestRegressor
import eli5
import shap
from sklearn.feature_selection import GenericUnivariateSelect, SelectPercentile, SelectKBest, f_classif, mutual_info_classif, RFE

from IPython.display import HTML
import json
import altair as alt

import artgor_utils

# setting up altair
workaround = artgor_utils.prepare_altair()
HTML("".join((
    "<script>",
    workaround,
    "</script>",
)))
os.listdir('../input/lanl-features')
train_features = pd.read_csv('../input/lanl-features/train_features.csv')
test_features = pd.read_csv('../input/lanl-features/test_features.csv')
train_features_denoised = pd.read_csv('../input/lanl-features/train_features_denoised.csv')
test_features_denoised = pd.read_csv('../input/lanl-features/test_features_denoised.csv')
train_features_denoised.columns = [f'{i}_denoised' for i in train_features_denoised.columns]
test_features_denoised.columns = [f'{i}_denoised' for i in test_features_denoised.columns]
y = pd.read_csv('../input/lanl-features/y.csv')
X = pd.concat([train_features, train_features_denoised], axis=1).drop(['seg_id_denoised', 'target_denoised'], axis=1)
X_test = pd.concat([test_features, test_features_denoised], axis=1).drop(['seg_id_denoised', 'target_denoised'], axis=1)
X = X[:-1]
y = y[:-1]
n_fold = 10
folds = KFold(n_splits=n_fold, shuffle=True, random_state=11)
params = {'num_leaves': 128,
          'min_child_samples': 79,
          'objective': 'gamma',
          'max_depth': -1,
          'learning_rate': 0.01,
          "boosting_type": "gbdt",
          "subsample_freq": 5,
          "subsample": 0.9,
          "bagging_seed": 11,
          "metric": 'mae',
          "verbosity": -1,
          'reg_alpha': 0.1302650970728192,
          'reg_lambda': 0.3603427518866501,
          'colsample_bytree': 0.1
         }
result_dict_lgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb',
                                                                                eval_metric='mae', plot_feature_importance=True)
submission = pd.read_csv('../input/LANL-Earthquake-Prediction/sample_submission.csv', index_col='seg_id')
submission['time_to_failure'] = result_dict_lgb['prediction']
print(submission.head())
submission.to_csv('submission.csv')
sub1 = pd.read_csv('../input/lanl-features/submission_1.csv')
sub1.to_csv('submission_1.csv', index=False)
scaler = StandardScaler()
scaler.fit(X)
X_train_scaled = pd.DataFrame(scaler.transform(X), columns=X.columns)
X_test_scaled = pd.DataFrame(scaler.transform(X_test), columns=X_test.columns)
### %time
n = 10
neigh = NearestNeighbors(n, n_jobs=-1)
neigh.fit(X_train_scaled)

dists, _ = neigh.kneighbors(X_train_scaled, n_neighbors=n)
mean_dist = dists.mean(axis=1)
max_dist = dists.max(axis=1)
min_dist = dists.min(axis=1)

X_train_scaled['mean_dist'] = mean_dist
X_train_scaled['max_dist'] = max_dist
X_train_scaled['min_dist'] = min_dist

test_dists, _ = neigh.kneighbors(X_test_scaled, n_neighbors=n)

test_mean_dist = test_dists.mean(axis=1)
test_max_dist = test_dists.max(axis=1)
test_min_dist = test_dists.min(axis=1)

X_test_scaled['mean_dist'] = test_mean_dist
X_test_scaled['max_dist'] = test_max_dist
X_test_scaled['min_dist'] = test_min_dist

params = {'num_leaves': 32,
          'min_data_in_leaf': 79,
          'objective': 'gamma',
          'max_depth': -1,
          'learning_rate': 0.01,
          "boosting": "gbdt",
          "bagging_freq": 5,
          "bagging_fraction": 0.8126672064208567,
          "bagging_seed": 11,
          "metric": 'mae',
          "verbosity": -1,
          'reg_alpha': 0.1302650970728192,
          'reg_lambda': 0.3603427518866501,
          'feature_fraction': 0.1
         }
result_dict_lgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb',
                                                                                eval_metric='mae', plot_feature_importance=True)
submission['time_to_failure'] = result_dict_lgb['prediction']
submission.to_csv('submission_nn.csv')
top_columns = ['iqr1_denoised', 'percentile_5_denoised', 'abs_percentile_90_denoised', 'percentile_95_denoised', 'ave_roll_std_10', 'num_peaks_10', 'percentile_roll_std_20',
               'ratio_unique_values_denoised', 'fftr_percentile_roll_std_75_denoised', 'num_crossing_0_denoised', 'percentile_95', 'ffti_percentile_roll_std_75_denoised',
               'min_roll_std_10000', 'percentile_roll_std_1', 'percentile_roll_std_10', 'fftr_percentile_roll_std_70_denoised', 'ave_roll_std_50', 'ffti_percentile_roll_std_70_denoised',
               'exp_Moving_std_300_mean_denoised', 'ffti_percentile_roll_std_30_denoised', 'mean_change_rate', 'percentile_roll_std_5', 'range_-1000_0', 'mad',
               'fftr_range_1000_2000_denoised', 'percentile_10_denoised', 'ffti_percentile_roll_std_80', 'percentile_roll_std_25', 'fftr_percentile_10_denoised',
               'ffti_range_-2000_-1000_denoised', 'autocorrelation_5', 'min_roll_std_100', 'fftr_percentile_roll_std_80', 'min_roll_std_500', 'min_roll_std_50', 'min_roll_std_1000',
               'ffti_percentile_20_denoised', 'iqr1', 'classic_sta_lta5_mean_denoised', 'classic_sta_lta6_mean_denoised', 'percentile_roll_std_10_denoised',
               'fftr_percentile_70_denoised', 'ffti_c3_50_denoised', 'ffti_percentile_roll_std_75', 'abs_percentile_90', 'range_0_1000', 'spkt_welch_density_50_denoised',
               'ffti_percentile_roll_std_40_denoised', 'ffti_range_-4000_-3000', 'mean_change_rate_last_50000']

X_train, X_valid, y_train, y_valid = train_test_split(X[top_columns], y, test_size=0.1)
model = lgb.LGBMRegressor(**params, n_estimators = 50000, n_jobs = -1, verbose=-1)
model.fit(X_train, y_train,
        eval_set=[(X_train, y_train), (X_valid, y_valid)], eval_metric='mae',
        verbose=10000, early_stopping_rounds=200)

perm = eli5.sklearn.PermutationImportance(model, random_state=1).fit(X_train, y_train)
eli5.show_weights(perm, top=50, feature_names=top_columns)
top_features = [i for i in eli5.formatters.as_dataframe.explain_weights_df(model).feature if 'BIAS' not in i][:40]
result_dict_lgb = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='lgb', plot_feature_importance=True, columns=top_features)
submission['time_to_failure'] = result_dict_lgb['prediction']
submission.to_csv('submission_eli5.csv')
params = {'num_leaves': 32,
          'min_child_samples': 79,
          'objective': 'gamma',
          'max_depth': -1,
          'learning_rate': 0.01,
          "boosting_type": "gbdt",
          "subsample_freq": 5,
          "subsample": 0.9,
          "bagging_seed": 11,
          "metric": 'mae',
          "verbosity": -1,
          'reg_alpha': 0.1302650970728192,
          'reg_lambda': 0.3603427518866501,
          'colsample_bytree': 1.0
         }
# %%time
# scores_dict = {'f_classif': [], 'mutual_info_classif': [], 'n_features': []}
# for i in range(5, 105, 5):
#     print(i)
#     s1 = SelectPercentile(f_classif, percentile=i)
#     X_train1 = s1.fit_transform(X, y.values.astype(int))
#     X_test1 = s1.transform(X_test)
#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)
#     scores_dict['f_classif'].append(np.mean(result_dict_lgb['scores']))

#     s2 = SelectPercentile(mutual_info_classif, percentile=i)
#     X_train1 = s2.fit_transform(X, y.values.astype(int))
#     X_test1 = s2.transform(X_test)
#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)
#     scores_dict['mutual_info_classif'].append(np.mean(result_dict_lgb['scores']))

#     scores_dict['n_features'].append(X_train1.shape[1])
scores_dict = {'f_classif': [2.0746468465171377, 2.0753843541953687, 2.062191535440333, 2.0654327826583034, 2.0643551320704936, 2.0617560048382675,
                             2.061565197738015, 2.0598878198917494, 2.0654865223333143, 2.0632788555735777, 2.058002635080971, 2.051075689018734,
                             2.0472543961304583, 2.052401474353084, 2.055924154798443, 2.0561794619762352, 2.0549680611994963, 2.057123777802326,
                             2.0591868861136904, 2.0577745274024553],
               'mutual_info_classif': [2.0866763775014006, 2.0745431497064324, 2.0564324832516427, 2.060125564781158, 2.067334544167612, 2.0665943783246448,
                                       2.063891669849029, 2.070194051004794, 2.0667490707700447, 2.0681653852378354, 2.0592743636982345, 2.061260741522344,
                                       2.05680667824411, 2.0565047875243003, 2.058252567141659, 2.0554927194831922, 2.0562776429736873, 2.0618179277444084,
                                       2.06364125584214, 2.0577745274024553],
               'n_features': [98, 196, 294, 392, 490, 588, 685, 783, 881, 979, 1077, 1175, 1273, 1370, 1468, 1566, 1664, 1762, 1860, 1958]}
scores_df = pd.DataFrame(scores_dict)
scores_df = scores_df.melt(id_vars=['n_features'], value_vars=['mutual_info_classif', 'f_classif'], var_name='metric', value_name='mae')
max_value = scores_df['mae'].max() * 1.05
min_value = scores_df['mae'].min() * 0.95
artgor_utils.render(alt.Chart(scores_df).mark_line().encode(
    y=alt.Y('mae:Q', scale=alt.Scale(domain=(min_value, max_value))),
    x='n_features:O',
    color='metric:N',
    tooltip=['metric:N', 'n:O', 'mae:Q']
).properties(
    title='Top N features by SelectPercentile vs CV'
).interactive())
# %%time
# scores_dict = {'f_classif': [], 'mutual_info_classif': [], 'n_features': []}
# for i in np.arange(10, 1958, 100):
#     print(i)
#     s1 = SelectKBest(f_classif, k=i)
#     X_train1 = s1.fit_transform(X, y.values.astype(int))
#     X_test1 = s1.transform(X_test)
#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)
#     scores_dict['f_classif'].append(np.mean(result_dict_lgb['scores']))

#     s2 = SelectKBest(mutual_info_classif, k=i)
#     X_train1 = s2.fit_transform(X, y.values.astype(int))
#     X_test1 = s2.transform(X_test)
#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)
#     scores_dict['mutual_info_classif'].append(np.mean(result_dict_lgb['scores']))

#     scores_dict['n_features'].append(X_train1.shape[1])
scores_dict = {'f_classif': [2.1495892622081354, 2.0778182269587147, 2.0716153738740006, 2.06152950679902, 2.0645162758752553, 2.0627705797004032, 2.0610992303725157,
                             2.057762113735462, 2.0618360883613627, 2.0603197111525984, 2.06081274633874, 2.0580767195278056, 2.0527646572747127, 2.0498353445032533,
                             2.052442594925, 2.0564456881902133, 2.0582284644115365, 2.0558612960548635, 2.0580900016350094, 2.058218782401599],
               'mutual_info_classif': [2.1235703196243687, 2.084958198672301, 2.0596822478390955, 2.053305869981444, 2.063468853227225, 2.0674399950434323, 2.0658618511287874,
                                       2.063003703200445, 2.0653174905858664, 2.0644340327023656, 2.0748993062333523, 2.0587602096358113, 2.0601495560836076, 2.0559629138548603,
                                       2.0553852701221134, 2.058022171415446, 2.060755947658241, 2.057916705462307, 2.056245795262636, 2.0580691870837056],
               'n_features': [10, 110, 210, 310, 410, 510, 610, 710, 810, 910, 1010, 1110, 1210, 1310, 1410, 1510, 1610, 1710, 1810, 1910]}
scores_df = pd.DataFrame(scores_dict)
scores_df = scores_df.melt(id_vars=['n_features'], value_vars=['mutual_info_classif', 'f_classif'], var_name='metric', value_name='mae')
max_value = scores_df['mae'].max() * 1.05
min_value = scores_df['mae'].min() * 0.95
artgor_utils.render(alt.Chart(scores_df).mark_line().encode(
    y=alt.Y('mae:Q', scale=alt.Scale(domain=(min_value, max_value))),
    x='n_features:O',
    color='metric:N',
    tooltip=['metric:N', 'n:O', 'mae:Q']
).properties(
    title='Top N features by SelectKBest vs CV'
).interactive())
# https://chrisalbon.com/machine_learning/feature_selection/drop_highly_correlated_features/
corr_matrix = X.corr().abs()

# Select upper triangle of correlation matrix
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))

# Find index of feature columns with correlation greater than 0.95
to_drop = [column for column in upper.columns if any(upper[column] > 0.99)]
X = X.drop(to_drop, axis=1)
X_test = X_test.drop(to_drop, axis=1)
result_dict_lgb_lgb = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='lgb', plot_feature_importance=True)
submission['time_to_failure'] = result_dict_lgb['prediction']
submission.to_csv('submission_no_corr.csv')
# %%time
# scores_dict = {'rfe_score': [], 'n_features': []}
# for i in np.arange(10, 1958, 100)[:3]:
#     print(i)
#     s1 = RFE(model, i, step=100)
#     X_train1 = s1.fit_transform(X, y.values.astype(int))
#     X_test1 = s1.transform(X_test)
#     result_dict_lgb = artgor_utils.train_model_regression(X_train1, X_test1, y.values.reshape(-1, ), params=params, folds=folds, model_type='lgb', plot_feature_importance=False)
#     scores_dict['rfe_score'].append(np.mean(result_dict_lgb['scores']))

#     scores_dict['n_features'].append(X_train1.shape[1])
scores_dict = {'rfe_score': [2.103586938061856, 2.052535910798748, 2.053228199447811], 'n_features': [10, 110, 210]}
scores_df = pd.DataFrame(scores_dict)
scores_df = scores_df.melt(id_vars=['n_features'], value_vars=['rfe_score'], var_name='metric', value_name='mae')
max_value = scores_df['mae'].max() * 1.05
min_value = scores_df['mae'].min() * 0.95
artgor_utils.render(alt.Chart(scores_df).mark_line().encode(
    y=alt.Y('mae:Q', scale=alt.Scale(domain=(min_value, max_value))),
    x='n_features:O',
    color='metric:N',
    tooltip=['metric:N', 'n:O', 'mae:Q']
).properties(
    title='Top N features by RFE vs CV'
).interactive())
### %time
rfr = RandomForestRegressor()

# parameter_grid = {'n_estimators': [50, 60],
#                   'max_depth': [5, 10]
#                  }

# grid_search = GridSearchCV(rfr, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
# grid_search.fit(X, y)
# print('Best score: {}'.format(grid_search.best_score_))
# print('Best parameters: {}'.format(grid_search.best_params_))
# rfr = RandomForestRegressor(**grid_search.best_params_)
rfr = RandomForestRegressor(n_estimators=50, max_depth=5)
result_dict_rfr = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=rfr)
# print(scores_rfr)
### %time
linreg = linear_model.LinearRegression(normalize=False, copy_X=True, n_jobs=-1)

result_dict_linreg = artgor_utils.train_model_regression(X, X_test, y, params=None, folds=folds, model_type='sklearn', model=linreg)
# print(scores_linreg)
### %time
ridge = linear_model.Ridge(normalize=True)

parameter_grid = {'alpha': [0.1, 1.0, 10.0]}

grid_search = GridSearchCV(ridge, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
grid_search.fit(X, y)
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
ridge = linear_model.Ridge(**grid_search.best_params_, normalize=True)
result_dict_ridge = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=ridge)
# print(scores_ridge)
### %time
knn = neighbors.KNeighborsRegressor()

parameter_grid = {'n_neighbors': [50, 100]}

grid_search = GridSearchCV(knn, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
grid_search.fit(X, y)
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
knn = neighbors.KNeighborsRegressor(**grid_search.best_params_)
result_dict_knn = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=knn)
### %time
lasso = linear_model.Lasso(normalize=True)

parameter_grid = {'alpha': [0.1, 1.0, 10.0]}

grid_search = GridSearchCV(lasso, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
grid_search.fit(X, y)
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
lasso = linear_model.Lasso(**grid_search.best_params_, normalize=True)
result_dict_lasso = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=lasso)
### %time
etr = ExtraTreesRegressor()

# parameter_grid = {'n_estimators': [500, 1000],
#                   'max_depth': [5, 10, 20]
#                  }

# grid_search = GridSearchCV(rfr, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
# grid_search.fit(X, y)
# print('Best score: {}'.format(grid_search.best_score_))
# print('Best parameters: {}'.format(grid_search.best_params_))
# etr = ExtraTreesRegressor(**grid_search.best_params_)
etr = ExtraTreesRegressor(n_estimators=1000, max_depth=10)
result_dict_etr = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=etr)
### %time
adr = AdaBoostRegressor()

parameter_grid = {'n_estimators': [10, 50],
                 }

grid_search = GridSearchCV(adr, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)
grid_search.fit(X, y)
print('Best score: {}'.format(grid_search.best_score_))
print('Best parameters: {}'.format(grid_search.best_params_))
adr = AdaBoostRegressor(**grid_search.best_params_)
result_dict_adr = artgor_utils.train_model_regression(X, X_test, y, params=params, folds=folds, model_type='sklearn', model=adr)
plt.figure(figsize=(12, 8));
scores_df = pd.DataFrame({'RandomForestRegressor': result_dict_rfr['scores']})
scores_df['ExtraTreesRegressor'] = result_dict_etr['scores']
scores_df['AdaBoostRegressor'] = result_dict_adr['scores']
scores_df['KNN'] = result_dict_knn['scores']
scores_df['LinearRegression'] = result_dict_linreg['scores']
scores_df['Ridge'] = result_dict_ridge['scores']
scores_df['Lasso'] = result_dict_lasso['scores']

sns.boxplot(data=scores_df);
plt.xticks(rotation=45);
params = {'num_leaves': 128,
          'min_child_samples': 79,
          'objective': 'gamma',
          'max_depth': -1,
          'learning_rate': 0.01,
          "boosting_type": "gbdt",
          "subsample_freq": 5,
          "subsample": 0.9,
          "bagging_seed": 11,
          "metric": 'mae',
          "verbosity": -1,
          'reg_alpha': 0.1302650970728192,
          'reg_lambda': 0.3603427518866501,
          'colsample_bytree': 0.2
         }
result_dict_lgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=params, folds=folds, model_type='lgb',
                                                                                eval_metric='mae', plot_feature_importance=True)
xgb_params = {'eta': 0.03,
              'max_depth': 9,
              'subsample': 0.85,
              'colsample_bytree': 0.3,
              'objective': 'reg:linear',
              'eval_metric': 'mae',
              'silent': True,
              'nthread': -1}
result_dict_xgb = artgor_utils.train_model_regression(X=X, X_test=X_test, y=y, params=xgb_params, folds=folds, model_type='xgb')
submission['time_to_failure'] = (result_dict_lgb['prediction'] + result_dict_etr['prediction'] + result_dict_xgb['prediction']) / 3
print(submission.head())
submission.to_csv('blending.csv')
plt.figure(figsize=(18, 8))
plt.subplot(2, 2, 1)
plt.plot(y, color='g', label='y_train')
plt.plot(result_dict_lgb['oof'], color='b', label='lgb')
plt.legend(loc=(1, 0.5));
plt.title('lgb');
plt.subplot(2, 2, 2)
plt.plot(y, color='g', label='y_train')
plt.plot(result_dict_etr['oof'], color='teal', label='xgb')
plt.legend(loc=(1, 0.5));
plt.title('xgb');
plt.subplot(2, 2, 3)
plt.plot(y, color='g', label='y_train')
plt.plot(result_dict_xgb['oof'], color='red', label='etr')
plt.legend(loc=(1, 0.5));
plt.title('Extratrees');
plt.subplot(2, 2, 4)
plt.plot(y, color='g', label='y_train')
plt.plot((result_dict_lgb['oof'] + result_dict_etr['oof'] + result_dict_xgb['oof']) / 3, color='gold', label='blend')
plt.legend(loc=(1, 0.5));
plt.title('blend');
train_stack = np.vstack([result_dict_rfr['oof'], result_dict_ridge['oof'], result_dict_knn['oof'], result_dict_lasso['oof'], result_dict_etr['oof'],
                         result_dict_adr['oof'], result_dict_lgb['oof'], result_dict_xgb['oof'], result_dict_etr['oof']]).transpose()
train_stack = pd.DataFrame(train_stack, columns=['rfr', 'ridge', 'knn', 'lasso', 'etr', 'adr', 'lgb', 'xgb', 'etr'])

test_stack = np.vstack([result_dict_rfr['prediction'], result_dict_ridge['prediction'], result_dict_knn['prediction'], result_dict_lasso['prediction'], result_dict_etr['prediction'],
                        result_dict_adr['prediction'], result_dict_lgb['prediction'], result_dict_xgb['prediction'], result_dict_etr['prediction']]).transpose()
test_stack = pd.DataFrame(test_stack, columns=['rfr', 'ridge', 'knn', 'lasso', 'etr', 'adr', 'lgb', 'xgb', 'etr'])
params = {'num_leaves': 8,
         #'min_data_in_leaf': 20,
         'objective': 'regression',
         'max_depth': 2,
         'learning_rate': 0.01,
         "boosting": "gbdt",
         "bagging_seed": 11,
         "metric": 'rmse',
        # "lambda_l1": 0.2,
         "verbosity": -1}
result_dict_lgb_stack = artgor_utils.train_model_regression(X=train_stack, X_test=test_stack, y=y, params=params, folds=folds, model_type='lgb',
                                                                                eval_metric='mae', plot_feature_importance=False,
                                                            columns=(list(train_stack.columns)))
submission['time_to_failure'] = result_dict_lgb_stack['prediction']
print(submission.head())
submission.to_csv('stacking.csv')


# Copyright 2019 Oihane Crucelaegui - AvanzOSC
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).

from ._common import _read_binary_file, _format_info
from eagle import _, exceptions, fields, models

class UploadEducationActivityType(models.TransientModel):
    _name = 'upload.education.activity_type'
    _description = 'Wizard to Upload Education Other Activity Types'

    file = fields.Binary(
        string='Other Activity Types File (T07)', filters='*.txt')

    def button_upload(self):
        lines = _read_binary_file(self.file)
        activity_type_obj = self.env[
            'education.activity_type'].with_context(active_test=False)
        if not lines:
            raise exceptions.Warning(_('Empty file.'))
        else:
            for line in lines:
                if len(line) > 0:
                    education_code = _format_info(line[0:3])
                    vals = {
                        'education_code': education_code,
                        'description': _format_info(line[3:103]),
                        'description_eu': _format_info(line[103:203]),
                        'short_description': _format_info(line[203:223]),
                        'short_description_eu': _format_info(line[223:243]),
                    }
                    activity_types = activity_type_obj.search([
                        ('education_code', '=', education_code)])
                    if activity_types:
                        activity_types.write(vals)
                    else:
                        activity_type_obj.create(vals)
        action = self.env.ref('education.action_education_activity_type')
        return action.read()[0]


'''
Created on Jan 10, 2019
@author: PDINDA
'''
'''
Created on Dec 24, 2018
@author: PDINDA
'''
import glob, os, subprocess, sys

def main():
    #path_tocreate_file = "/root/infracloud/mapperrules_config.properties"

    # f = open("find_bad_atoms.py", "r")
    # print(f.read())

    path_tocreate_file = os.environ.get("MAPPER_CONFIG_PATH")
    if path_tocreate_file is None:
        print(
            "Please provide path where the final mapper configuration should be created, as MAPPER_CONFIG_PATH env variable.")
        return "Please provide path where the final mapper configuration should be created, as MAPPER_CONFIG_PATH env variable."
    file_w = open(path_tocreate_file, "a")
    print(file_w)
    # file_r = open("find_bad_atoms.py", "a")
    # print(file_r)
    file_list = glob.glob(os.path.abspath("../python_fission/python/atoms/**/*-mapper.prop"), recursive=True)
    #file_list = glob.glob(os.path.abspath("../python_fission/python/atoms/**/app_server_status_check.py"), recursive=True)
    # file_w=open("test.py","w")
    # for line in file_r.readlines():
    #     file_w.write(line)
    for file in file_list:
        path, filename = os.path.split(file)
        '''
        VIVEK : YOU HAVE THE prop file here of each atoms
        The prop names are as follows : ATOM_DIRECTORY_NAME--mapper.prop
        '''
        file_path = path + "/" + filename
        file_content = open(file_path, "r")
        for aline in file_content:
            file_w.write(aline)

if __name__ == "__main__":
    main()


from pkg_resources import parse_requirements
from setuptools import find_packages, setup

def load_requirements(fname: str) -> list:
    requirements = []
    with open(fname, 'r') as fp:
        for req in parse_requirements(fp.read()):
            extras = '[{}]'.format(','.join(req.extras)) if req.extras else ''
            requirements.append('{}{}{}'.format(req.name, extras, req.specifier))
    return requirements

setup(
    name='gokgs',
    version='1.0.0',
    author='Daniil Novoselov',
    author_email='gudnmail@gmail.com',
    python_requires='>=3.9',
    packages=find_packages(exclude=['tests']),
    install_requires=load_requirements('requirements.txt'),
    entry_points={
        'console_scripts': [
            'gokgs = gokgs.__main__:main',
        ],
    },
    include_package_data=True,
)


from django.shortcuts import render
from django.contrib.auth.models import User
from .models import *

def index(request):
    owner = User.objects.get(id=1)
    data = UserData.objects.get(user=owner)
    sections = Section.objects.filter(position__gt=0).order_by('position')
    context = {
        'name': owner.first_name + ' ' + owner.last_name,
        'first_name': owner.first_name,
        'last_name': owner.last_name,
        'mail': owner.email,
        'user_data': data,
        'sections': sections}
    return render(request, 'index.html', context)


#Write your code here
class Calculator():
        def power(self,n,p):
            if n >= 0 and p >= 0:
                power = n**p
            else:
                power = 'n and p should be non-negative'
            return power

myCalculator=Calculator()