#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 15 16:43:31 2017

@author: ruess
"""

import pytest
from aiida_kkr.tools.kkr_params import kkrparams

# helper functions

def check_full_dict(p,p0):
    """
    helper function that compares full dictionary
    """
    from numpy import ndarray, array
    for key in [i[0] for i in p.get_set_values()]:
        v = p.get_value(key)
        v0 = p0.get_value(key)
        if type(v) != list and type(v) != ndarray:
            if v!=v0:
                print(key, v, v0)
            assert v==v0
        elif type(v[0]) != str:
            if abs(array(v)-array(v0)).max()>=10**-14:
                print(key, abs(array(v)-array(v0)).max())
            assert abs(array(v)-array(v0)).max()<10**-14
        else:
            if set(v)-set(v0)!=set():
                print(key, set(v)-set(v0))
            assert set(v)-set(v0)==set()

# tests

class Test_create_and_set_keys():
    def test_create_params_with_inital_values(self):
        p = kkrparams(RBASIS=[0,0,0], params_type='voronoi')
        assert type(p)==kkrparams
        assert p.values['<RBASIS>'] == [0,0,0]

    def test_default_values(self):
        p = kkrparams()
        assert p.values['EMIN'] is None

    def test_set_single_value(self):
        p = kkrparams()
        p.set_value('EMIN', 2)
        assert p.values['EMIN'] == 2.
        assert p.values['EMAX'] is None

    def test_set_multiple_values(self):
        p = kkrparams()
        p.set_multiple_values(EMIN=1, EMAX=2)
        assert p.values['EMIN']== 1.
        assert p.values['EMAX']== 2.

class Test_capture_wrong_input():
    def test_wrong_input_type(self):
        p = kkrparams()
        known_error = False
        try:
            p.set_value('EMIN', '2')
        except TypeError:
            known_error = True
        assert known_error

        known_error = False
        try:
            p.set_value('EMIN', False)
        except TypeError:
            known_error = True
        assert known_error

    def test_wrong_input_array_dimension(self):
        p = kkrparams()
        from numpy import array, sqrt
        bravais = array([[0.7071067812, -0.5, 0.0], [0.7071067812, 0.5, 0.0], [sqrt(2), 0.0, 0.866025404]])

        # atom positions in relative coordinates
        basis_vectors = []
        for iatom in range(6):
            tmp = array([0, 0, 0])+iatom*array([0.5, 0.5, bravais[2, 2]])
            tmp[0] = tmp[0]%1
            tmp[1] = tmp[1]%1
            print(iatom, tmp)
            basis_vectors.append(tmp)
        basis_vectors = array(basis_vectors)
        p.set_value('INTERFACE', True)
        p.set_value('<RBLEFT>', array([[1,1],[0,1]]))

    def test_input_consistency_check_fail(self):
        knownError = False
        try:
            p = kkrparams(ZATOM=29., LMAX=2, NAEZ=1, BRAVAIS=[[1,0,0],[0,1,0],[0,0,1]], RMAX=7, GMAX=65, NSPIN=2, RBASIS=[0,0,0], ALATBASIS=1)
            p.set_value('LDAU_PARA', [1,2])
            p._check_input_consistency()
        except TypeError:
           knownError = True
        assert knownError

    def test_inconsistency_bulk_mode_bravais(self):
        p = kkrparams(LMAX=2, NAEZ=1, BRAVAIS=[[1,0,0],[0,1,0],[0,0,0]], NSPIN=2, RBASIS=[0,0,0], ALATBASIS=1, RMAX=7, GMAX=65, ZATOM=29.)
        knownError = False
        try:
            p.fill_keywords_to_inputfile()
        except ValueError:
            knownError = True
        assert knownError

class Test_get_info():
    def test_get_mandatory(self):
        p = kkrparams()
        manlist = p.get_all_mandatory()
        assert set(manlist)==set(['LMAX', 'NAEZ', 'BRAVAIS', 'RMAX', 'GMAX', 'NSPIN', '<RBASIS>', 'ALATBASIS', '<ZATOM>'])

    def test_get_set_values(self):
        p = kkrparams()
        setlist = p.get_set_values()
        assert setlist==[]

    def test_get_set_values2(self):
        from numpy import array
        p = kkrparams()
        p.set_multiple_values(EMIN=1, EMAX=2)
        setlist = p.get_set_values()
        assert set(array(setlist).flatten()) == set(array([['EMIN', 1.], ['EMAX', 2.]]).flatten())

    def test_get_description(self):
        p = kkrparams()
        desc = p.get_description('EMIN')
        assert desc=='Accuracy, Valence energy contour: Lower value (in Ryd) for the energy contour'

    def test_get_type(self):
        p = kkrparams()
        tlist = p.get_type('BRAVAIS')
        assert tlist == [float, float, float, float, float, float, float, float, float]

    def test_is_mandatory(self):
        p = kkrparams()
        man = p.is_mandatory('EMAX')
        assert (not man)

    def test_get_value(self):
        p = kkrparams(LMAX=3)
        # check for KeyError if wrong key is checked
        known_error = False
        try:
            p.get_value('something_wrong')
        except KeyError:
            known_error = True
        assert known_error
        # check for returning unset value
        npol = p.get_value('NPOL')
        assert npol == None
        # check correct LMAX value
        lmax = p.get_value('LMAX')
        assert lmax == 3
        # check for returning lists for RUNOPT and TESTOPT
        runopt = p.get_value('RUNOPT')
        testopt = p.get_value('TESTOPT')
        assert runopt == []
        assert testopt == []
        p = kkrparams(TESTOPT=['test1', 'test2'], RUNOPT=['NEWSOSOL'])
        runopt = p.get_value('RUNOPT')
        testopt = p.get_value('TESTOPT')
        assert runopt == ['NEWSOSOL']
        assert set(testopt) == set(['test1', 'test2'])

class Test_fill_inputfile():
    def test_fill_inputfile_minimal_Voronoi(self):
        p = kkrparams(ZATOM=29., LMAX=2, NAEZ=1, BRAVAIS=[[1,0,0],[0,1,0],[0,0,1]], RCLUSTZ=1.5, NSPIN=2, RBASIS=[0,0,0], ALATBASIS=1)
        p.fill_keywords_to_inputfile(is_voro_calc=True)
        txt = open('inputcard').readlines()
        ref = ['ALATBASIS= 1.00000000000000\n', 'BRAVAIS\n', '1.00000000000000 0.00000000000000 0.00000000000000\n',
               '0.00000000000000 1.00000000000000 0.00000000000000\n', '0.00000000000000 0.00000000000000 1.00000000000000\n', 'NAEZ= 1\n', '<RBASIS>\n',
               '0.00000000000000 0.00000000000000 0.00000000000000\n', '<ZATOM>\n', '29.00000000000000\n', 'NSPIN= 2\n', 'LMAX= 2\n', 'RCLUSTZ= 1.50000000000000\n']
        done=False
        while not done:
            try:
                txt.remove('\n')
            except ValueError:
                done = True
        assert len(txt)==len(ref)
        txt.sort()
        ref.sort()
        print(txt, ref)
        for i in range(len(txt)):
            print(i, txt[i], ref[i])
            assert set(txt[i].split())==set(ref[i].split())

    def test_fill_inputfile_KKR(self):
        reffile = ['ALATBASIS= 1.00000000000000\n', 'BRAVAIS\n', '1.00000000000000 0.00000000000000 0.00000000000000\n', '<ZATOM>\n', '29.00000000000000\n',
                   '0.00000000000000 1.00000000000000 0.00000000000000\n', '0.00000000000000 0.00000000000000 1.00000000000000\n', 'NAEZ= 1\n',
                   '<RBASIS>\n', '0.00000000000000 0.00000000000000 0.00000000000000\n', 'NSPIN= 2\n', 'LMAX= 2\n',
                   'RCLUSTZ= 1.50000000000000\n', 'RMAX=      7.00000000000000\n', 'GMAX=      65.00000000000000\n']
        p = kkrparams(ZATOM=29., LMAX=2, NAEZ=1, BRAVAIS=[[1,0,0],[0,1,0],[0,0,1]], RMAX=7, GMAX=65, RCLUSTZ=1.5, NSPIN=2, RBASIS=[0,0,0], ALATBASIS=1)
        p.fill_keywords_to_inputfile()
        txt = open('inputcard').readlines()
        done=False
        while not done:
            try:
                txt.remove('\n')
            except ValueError:
                done = True
        assert len(txt)==len(reffile)
        txt.sort()
        reffile.sort()
        for i in range(len(txt)):
            assert set(txt[i].split())==set(reffile[i].split())

    def test_fill_inputfile_empty_check(self):
        p = kkrparams(LMAX=2, NAEZ=1)
        known_error = False
        try:
            p.fill_keywords_to_inputfile()
        except ValueError:
            known_error = True
        assert known_error

    def test_fill_inputfile_all_keys(self):
        """Example filling all keys"""
        from numpy import array, sqrt

        alat=5.416871386
        naez=6
        bravais=array([[0.7071067812, -0.5, 0.0], [0.7071067812, 0.5, 0.0], [sqrt(2), 0.0, 0.866025404]])
        lmax=2
        nspin=2
        nucl_numbers=[0,0,26,27,26,27,0,0]
        cpa_info = [naez+2, [1., 1., 0.98, 0.02, 0.98, 0.02, 1., 1.], [1, 2, 3, 3, 4, 4, 5, 6]]
        npol=4
        npt1, npt2, npt3 = 3, 10, 3
        tempr= 800
        basis_vectors = []
        for iatom in range(naez):
            tmp = array([0, 0, 0])+iatom*array([0.5, 0.5, bravais[2, 2]])
            tmp[0] = tmp[0]%1
            tmp[1] = tmp[1]%1
            print(iatom, tmp)
            basis_vectors.append(tmp)
        basis_vectors = array(basis_vectors)
        natyp = cpa_info[0]
        cpa_conc = cpa_info[1]
        cpa_sites = cpa_info[2]
        ins=1
        kshape=ins
        rmax, gmax= 7, 65
        rcls=1.5
        bzdivide=[10,10,0]
        emin=-0.4
        p = kkrparams()
        p.set_multiple_values(ZATOM=nucl_numbers, RBASIS=basis_vectors, BRAVAIS=bravais, NAEZ=naez, ALATBASIS=alat)
        p.set_multiple_values(NSPIN=nspin, LMAX=lmax, NPOL=npol, NPT1=npt1, NPT2=npt2, NPT3=npt3, TEMPR=tempr)
        p.set_multiple_values(RMAX=rmax, GMAX=gmax)
        p.set_multiple_values(RCLUSTZ=rcls, BZDIVIDE=bzdivide, EMIN=emin)
        p.set_multiple_values(INS=ins, KSHAPE=kshape)
        p.set_multiple_values(INTERFACE=True, NLBASIS=1, NRBASIS=1,
                              ZPERIODL=array([-0.5, -0.5, -bravais[2, 2]]),
                              ZPERIODR=array([0.5, 0.5, bravais[2, 2]]),
                              RBLEFT=basis_vectors[0]+array([-0.5, -0.5, -bravais[2, 2]]),
                              RBRIGHT=basis_vectors[naez-1]+array([0.5, 0.5, bravais[2, 2]]))
        p.set_value('LINIPOL', True)
        p.set_value('XINIPOL', [1 for i in range(natyp)])
        p.set_value('HFIELD', 0.02)
        p.set_value('NSTEPS', 1)
        p.set_value('IMIX', 0)
        p.set_value('STRMIX', 0.01)
        p.set_value('CARTESIAN', False)
        p.set_multiple_values(KAOEZR=1, KAOEZL=1, FPRADIUS=[-1 for i in range(natyp)], RCLUSTXY=rcls,
                              TKSEMI=800,EMAX=1, NPOLSEMI=0, N2SEMI=0, N1SEMI=0, N3SEMI=0, FSEMICORE=0,
                              KVREL=1, NCHEB=7, VCONST=0, SOCSCL=[1 for i in range(natyp)],
                              LAMBDA_XC=1, FCM=20, ITDBRY=20, KREADLDAU=0, RUNOPT=['LDAU', 'SEMICORE', 'IRGENDWAS FALSCHES'],
                              TESTOPT=['TSTOPTX0', 'TSTOPTX1', 'TSTOPTX2', 'TSTOPTX3', 'TSTOPTX4', 'TSTOPTX5', 'TSTOPTX6', 'TSTOPTX7', 'TSTOPTX8', 'TSTOPTXYZZZZZZ'], QBOUND=10**-3,
                              NPAN_LOG=3, NPAN_EQ=4, CPAINFO=[10**-3, 20], LLOYD=0, EMUSEMI=0, ICST=2,
                              TOLRDIF=0.01, BRYMIX=0.01, EBOTSEMI=0, NRIGHTHO=10, KEXCOR=2, NLEFTHOS=10,
                              R_LOG=0.4, LDAU_PARA=[1, 2, 0, 0, 0], NAT_LDAU=0,
                              RMTREFL=2.3, RMTREFR=2.3, DELTAE=[10**-5, 0], RMTREF=[2.3 for i in range(natyp)])
        p.set_value('<SHAPE>', [1 for i in range(natyp)])
        p.set_multiple_values(NATYP=natyp, SITE=cpa_sites)
        p.set_value('<CPA-CONC>', cpa_conc)
        p.set_value('FILES', ['output.pot', ''])
        p.fill_keywords_to_inputfile(is_voro_calc=True)

    def test_set_rmtcore(self):
        #test rmtcore
        from numpy import array
        from aiida_kkr.tools.common_functions import search_string

        para_dict = dict([(u'INS', 0),
            (u'RCLUSTZ', 1.69),
            (u'LMAX', 2),
            (u'GMAX', 65.0),
            (u'<RMTCORE>', [0.3535533906, 0.3535533906, 0.3535533906, 0.3535533906]),
            (u'RMAX', 7.0),
            (u'NSPIN', 1)])
        zatom = array([ 47.,  47.,  47.,  47.])
        alat = 7.8692316414074615
        natom = 4
        positions = array([[ 0. ,  0. ,  0. ],
                           [ 0. ,  0.5,  0.5],
                           [ 0.5,  0. ,  0.5],
                           [ 0.5,  0.5,  0. ]])
        bravais = array([[ 1.,  0.,  0.],
                         [ 0.,  1.,  0.],
                         [ 0.,  0.,  1.]])
        k =kkrparams(**para_dict)
        k.set_multiple_values(ZATOM=zatom, NAEZ=natom, ALATBASIS=alat, RBASIS=positions, BRAVAIS=bravais)
        k.fill_keywords_to_inputfile()

        txt = open('inputcard').readlines()
        naez = int(txt[search_string('NAEZ', txt)].split()[-1])
        rmtcore = []
        l_offset = search_string('RMTCORE', txt)
        for iatom in range(naez):
            rmtcore_at = float(txt[l_offset+1+iatom].split()[-1])
            rmtcore.append(rmtcore_at)
        maxdiff = (max(abs(array(para_dict['<RMTCORE>']) - array(rmtcore))))
        assert maxdiff < 10**-6

    def test_set_kkrimp_params_full(self):
        p = kkrparams(params_type='kkrimp')
        p.set_multiple_values(CALCORBITALMOMENT=0, RUNFLAG='', QBOUND=10**-7, NSPIN=1,
                              TESTFLAG='', NPAN_EQ=7, CALCFORCE=0, NPAN_LOGPANELFAC=2,
                              SPINORBIT=0, ITDBRY=20, NPAN_LOG=5, INS=1, ICST=2,
                              CALCJIJMAT=0, NCHEB=10, HFIELD=[0.00, 0], BRYMIX=0.05,
                              KVREL=1, IMIX=0, RADIUS_MIN=-1, NCOLL=0, RADIUS_LOGPANELS=0.6,
                              MIXFAC=0.05, SCFSTEPS=1, XC='LDA-VWN')
        p.fill_keywords_to_inputfile(output='config.cfg')
        reftxt = ['RUNFLAG=\n', 'TESTFLAG=\n', '\n', 'INS= 1\n', 'KVREL= 1\n', 'NSPIN= 1\n', '\n', 'SCFSTEPS= 1\n', 'IMIX= 0\n', 'ITDBRY= 20\n', 'MIXFAC=      0.05000000000000\n', 'BRYMIX=      0.05000000000000\n', 'QBOUND= 1.000000e-07\n', '\n', 'XC= LDA-VWN\n', 'ICST= 2\n', 'SPINORBIT= 0\n', 'NCOLL= 0\n', 'NPAN_LOGPANELFAC= 2\n', 'RADIUS_LOGPANELS=      0.60000000000000\n', 'RADIUS_MIN= -1\n', 'NPAN_LOG= 5\n', 'NPAN_EQ= 7\n', 'NCHEB= 10\n', '\n', 'HFIELD=      0.00000000000000 0\n', '\n', 'CALCORBITALMOMENT= 0\n', 'CALCFORCE= 0\n', 'CALCJIJMAT= 0\n']
        txt = open('config.cfg').readlines()
        assert txt==reftxt

class Test_read_inputfile():
    def test_read_minimal_inputfile(self):
        p = kkrparams(ZATOM=26., LMAX=2, NAEZ=1, BRAVAIS=[[1,0,0],[0,1,0],[0,0,1]], RCLUSTZ=1.5, NSPIN=2, RBASIS=[0,0,0], ALATBASIS=1)
        p.fill_keywords_to_inputfile(is_voro_calc=True)
        p2 = kkrparams(params_type='voronoi')
        p2.read_keywords_from_inputcard()
        check_full_dict(p,p2)

    def test_read_unsorted_inputfile(self):
        p = kkrparams(ZATOM=26., LMAX=2, NAEZ=1, BRAVAIS=[[1,0,0],[0,1,0],[0,0,1]], RCLUSTZ=1.5, NSPIN=2, RBASIS=[0,0,0], ALATBASIS=1, RMAX=7, GMAX=65)
        p.fill_keywords_to_inputfile(output='input.temp.txt')
        txt = open('input.temp.txt', 'r').readlines()
        # exchange some lines
        tmp = txt[0]; txt[0] = txt[5]; txt[5]=tmp
        tmp = txt[-1]; txt[-1] = txt[-2]; txt[-2]=tmp
        tmp = txt[-2]; txt[-2] = txt[-4]; txt[-4]=tmp
        tmp = txt[-3]; txt[-3] = txt[-1]; txt[-1]=tmp
        open('input.temp_unsorted.txt', 'w').writelines(txt)
        p2 = kkrparams()
        p2.read_keywords_from_inputcard(inputcard='input.temp_unsorted.txt')
        print(p2.get_dict())
        print(dict(p2.get_set_values()))
        check_full_dict(p,p2)

    def test_read_slab(self):
        from numpy import array
        from aiida_kkr.tools.common_functions import get_Ang2aBohr
        p = kkrparams(params_type='kkr')

        # automatically read keywords from inpucard
        p.read_keywords_from_inputcard(inputcard='../tests/files/kkr/import_calc_old_style/inputcard')
        # convert some read-in stuff back from Ang. units to alat units
        rbl = p.get_value('<RBLEFT>')
        rbr = p.get_value('<RBRIGHT>')
        zper_l = p.get_value('ZPERIODL')
        zper_r = p.get_value('ZPERIODR')
        ang2alat = get_Ang2aBohr()/p.get_value('ALATBASIS')
        if rbl is not None: p.set_value('<RBLEFT>', array(rbl)*ang2alat)
        if rbr is not None: p.set_value('<RBRIGHT>', array(rbr)*ang2alat)
        if zper_l is not None: p.set_value('ZPERIODL', array(zper_l)*ang2alat)
        if zper_r is not None: p.set_value('ZPERIODR', array(zper_r)*ang2alat)

        # set parameters of expected values manually
        p0 = kkrparams(RUNOPT=['xigid-ef','LLOYD', 'ewald2d', 'NEWSOSOL', 'DOS'], TESTOPT=['ie','RMESH','clusters','MPIenerg','fullBZ','DOS'], LMAX=3, NSPIN=2, NATYP=80, NAEZ=80, CARTESIAN=True, ALATBASIS=20.156973053, BRAVAIS=[[0.38437499, 0., 0.], [0.19218749, -0.33287851, 0.], [0.19218749, -0.11095950, 1.]], INTERFACE=True, NRIGHTHO=10, NLEFTHOS=10, NLBASIS=10, NRBASIS=10, ZPERIODL=[-1.92187500000000e-01, 1.10959504859881e-01, -1.00000000000000e+00], ZPERIODR=[1.92187500000000e-01, -1.10959504859881e-01, 1.00000000000000e+00], RCLUSTZ=0.65, RCLUSTXY=0.65, EMIN=-1.2, EMAX=1.2, TEMPR=473., NPOL=7, NPT1=7, NPT2=40, NPT3=6, KSHAPE=2, INS=1, ICST=2, KEXCOR=2, HFIELD=0, VCONST=0, NPAN_LOG=17, NPAN_EQ=7, NCHEB=12, R_LOG=0.8, BZDIVIDE=[40, 40, 1], NSTEPS=500, IMIX=5, STRMIX=0.02, FCM=20., QBOUND=10**-7, BRYMIX=0.02, ITDBRY=30, LINIPOL=False, FILES=['potential', 'shapefun'], RMAX=15., GMAX=900.)
        p0.set_value('<ZATOM>', [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 51.0, 0.0, 52.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0])
        p0.set_value('<SHAPE>', [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])
        p0.set_multiple_values(KAOEZR=[i for i in range(1,11)], KAOEZL=[i for i in range(1,11)], KVREL=1, RMTREFL=[2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000], RMTREFR=[2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000])
        p0.set_multiple_values(RMTREF=[2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000, 2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000, 2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000, 2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000, 2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000, 2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000, 2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000, 2.2671000, 2.2671000, 2.4948000, 2.3562000, 2.3562000, 2.3562000, 2.4948000, 2.2671000, 2.2671000, 2.5740000])
        p0.set_multiple_values(RBLEFT=[[-1.92187500000000e-01,  1.10959504859881e-01, -1.00000000000000e+00], [ 8.32667268468867e-17,  2.77555756156289e-17, -9.49500000000000e-01], [ 1.92187500000000e-01, -1.10959504859881e-01, -8.33000000000000e-01], [ 3.84375000000000e-01, -2.21919009719762e-01, -7.16500000000000e-01], [ 8.32667268468867e-17,  0.00000000000000e+00, -6.33000000000000e-01], [ 1.92187500000000e-01, -1.10959504859881e-01, -5.49500000000000e-01],  [ 3.84375000000000e-01, -2.21919009719762e-01, -4.33000000000000e-01], [ 2.77555756156289e-17,  1.38777878078145e-17, -3.16500000000000e-01], [ 1.92187500000000e-01, -1.10959504859881e-01, -2.66000000000000e-01], [ 3.84375000000000e-01, -2.21919009719762e-01, -1.33000000000000e-01]],
                               RBRIGHT=[[1.53750000000000e+00, -8.87676038879049e-01,  8.00000000000000e+00], [1.72968750000000e+00, -9.98635543738930e-01,  8.05050000000000e+00], [1.92187500000000e+00, -1.10959504859881e+00,  8.16700000000000e+00], [2.11406250000000e+00, -1.22055455345869e+00,  8.28350000000000e+00], [1.72968750000000e+00, -9.98635543738930e-01,  8.36700000000000e+00], [1.92187500000000e+00, -1.10959504859881e+00,  8.45050000000000e+00], [2.11406250000000e+00, -1.22055455345869e+00,  8.56700000000000e+00], [1.72968750000000e+00, -9.98635543738930e-01,  8.68350000000000e+00], [1.92187500000000e+00, -1.10959504859881e+00,  8.73400000000000e+00], [2.11406250000000e+00, -1.22055455345869e+00,  8.86700000000000e+00]],
                               RBASIS=[[0.0, 0.0, 0.0], [0.1921875, -0.110959504859881, 0.0505000000000001], [0.384375, -0.221919009719762, 0.167], [0.5765625, -0.332878514579644, 0.2835], [0.1921875, -0.110959504859881, 0.367], [0.384375, -0.221919009719762, 0.4505], [0.5765625, -0.332878514579644, 0.567], [0.1921875, -0.110959504859881, 0.6835], [0.384375, -0.221919009719762, 0.734], [0.5765625, -0.332878514579644, 0.867], [0.1921875, -0.110959504859881, 1.0], [0.384375, -0.221919009719762, 1.0505], [0.5765625, -0.332878514579643, 1.167], [0.76875, -0.443838019439525, 1.2835], [0.384375, -0.221919009719762, 1.367], [0.5765625, -0.332878514579643, 1.4505], [0.76875, -0.443838019439525, 1.567], [0.384375, -0.221919009719762, 1.6835], [0.5765625, -0.332878514579643, 1.734], [0.76875, -0.443838019439525, 1.867], [0.384375, -0.221919009719762, 2.0], [0.5765625, -0.332878514579643, 2.0505], [0.76875, -0.443838019439525, 2.167], [0.9609375, -0.554797524299406, 2.2835], [0.5765625, -0.332878514579643, 2.367], [0.76875, -0.443838019439525, 2.4505], [0.9609375, -0.554797524299406, 2.567], [0.5765625, -0.332878514579643, 2.6835], [0.76875, -0.443838019439525, 2.734], [0.9609375, -0.554797524299406, 2.867], [0.5765625, -0.332878514579643, 3.0], [0.76875, -0.443838019439525, 3.0505], [0.9609375, -0.554797524299406, 3.167], [1.153125, -0.665757029159287, 3.2835], [0.76875, -0.443838019439525, 3.367], [0.9609375, -0.554797524299406, 3.4505], [1.153125, -0.665757029159287, 3.567], [0.76875, -0.443838019439525, 3.6835], [0.9609375, -0.554797524299406, 3.734], [1.153125, -0.665757029159287, 3.867], [0.76875, -0.443838019439525, 4.0], [0.9609375, -0.554797524299406, 4.0505], [1.153125, -0.665757029159287, 4.167], [1.3453125, -0.776716534019168, 4.2835], [0.9609375, -0.554797524299406, 4.367], [1.153125, -0.665757029159287, 4.4505], [1.3453125, -0.776716534019168, 4.567], [0.9609375, -0.554797524299406, 4.6835], [1.153125, -0.665757029159287, 4.734], [1.3453125, -0.776716534019168, 4.867], [0.9609375, -0.554797524299406, 5.0], [1.153125, -0.665757029159287, 5.0505], [1.3453125, -0.776716534019168, 5.167], [1.5375, -0.887676038879049, 5.2835], [1.153125, -0.665757029159287, 5.367], [1.3453125, -0.776716534019168, 5.4505], [1.5375, -0.887676038879049, 5.567], [1.153125, -0.665757029159287, 5.6835], [1.3453125, -0.776716534019168, 5.734], [1.5375, -0.887676038879049, 5.867], [1.153125, -0.665757029159287, 6.0], [1.3453125, -0.776716534019168, 6.0505], [1.5375, -0.887676038879049, 6.167], [1.7296875, -0.99863554373893, 6.2835], [1.3453125, -0.776716534019168, 6.367], [1.5375, -0.887676038879049, 6.4505], [1.7296875, -0.99863554373893, 6.567], [1.3453125, -0.776716534019168, 6.6835], [1.5375, -0.887676038879049, 6.734], [1.7296875, -0.99863554373893, 6.867], [1.3453125, -0.776716534019168, 7.0], [1.5375, -0.887676038879049, 7.0505], [1.7296875, -0.99863554373893, 7.167], [1.921875, -1.10959504859881, 7.2835], [1.5375, -0.887676038879049, 7.367], [1.7296875, -0.99863554373893, 7.4505], [1.921875, -1.10959504859881, 7.567], [1.5375, -0.887676038879049, 7.6835], [1.7296875, -0.99863554373893, 7.734], [1.921875, -1.10959504859881, 7.867]])

        # check all values
        check_full_dict(p,p0)

class Test_other():
    def test_get_missing_keys(self):
        p = kkrparams()
        missing = p.get_missing_keys()
        assert set(missing)==set(['<ZATOM>', 'BRAVAIS', 'LMAX', 'GMAX', 'RMAX', 'NAEZ', '<RBASIS>', 'NSPIN', 'ALATBASIS'])
        missing = p.get_missing_keys(use_aiida=True)
        assert set(missing)==set(['LMAX', 'GMAX', 'RMAX', 'NSPIN'])

        p = kkrparams(params_type='voronoi', EMIN=-2, LMAX=3)
        missing = p.get_missing_keys()
        assert set(missing)==set(['<ZATOM>', 'BRAVAIS', 'RCLUSTZ', 'NAEZ', '<RBASIS>', 'NSPIN', 'ALATBASIS'])

    def test_set_value_None(self):
        p = kkrparams()
        p.set_value('EMIN', -1)
        assert p.values['EMIN'] == -1

        p.set_value('EMIN',None)
        assert p.values['EMIN'] == -1

        p.remove_value('EMIN')
        assert p.values['EMIN'] is None

    def test_set_potname_empty(self):
        p = kkrparams()
        p.set_multiple_values(RMAX=1, GMAX=1, NSPIN=1, RBASIS=[0,0,0], LMAX=2, RCLUSTZ=1.2, NAEZ=1, ZATOM=[0], BRAVAIS=[[1,0,0],[0,1,0],[0,0,1]], ALATBASIS=1, FILES=['','shapenew'])
        p.fill_keywords_to_inputfile()
        from aiida_kkr.tools.common_functions import search_string
        txt = open('inputcard').readlines()
        itmp = search_string('FILES', txt)
        potname = txt[itmp+2].split()[0]
        shapename = txt[itmp+4].split()[0]
        assert 'potential' == potname
        assert 'shapenew' == shapename

    def test_get_dict(self):
        d0 = {'<RMTCORE>':  None, 'ICST': None, '<RMTREF>': None, 'N1SEMI': None, '<FPRADIUS>': None, '<NRBASIS>': None, '<SOCSCL>': None, 'XINIPOL': None, 'EMAX': None, '<RBLEFT>': None, 'NLEFTHOS': None, '<ZATOM>': [0.0], 'RCLUSTXY': None, 'NPAN_EQ': None, '<RBRIGHT>': None, 'BRAVAIS': [[1, 0, 0], [0, 1, 0], [0, 0, 1]], 'INS': None, 'NAT_LDAU': None, '<RMTREFR>': None, 'ZPERIODL': None, 'TESTOPT': None, 'KEXCOR': None, '<TOLRDIF>': None, 'TEMPR': None, 'EBOTSEMI': None, 'NATYP': None, 'RUNOPT': None, 'HFIELD': None, 'NPOL': None, 'RCLUSTZ': 1.2, 'ZPERIODR': None, 'N3SEMI': None, 'LMAX': 2, 'ITDBRY': None, '<KAOEZR>': None, '<LLOYD>': None, 'STRMIX': None, 'CPAINFO': None, 'FCM': None, '<SHAPE>': None, 'NPAN_LOG': None, 'CARTESIAN': None, 'FSEMICORE': None, 'LAMBDA_XC': None, 'GMAX': None, '<CPA-CONC>': None, 'RMAX': None, 'NCHEB': None, 'EMIN': None, 'NAEZ': 1, '<DELTAE>': None, 'KREADLDAU': None, '<RBASIS>': [0, 0, 0], '<SITE>': None, 'NPT2': None, 'NPT3': None, 'NPT1': None, 'N2SEMI': None, 'NPOLSEMI': None, '<RMTREFL>': None, 'FILES': ['', 'shapenew'], 'LDAU_PARA': None, 'NSPIN': 1, 'QBOUND': None, 'NRIGHTHO': None, 'KVREL': None, 'TKSEMI': None, '<KAOEZL>': None, 'NSTEPS': None, 'KSHAPE': None, '<NLBASIS>': None, 'LINIPOL': None, 'BZDIVIDE': None, 'INTERFACE': None, 'BRYMIX': None, 'EMUSEMI': None, 'ALATBASIS': 1.0, 'R_LOG': None, 'IMIX': None, 'VCONST': None}
        p = kkrparams()
        p.set_multiple_values(RMAX=1, GMAX=1, NSPIN=1, RBASIS=[0,0,0], LMAX=2, RCLUSTZ=1.2, NAEZ=1, ZATOM=[0], BRAVAIS=[[1,0,0],[0,1,0],[0,0,1]], ALATBASIS=1, FILES=['','shapenew'])
        assert set(d0.keys()) == set(p.get_dict().keys())

        l0 = ['<SHAPE>', 'KSHAPE', 'ZPERIODL', '<NRBASIS>', '<NLBASIS>', '<RBASIS>', 'NAEZ', 'CARTESIAN', '<RBRIGHT>', '<RBLEFT>', 'INTERFACE', 'BRAVAIS', 'ALATBASIS', 'ZPERIODR']
        assert p.get_dict(group='lattice').keys() == l0

        l0 = ['ZPERIODL', '<NRBASIS>', '<NLBASIS>', '<RBRIGHT>', '<RBLEFT>', 'INTERFACE', 'ZPERIODR']
        assert l0 == p.get_dict(group='lattice', subgroup='2D mode').keys()

    def test_get_KKRcalc_parameter_defaults(self):
        d = kkrparams.get_KKRcalc_parameter_defaults()
        from aiida_kkr.tools.kkr_params import __kkr_default_params__
        d0 = __kkr_default_params__
        assert d[0]==d0


# -*- coding: utf-8 -*-
# _simulateDSM.py
# Module providing the simulateDSM function,
# a switch to select the fastest simulation routine.
# This file is part of python-deltasigma.
#
# python-deltasigma is a 1:1 Python replacement of Richard Schreier's
# MATLAB delta sigma toolbox (aka "delsigma"), upon which it is heavily based.
# The delta sigma toolbox is (c) 2009, Richard Schreier.
#
# python-deltasigma is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# LICENSE file for the licensing terms.
#
# This file was originally from `pydsm`, then modified quite a bit.
# Many thanks to the original author.
#
# The original file is
# Copyright (c) 2012, Sergio Callegari
# All rights reserved.
#
# The modifications are mine.
# Copyright (c) 2014, G. Venturini and the python-deltasigma contributors
#

from __future__ import print_function

import os
from warnings import warn

import numpy as np

from ._config import _debug, setup_args
from ._simulateDSM_python import simulateDSM as _simulateDSM_python
from ._utils import _get_zpk, _is_zpk

warned = False

# Code to compile the Cython extensions
# Extensions tested on Linux and Mac OS X, but not on Windows
# please report any bug (or patches!) on
# https://github.com/ggventurini/python-deltasigma/issues

try:
    import pyximport
    pyximport.install(setup_args=setup_args)
    from ._simulateDSM_cblas import simulateDSM as _simulateDSM_cblas
except ImportError as e:
    if _debug:
        print(str(e))
    _simulateDSM_cblas = None

try:
    import pyximport
    pyximport.install(setup_args=setup_args)
    from ._simulateDSM_scipy_blas import simulateDSM as _simulateDSM_scipy_blas
except ImportError as e:
    if _debug:
        print(str(e))
    _simulateDSM_scipy_blas = None

# fall back to CPython

simulation_backends = {'CBLAS':(_simulateDSM_cblas is not None),
                       'Scipy_BLAS':(_simulateDSM_scipy_blas is not None),
                       'CPython':True}

def simulateDSM(u, arg2, nlev=2, x0=0.):
    """Simulate a delta-sigma modulator.

    Compute the output of a general delta-sigma modulator with input ``u``,
    a structure described by ``ABCD``, an initial state ``x0`` (default zero) and
    a quantizer with a number of levels specified by ``nlev``.

    **Syntax:**

     * ``[v, xn, xmax, y] = simulateDSM(u, ABCD, nlev=2, x0=0)``
     * ``[v, xn, xmax, y] = simulateDSM(u, ntf, nlev=2, x0=0)``

    **Parameters:**

    u : ndarray or sequence
        The input vector to be used in the simulation. Multiple inputs
        are implied by the number of rows in ``u``.
    arg2 : 2D ndarray or a supported LTI description
        The second argument may be either the ABCD matrix describing the
        modulator or its NTF. In the latter case, the NTF is converted to
        a ZPK description and the structure that is simulated is the
        block-diagonal structure used by scipy's ``zpk2ss()``.
        The STF is assumed to be 1.
    nlev : int or sequence or ndarray
        Number of levels in the quantizers. Set ``nlev`` to a scalar for a
        single quantizer modulator. Multiple quantizers are implied by
        making ``nlev`` an array.
    x0 : float or sequence or ndarray
        The initial status of the modulator. If ``x0`` is set to float, its
        value will be used for all the states. If it is set to a sequence of
        floats, each of its values will be assigned to a state variable.

    **Returns:**

    v : ndarray
        The quantizer output.
    xn : ndarray
        The modulator states.
    xmax : ndarray
        The maximum value that each state reached during simulation.
    y : ndarray
        The quantizer input (ie the modulator output).

    **Notes:**

    Three implementations of this function are (potentially) available to the
    user, in order of ascending execution speed:

    * A CPython implementation, always available.
    * A Cython-based implementation requiring the BLAS headers and a compatible
      compiler.
    * A Cython-based implementation accessing the BLAS library pre-compiled
      through scipy, requiring only a compatible compiler.

    The difference in execution time from the first implementation -- dynamically
    interpreted -- to the latter two -- statically compiled automatically before
    execution -- is a factor 20.

    The fastest available implementation is automatically selected.

    To assess which implementations are available in your installation, check
    the ``simulation_backends`` variable, for example::

       from __future__ import print_function
       import deltasigma as ds
       print(ds.simulation_backends)

    Example output::

        {'Scipy_BLAS': True, 'CBLAS': True, 'CPython': True}

    **Example:**

    Simulate a 5th-order binary modulator with a half-scale sine-wave input and
    plot its output in the time and frequency domains.::

        import numpy as np
        from deltasigma import *
        OSR = 32
        H = synthesizeNTF(5, OSR, 1)
        N = 8192
        fB = np.ceil(N/(2*OSR))
        f = 85
        u = 0.5*np.sin(2*np.pi*f/N*np.arange(N))
        v = simulateDSM(u, H)[0]

    Graphical display of the results:

    .. plot::

        import numpy as np
        import pylab as plt
        from numpy.fft import fft
        from deltasigma import *
        OSR = 32
        H = synthesizeNTF(5, OSR, 1)
        N = 8192
        fB = int(np.ceil(N/(2*OSR)))
        f = 85
        u = 0.5*np.sin(2*np.pi*f/N*np.arange(N))
        v = simulateDSM(u, H)[0]
        plt.figure(figsize=(10, 7))
        plt.subplot(2, 1, 1)
        t = np.arange(85)
        # the equivalent of MATLAB 'stairs' is step in matplotlib
        plt.step(t, u[t], 'g', label='u(n)')
        plt.step(t, v[t], 'b', label='v(n)')
        plt.axis([0, 85, -1.2, 1.2]);
        plt.ylabel('u, v');
        plt.xlabel('sample')
        plt.legend()
        plt.subplot(2, 1, 2)
        spec = fft(v*ds_hann(N))/(N/4)
        plt.plot(np.linspace(0, 0.5, N/2 + 1), dbv(spec[:int(N/2) + 1]))
        plt.axis([0, 0.5, -120, 0])
        plt.grid(True)
        plt.ylabel('dBFS/NBW')
        snr = calculateSNR(spec[:fB], f)
        s = 'SNR = %4.1fdB' % snr
        plt.text(0.25, -90, s)
        s =  'NBW = %7.5f' % (1.5/N)
        plt.text(0.25, -110, s)
        plt.xlabel("frequency $1 \\\\rightarrow f_s$")

    Click on "Source" above to see the source code.
    """
    global warned
    if _simulateDSM_cblas or _simulateDSM_scipy_blas:
        if not _is_zpk(arg2) and not isinstance(arg2, np.ndarray):
            arg2 = _get_zpk(arg2)
        if _simulateDSM_cblas:
            return _simulateDSM_cblas(u, arg2, nlev, x0, store_xn=True,
                                      store_xmax=True, store_y=True)
        return _simulateDSM_scipy_blas(u, arg2, nlev, x0, store_xn=True,
                                       store_xmax=True, store_y=True)
    else:
        if not warned:
            warn('Using a slow implementation of simulateDSM\n' +
                 'Refer to the docs for how to switch to a fast one')
            warned = True
        return _simulateDSM_python(u, arg2, nlev, x0)


import numpy as np
import pylab as plt
np.random.seed(17)
plt.rc('font', size=8)
nproblem = 8 # magic
nstudent = 150 # magic

problems = [r"""\begin{problem}
(From Problem Set 6)
What would be the radius of a black hole with the mass of the Earth?
\end{problem}""", r"""\begin{problem}
(From Problem Set 6)
What is the distance between Earth and the source of GW170817?
\end{problem}""", r"""\begin{problem}
(From Problem Set 6)
What is the mean orbital speed $v$ of the star S2 in the Galactic Center?
\end{problem}""", r"""\begin{problem}
(From Problem Set 6)
In event GW150914, what were the three black-hole masses $M_1,M_2,M_3$?
\end{problem}""", r"""\begin{problem}
(From the reading)
In what year did LIGO make its first discovery of a black-hole merger?
\end{problem}""", r"""\begin{problem}
(From Lecture)
Roughly what is the orbital period of the International Space Station?
\end{problem}""", r"""\begin{problem}
(From the reading)
What happens when something crosses the horizon of the black hole?
\end{problem}""", r"""\begin{problem}
(From Lecture)
What---very roughly---is the (dimensionless) ratio of the mass of the black
hole at the center of the Milky Way to the total mass of the Milky Way? That is,
what fraction of the total mass of the Milky Way is the black hole?
\end{problem}"""]
assert len(problems) == nproblem

print(r"""
\documentclass[12pt, letterpaper]{article}
\include{eu}
\pagestyle{empty}

\begin{document}

""")

for student in range(nstudent):
    print(r"""
\examheader{Term Exam 6}

""")
    pindx = np.argsort(np.random.uniform(size=nproblem))
    for problem, indx in enumerate(pindx):
        print(problems[indx])
        print(r"""
\vfill ~
""")
        if problem == 3:
            print(r"""
\clearpage

""")
    print(r"""
\cleardoublepage

""")

print(r"""
\end{document}
""")


#!/usr/bin/env python
u"""
reanalysis_geopotential_heights.py
Written by Tyler Sutterley (10/2021)
Reads temperature and specific humidity data to calculate geopotential height
    and pressure difference fields at half levels from reanalysis

INPUTS:
    Reanalysis model to run
    ERA-Interim: http://apps.ecmwf.int/datasets/data/interim-full-moda
    ERA5: http://apps.ecmwf.int/data-catalogues/era5/?class=ea
    MERRA-2: https://gmao.gsfc.nasa.gov/reanalysis/MERRA-2/

COMMAND LINE OPTIONS:
    -D X, --directory X: Working data directory
    -Y X, --year X: years to run
    -V, --verbose: Output information for each output file
    -M X, --mode X: Permission mode of directories and files

PYTHON DEPENDENCIES:
    numpy: Scientific Computing Tools For Python
        https://numpy.org
        https://numpy.org/doc/stable/user/numpy-for-matlab-users.html
    netCDF4: Python interface to the netCDF C library
        https://unidata.github.io/netcdf4-python/netCDF4/index.html

PROGRAM DEPENDENCIES:
    utilities.py: download and management utilities for files

UPDATE HISTORY:
    Updated 10/2021: using python logging for handling verbose output
    Updated 07/2021: can use input files to define command line arguments
        added check for ERA5 expver dimension (denotes mix of ERA5 and ERA5T)
    Updated 05/2021: define int/float precision to prevent deprecation warning
    Updated 03/2021: automatically update years to run based on current time
    Updated 01/2021: read from netCDF4 file in slices to reduce memory load
    Updated 12/2020: using argparse to set command line options
    Updated 01/2020: outputs variables as 32-bit floats instead of 64-bit floats
        clear variables and iterate years to reduce required memory
        iterate over time variable to calculate heights using incomplete files
    Updated 08/2019: adjust time scale variable for MERRA-2
    Updated 07/2018: added parameters for ERA5
    Written 03/2018
"""
from __future__ import print_function

import os
import re
import time
import logging
import netCDF4
import argparse
import numpy as np
import gravity_toolkit.utilities as utilities

#-- PURPOSE: reads temperature and specific humidity data to calculate
#-- geopotential height fields at half levels from reanalysis
def reanalysis_geopotential_heights(base_dir, MODEL, YEAR=None,
    VERBOSE=False, MODE=0o775):

    #-- create logger for verbosity level
    loglevel = logging.INFO if VERBOSE else logging.CRITICAL
    logging.basicConfig(level=loglevel)

    #-- directory setup
    ddir = os.path.join(base_dir,MODEL)
    #-- set model specific parameters
    if (MODEL == 'ERA-Interim'):
        #-- invariant parameters file
        input_invariant_file = 'ERA-Interim-Invariant-Parameters.nc'
        #-- coordinate parameters file
        input_coordinate_file = 'ERA-Interim_coordvars.nc'
        #-- surface pressure file format
        input_pressure_file = 'ERA-Interim-Monthly-SP-{0:4d}.nc'
        #-- regular expression pattern for finding files
        regex_pattern = r'ERA\-Interim\-Monthly\-Levels\-({0})\.nc$'
        #-- output file format
        output_file_format = 'ERA-Interim-GPH-Levels-{0:4d}.nc'
        SURFNAME = 'z'
        ZNAME = 'z'
        VARNAME = 'sp'
        TNAME = 't'
        QNAME = 'q'
        DIFFNAME = 'dp'
        LONNAME = 'longitude'
        LATNAME = 'latitude'
        TIMENAME = 'time'
        LEVELNAME = 'lvl'
        ANAME,BNAME = ('a_model_alt','b_model_alt')
        AINTERFACE,BINTERFACE = ('a_interface','b_interface')
        #-- hours since 1900-01-01 00:00:0.0
        TIME_LONGNAME = 'Time'
        UNITS = 'm**2 s**-2'
        GRAVITY = 1.0
    elif (MODEL == 'ERA5'):
        #-- invariant parameters file
        input_invariant_file = 'ERA5-Invariant-Parameters.nc'
        #-- coordinate parameters file
        input_coordinate_file = 'ERA5_coordvars.nc'
        #-- surface pressure file format
        input_pressure_file = 'ERA5-Monthly-SP-{0:4d}.nc'
        #-- regular expression pattern for finding files
        regex_pattern = r'ERA5\-Monthly\-Levels\-({0})\.nc$'
        #-- output file format
        output_file_format = 'ERA5-GPH-Levels-{0:4d}.nc'
        SURFNAME = 'z'
        ZNAME = 'z'
        VARNAME = 'sp'
        TNAME = 't'
        QNAME = 'q'
        DIFFNAME = 'dp'
        LONNAME = 'longitude'
        LATNAME = 'latitude'
        TIMENAME = 'time'
        LEVELNAME = 'lvl'
        ANAME,BNAME = ('a_half','b_half')
        AINTERFACE,BINTERFACE = ('a_interface','b_interface')
        #-- hours since 1900-01-01 00:00:0.0
        TIME_LONGNAME = 'Time'
        UNITS = 'm**2 s**-2'
        GRAVITY = 1.0
    elif (MODEL == 'MERRA-2'):
        #-- invariant parameters file
        input_invariant_file = 'MERRA2_101.const_2d_asm_Nx.00000000.nc4'
        #-- coordinate parameters file
        input_coordinate_file = 'MERRA2_101.Coords_Nx.00000000.nc'
        #-- regular expression pattern for finding files
        regex_pattern = r'MERRA2_(\d+).instM_3d_ana_Nv.({0})(\d{{2}}).SUB.nc$'
        #-- output file format
        output_file_format='MERRA2_{0:0.0f}.GPH_levels.{1:4.0f}{2:02.0f}.SUB.nc'
        SURFNAME = 'PHIS'
        ZNAME = 'PHIS'
        VARNAME = 'PS'
        TNAME = 'T'
        QNAME = 'QV'
        DIFFNAME = 'dP'
        LONNAME = 'lon'
        LATNAME = 'lat'
        TIMENAME = 'time'
        LEVELNAME = 'lev'
        ANAME,BNAME = ('a_half','b_half')
        AINTERFACE,BINTERFACE = ('a_interface','b_interface')
        #-- minutes since start of file
        TIME_LONGNAME = 'Time'
        UNITS = 'm+2 s-2'
        GRAVITY = 1.0

    #-- read model orography for dimensions
    geopotential,lon,lat=ncdf_invariant(os.path.join(ddir,input_invariant_file),
        LONNAME,LATNAME,SURFNAME)
    #-- read parameters for calculating pressures at levels
    lev,A,B,AI,BI=ncdf_coordinates(os.path.join(ddir,input_coordinate_file),
        LEVELNAME,ANAME,BNAME,AINTERFACE,BINTERFACE)
    #-- Gas constant for dry air
    R_dry = 287.06

    #-- read each reanalysis pressure field for each year
    regex_years = r'\d{4}' if (YEAR is None) else '|'.join(map(str,YEAR))
    rx = re.compile(regex_pattern.format(regex_years), re.VERBOSE)
    input_files = [fi for fi in os.listdir(ddir) if rx.match(fi)]
    #-- for each reanalysis file
    for fi in sorted(input_files):
        #-- read input temperature and specific humidity data
        fid1 = netCDF4.Dataset(os.path.join(ddir,fi),'r')
        #-- extract shape from temperature variable
        ntime,nlevels,nlat,nlon = fid1.variables[TNAME].shape
        #-- invalid value
        fill_value = fid1.variables[TNAME]._FillValue
        #-- save output variables into a python dictionary.
        dinput = {}
        dinput[ZNAME] = np.zeros((ntime,nlevels,nlat,nlon),dtype=np.float32)
        dinput[DIFFNAME] = np.zeros((ntime,nlevels,nlat,nlon),dtype=np.float32)
        #-- model levels in reverse order
        dinput[LEVELNAME] = lev[::-1].copy()
        #-- extract time and time units
        dinput[TIMENAME] = np.copy(fid1.variables[TIMENAME][:])
        TIME_UNITS = fid1.variables[TIMENAME].units
        dinput[LONNAME] = lon.copy()
        dinput[LATNAME] = lat.copy()

        if MODEL in ('MERRA-2'):
            #-- extract date from monthly files
            MOD,YEAR,MONTH = np.array(rx.findall(fi).pop(), dtype=np.float64)
            #-- output monthly filename
            FILE = os.path.join(ddir,output_file_format.format(MOD,YEAR,MONTH))
            #-- read surface pressure
            surface_pressure = np.copy(fid1.variables[VARNAME][:])
        elif MODEL in ('ERA-Interim','ERA5'):
            #-- extract year from file name
            YEAR, = np.array(rx.findall(fi),dtype=np.int64)
            #-- output yearly filename
            FILE = os.path.join(ddir,output_file_format.format(YEAR))
            #-- read input surface pressure data
            pressure_file = input_pressure_file.format(YEAR)
            with netCDF4.Dataset(os.path.join(ddir,pressure_file),'r') as fid2:
                surface_pressure = np.copy(fid2.variables[VARNAME][:])

        #-- iterate over dates
        for t in range(ntime):
            #-- check dimensions for expver slice
            if (fid1.variables[VARNAME].ndim == 5):
                t_time,q_time = ncdf_expver(fid1,t,TNAME,QNAME)
            else:
                #-- temperature and specific humidity
                #-- reverse layers so bottom=0
                t_time = fid1.variables[TNAME][t,::-1,:,:]
                q_time = fid1.variables[QNAME][t,::-1,:,:]
            #-- calculate geopotential over model levels
            geopotential_height = np.empty((nlat,nlon),dtype=np.float32)
            #-- start with surface geopotential converted to units (m^2/s^2)
            geopotential_height[:,:] = geopotential*GRAVITY
            #-- Integrate the model layers in the atmosphere
            for k in range(nlevels):
                #-- calculate virtual temperature
                virtual_temp = (1.0 + 0.609133*q_time[k,:,:])*t_time[k,:,:]
                #-- calculate numerator and denominator for pressure ratio
                Pnum = A[k] + B[k]*surface_pressure[t,:,:]
                if ((k+1) == nlevels):
                    Pdom = 0.1
                else:
                    Pdom = A[k+1] + B[k+1]*surface_pressure[t,:,:]
                #-- add level to geopotential_levels
                geopotential_height[:,:] += R_dry*virtual_temp*np.log(Pnum/Pdom)
                #-- save level to output variable and convert to output units
                dinput[ZNAME][t,k,:,:] = geopotential_height/GRAVITY
                #-- calculate pressure difference between levels (at interfaces)
                Plower = AI[k] + BI[k]*surface_pressure[t,:,:]
                Pupper = AI[k+1] + BI[k+1]*surface_pressure[t,:,:]
                dinput[DIFFNAME][t,k,:,:] = Pupper - Plower

        #-- save to file
        ncdf_geopotential_write(dinput, fill_value, FILENAME=FILE, ZNAME=ZNAME,
            LEVELNAME=LEVELNAME, DIFFNAME=DIFFNAME, LONNAME=LONNAME,
            LATNAME=LATNAME, TIMENAME=TIMENAME, TIME_UNITS=TIME_UNITS,
            TIME_LONGNAME=TIME_LONGNAME, UNITS=UNITS)
        #-- set the permissions level of the output file to MODE
        os.chmod(FILE, MODE)
        #-- clear dinput dictionary variable
        dinput = None
        #-- close the input netCDF4 file
        fid1.close()

#-- PURPOSE: Compute the Specific Humidity from parameters (Bolton 1980)
#-- http://cires1.colorado.edu/~voemel/vp.html
#-- https://www.eol.ucar.edu/projects/ceop/dm/documents/refdata_report/eqns.html
#-- https://github.com/NCAR/ncl/blob/master/ni/src/lib/nfpfort/mixhum_ptrh.f
def calculate_specific_humidity(P, T, RH):
    #-- ratio of the molecular weights of water vapor to dry air
    epsilon = 0.622
    #-- calibration pressure and temperature
    pc = 6.112
    tc = 243.5
    #-- calculate the saturation vapor pressure in mb
    Es = pc * np.exp((17.67 * T)/(T + tc))
    #-- calculate the vapor pressure in mb
    Ev = Es * (RH/100.0)
    #-- calculate the dew point temperature
    Td = np.log(Ev/pc) * tc/(17.67 - np.log(Ev/pc))
    #-- calculate the specific humidity
    Q = (epsilon * Ev)/(P/100.0 - (0.378 * Ev))
    return (Q,Td)

#-- PURPOSE: extract temperature and specific humidity variables
#-- from a 5d netCDF4 dataset
#-- ERA5 expver dimension (denotes mix of ERA5 and ERA5T)
def ncdf_expver(fileID, slice, TNAME, QNAME):
    ntime,nexp,nlevel,nlat,nlon = fileID.variables[TNAME].shape
    fill_value = fileID.variables[TNAME]._FillValue
    #-- reduced temperature and specific humidity for time
    temperature = np.ma.zeros((nlevel,nlat,nlon))
    temperature.fill_value = fill_value
    humidity = np.ma.zeros((nlevel,nlat,nlon))
    humidity.fill_value = fill_value
    #-- iterate over expver slices to find valid outputs
    for j in range(nexp):
        #-- check if any are valid for expver
        if np.any(fileID.variables[TNAME][slice,j,:,:,:]):
            #-- reverse layers so bottom=0
            temperature[:,:,:] = fileID.variables[TNAME][slice,j,::-1,:,:]
            humidity[:,:,:] = fileID.variables[QNAME][slice,j,::-1,:,:]
    #-- update mask variables
    temperature.mask = (temperature.data == temperature.fill_value)
    humidity.mask = (humidity.data == humidity.fill_value)
    #-- return the reduced temperature and specific humidity variables
    return (temperature,humidity)

#-- PURPOSE: read reanalysis invariant parameters (geopotential,lat,lon)
def ncdf_invariant(FILENAME,LONNAME,LATNAME,ZNAME):
    with netCDF4.Dataset(FILENAME,'r') as fileID:
        geopotential = fileID.variables[ZNAME][:].squeeze()
        longitude = fileID.variables[LONNAME][:].copy()
        latitude = fileID.variables[LATNAME][:].copy()
    return (geopotential,longitude,latitude)

#-- PURPOSE: read reanalysis coordinate parameters
#-- reverse order to go from surface to top-of-atmosphere
def ncdf_coordinates(FILENAME,LEVELNAME,ANAME,BNAME,AINTERFACE,BINTERFACE):
    with netCDF4.Dataset(FILENAME,'r') as fileID:
        levels = fileID.variables[LEVELNAME][:].copy()
        A = fileID.variables[ANAME][::-1].copy()
        B = fileID.variables[BNAME][::-1].copy()
        AI = fileID.variables[AINTERFACE][::-1].copy()
        BI = fileID.variables[BINTERFACE][::-1].copy()
    return (levels,A,B,AI,BI)

#-- PURPOSE: write output geopotential fields data to file
def ncdf_geopotential_write(dinput, fill_value, FILENAME=None, ZNAME=None,
    DIFFNAME=None, LEVELNAME=None, LONNAME=None, LATNAME=None, TIMENAME=None,
    TIME_UNITS=None, TIME_LONGNAME=None, UNITS=None):
    #-- opening NetCDF file for writing
    fileID = netCDF4.Dataset(FILENAME, 'w', format="NETCDF4")

    #-- Defining the NetCDF dimensions
    for key in [LONNAME,LATNAME,TIMENAME,LEVELNAME]:
        fileID.createDimension(key, len(dinput[key]))

    #-- defining the NetCDF variables
    nc = {}
    nc[LATNAME]=fileID.createVariable(LATNAME,dinput[LATNAME].dtype,(LATNAME,))
    nc[LONNAME]=fileID.createVariable(LONNAME,dinput[LONNAME].dtype,(LONNAME,))
    nc[TIMENAME]=fileID.createVariable(TIMENAME,dinput[TIMENAME].dtype,(TIMENAME,))
    nc[LEVELNAME]=fileID.createVariable(LEVELNAME,dinput[LEVELNAME].dtype,(LEVELNAME,))
    nc[DIFFNAME] = fileID.createVariable(DIFFNAME, dinput[DIFFNAME].dtype,
        (TIMENAME,LEVELNAME,LATNAME,LONNAME,), fill_value=fill_value, zlib=True)
    nc[ZNAME] = fileID.createVariable(ZNAME, dinput[ZNAME].dtype,
        (TIMENAME,LEVELNAME,LATNAME,LONNAME,), fill_value=fill_value, zlib=True)
    #-- filling NetCDF variables
    for key,val in dinput.items():
        nc[key][:] = dinput[key].copy()
        dinput[key] = None

    #-- Defining attributes for longitude and latitude
    nc[LONNAME].long_name = 'Longitude'
    nc[LONNAME].units = 'degrees_east'
    nc[LATNAME].long_name = 'Latitude'
    nc[LATNAME].units = 'degrees_north'
    #-- Defining attributes for time
    nc[TIMENAME].units = TIME_UNITS
    nc[TIMENAME].long_name = TIME_LONGNAME
    #-- Definining attributes for model levels
    nc[LEVELNAME].long_name = 'Model_Level_Number'
    #-- Defining attributes for geopotential height
    nc[ZNAME].long_name = 'Geopotential_Heights_on_Model_Levels'
    nc[ZNAME].units = UNITS
    #-- Defining attributes for pressure differences
    nc[DIFFNAME].long_name = 'Pressure_Differences_between_Levels'
    nc[DIFFNAME].units = 'Pa'

    #-- Output NetCDF structure information
    logging.info(os.path.basename(FILENAME))
    logging.info(list(fileID.variables.keys()))

    #-- Closing the NetCDF file
    fileID.close()
    #-- clear nc dictionary variable
    nc = None

#-- Main program that calls reanalysis_geopotential_heights()
def main():
    #-- Read the system arguments listed after the program
    parser = argparse.ArgumentParser(
        description="""Reads temperature and specific humidity data
            to calculate geopotential height and pressure difference
            fields at half levels from reanalysis
            """,
        fromfile_prefix_chars="@"
    )
    parser.convert_arg_line_to_args = utilities.convert_arg_line_to_args
    #-- command line parameters
    choices = ['ERA-Interim','ERA5','MERRA-2']
    parser.add_argument('model',
        metavar='MODEL', type=str, nargs='+',
        default=['ERA5','MERRA-2'], choices=choices,
        help='Reanalysis Model')
    #-- working data directory
    parser.add_argument('--directory','-D',
        type=lambda p: os.path.abspath(os.path.expanduser(p)),
        default=os.getcwd(),
        help='Working data directory')
    #-- years to run
    now = time.gmtime()
    parser.add_argument('--year','-Y',
        type=int, nargs='+', default=range(2000,now.tm_year+1),
        help='Years of model outputs to run')
    #-- print information about each input and output file
    parser.add_argument('--verbose','-V',
        default=False, action='store_true',
        help='Verbose output of run')
    #-- permissions mode of the local directories and files (number in octal)
    parser.add_argument('--mode','-M',
        type=lambda x: int(x,base=8), default=0o775,
        help='Permission mode of directories and files')
    args,_ = parser.parse_known_args()

    #-- for each reanalysis model
    for MODEL in args.model:
        #-- run program
        reanalysis_geopotential_heights(args.directory, MODEL, YEAR=args.year,
            VERBOSE=args.verbose, MODE=args.mode)

#-- run main program
if __name__ == '__main__':
    main()


# coding: utf-8

# In[1]:

import pandas as pd
import os
import matplotlib.pyplot as plt

import re
import numpy as np
import pandas as pd
from scipy.stats import mode

from nltk import skipgrams

from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize
import itertools

import lightgbm as lgb
from lightgbm import LGBMClassifier
from sklearn.model_selection import cross_val_score, RandomizedSearchCV
from sklearn.metrics import accuracy_score

from sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.model_selection import train_test_split, StratifiedKFold
from sklearn import decomposition, ensemble

from sklearn.pipeline import Pipeline
from sklearn.multiclass import OneVsRestClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.neighbors import KNeighborsClassifier
from xgboost import XGBClassifier
from rgf.sklearn import FastRGFClassifier
from sklearn.model_selection import GridSearchCV

from nltk.corpus import stopwords
stop_words = set(stopwords.words('english'))

SEED = 42

join = os.path.join

# In[68]:

data = pd.read_csv('Devex_train.csv', encoding="latin-1")

# In[72]:

df_train = pd.read_csv('Devex_train.csv', low_memory=False, encoding='latin1')
df_submission = pd.read_csv('Devex_submission_format.csv', low_memory=False, encoding='latin1')

df_train.fillna(0, inplace=True)
df_train_clean = df_train.drop(columns=df_train.columns[3:15])

# In[76]:

labels = df_submission.columns[1:]
df_train_clean = pd.concat([pd.DataFrame(columns=labels),df_train_clean])
df_train_clean.fillna(0, inplace=True)

# In[77]:

unique_id_col = df_train_clean.pop('Unique ID')
type_col = df_train_clean.pop('Type')
text_col = df_train_clean.pop('Text')

df_train_clean.insert(0, 'Unique ID', unique_id_col)
df_train_clean.insert(1, 'Type', type_col)
df_train_clean.insert(2, 'Text', text_col)

# In[78]:

cleanr = re.compile('<.*?>')
REPLACE_BY_SPACE_RE = re.compile('[/(){}\[\]\|@,;.-]')
BAD_SYMBOLS_RE = re.compile('[^0-9a-zA-Z #+_]')
STOPWORDS = set(stopwords.words('english'))

from nltk.stem import WordNetLemmatizer, PorterStemmer
word_lemma = WordNetLemmatizer()
stem = PorterStemmer()
def remove_html(raw_html):
    cleantext = re.sub(cleanr, '', raw_html)
    cleantext = cleantext.lower()
    cleantext = re.sub(' ', ' ', cleantext)
    cleantext = re.sub('•', ' ', cleantext)

    cleantext = re.sub(REPLACE_BY_SPACE_RE, " ", cleantext)
    cleantext = re.sub(BAD_SYMBOLS_RE, "", cleantext)

    cleantext = " ".join([word_lemma.lemmatize(w) for w in cleantext.split(" ") if w not in STOPWORDS])
    #cleantext = " ".join([w for w in cleantext.split(" ") if w not in STOPWORDS])

    cleantext = cleantext + ' '.join([' '.join(x) for x in (list(skipgrams(itertools.islice(cleantext.split(), 50), 3, 1)))])

    return cleantext

# In[79]:

df_train_clean = df_train_clean.replace({r'\x0D': ' '}, regex=True) #removing carriage returns
df_train_clean['Text'] = df_train_clean['Type'] + " " + df_train_clean['Text']
df_train_clean['Text'] = df_train_clean['Text'].apply(remove_html)

for i in range(len(df_train)):
    for j in range(3,15):
        if df_train.iloc[i,j]!=0:
            label = df_train.iloc[i,j][0:5] #first 5 characters of the string is a label  (e.g. 3.8.1)
            df_train_clean.at[i,label] = 1

df_test = pd.read_csv('Devex_test_questions.csv', low_memory=False, encoding='latin1')

# In[83]:

train_x, test_x = model_selection.train_test_split(df_train_clean[['Text', '3.1.1', '3.1.2', '3.2.1', '3.2.2', '3.3.1', '3.3.2', '3.3.3', '3.3.4', '3.3.5', '3.4.1', '3.4.2', '3.5.1',
       '3.5.2', '3.6.1', '3.7.1', '3.7.2', '3.8.1', '3.8.2', '3.9.1', '3.9.2',
       '3.9.3', '3.a.1', '3.b.1', '3.b.2', '3.b.3', '3.c.1', '3.d.1']], test_size=0.3, shuffle=True, random_state=42)

# In[88]:

labels = ['3.1.1', '3.1.2', '3.2.1', '3.2.2', '3.3.1', '3.3.2', '3.3.3', '3.3.4', '3.3.5', '3.4.1', '3.4.2', '3.5.1',
       '3.5.2', '3.6.1', '3.7.1', '3.7.2', '3.8.1', '3.8.2', '3.9.1', '3.9.2',
       '3.9.3', '3.a.1', '3.b.1', '3.b.2', '3.b.3', '3.c.1', '3.d.1']

# In[91]:

df_test = pd.read_csv('Devex_test_questions.csv', encoding='latin-1')
df_test['Text'] =  df_test['Type'] + " " + df_test['Text']
df_test['Text'] =  df_test['Text'].apply(remove_html)

# In[ ]:

nb_pipeline = Pipeline([
                ('tfidf', CountVectorizer(stop_words=stop_words, ngram_range=(1, 1), max_features=20000, max_df=0.98)),
                ('clf', OneVsRestClassifier(MultinomialNB(alpha=1.6,
                    fit_prior=True, class_prior=None))),
            ])

dt_pipeline = Pipeline([
                ('tfidf', CountVectorizer(stop_words=stop_words, min_df=4, ngram_range=(1, 1), max_features=22000, max_df=0.98)),
                ('clf', OneVsRestClassifier(DecisionTreeClassifier(max_depth=10, random_state=SEED))),
            ])

knn_pipeline = Pipeline([
                ('tfidf', CountVectorizer(stop_words=stop_words, min_df=4, ngram_range=(1, 1), max_features=22000, max_df=0.98)),
                ('clf', OneVsRestClassifier(KNeighborsClassifier(n_neighbors=20))),
            ])

lg_pipeline = Pipeline([
                ('tfidf', CountVectorizer(stop_words=stop_words, min_df=4, ngram_range=(1, 1), max_features=22000, max_df=0.98)),
                ('clf', OneVsRestClassifier(LogisticRegression(C=0.8))),
            ])

lgb_model = LGBMClassifier(metric="accuracy", n_estimators=100,  num_leaves=31, boosting_type="dart",
                       learning_rate=0.15, max_depth=15)
lgb_pipeline_cnt = Pipeline([
                ('cntvec', CountVectorizer(stop_words=stop_words, min_df=4, max_features=22000, max_df=.99, dtype=np.float32)),
                ('clf', OneVsRestClassifier(lgb_model)),
            ])

lgb_pipeline_tfidf = Pipeline([
                ('cntvec', TfidfVectorizer(stop_words=stop_words, min_df=4, max_features=22000, max_df=.99, dtype=np.float32)),
                ('clf', OneVsRestClassifier(lgb_model)),
            ])

rnd_pipeline = Pipeline([
                ('tfidf', CountVectorizer(stop_words=stop_words,min_df=4, ngram_range=(1, 1), max_features=22000, max_df=0.98)),
                ('clf', OneVsRestClassifier(RandomForestClassifier(n_estimators=200, max_depth=15, n_jobs=8))),
            ])

xgb_pipeline_cnt = Pipeline([
                ('tfidf', TfidfVectorizer(stop_words=stop_words, min_df=4, max_features=22000, max_df=.98)),
                ('clf', OneVsRestClassifier(XGBClassifier(n_jobs=8,
                                                          n_estimators=200,
                                                          learning_rate=0.2,
                                                          max_depth=15,
                                                          scale_pos_weight=1.5,
                                                          gamma=1
                                                         ))),
            ])

xgb_pipeline_tfidf = Pipeline([
                ('tfidf', TfidfVectorizer(stop_words=stop_words, min_df=4, max_features=22000, max_df=.98)),
                ('clf', OneVsRestClassifier(XGBClassifier(n_jobs=8,
                                                          n_estimators=200,
                                                          learning_rate=0.2,
                                                          max_depth=15,
                                                          scale_pos_weight=1.5,
                                                          gamma=1
                                                         ))),
            ])

rgf_pipeline_cnt = Pipeline([
                ('tfidf', TfidfVectorizer(stop_words=stop_words, min_df=4, max_features=30000, max_df=.99)),
                ('clf', OneVsRestClassifier(FastRGFClassifier(n_estimators=500, max_depth=6, min_samples_leaf=10))),
            ])

rgf_pipeline_tfidf = Pipeline([
                ('tfidf', TfidfVectorizer(stop_words=stop_words, min_df=4, max_features=30000, max_df=.99)),
                ('clf', OneVsRestClassifier(FastRGFClassifier(n_estimators=500, max_depth=6, min_samples_leaf=10))),
            ])

# In[92]:

def model_fit_predict(model, X_train, y_train, X_test, sub_data):
    model.fit(X_train, y_train)
    pred = model.predict(X_test)
    pred_prob = model.predict_proba(X_test)
    pred_sub = model.predict(sub_data)
    prob_sub = model.predict_proba(sub_data)

    return pred, pred_prob, pred_sub, prob_sub

# In[97]:

print("Training Starts!!")
#nb_preds, nb_probs, nb_pred_sub, nb_prob_sub = model_fit_predict(nb_pipeline, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])
X_test = test_x['Text']
lr_preds, lr_probs, lr_pred_sub, lr_prob_sub = model_fit_predict(lg_pipeline, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])

dt_preds, dt_probs, dt_pred_sub, dt_prob_sub = model_fit_predict(dt_pipeline, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])
knn_preds, knn_probs, knn_pred_sub, knn_prob_sub = model_fit_predict(knn_pipeline, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])
rf_preds, rf_probs, rf_pred_sub, rf_prob_sub = model_fit_predict(rnd_pipeline, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])

lgb_preds_cnt, lgb_probs_cnt, lgb_pred_sub_cnt, lgb_prob_sub_cnt = model_fit_predict(lgb_pipeline_cnt, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])
lgb_preds_tf, lgb_probs_tf, lgb_pred_sub_tf, lgb_prob_sub_tf = model_fit_predict(lgb_pipeline_tfidf, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])

xgb_preds_cnt, xgb_probs_cnt, xgb_pred_sub_cnt, xgb_prob_sub_cnt = model_fit_predict(xgb_pipeline_cnt, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])
xgb_preds_tf, xgb_probs_tf, xgb_pred_sub_tf, xgb_prob_sub_tf = model_fit_predict(xgb_pipeline_tfidf, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])

rgf_preds_cnt, rgf_probs_cnt, rgf_pred_sub_cnt, rgf_prob_sub_cnt = model_fit_predict(rgf_pipeline_cnt, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])
rgf_preds_tf, rgf_probs_tf, rgf_pred_sub_tf, rgf_prob_sub_tf = model_fit_predict(rgf_pipeline_tfidf, df_train_clean['Text'], df_train_clean[labels], X_test, df_test['Text'])

# In[98]:

"""print('Accuracy LR {}'.format(accuracy_score(test_x[labels].values, lr_preds)))
print('Accuracy DT {}'.format(accuracy_score(test_x[labels].values, dt_preds)))
print('Accuracy KNN {}'.format(accuracy_score(test_x[labels].values, knn_preds)))
print('Accuracy RF {}'.format(accuracy_score(test_x[labels].values, rf_preds)))
print('Accuracy XGB {}'.format(accuracy_score(test_x[labels].values, xgb_preds_cnt)))
print('Accuracy XGB {}'.format(accuracy_score(test_x[labels].values, xgb_preds_tf)))
print('Accuracy LGB {}'.format(accuracy_score(test_x[labels].values, lgb_preds_cnt)))
print('Accuracy LGB {}'.format(accuracy_score(test_x[labels].values, lgb_preds_tf)))
print('Accuracy RGF {}'.format(accuracy_score(test_x[labels].values, rgf_preds_cnt)))
print('Accuracy RGF {}'.format(accuracy_score(test_x[labels].values, rgf_preds_tf)))"""

print("Training Done!!")
# In[134]:

temp_pred = (lr_prob_sub*0.2+xgb_prob_sub_cnt*0.6
                +rgf_pred_sub_cnt*0.1+lgb_pred_sub_tf*0.1)

#temp_pred = (lr_prob_sub*0.2+xgb_prob_sub_cnt*0.5+lgb_prob_sub_tf*.2+dt_prob_sub*0.05+rf_prob_sub*0.025+knn_prob_sub*0.025)
temp_pred = np.where(temp_pred >=0.49, 1, 0 )

save_comb = """temp_pred = (lr_prob_sub*0.2+dt_prob_sub*0.05+knn_prob_sub*0.025+rf_prob_sub*0.025+xgb_prob_sub*0.5+lgb_prob_sub*.2)
temp_pred = np.where(temp_pred >=0.49, 1, 0 )    This gives 0.3866 accuracy LB"""

df_submission['ID'] = df_test['Unique ID']; df_submission.iloc[:, 1:] = temp_pred
df_submission.to_csv('sub_voting.csv', index=False)

print("Prediction are generate in sub_voting.csv !!")

Competition-Solutions/Text/Sustainable Development Goals (SDGs) Text Classification Challenge/Solution 2/ensemble.py


import pandas as pd
import numpy as np
from pandas import DataFrame

def log_column(data: DataFrame, column: str) -> DataFrame:
    """

    :param data: DataFrame with values to log
    :param column: Column to log
    :return: DataFrame with log column
    """
    data = np.log1p(data[column])
    return data

def drop_column(data: DataFrame, column: str, axis: int) -> DataFrame:
    """

    :param data: DataFrame with values to drop column
    :param column: Column to drop
    :param axis: Axis to drop. Column = -1 Row = 0
    :return: DataFrame without chosen column.
    """
    data = data.drop(column, axis=axis)
    return data


"""
This .py file contains two functions for common data cleaning tasks. These functions can be used with
data loaded into a pandas DataFrame.
"""

import pandas as pd
import numpy as np
from sklearn.utils import shuffle

# Confirms whether or not a DataFrame contains missing values
def null_count(df):
    """
    This function will return the number of null values contained
    within a DataFrame
    """
    return df.isnull().sum().sum()

# Randomizer
def randomize(df, seed):
    """
    Eponymous Comment. Function will return a reproducible
    randomized varibale via a seed. Shuffle from sklearn and it's
    random_state parameter do the heavy lifting
    """
    randomized = shuffle(
                         df,
                         random_state=seed
                         )
    return randomized


# Copyright 2017 J Carruthers jbc@bu.edu
# Solution to HW 5: collision_tester

import unittest
import subprocess
import random
import math
import numpy

AUTHORS = ['jbc@bu.edu']

PROGRAM_TO_TEST = "collisionc_0"

# r = random.randint
# random_twenty=[ (1000+x,r(-2000,2000),r(-2000,2000),
#                  r(-10,10),r(-10,10)) for x in range(20)]
# print(random_twenty)

BAD_ARGS_RC = 2
BAD_INPUT_RC = 1

random_twenty = [(1000, -1663, -1068, 4, 3), (1001, 1771, 1241, 4,
                                              -9), (1002, 531, -1842, 5, -7),
                 (1003, -1999, 576, 9, 7), (1004, 282, -338, 0,
                                            -5), (1005, 1432, 1146, -3, -10),
                 (1006, 1818, 1972, 9,
                  10), (1007, -1819, 365, -6,
                        6), (1008, 1896, 1553, 4,
                             -8), (1009, 1561, 952, 5,
                                   4), (1010, -1937, -1948, -2,
                                        -6), (1011, 110, 1424, 1,
                                              7), (1012, 1754, 1794, -2, -7),
                 (1013, 1042, -193, 5,
                  -4), (1014, 563, 1074, 8,
                        -9), (1015, 243, 62, 5,
                              10), (1016, 1749, 923, -10,
                                    -5), (1017, -1365, -346, -5,
                                          7), (1018, 495, -769, -10,
                                               -1), (1019, -548, -841, 5, -2)]

basic_input = """a 10 20 -1.5 2
b 90 90 -3 -3
c 100 100 1 1
"""

basic_output = """1
a 8.5 22 -1.5 2
b 87 87 -3 -3
c 101 101 1 1
2
a 7 24 -1.5 2
b 84 84 -3 -3
c 102 102 1 1
3
a 5.5 26 -1.5 2
b 81 81 -3 -3
c 103 103 1 1
"""

bad_args = [('-4', "-5"), ("one", "two"), ("one", "4", "5"), ("4", "5",
                                                              "alpha")]

bad_inputs = [
    """a
b 0 0 1 1
c 20 20 1 1
""", """a 0 0 0 0
b 10 10 10 10 10
""", """a b c 0 0
d 10 10 10 10
e 30 30 30 30
""", """a 10 10 1 1

"""
]

collision_input_one = """one -10 -10 2 2
two 10 10 -1 -1
"""

collision_output_one = """12
one -9.0710678 -9.0710678 -1 -1
two 21.071068 21.071068 2 2
"""
large_time_input = "a 0 0 0 0\n"

large_time_list = ['1000', '100000']
large_time_output = """1000
a 0 0 0 0
100000
a 0 0 0 0
"""
runners = [("for", 100, -100, 100, 100), ('back', 0, 0, -100, 100)]

big_locations_input = """one 1000000 1000000 -100 -100
two -1000000 1000000 100 -100
three 1000000 -1000000 -100 100
four -1000000 -1000000 100 100
"""
big_locations_output = """9999
one 100 100 -100 -100
two -100 100 100 -100
three 100 -100 -100 100
four -100 -100 100 100
"""

dup_names = """one 0 0 1 1
one 10 10 10 10
two 20 20 20 20
"""

dup_names_out = """1
one 1 1 1 1
one 20 20 10 10
two 40 40 20 20
2
one 2 2 1 1
one 30 30 10 10
two 60 60 20 20
"""

def runprogram(program, args, inputstr):
    coll_run = subprocess.run(
        [program, *args],
        input=inputstr.encode(),
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        timeout=1)
    "run a program and get result: wrapper for subprocess.run"

    ret_code = coll_run.returncode
    program_output = coll_run.stdout.decode()
    program_errors = coll_run.stderr.decode()
    return (ret_code, program_output, program_errors)

class CollisionTestCase(unittest.TestCase):
    def check_collision_output(self, bad, good):
        "process and numerically compare two outputs (helper function)"
        badlines = bad.splitlines()
        goodlines = good.splitlines()
        self.assertEqual(len(badlines), len(goodlines))
        for badline, goodline in zip(badlines, goodlines):
            goodvals = goodline.split()
            badvals = badline.split()
            if len(goodvals) != len(badvals):
                self.fail('improper line format')
            elif len(goodvals) == 1:  # time line
                self.assertTrue(
                    math.isclose(float(goodvals[0]), float(badvals[0])))
            else:
                self.assertEqual(goodvals[0], badvals[0])
                self.assertTrue(
                    numpy.allclose([float(x) for x in goodvals[1:]],
                                   [float(x) for x in badvals[1:]]))

    def test_collision_one(self):
        "test a simple collision event"
        (rc, out, errs) = runprogram(PROGRAM_TO_TEST, ["12"],
                                     collision_input_one)
        self.assertEqual(rc, 0)
        self.check_collision_output(out, collision_output_one)
        self.assertEqual(errs, "")

    def test_twenty(self):
        "twenty objects with random motion, no collisions"
        strin = "\n".join(" ".join(str(n) for n in x) for x in random_twenty)
        correct_out = "12\n" + "\n".join("{} {} {} {} {}".format(
            x[0], x[1] + 12 * x[3], x[2] + 12 * x[4], x[3], x[4])
                                         for x in random_twenty) + "\n"
        (rc, out, errs) = runprogram(PROGRAM_TO_TEST, ["12"], strin)
        self.assertEqual(rc, 0)
        self.check_collision_output(out, correct_out)
        self.assertEqual(errs, "")

    def test_three_args(self):
        """three arguments, out of order, and with negatives.
        use non-integer values for velocity."""
        (rc, out, errs) = runprogram(PROGRAM_TO_TEST, ["1", "-2", "3", "2"],
                                     basic_input)
        self.assertEqual((rc, errs), (0, ""))
        self.check_collision_output(out, basic_output)

    def test_bad_args(self):
        "bad argument examples: negative or alpha"
        for bad_arg in bad_args:
            with self.subTest(CASE=repr(bad_arg)):
                (rc, out, errs) = runprogram(PROGRAM_TO_TEST, bad_arg,
                                             basic_input)
                if rc != BAD_ARGS_RC:
                    self.fail('rc')

    def test_bad_inputs(self):
        "bad input formatting."
        for bad_in in bad_inputs:
            with self.subTest(CASE=repr(bad_in)):
                (rc, out, errs) = runprogram(PROGRAM_TO_TEST, ["1", "2"],
                                             bad_in)
                if rc != BAD_INPUT_RC:
                    self.fail('rc')

    def test_many_args(self):
        "handle 100+ arguments and large distances"
        many_args = list(range(1, 110))
        strin = "\n".join(" ".join(str(n) for n in x) for x in runners)
        outlines = []
        for time in many_args:
            outlines.append("{}\n".format(time))
            for name, x, y, vx, vy in runners:
                outlines.append("{} {} {} {} {}\n".format(
                    name, x + vx * time, y + vy * time, vx, vy))
        correct_out = "".join(outlines)

        (rc, out, errs) = runprogram(PROGRAM_TO_TEST,
                                     [str(x) for x in many_args], strin)
        self.assertEqual(rc, 0)
        self.check_collision_output(out, correct_out)
        self.assertEqual(errs, "")

    def test_large_time(self):
        "large time values"
        (rc, out, errs) = runprogram(PROGRAM_TO_TEST, large_time_list,
                                     large_time_input)
        self.check_collision_output(out, large_time_output)

    def test_big_locations(self):
        "wide field of motion"
        (rc, out, errs) = runprogram(PROGRAM_TO_TEST, ['9999'],
                                     big_locations_input)
        self.check_collision_output(out, big_locations_output)

    def test_dup_names(self):
        "handle duplicate named objects"
        (rc, out, errs) = runprogram(PROGRAM_TO_TEST, ['1', "2"], dup_names)
        self.check_collision_output(out, dup_names_out)
        self.assertEqual(rc, 0)
        self.assertEqual(errs, "")

    def test_many_collisions(self):
        "handle multiple collisions"
        inlines = ["mover 0 0 0 1\n"]
        outlines = ["2000\n", "mover 0 10 0 0\n"]
        for i, ypos in enumerate(range(20, 3000, 20)):
            inlines.append("mover{} 0 {} 0 0\n".format(i, ypos))
            outlines.append("mover{} 0 {} 0 0\n".format(i, ypos + 10))

        input_str = "".join(inlines)
        outlines[-1] = "mover148 0 3490 0 1\n"

        correct_out = "".join(outlines)
        (rc, out, errs) = runprogram(PROGRAM_TO_TEST, ['2000'], input_str)
        self.check_collision_output(out, correct_out)

def main():
    unittest.main()

if __name__ == '__main__':
    main()


""" Module Docsting: Documentation for "Add Timedelta to Index"

# Add Timedelta to Index

## Description
This component adds the provided timedelta to each of the indices of the provided dataframe or series.

## Inputs
* **df_or_series** (Any): Both dataframe and series are accepted, the indices must be datetimes.
* **timedelta** (String): Timedelta to be added (may be negative) to each of the indices, e.g. '3s', '-1min', or '2days'.

## Outputs
* **df_or_series** (Any): Dataframe or series same as the input just with the provided timedelta added to each of the indices.

## Details
This component adds the provided timedelta to each of the indices of the provided dataframe or series.

## Examples
The json input of a typical call of this component, adding a timedelta of 2 days to each of the indices is
```
{
	"df_or_series": {
		"2019-08-01T15:20:00": 1.0,
		"2019-08-02T15:20:15": 7.0,
		"2019-08-04T15:19:20": 5.0
	},
	"timedelta": "2days"
}
```
The expected output is
```
	"df_or_series": {
		"2019-08-03T15:20:00": 1.0,
		"2019-08-04T15:20:15": 7.0,
		"2019-08-06T15:19:20": 5.0
	}
```

The json input of a call of this component with the same series, adding a timedelta of -1 minute
```
{
	"df_or_series": {
		"2019-08-03T15:20:00": 1.0,
		"2019-08-04T15:20:15": 7.0,
		"2019-08-06T15:19:20": 5.0
	},
	"timedelta": "-1min"
}
```
The expected output is
```
	"df_or_series": {
		"2019-08-03T15:19:00": 1.0,
		"2019-08-04T15:19:15": 7.0,
		"2019-08-06T15:18:20": 5.0
	}
```

"""

from hetdesrun.component.registration import register
from hetdesrun.datatypes import DataType

import pandas as pd
import numpy as np

# ***** DO NOT EDIT LINES BELOW *****
# These lines may be overwritten if component details or inputs/outputs change.
@register(
    inputs={"df_or_series": DataType.Any, "timedelta": DataType.String},
    outputs={"df_or_series": DataType.Any},
    name="Add Timedelta to Index",
    description="Add a timedelta to the index of a frame or series",
    category="Time length operations",
    id="c587ced1-7841-4208-8f88-9a9bd6a28f20",
    revision_group_id="3b838621-8d8e-493a-a91a-5a7680385ed9",
    version_tag="1.0.0"
)
def main(*, df_or_series, timedelta):
    # entrypoint function for this component
    # ***** DO NOT EDIT LINES ABOVE *****
    """ Usage example:
    >>> main(
    ...     df_or_series=pd.Series(
    ...             [10.0, 22.0, 18.0, 2.0],
    ...             index=pd.to_datetime(["2019-08-01T15:20:10", "2019-08-01T15:20:11", "2019-08-01T15:20:14", "2019-08-01T15:20:16"])
    ...     ),
    ...     timedelta = "-4s",
    ... )["df_or_series"]
    2019-08-01 15:20:06    10.0
    2019-08-01 15:20:07    22.0
    2019-08-01 15:20:10    18.0
    2019-08-01 15:20:12     2.0
    dtype: float64
    """
    # write your function code here.
    df_or_series = pd.DataFrame.from_dict(df_or_series, orient="index")
    df_or_series.index = pd.to_datetime(df_or_series.index)
    if df_or_series.columns.size < 2:
        df_or_series = df_or_series.squeeze("columns")
    new_index = df_or_series.index + pd.Timedelta(timedelta)
    df_or_series.index = new_index
    return {"df_or_series": df_or_series}


#
# Copyright 2019 Gianluca Frison, Dimitris Kouzoupis, Robin Verschueren,
# Andrea Zanelli, Niels van Duijkeren, Jonathan Frey, Tommaso Sartor,
# Branimir Novoselnik, Rien Quirynen, Rezart Qelibari, Dang Doan,
# Jonas Koenemann, Yutao Chen, Tobias Schöls, Jonas Schlagenhauf, Moritz Diehl
#
# This file is part of acados.
#
# The 2-Clause BSD License
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# 1. Redistributions of source code must retain the above copyright notice,
# this list of conditions and the following disclaimer.
#
# 2. Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.;
#

from acados_template import *
import acados_template as at
from export_ode_model import *
import numpy as np
import scipy.linalg
from ctypes import *

# create render arguments
ocp = AcadosOcp()

# export model
model = export_ode_model()

# set model_name
ocp.model_name = model.name

Tf = 1.0
nx = model.x.size()[0]
nu = model.u.size()[0]
ny = nx + nu
ny_e = nx
N = 100

# set ocp_nlp_dimensions

ocp.set('dims_nx', nx)
ocp.set('dims_ny', ny)
ocp.set('dims_ny_e', ny_e)
ocp.set('dims_nbx', 0)
ocp.set('dims_nbu', nu)
ocp.set('dims_nu', model.u.size()[0])
ocp.set('dims_N', N)

# set weighting matrices
Q = np.eye(4)
Q[0,0] = 1e3
Q[1,1] = 1e-2
Q[2,2] = 1e3
Q[3,3] = 1e-2

R = np.eye(1)
R[0,0] = 1e-2

ocp.set('cost_W', scipy.linalg.block_diag(Q, R))

Vx = np.zeros((ny, nx))
Vx[0,0] = 1.0
Vx[1,1] = 1.0
Vx[2,2] = 1.0
Vx[3,3] = 1.0

ocp.set('cost_Vx', Vx)

Vu = np.zeros((ny, nu))
Vu[4,0] = 1.0
ocp.set('cost_Vu', Vu)

ocp.set('cost_W_e', Q)

Vx_e = np.zeros((ny_e, nx))
Vx_e[0,0] = 1.0
Vx_e[1,1] = 1.0
Vx_e[2,2] = 1.0
Vx_e[3,3] = 1.0

ocp.set('cost_Vx_e', Vx_e)

ocp.set('cost_yref', np.zeros((ny, )))
ocp.set('cost_yref_e', np.zeros((ny_e, )))

# setting bounds
Fmax = 80.0
ocp.set('constraints_lbu', np.array([-Fmax]))
ocp.set('constraints_ubu', np.array([-Fmax]))
ocp.set('constraints_x0', np.array([0.0, 0.0, 3.14, 0.0])
ocp.set('constraints_idxbu', np.array([0])

# set constants
# ocp.constants['PI'] = 3.1415926535897932

# set QP solver
# ocp.solver_options.qp_solver = 'PARTIAL_CONDENSING_HPIPM'
ocp.set('solver_options_qp_solver', 'FULL_CONDENSING_QPOASES')
ocp.set('solver_options_hessian_approx', 'GAUSS_NEWTON')
ocp.set('solver_options_integrator_type', 'ERK')

# set prediction horizon
ocp.set('solver_options_tf', Tf)
ocp.set('solver_options_nlp_solver_type', 'SQP')

# set header path
ocp.set('acados_include_path', '/usr/local/include')
ocp.set('acados_lib_path', '/usr/local/lib')

# json_layout = acados_ocp2json_layout(ocp)
# with open('acados_layout.json', 'w') as f:
#     json.dump(json_layout, f, default=np_array_to_list)
# exit()

acados_solver = generate_solver(model, ocp, json_file = 'acados_ocp.json')

Nsim = 100

simX = np.ndarray((Nsim, nx))
simU = np.ndarray((Nsim, nu))

for i in range(Nsim):
    status = acados_solver.solve()

    # get solution
    x0 = acados_solver.get(0, "x")
    u0 = acados_solver.get(0, "u")

    for j in range(nx):
        simX[i,j] = x0[j]

    for j in range(nu):
        simU[i,j] = u0[j]

    # update initial condition
    x0 = acados_solver.get(1, "x")

    acados_solver.set(0, "lbx", x0)
    acados_solver.set(0, "ubx", x0)

# plot results
import matplotlib
import matplotlib.pyplot as plt
t = np.linspace(0.0, Tf/N, Nsim)
plt.subplot(2, 1, 1)
plt.step(t, simU, 'r')
plt.title('closed-loop simulation')
plt.ylabel('u')
plt.xlabel('t')
plt.grid(True)
plt.subplot(2, 1, 2)
plt.plot(t, simX[:,2])
plt.ylabel('theta')
plt.xlabel('t')
plt.grid(True)
plt.show()


import numpy as np
import torch
from torch.utils import data

class SimpleAutoDataset(data.Dataset):
	def __init__(self,numpyFeatures,full_gpu=False):
		self.data = numpyFeatures
		self.full_gpu = full_gpu #if true, push everything to gpu

	def __getitem__(self,index):
		if self.full_gpu:
			return self.getItemTorch(index)
		else:
			return self.getItemReg(index)

	def __len__(self):
		return self.data.shape[0]

	def activate(self):
		if self.full_gpu: #push everything to gpu
			self.torchData = torch.from_numpy(self.data).cuda().float()

	def deactivate(self):
		self.torchData = None

	def getItemTorch(self,index):
		return (self.torchData[index,:].flatten().float(),self.torchData[index,:].flatten().float())

	def getItemReg(self,index):
		return (torch.from_numpy(self.data[index,:]).flatten().float(),torch.from_numpy(self.data[index,:]).flatten().float())