#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 11 14:27:16 2019
@author: AndreaLøvemærke
Applying scripts authored by Katrine Tjell Mølgaard
The protocol is developed by Niek J. Bouman & Niels de Vreede
"""

import numpy as np
from scipy import linalg
from scipy.linalg import toeplitz
from numpy.linalg import inv
import random
from numpy.linalg import matrix_rank
from threading import Thread
import FFArithmetic as field
import shamir_scheme as ss
import time
import proc

# script simulates multiple party computation
# consists of 3 perties and 1 data owner (the car)

# car produces:
    # Beaver's Triplets
    # random variable for equality tests
    # shares of A, b, I_n matrices securely
    # generates random Toeplitz matrices U and L openly

# parties computes Pivo-free Gaussian Elimination
# according to Niek's protocol

# parties distribute shares of the result to the car
# the car reconstructs the result, only the car can know the result
# the result is the solution to the linear system, Ax= b

# F = IntegerModP
# n= number of parties
# t= shamir degree
#time_start = time.clock()
for abe in range (0,10):
    class server:
        securecom = {}
        broadcasts = {}
        def __init__(self,F, n, t, numTrip):
            self.b = ss.share(F,np.random.choice([-1,1]), t, n)
            self.triplets = [proc.triplet(F,n,t) for i in range(numTrip)]

    class part(Thread):

        def __init__(self, F, n, t, i, s):
            Thread.__init__(self)
            self.c = 0
            self.comr = 0
            self.F = F
            self.n = n
            self.t = t
            self.i = i
            self.server = s
            self.comtime = 0

        def distribute_shares(self,var, name):
            shares = ss.share(self.F, var, self.t, self.n)  # create shares

            s = name                                        # identify share
            st = time.time()
            self.server.securecom[s] = shares

        def get_share(self, name):
            st = time.time()
            while True:
                try:
                    res =  (self.server.securecom[name][self.i])
                    break
                except:
                    continue
            sl = time.time()
            self.comtime +=(sl-st)
            return res

        def add_shares(self,a,b):
            return a+b

        def mult_shares(self, a, b):
            r = self.server.triplets[self.c][self.i]
            self.c += 1

            d_local = a - r[0]
            self.broadcast('d' + str(self.comr), d_local)
            d_pub = self.reconstruct_secret('d' + str(self.comr))
            self.comr +=1
            e_local = b - r[1]
            self.broadcast('e' + str(self.comr), e_local)
            e_pub = self.reconstruct_secret('e' + str(self.comr))
            self.comr+=1
            return d_pub * e_pub + d_pub*r[1] + e_pub*r[0] + r[2]

        def reconstruct_secret(self, c):
            res = []
            for i in range(self.n):
                res.append(self.get_broadcast(c + str(i)))
            return ss.rec(self.F, res)

        def broadcast(self, name, c):
            st = time.time()
            self.server.broadcasts[name + str(self.i)] = c
            sl = time.time()
            self.comtime += sl-st

        def get_broadcast(self, c):
            st = time.time()
            while True:
                try:
                    res = self.server.broadcasts[c]
                    break
                except:
                    continue
            sl = time.time()
            self.comtime += (sl-st)
            return res

        def run(self):
            #print(self.i)
            #print(self.server.securecom)

            # recieve shares for A, b, I_n, h, t, random variable

            input_sharesa00=self.get_share('a00').n         # reconstruct OK
            input_sharesa01=self.get_share('a01').n
            input_sharesa10=self.get_share('a10').n
            input_sharesa11=self.get_share('a11').n
            input_sharesb0=self.get_share('b0').n
            input_sharesb1=self.get_share('b1').n

            input_sharesI00=self.get_share('iden00').n      # reconstruct OK
            input_sharesI01=self.get_share('iden01').n
            input_sharesI10=self.get_share('iden10').n
            input_sharesI11=self.get_share('iden11').n

            h_share=self.get_share('h_shares').n
            t_share=self.get_share('t_shares').n
            ra_share=self.get_share('ran_shares').n   # shares of random variable
                                                      # for equality test

            # arrange arrays corresponding to original input
            AA=np.array([[input_sharesa00, input_sharesa01],[input_sharesa10, input_sharesa11]])
            bb= np.array([[input_sharesb0],[input_sharesb1]])
            I_n=np.array([[input_sharesI00, input_sharesI01],[input_sharesI10, input_sharesI11]])

            e11= np.array(U@AA@L) # OK
            e12=np.array(U@bb)
            e21= np.array(I_n)
            e22= np.array(np.zeros((nn,l)))

            C_shares=np.array(([e11[0,0], e11[0,1], e12[0,0]],[e11[1,0], e11[1,1], e12[1,0]], [e21[0,0], e21[0,1], 0],[ e21[1,0], e21[1,1], 0]))
            C_shares=C_shares.astype(int)

            f = []
            r_temp = []
            r = []

    #
            for k in range(0, mu):

                broad_ckk= self.mult_shares(ra_share,C_shares[k,k]).n
                self.broadcast('c_kk' + str(self.comr), broad_ckk)

                # protocol line 5
                r_temp.append(self.reconstruct_secret('c_kk'+str(self.comr)))

                if r_temp[k] == 0:
                    r.append(0)
                elif r_temp[k] != 0:
                    r.append(1)
                else:
                    print('error message')

                # protocol line 6:
                C_shares[mu+k,k] = h_share

                #self.broadcast('rec_C' + str(self.comr), C_shares[mu+k,k])
                #C_rec= self.reconstruct_secret('rec_C'+str(self.comr)).n
                #print('C reconstruct is:', C_rec)

                f.append(h_share)

                t_share = self.mult_shares(t_share,h_share).n     # mult shares med Beavers

                # protocol line 9
                c_kk = (C_shares[k,k]+1-r[k])    # when c_kk !=0 then r will be 1
                                                 # and 1-r will cancel out

                h_share = self.mult_shares(h_share,c_kk).n

                for i in range(0, mu+k):
                    for j in range(k+1, nn+l):
                        if i!= k and (i<= mu or j <= nn-1):
                            # protocol line 13:
                            dummy=np.matrix( np.vstack((C_shares[i,j],-(C_shares[i,k]))))
                            temp= np.matrix(np.hstack(((C_shares[k,k]+1-r[k]), C_shares[k,j])))
                            temp_C1=self.mult_shares(temp[0,0],dummy[0,0]).n
                            temp_C2=self.mult_shares(temp[0,1],dummy[1,0]).n
                            C_shares[i,j]=temp_C1 +temp_C2 #manuel computation 1x2 2x1 = 1x1

            # protocol line 14 and 15 not implemented

            g = []
            ss = []

            # protocol line 16:
            X = np.asarray(C_shares[(0,1), -1]) # array (ligger ned)

            # protocol line 17 not implemented

            # protocol line 18:
            inv_temp=self.mult_shares(t_share,h_share).n

            # inverting securely, [BIB89]
            test_in=self.mult_shares(ra_share,inv_temp).n

            self.broadcast('yinv' + str(self.comr), test_in)
            ww= self.reconstruct_secret('yinv'+str(self.comr)).n

            w_inv=1/ww
            #print(ww)

            s_w_inv= w_inv*10E10
            s_w_inv=int(s_w_inv)

            if self.i ==0:
                self.distribute_shares(s_w_inv,'inv_y')

            sw_inv_share=self.get_share('inv_y').n # ok, reconstruction is true to original

            g=self.mult_shares(sw_inv_share, ra_share).n

#            ans=1/th_product
#            ans=int(ans*10E10)
#
           # print('Original value scales: ',ans, ':')

#            if self.i ==0:
#                self.distribute_shares(ans,'th_inv')
#
#            g=self.get_share('th_inv').n # ok, reconstruction is true to original

            # protocol line 19:
            f_diag=np.diag(f)

            gt_temp=self.mult_shares(g,t_share).n  # OKK # previously made with g

            gtL=gt_temp * L
            # OKK

            fx=np.zeros(2, dtype=int)  #np.matrix(np.zeros((2,1)))

            for k in range(0, mu):
                fx[k] = self.mult_shares(f[k], X[k]).n

            #print('fx original:', fx[0])
            fx=fx.astype(int)

            #print('fx int', fx[0])
            self.broadcast('ok' + str(self.comr), fx[0])
            ook=self.reconstruct_secret('ok'+str(self.comr)).n
         #   print('reconstruction check', ook)

            # OKK!

            [ra,ca]=gtL.shape    # dimension OK
            [rb]=fx.shape
            cb=1

            for ii in range(0,ra):
                for jj in range(0,cb):
                    for kk in range(0,ca):
                        X[ii]=X[ii]+self.mult_shares(gtL[ii,kk],fx[kk]).n
             # OKK

            X=np.reshape(X, (2, 1))
            #X= X/10E10
            #X=X.astype(int)

    #

            # protocol line 20:
    #        AX_temp = []
    #
    #        [ra,ca]=AA.shape
    #        [rb, cb]=X.shape
    #
    #        res_AX=np.zeros((ra,cb))
    #
    #        for ii in range(0,ra):
    #            for jj in range(0,cb):
    #                for kk in range(0,ca):
    #                    res_AX[ii,jj]= res_AX[ii,jj]+ self.mult_shares(AA[ii,kk],X[kk,jj]).n
    #                    # OKK
    #
    #
    #        zAX=z1@res_AX # OKK
    #        zAX=zAX
    #
    #
    #        zb=z1@bb*10E10 # OKK
    #
    #        ko= zAX-zb
    #        zbR=self.mult_shares(zb,ra_share).n
    #        self.broadcast('zbran' + str(self.comr), zbR)# OKK
    #        zbRAN=self.reconstruct_secret('zbran'+str(self.comr)).n
    ##

            #ss=zAX-zb # OKK

    #        self.broadcast('okay1' + str(self.comr), zAX[0,0])
    #        test1=self.reconstruct_secret('okay1'+str(self.comr)).n
    #        print('zAX :', test1)
    #
    #        self.broadcast('okay2' + str(self.comr), zb[0,0])
    #        test2=self.reconstruct_secret('okay2'+str(self.comr)).n
    #        print('zb :', test2)
    #
    #        self.broadcast('okay3' + str(self.comr), ko[0,0])
    #        test3=self.reconstruct_secret('okay3'+str(self.comr)).n
    #        print('ss :', test3)
    #

            # protocol line 22:
    #        # i loop, når eksemples udvides, for nu blot en skalar
    #
    #        broad_sj= self.mult_shares(ra_share,ss[0,0]).n
    #        self.broadcast('s_ran' + str(self.comr), broad_sj)
    #
    #        s_update=self.reconstruct_secret('s_ran'+str(self.comr)).n
    ##       s_temp = []
    #       s_temp.append(self.reconstruct_secret('s_scalar'+str(self.comr)))

            # print("i {}: s_updtae{}".format(self.i, s_update))
    #        if s_update <10E9:
    #
    #            if self.i ==0:
    #                self.distribute_shares(1,'one') # must be a share, 1 can be a share of 1 if other coefficients are chosen to be zero
    #            ss= self.get_share('one').n
    #
    #
    #
                # when example is expanded
    ##            for k in range(0, mu):
    ##            if s_temp[k] == 0:
    ##                print(s_temp[k])#r.append(0)
    ##            elif s_temp[k] != 0:
    ##                print(s_temp[k])# s.append(1)
    ##            else:
    ##                print('error message')

    ##        # loop not necessary, s = scalar, loop only happens once for this example at least
    ##        for j in range(0, l):
    ##            print(j)
    ##            equality test, must be done securely, ideally
    ##            self.broadcast('sj' + str(self.comr), s[k])

            # protocol line 23
    #       #### #X=np.dot(X,ss)
    #
           # for k in range(0,mu):
             #   X[k,0] = self.mult_shares(X[k,0],s_update).n

    #        self.broadcast('okay' + str(self.comr), X[1,0])
    #        test=self.reconstruct_secret('okay'+str(self.comr)).n
    #        print('XX res', test)

            # protocol line 26:
            self.X = X

    class ground:

        def __init__(self,F, A, b, hh, tt, iden, ran, n, t,s): #hh, tt
            self.A = A
            self.b = b
            self.server = s
            self.F = F
            self.t = t
            self.n = n
            self.hh=hh
            self.tt=tt
            self.iden= iden
            self.ran= ran

        def distribute_shares(self,var, name):
            shares = ss.share(self.F, var, self.t, self.n)  # create shares

            s = name                                        # identify share
            st = time.time()
            self.server.securecom[s] = shares

        def reconstruct_secret1(self, c):
            res = []
            for i in range(self.n):
                res.append(self.get_broadcast(c + str(i)))
            return ss.rec(self.F, res)

        @staticmethod
        def reconstruct_secret(F, c):
             return ss.rec(F, c)

    ##############################################################################
    # Define field, number of parties, polynomial order, input matrices and more #
    ##############################################################################

    F = field.GF(792606555396977)   # prime, if larger the variables overfloat
                                    # if smaller, the scaling must be smaller to
                                    # to be within the field and the precision
                                    # decreases too much to return correct output
                                    # 792606555396977

    n = 3
    t = 1
    serv = server(F, n, t, 1500)  # minimum of 24  # been running with 1500

    # Matrix dimensions
    m = 2   # number of A rows
    nn = 2  # number of A coloums
    l = 1   # number of b rows
    mu=min(nn,m)

    # Protocol inputs
    A= 0 # initialize
    b= 0 # initialize

    tt = 1      # secret
    hh = 1      # secret

    # Secret matrices, input
    H= np.array([[2, 3], [4 , 9]])      # A, secret
    G= np.array([[6],[15]])             # b, secret
    iden_n=np.identity(nn)

    AB=np.hstack((H,G))

    rankAB=np.array(matrix_rank(AB))
    rankA= np.array(matrix_rank(H))

# =============================================================================
#     if rankA==rankAB:
#         print('System is solvable')
#     else:
#         print('Preconditioning fails, system not solvable')
# #
# =============================================================================
    # Generate Upper/Lower Toeplitz matrices (open)
    # OBS: random variables small range due to overfloat caused by scaling
    # to avoid floats when inverting

    L = toeplitz([1,random.randint(1,10)])
    U = toeplitz([1,random.randint(1,10)])

    L= np.array(np.tril(L) )               # remove upper triangular entries
    U= np.array(np.triu(U) )               # remove lower triangular entires

    #ran=7
    ran= random.randint(1,10)

    iden= 0

    #L= np.array(([1, 0],[11, 1]))
   # U= np.array(([1, 8],[0, 1]))

    #rank= np.array(matrix_rank(U@H@L))

    ## check i A'=UAL has generic rank profile

    #print('First square of C',(np.eye(2)@U@H@L))    # OK
    #print('Second square of C',(np.eye(2)@U@G))     # OK
    #if rank == nn:
    #    print('Generic rank OK')
    #else :
    #    print('Preconditioning fails')

    # create shares of each matrix entry
    gr=ground(F, A, b, hh, tt, iden, ran, n, t, serv) # hh, tt)

    for p in range(0, m):
        b=G[p]
        b=int(b)

        gr.distribute_shares(b,'b'+str(p))

        for q in range(0, nn):
            A=H[p,q]
            A=int(A)
            iden_ny=iden_n[p,q]
            iden_ny=int(iden_ny)

            gr.distribute_shares(A,'a'+str(p)+str(q))
            gr.distribute_shares(iden_ny,'iden'+str(p)+str(q))

    gr.distribute_shares(gr.hh,'h_shares') #hh
    gr.distribute_shares(gr.tt,'t_shares') #tt
    gr.distribute_shares(gr.ran,'ran_shares') #tt

    # all shares have been constructed

    # get shares in each party
    part1 = part(F,n,t,0, serv)
    part2 = part(F,n,t,1, serv)
    part3 = part(F,n,t,2, serv)

    part1.start()
    part2.start()
    part3.start()

    part1.join()  # wait for threats to finnish (syncronizing)
    part2.join()
    part3.join()

    a_p=part1.X
    b_p=part2.X
    c_p=part3.X

    # reconstruct solution
    X_fi1=gr.reconstruct_secret(F,[a_p[0,0],b_p[0,0],c_p[0,0]])
    X_fi2=gr.reconstruct_secret(F,[a_p[1,0],b_p[1,0],c_p[1,0]])

    #test= gr.reconstruct_secret(F, [75961882674271977, 75833536030433303, 75705189386594629])
    #print('rec:', test)

    res1=int(str(X_fi1))
    res2 = int(str(X_fi2))

    dummy =10E13

    # differentiate between positive and negativ numbers
    if res1 > dummy:
        res1 = res1 -792606555396977      # subtracting field due to wrap around
                                          # caused by negative value

    if res2 > dummy:
        res2= res2 -792606555396977

    # final result
    finalX1=res1/10E10
    finalX2 =res2/10E10
    #time_elapsed = (time.clock() - time_start)
    print(finalX1, finalX2)
    #print('Solution, X= [',finalX1, finalX2, ']')


import logging
from functools import reduce
from typing import Any, Dict, Type
from sqlalchemy.orm.attributes import flag_modified
from sqlalchemy.orm import reconstructor, relationship, backref
from datetime import datetime
import base64
import io
from PIL import Image
from sqlalchemy.sql.schema import Column, ForeignKey
from sqlalchemy.sql.sqltypes import DateTime, Integer, JSON, String

from app.database import Base

from app.character.core import CharacterMechanics, MECHANICS
from app.content.mixins import BaseContent

logger = logging.getLogger(__name__)

def fix_image(imagedata: str) -> str:
    imagetype, imagedata = imagedata.split(',')
    decoded = base64.b64decode(imagedata)
    buf = io.BytesIO(decoded)
    img = Image.open(buf).convert('RGB')
    img.thumbnail((192, 192))
    buf = io.BytesIO()
    img.save(buf, format='JPEG')
    return base64.b64encode(buf.getvalue()).decode('utf-8')

class Character(BaseContent, Base):
    __tablename__ = 'charactersheet'
    id = Column(Integer, primary_key=True)
    title = Column(String(256))
    body = Column(JSON)
    timestamp = Column(DateTime, index=True, default=datetime.utcnow,
                       onupdate=datetime.utcnow)
    user_id = Column(Integer, ForeignKey('user_profile.id'))
    player = relationship('UserProfile', backref=backref(
        'characters', lazy='dynamic'))

    folder = relationship('Folder', backref='characters')

    def __repr__(self):
        return '<Character {}>'.format(self.title)

    def __init__(self,
                 mechanics: Type[CharacterMechanics] = CharacterMechanics,
                 *args, **kwargs):
        super(Character, self).__init__(*args, **kwargs)
        self._data = None
        # Add a subclass or something that
        # has the mechanics of the character.
        self.mechanics = mechanics(self)

    @reconstructor
    def init_on_load(self):
        system = self.data.get('system', '')
        logger.debug(f"Loading character of type {system}")
        system = self.system
        self.mechanics = MECHANICS.get(system, CharacterMechanics)(self)

    @property
    def data(self) -> Dict[str, Any]:
        if isinstance(self.body, dict):
            return self.body
        raise TypeError("Body is not a dictionary")

    @property
    def system(self) -> str:
        s = self.data.get('system', None)
        if s is not None:
            return s

        logger.warning("Deprecation: Outdated character data")
        default = "Call of Cthulhu TM"
        if self.data.get('meta', {}).get('GameName') == default:
            logger.warning("Trying old CoC stuff.")
            return "coc7e"

        return "Unknown"

    @property
    def version(self):
        v = self.data.get('version', None)
        return v

    @property
    def game(self):
        return self.mechanics.game()

    def validate(self):
        return self.mechanics.validate()

    def to_dict(self):
        return {
            'id': self.id,
            'title': self.title,
            'body': self.data,
            'timestamp': self.timestamp,
            'user_id': self.user_id
        }

    def get_sheet(self):
        return self.data

    @property
    def name(self):
        return self.mechanics.name

    @property
    def age(self):
        return self.mechanics.age

    @property
    def portrait(self):
        return self.mechanics.portrait()

    @property
    def description(self):
        return self.mechanics.description

    def attribute(self, *args):

        path = args[0]

        val = reduce(lambda x, y: x.get(y, None) if x is not None else None,
                     path.split("."),
                     self.data)

        return val

    def set_attribute(self, attribute: Dict):
        """Set a specific attribute."""

        if attribute.get('category', None) == 'skill':
            logger.debug("Set a skill")
            datatype = attribute.get('type', 'string')
            skill = attribute['field']
            subfield = attribute.get('subfield', None)
            value = attribute.get('value')

            if datatype == 'number' and not isinstance(value, int):
                value = None

            skill = self.skill(skill, subfield)
            skill['value'] = value

        elif attribute.get('type', None) == 'skillcheck':
            logger.debug("Check a skill")
            skill = attribute['field']
            subfield = attribute.get('subfield', None)
            check = attribute.get('value', False)
            skill = self.skill(skill, subfield)
            skill['checked'] = check

        elif attribute.get('type', None) == 'occupationcheck':
            logger.debug("Mark occupation skill")
            skill = attribute['field']
            subfield = attribute.get('subfield', None)
            check = attribute.get('value', False)
            skill = self.skill(skill, subfield)
            skill['occupation'] = check

        elif attribute.get('type', None) == 'portrait':
            logger.debug("Set portrait")
            data = attribute.get('value', None)
            if data is not None:
                self.mechanics.set_portrait(fix_image(data))
        else:
            logger.debug(
                f"Set '{attribute['field']}' to '{attribute['value']}'")
            s = reduce(lambda x, y: x[y], attribute['field'].split(".")[:-1],
                       self.data)
            s[attribute['field'].split(".")[-1]] = attribute['value']

    def store_data(self):
        """Mark data as modified."""
        flag_modified(self, "body")

    def skill(self, *args, **kwargs):
        return self.mechanics.skill(*args, **kwargs)

    def skills(self, *args):
        """Return a list of skills."""
        return self.data['skills']

    def add_skill(self, skillname: str, value: int = 1):
        if self.skill(skillname) is not None:
            raise ValueError(f"Skill {skillname} already exists.")

        self.data['skills'].append({"name": skillname,
                                    "value": value,
                                    "start_value": value})
        if isinstance(self.data['skills'], list):
            self.data['skills'].sort(key=lambda x: x['name'])

    def add_subskill(self, name: str, parent: str):
        value = self.skill(parent)['value']
        start_value = self.skill(parent)['start_value']
        logger.debug("Try to add subskill")
        logger.debug(f"Name: {name}, parent {parent}, value {value}")
        if self.skill(parent, name) is not None:
            raise ValueError(f"Subskill {name} in {parent} already exists.")

        skill = self.skill(parent)
        if 'subskills' not in skill:
            skill['subskills'] = []
        skill['subskills'].append({
            'name': name,
            'value': value,
            'start_value': start_value
        })

    @property
    def schema_version(self):
        return self.data['meta']['Version']


from dagan_networks_wgan import *
import tensorflow as tf
from tensorflow.python.training import session_run_hook
from tensorflow.python.training import training_util
from tensorflow.python.training.session_run_hook import SessionRunArgs
import glob
import logging
import numpy as np
import pandas as pd
import PIL.Image
import os
import utils.interpolations

IMAGE_SIZE = (64, 64)

def null_dataset():
    def _input_fn():
        return None

    return _input_fn

def eval_fn():
    inputs = ['0']

    def _input_fn():
        ds = tf.data.Dataset.from_tensor_slices(inputs)

        def _fake_image(_):
            z_vectors = np.random.normal(size=(1, 1))
            z = tf.zeros([1, 1, 1, 3], dtype=tf.float32)
            return {'i': z, 'j': z, 'z': z_vectors}, 0

        ds = ds.map(_fake_image, num_parallel_calls=1)
        return ds

    return _input_fn

def test_fn(params):
    num_generations = params['num_generations']
    batch = params['batch_size']
    z_vectors = utils.interpolations.create_mine_grid(rows=num_generations, cols=num_generations,
                                                      dim=params['z_dim'], space=3, anchors=None,
                                                      spherical=True, gaussian=True)
    im = params['test_image']
    im = PIL.Image.open(im)
    im = im.resize(IMAGE_SIZE)
    im = np.asarray(im)
    im = im / 127.5 - 1
    im = np.reshape(im, (1, IMAGE_SIZE[0], IMAGE_SIZE[1], 3))
    im = np.repeat(im, batch, axis=0)
    l0 = np.zeros((params['batch_size']), dtype=np.int32)

    def _input_fn():
        def _gen():
            for i in range(0, num_generations * num_generations, batch):
                z_vector = z_vectors[i:i + batch]
                if len(z_vector) < batch:
                    z_vector = np.pad(z_vector, ((0, batch - len(z_vector)), (0, 0)), 'constant')
                yield ({'i': im, 'j': im, 'z': z_vector}, l0)

        cshape = tf.TensorShape([batch, IMAGE_SIZE[0], IMAGE_SIZE[1], 3])
        ds = tf.data.Dataset.from_generator(_gen, ({'i': tf.float32, 'j': tf.float32, 'z': tf.float32}, tf.int32), (
            {'i': cshape, 'j': cshape, 'z': tf.TensorShape([batch, params['z_dim']])},
            tf.TensorShape([batch])))
        return ds

    return _input_fn

def input_fn(params, is_training):
    limit = params['limit_train']
    if limit is None:
        limit = -1

    class_files = []
    all_files = 0
    for i in glob.glob(params['data_set'] + '/*'):
        files = glob.glob(i + '/*.jpg')
        class_files.append(files)
        all_files += len(files)

    if limit < 1:
        limit = all_files

    z = np.zeros((params['batch_size'], params['z_dim']), dtype=np.float32)
    l0 = np.zeros((params['batch_size']), dtype=np.int32)
    limit = limit * params['epoch'] // params['batch_size'] + 1
    logging.info('Steps number: {}'.format(limit))

    def _input_fn():
        def _gen():
            for _ in range(limit):
                sc = np.random.choice(len(class_files), params['batch_size'])
                a_batch = []
                b_batch = []
                for i in range(params['batch_size']):
                    samples = np.random.choice(class_files[sc[i]], 2)
                    im = PIL.Image.open(samples[0])
                    im = im.resize(IMAGE_SIZE)
                    a = np.asarray(im)
                    im = PIL.Image.open(samples[1])
                    im = im.resize(IMAGE_SIZE)
                    b = np.asarray(im)
                    a_batch.append(a)
                    b_batch.append(b)
                a_batch = np.stack(a_batch) / 127.5 - 1
                b_batch = np.stack(b_batch) / 127.5 - 1
                yield ({'i': a_batch, 'j': b_batch, 'z': z}, l0)

        cshape = tf.TensorShape([params['batch_size'], IMAGE_SIZE[0], IMAGE_SIZE[1], 3])
        ds = tf.data.Dataset.from_generator(_gen, ({'i': tf.float32, 'j': tf.float32, 'z': tf.float32}, tf.int32), (
            {'i': cshape, 'j': cshape, 'z': tf.TensorShape([params['batch_size'], params['z_dim']])},
            tf.TensorShape([params['batch_size']])))
        ds = ds.prefetch(4)
        return ds

    return _input_fn

def _encoder_model_fn(features, labels, mode, params=None, config=None):
    generator_layers = [64, 64, 128, 128]
    discriminator_layers = [64, 64, 128, 128]
    gen_depth_per_layer = params['generator_inner_layers']
    discr_depth_per_layer = params['discriminator_inner_layers']
    gen_inner_layers = [gen_depth_per_layer, gen_depth_per_layer, gen_depth_per_layer, gen_depth_per_layer]
    discr_inner_layers = [discr_depth_per_layer, discr_depth_per_layer, discr_depth_per_layer,
                          discr_depth_per_layer]
    generator_layer_padding = ["SAME", "SAME", "SAME", "SAME"]
    a = features['i']
    z = features['z']
    b = None
    if (mode == tf.estimator.ModeKeys.TRAIN):
        b = features['j']
    dagan = DAGAN(batch_size=params['batch_size'], input_x_i=a, input_x_j=b,
                  dropout_rate=params['dropout_rate'], generator_layer_sizes=generator_layers,
                  generator_layer_padding=generator_layer_padding, num_channels=a.shape[3],
                  is_training=(mode == tf.estimator.ModeKeys.TRAIN),
                  augment=tf.constant(params['random_rotate'], dtype=tf.bool),
                  discriminator_layer_sizes=discriminator_layers,
                  discr_inner_conv=discr_inner_layers,
                  gen_inner_conv=gen_inner_layers, z_dim=params['z_dim'], z_inputs=z,
                  use_wide_connections=params['use_wide_connections'])

    if (mode == tf.estimator.ModeKeys.TRAIN):
        generated = None
        losses, graph_ops = dagan.init_train()
        accumulated_d_loss = tf.Variable(0.0, trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES])
        acc_d_loss_op = accumulated_d_loss.assign_add(losses["d_losses"])
        acc_d_loss_zero_op = accumulated_d_loss.assign(tf.zeros_like(accumulated_d_loss))
        accumulated_g_loss = tf.Variable(0.0, trainable=False, collections=[tf.GraphKeys.LOCAL_VARIABLES])
        acc_g_loss_op = accumulated_g_loss.assign_add(losses["g_losses"])
        acc_g_loss_zero_op = accumulated_g_loss.assign(tf.zeros_like(accumulated_g_loss))
        reset = tf.group(acc_d_loss_zero_op, acc_g_loss_zero_op)
        d_train = tf.group(graph_ops['d_opt_op'], acc_d_loss_op)
        g_train = tf.group(graph_ops['g_opt_op'], acc_g_loss_op)
        train_hooks = [
            MultiStepOps(params['discriminator_inner_steps'], params['generator_inner_steps'], d_train, g_train, reset)]
        total_loss = accumulated_d_loss + accumulated_g_loss
        train_op = training_util._increment_global_step(1)  # pylint: disable=protected-access
    else:
        _, generated = dagan.sample_same_images()
        output = tf.identity(generated,name='output')
        export_outputs = {
            tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: tf.estimator.export.PredictOutput(
                output)}
        total_loss = None
        train_op = None
        train_hooks = None

    return tf.estimator.EstimatorSpec(
        mode=mode,
        loss=total_loss,
        predictions=generated,
        training_hooks=train_hooks,
        export_outputs=export_outputs,
        train_op=train_op)

class MultiStepOps(session_run_hook.SessionRunHook):
    def __init__(self, d_steps, g_steps, d_train, g_train, reset):
        self._global_step_tensor = None
        self._d_train = d_train
        self._g_train = g_train
        self._d_steps = d_steps
        self._g_steps = g_steps
        self._reset = reset

    def begin(self):
        None

    def before_run(self, run_context):  # pylint: disable=unused-argument
        for n in range(self._d_steps):
            run_context.session.run([self._d_train])
        for n in range(self._g_steps):
            run_context.session.run([self._g_train])
        return None

    def after_run(self, run_context, run_values):
        run_context.session.run([self._reset])

class DAGANEstimator(tf.estimator.Estimator):
    def __init__(
            self,
            params=None,
            model_dir=None,
            config=None,
            warm_start_from=None
    ):
        def _model_fn(features, labels, mode, params, config):
            return _encoder_model_fn(
                features=features,
                labels=labels,
                mode=mode,
                params=params,
                config=config)

        super(DAGANEstimator, self).__init__(
            model_fn=_model_fn,
            model_dir=model_dir,
            config=config,
            params=params,
            warm_start_from=warm_start_from
        )


import pygame, sys, ng, st,load, random
#############################################################################################

pygame.init()
try:
    settingsfile=open('settings.txt', 'r')
    lines=settingsfile.readlines()
    lenght=len(lines)
    for l in range(lenght):
        lines[l]=lines[l].replace('\n','')
    window_width = int(lines[0])
    window_height = int(lines[1])
    fullscreen = int(lines[2])
    settingsfile.close()
    if fullscreen==0:
        window = pygame.display.set_mode((window_width, window_height))
    elif fullscreen==1:
        window = pygame.display.set_mode((window_width, window_height),pygame.FULLSCREEN)
except FileNotFoundError:
    settingsfile = open('settings.txt', 'w')
    window_width = 1280
    window_height = 720
    fullscreen=0
    settingsfile.write("%s\n"%window_width)
    settingsfile.write("%s\n"%window_height)
    settingsfile.write("%s"%fullscreen)
    window = pygame.display.set_mode((window_width, window_height))
pygame.display.set_caption('MFF&K')
#Visual evects boxes
box1 = pygame.Rect(0, 0, 2, window_height)
box2 = pygame.Rect(window_width, window_height, window_width, window_height)
# Drawing
pygame.draw.rect(window, (19, 31, 109), pygame.Rect(0, 0, window_width, window_height))
#Variables
clock=pygame.time.Clock()
time=0.0
ticks=0
maxfps = 60
font=pygame.font.SysFont("Times New Roman, Arial", 40)
while True:
    #Coursor position
    pos=pygame.mouse.get_pos()

    #Ticking
    time += clock.tick()/1000.0
    while time >1/maxfps and ticks<=window_height/4:
        pygame.event.set_grab(False)
        time-= 1/maxfps
        ticks+=1
    #Visual Evects
        box1.width += 2
        box2.y -= 2
        box2.x -= 2
    #Display
    pygame.draw.rect(window, (19, 31, 109), pygame.Rect(0, 0, window_width, window_height))
    newgame = font.render('New Game', True, [0, 0, 0])
    loadgame = font.render('Load Game', True, [0, 0, 0])
    settings = font.render('Settings', True, [0, 0, 0])
    quitgame = font.render('Quit', True, [0, 0, 0])
    logo = pygame.image.load('logo.png')
    newgamebox = pygame.Rect(window_width/2-newgame.get_rect().width/2 -5, (window_height/2)-105, newgame.get_rect().width + 10, 50)
    loadgamebox =pygame.Rect(window_width/2-newgame.get_rect().width/2 -5, (window_height/2)-5, newgame.get_rect().width + 10, 50)
    settingsbox= pygame.Rect(window_width/2-newgame.get_rect().width/2 -5, (window_height/2)+95, newgame.get_rect().width + 10, 50)
    quitgamebox = pygame.Rect(window_width/2-newgame.get_rect().width/2 -5, (window_height/2)+195, newgame.get_rect().width + 10, 50)
    pygame.draw.rect(window, (0, 178, 255), box1)
    pygame.draw.rect(window, (0, 178, 255), box2)

    if ticks>=window_height/4:
        # Handle events
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit(0)
            if event.type == pygame.MOUSEBUTTONDOWN:
                if quitgamebox.collidepoint(event.pos):
                    sys.exit(0)
                if loadgamebox.collidepoint(event.pos):
                    load.loadf(window, box1, box2, logo, window_height,window_width,font)
                # New game
                if newgamebox.collidepoint(event.pos):
                    ######option=0 back, option=1 easy, option=2 normal option=3 hard######
                    option = ng.newgamef(window, box1, box2, logo, window_height, window_width, font, ticks, time)
                    if option == 0:
                        continue
                    else:
                        option1 = ng.createpartyf(window, box1, box2, window_height, window_width, clock, font, maxfps,
                                                  option)
                        if option1 == 0:
                            ticks = 0
                            time = 0
                            break
                if settingsbox.collidepoint(event.pos):
                    apply = st.settings(window, box1, box2, logo, window_height, window_width, font, ticks)
                    if apply == 1:
                        time = 0
                        ticks = 0
                        settingsfile = open('settings.txt', 'r')
                        lines = settingsfile.readlines()
                        lenght = len(lines)
                        for l in range(lenght):
                            lines[l] = lines[l].replace('\n', '')
                        window_width = int(lines[0])
                        window_height = int(lines[1])
                        fullscreen = int(lines[2])
                        settingsfile.close()
                        box1 = pygame.Rect(0, 0, 2, window_height)
                        box2 = pygame.Rect(window_width, window_height, window_width, window_height)
                        if fullscreen == 0:
                            window = pygame.display.set_mode((window_width, window_height))
                        elif fullscreen == 1:
                            window = pygame.display.set_mode((window_width, window_height), pygame.FULLSCREEN)
        window.blit(logo, (window_width/2-logo.get_rect().width/2, 0))
        pygame.draw.rect(window, (0, 178, 255), newgamebox)
        pygame.draw.rect(window, (0, 178, 255), loadgamebox)
        pygame.draw.rect(window, (0, 178, 255), settingsbox)
        pygame.draw.rect(window, (0, 178, 255), quitgamebox)
        window.blit(newgame, (window_width / 2 - newgame.get_rect().width / 2, window_height / 2 - 100))
        window.blit(loadgame, (window_width / 2 - loadgame.get_rect().width / 2, window_height / 2))
        window.blit(settings, (window_width / 2 - settings.get_rect().width / 2, window_height / 2+100))
        window.blit(quitgame, (window_width / 2 - quitgame.get_rect().width / 2, window_height / 2 + 200))
    #Mouse in rectangle events

        if newgamebox.collidepoint(pos):
            pygame.draw.rect(window, (240, 240, 0), newgamebox)
            window.blit(newgame, (window_width / 2 - newgame.get_rect().width / 2, window_height / 2 - 100))
        elif loadgamebox.collidepoint(pos):
            pygame.draw.rect(window, (240, 240, 0), loadgamebox)
            window.blit(loadgame, (window_width / 2 - newgame.get_rect().width / 2, window_height / 2 ))
        elif settingsbox.collidepoint(pos):
            pygame.draw.rect(window, (240, 240, 0), settingsbox)
            window.blit(settings, (window_width / 2 - settings.get_rect().width / 2, window_height / 2 + 100))
        elif quitgamebox.collidepoint(pos):
            pygame.draw.rect(window, (240, 240, 0), quitgamebox)
            window.blit(quitgame, (window_width / 2 - quitgame.get_rect().width / 2, window_height / 2 + 200))
    pygame.display.update()


#!/usr/local/bin/python3.6
# -*- coding: utf-8 -*-

import datetime
import smtplib
from email.mime.text import MIMEText
import sqlite3

mail_from = ''
conn = sqlite3.connect('db.sqlite3')
c = conn.cursor()

# ┌───────────── minute (0 - 59)
# │ ┌───────────── hour (0 - 23)
# │ │ ┌───────────── day of month (1 - 31)
# │ │ │ ┌───────────── month (1 - 12)
# │ │ │ │ ┌───────────── day of week (0 - 6) (Sunday to Saturday;
# │ │ │ │ │                                       7 is also Sunday on some systems)
# │ │ │ │ │
# │ │ │ │ │
# * * * * *  command to execute

#weekends = json.loads('{"1":[1,2,3,4,5,6,7,8,14,15,21,22,28,29],"2":[4,5,11,12,18,19,23,24,25,26],"3":[4,5,8,11,12,18,19,25,26],"4":[1,2,8,9,15,16,22,23,29,30],"5":[1,6,7,8,9,13,14,20,21,27,28],"6":[3,4,10,11,12,17,18,24,25],"7":[1,2,8,9,15,16,22,23,29,30],"8":[5,6,12,13,19,20,26,27],"9":[2,3,9,10,16,17,23,24,30],"10":[1,7,8,14,15,21,22,28,29],"11":[4,5,6,11,12,18,19,25,26],"12":[2,3,9,10,16,17,23,24,30,31]}')
#short_days = json.loads('{"1":[],"2":[22],"3":[7],"4":[],"5":[],"6":[],"7":[],"8":[],"9":[],"10":[],"11":[3],"12":[]}')

def check_cron(cur):
    year, month, day, hour, minute, second, day_of_week, year_day, x = datetime.datetime.now().timetuple()
    select = cur.execute('SELECT id, schedule FROM mailing_email ORDER BY id')
    id_list = []
    for row in select:
        mail_id, cron = row
        if str(minute) == cron.split()[0] or '*' == cron.split()[0]:
            if str(hour) == cron.split()[1] or '*' == cron.split()[1]:
                if str(day) == cron.split()[2] or '*' == cron.split()[2]:
                    if str(month) == cron.split()[3] or '*' == cron.split()[3]:
                        if str(day_of_week) == cron.split()[4] or '*' == cron.split()[4]:
                            id_list.append(mail_id)
    if len(id_list) > 0:
        return id_list
    else:
        return False

def send_mails(data, cur, mail_from, email):
    if data:
        for id in data:
            to, cc, subject, body = cur.execute('SELECT to, cc, subject, body FROM mailing_email WHERE id={0}'.format(id))
            msg = MIMEText(email.format(body), 'html')
            msg['Subject'] = subject
            msg['From'] = mail_from
            msg['To'] = to
            msg['CC'] = cc
            server = smtplib.SMTP('localhost')
            server.sendmail(mail_from, to, msg.as_string())
            server.quit()

email = '''<!doctype html>

<meta name="viewport" content="width=device-width">
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<title>HTML Email</title>

{0}

'''

data = check_cron(c)
if data:
    send_mails(data, c, mail_from, email)


import numpy as np
import matplotlib.pyplot as plt
from scipy.spatial import ConvexHull as ch

if __name__ == '__main__':
    losses = np.array([
        [2.0, 3.0],
        [4.0, 0.0],
        [1.0, 1.0],
        [9.0, 8.0],
    ])
    grid_dot_num = 1000
    min_val, max_val = -1.0, 10.0
    lambda_coeff = 0.6
    hull = ch(losses)
    figsize = tuple(1.0 * np.array([7.5, 7.5]))
    plt.figure(figsize=figsize)
    plt.xlabel(r'$G_1$')
    plt.ylabel(r'$G_2$')
    plt.plot(losses[:, 0], losses[:, 1], 'ko')
    for i in range(losses.shape[0]):
        plt.annotate(r'$\alpha_%i$' % (i + 1), losses[i] + [-0.2, 0.3])
    for simplex in hull.simplices:
        plt.plot(losses[simplex, 0], losses[simplex, 1], 'k-')
    x, y = np.linspace(min_val, max_val, grid_dot_num), np.linspace(min_val, max_val, grid_dot_num)
    plt.plot(x, y, 'k--')
    plt.axis([min_val, max_val, min_val, max_val])
    xx, yy = np.meshgrid(x, y)
    wedge = lambda_coeff * np.minimum(xx, yy) + (1 - lambda_coeff) * np.maximum(xx, yy)
    plt.contour(xx, yy, wedge, [1.0], colors=['black'])
    plt.grid(True)
    plt.show()
    plt.close()


# pylint: skip-file
import os
# MXNET_CPU_WORKER_NTHREADS must be greater than 1 for custom op to work on CPU
os.environ["MXNET_CPU_WORKER_NTHREADS"] = "8"
import mxnet as mx
import numpy as np
import logging
from ast import literal_eval as make_tuple

class MultiBoxTarget2(mx.operator.CustomOp):
    """
    Python implementation of MultiBoxTarget layer.
    """
    def __init__(self, n_class, img_wh, th_iou, th_iou_neg, th_nms_neg, max_pos_sample,
            reg_sample_ratio, hard_neg_ratio, ignore_label, #box_ratios,
            variances, per_cls_reg, normalization):
        #
        super(MultiBoxTarget2, self).__init__()
        self.n_class = n_class
        self.img_shape = (0, 0, img_wh[0], img_wh[1])
        self.th_iou = th_iou
        self.th_iou_neg = th_iou_neg
        self.th_nms_neg = th_nms_neg
        self.max_pos_sample = max_pos_sample
        self.reg_sample_ratio = reg_sample_ratio
        self.hard_neg_ratio = hard_neg_ratio
        self.ignore_label = ignore_label
        # self.box_ratios = box_ratios
        self.variances = variances
        self.per_cls_reg = per_cls_reg
        self.normalization = normalization

        # precompute nms candidates
        self.nidx_neg = None
        self.anchors_t = None
        self.area_anchors_t = None

        self.th_anc_overlap = 0.6
        self.max_sample = int(self.max_pos_sample + \
                          self.reg_sample_ratio * self.max_pos_sample + \
                          self.hard_neg_ratio * self.max_pos_sample)

    def forward(self, is_train, req, in_data, out_data, aux):
        """
        Compute IOUs between valid labels and anchors.
        Then sample positive and negatives.
        """
        # assert len(in_data) == 2
        # inputs: ['preds_cls', 'preds_reg', 'anchors', 'label']
        # outs: ['sample_cls', 'sample_reg', 'target_cls', 'target_reg', 'mask_reg']
        n_batch, n_anchors, nch = in_data[0].shape

        preds_cls = in_data[0] # (n_batch, n_anchor, nch)
        preds_reg = in_data[1]
        labels_all = in_data[3].asnumpy().astype(np.float32) # (batch, num_label, 5)
        # probs_cls = mx.nd.slice_axis(in_data[4], axis=1, begin=1, end=None)
        # max_probs_cls = mx.nd.slice_axis(in_data[4], axis=1, begin=0, end=1).asnumpy()
        # max_probs_cls = 1 - np.reshape(max_probs_cls, (n_batch, -1))
        # probs_cls = in_data[4].asnumpy()
        max_cids = mx.nd.argmax(in_data[4], axis=1).asnumpy().astype(int)
        probs_bg_cls = mx.nd.slice_axis(in_data[4], axis=1, begin=0, end=1).asnumpy()
        probs_bg_cls = 1 - np.reshape(probs_bg_cls, (n_batch, -1))

        # precompute some data
        if self.anchors_t is None:
            self.anchors = in_data[2].asnumpy()
            self.anchors_t = mx.nd.transpose(in_data[2], (1, 0)).copy()
            self.area_anchors_t = \
                    (self.anchors_t[2] - self.anchors_t[0]) * (self.anchors_t[3] - self.anchors_t[1])
            self.nidx_neg = [[]] * n_anchors
            # self.nidx_pos = [[]] * n_anchors
            overlaps = _compute_overlap(self.anchors_t, self.area_anchors_t, self.img_shape)
            self.oob_mask = (overlaps <= self.th_anc_overlap)
        else:
            assert self.anchors.shape == in_data[2].shape

        # process each batch
        sample_cls = mx.nd.zeros((self.max_sample, nch), ctx=in_data[0].context)
        target_cls = np.full((self.max_sample), -1, dtype=np.float32)

        nch_reg = 4 * self.n_class if self.per_cls_reg else 4
        sample_reg = mx.nd.zeros((self.max_sample, nch_reg), ctx=in_data[0].context)
        target_reg = np.full((self.max_sample, nch_reg), -1, dtype=np.float32)
        mask_reg = np.zeros((self.max_sample, nch_reg), dtype=np.float32)

        anchor_locs_all = np.full((self.max_sample, 2), -1, dtype=np.int32)

        # positive samples
        anchor_locs_pos = np.empty((0, 2), dtype=int)
        tc_pos = np.empty((0,), dtype=np.int32)
        tr_pos = np.empty((0, nch_reg))
        mr_pos = np.empty((0, nch_reg))
        # probs_pos = []
        max_iou_pos = []
        n_pos_sample = 0
        for i in range(n_batch):
            anchor_locs, tc, tr, mr, max_iou = \
                    self._forward_batch_pos(labels_all[i], max_cids[i], i)
            anchor_locs_pos = np.vstack((anchor_locs_pos, anchor_locs))
            tc_pos = np.append(tc_pos, tc)
            tr_pos = np.vstack((tr_pos, tr))
            mr_pos = np.vstack((mr_pos, mr))
            # probs_pos += p
            max_iou_pos.append(max_iou)
            n_pos_sample = np.maximum(n_pos_sample, np.sum(np.array(tc) > 0))

        # negative samples
        n_neg_sample = np.maximum(1, np.round(self.hard_neg_ratio * n_pos_sample))
        anchor_locs_neg = []
        probs_neg = []
        # gather per batch samples
        for i in range(n_batch):
            anchor_locs, p = self._forward_batch_neg(n_neg_sample, probs_bg_cls[i], max_iou_pos[i], i)
            anchor_locs_neg += anchor_locs
            probs_neg += p

        # subsample pos/reg samples if we have too many
        max_sample = int(1 + self.reg_sample_ratio) * self.max_pos_sample
        if len(tc_pos) > max_sample:
            pidx = np.random.choice(np.arange(len(tc_pos)), max_sample, replace=False)
            anchor_locs_pos = anchor_locs_pos[pidx]
            tc_pos = tc_pos[pidx]
            tr_pos = tr_pos[pidx, :]
            mr_pos = mr_pos[pidx, :]

        # hard negative sampling with whole batchs
        n_max_sample_neg = np.sum(tc_pos > 0) * self.hard_neg_ratio
        sidx = np.argsort(np.array(probs_neg))[::-1]
        if len(sidx) > n_max_sample_neg:
            sidx = sidx[:n_max_sample_neg]
        anchor_locs_neg = np.array(anchor_locs_neg)[sidx, :]

        # subsample neg samples if we have too many
        max_sample = int(self.hard_neg_ratio) * self.max_pos_sample
        if len(anchor_locs_neg) > max_sample:
            assert False
            pidx = np.random.choice(np.arange(len(anchor_locs_neg)), max_sample, replace=False)
            anchor_locs_neg = anchor_locs_neg[pidx, :]

        # gather and arrange samples we will use
        k = 0
        for i, aloc in enumerate(anchor_locs_pos):
            if k >= self.max_sample:
                break
            bid = aloc[0]
            aid = aloc[1]

            target_cls[k] = tc_pos[i]
            sample_cls[k] = preds_cls[bid][aid]
            target_reg[k] = tr_pos[i]
            mask_reg[k] = mr_pos[i]
            sample_reg[k] = preds_reg[bid][aid]
            anchor_locs_all[k] = aloc
            k += 1

        for aloc in anchor_locs_neg:
            if k >= self.max_sample:
                break
            bid = aloc[0]
            aid = aloc[1]

            target_cls[k] = 0
            sample_cls[k] = preds_cls[bid][aid]
            sample_reg[k] = preds_reg[bid][aid]
            anchor_locs_all[k] = aloc
            k += 1

        # mask_reg = np.reshape(mask_reg, (-1, 1))

        self.assign(aux[0], 'write', mx.nd.array(anchor_locs_all))
        self.assign(out_data[0], req[0], sample_cls)
        self.assign(out_data[1], req[1], sample_reg)
        self.assign(out_data[2], req[2], mx.nd.array(target_cls, ctx=in_data[0].context))
        self.assign(out_data[3], req[3], mx.nd.array(target_reg, ctx=in_data[0].context))
        self.assign(out_data[4], req[4], mx.nd.array(mask_reg, ctx=in_data[0].context))

    def backward(self, req, out_grad, in_data, out_data, in_grad, aux):
        # pass the gradient to their corresponding positions
        grad_cls = mx.nd.zeros(in_grad[0].shape, ctx=in_grad[0].context) # (n n_anchor nch)
        grad_reg = mx.nd.zeros(in_grad[1].shape, ctx=in_grad[1].context) # (n n_anchor nch)
        n_batch = grad_cls.shape[0]
        target_locs = aux[0].asnumpy().astype(int)
        sample_cls = out_data[2].asnumpy().astype(int)

        if self.normalization == True:
            norm_cls = 1.0 / mx.nd.maximum(1.0, mx.nd.sum(out_data[2] >= 0))
            norm_reg = 1.0 / mx.nd.maximum(1.0, mx.nd.sum(out_data[4] > 0) / 4.0)
        else:
            norm_cls = 1.0
            norm_reg = 1.0

        for i, tloc in enumerate(target_locs):
            bid = tloc[0]
            aid = tloc[1]
            if bid == -1 or aid == -1:
                continue
            sf = 1.0 / self.hard_neg_ratio if sample_cls[i] == 0 else 1
            grad_cls[bid][aid] = out_grad[0][i] * norm_cls * sf
            grad_reg[bid][aid] = out_grad[1][i] * norm_reg * sf

        self.assign(in_grad[0], req[0], grad_cls)
        self.assign(in_grad[1], req[1], grad_reg)
        self.assign(in_grad[2], req[2], 0)
        self.assign(in_grad[3], req[3], 0)
        self.assign(in_grad[4], req[4], 0)

    def _forward_batch_pos(self, labels, max_cids, batch_id):
        n_anchors = self.anchors_t.shape[1]

        cls_target = np.zeros((n_anchors,), dtype=np.int32)
        nch_reg = 4 * self.n_class if self.per_cls_reg else 4
        reg_target = np.zeros((n_anchors, nch_reg))
        reg_mask = np.zeros((n_anchors, nch_reg))

        labels = _get_valid_labels(labels)
        max_iou = np.zeros(n_anchors)

        for label in labels:
            gt_cls = int(label[0])
            # import ipdb
            # ipdb.set_trace()
            # lsq = _fit_box_ratio(label[1:], self.box_ratios)
            iou = _compute_iou(label[1:], self.anchors_t, self.area_anchors_t)

            # skip already occupied ones
            iou_mask = iou > max_iou
            max_iou = np.maximum(iou, max_iou)
            if label[0] == -1:
                continue
            # skip oob boxes
            iou_mask = np.logical_and(iou_mask, self.oob_mask == False)

            # positive and regression samples
            pidx = np.where(np.logical_and(iou_mask, iou > self.th_iou))[0]
            ridx = np.where(np.logical_and(iou_mask, iou > self.th_iou_neg))[0]
            ridx = np.setdiff1d(ridx, pidx)

            if pidx.size > 5:
                pidx = np.random.choice(pidx, 5, replace=False)
            cls_target[pidx] = gt_cls
            rt, rm = _compute_loc_target(label[1:], self.anchors[pidx, :], self.variances)
            if self.per_cls_reg:
                rt, rm = _expand_target(rt, gt_cls, self.n_class)
            reg_target[pidx, :] = rt
            reg_mask[pidx, :] = rm

            if not self.per_cls_reg:
                ridx = ridx[max_cids[ridx] == gt_cls]
            if ridx.size > pidx.size * self.reg_sample_ratio:
                ridx = np.random.choice(ridx, pidx.size * self.reg_sample_ratio, replace=False)
            cls_target[ridx] = -1
            rt, rm = _compute_loc_target(label[1:], self.anchors[ridx, :], self.variances)
            if self.per_cls_reg:
                rt, rm = _expand_target(rt, gt_cls, self.n_class)
            reg_target[ridx, :] = rt
            reg_mask[ridx, :] = rm

        # gather data
        anc_idx = np.where(cls_target != 0)[0]
        cls_target = cls_target[anc_idx]
        reg_target = reg_target[anc_idx, :]
        reg_mask = reg_mask[anc_idx, :]
        anchor_locs = np.zeros((len(anc_idx), 2), dtype=np.int32)
        anchor_locs[:, 0] = batch_id
        anchor_locs[:, 1] = anc_idx
        return anchor_locs, cls_target, reg_target, reg_mask, max_iou

    def _forward_batch_neg(self, n_neg_sample, max_probs, neg_iou, batch_id):
        # first remove positive samples from mining
        pidx = np.where(neg_iou > self.th_iou_neg)[0]
        max_probs[pidx] = -1.0

        neg_probs = []
        neg_anchor_locs = []

        eidx = np.argsort(max_probs)[::-1]

        # pick hard samples one by one, with nms
        for ii in eidx:
            if max_probs[ii] < 0.0 or self.oob_mask[ii]:
                continue

            neg_probs.append(max_probs[ii])
            neg_anchor_locs.append((batch_id, ii))
            # apply nms
            if len(self.nidx_neg[ii]) == 0:
                self.nidx_neg[ii] = _compute_nms_cands( \
                        self.anchors[ii], self.anchors_t, self.area_anchors_t, self.th_nms_neg)
            nidx = self.nidx_neg[ii]
            # nidx = ii
            max_probs[nidx] = -1
            if len(neg_anchor_locs) >= n_neg_sample:
                break

        return neg_anchor_locs, neg_probs

def _get_valid_labels(labels):
    #
    n_valid_label = 0
    for label in labels:
        if np.all(label == -1.0):
            break
        n_valid_label += 1
    return labels[:n_valid_label, :]

# def _fit_box_ratio(bb, bb_ratios):
#     #
#     logr = np.log((bb[2] - bb[0]) / (bb[3] - bb[1]))
#     diff_a = np.abs(logr - np.log(bb_ratios))
#     midx = np.argmin(diff_a)
#     target_r = bb_ratios[midx]
#
#     cx = (bb[0] + bb[2]) / 2.0
#     cy = (bb[1] + bb[3]) / 2.0
#
#     ww = (bb[2] - bb[0])
#     hh = (bb[3] - bb[1])
#     ww2 = np.maximum(ww, hh * target_r) / 2.0
#     hh2 = np.maximum(hh, ww * target_r) / 2.0
#     return np.array((cx - ww2, cy - hh2, cx + ww2, cy + hh2))

def _compute_nms_cands(anc, anchors_t, area_anchors_t, th_nms):
    #
    iou = _compute_iou(anc, anchors_t, area_anchors_t)
    iidx = np.where(iou > th_nms)[0]
    return iidx

def _compute_iou(label, anchors_t, area_anchors_t):
    #
    iw = mx.nd.minimum(label[2], anchors_t[2]) - mx.nd.maximum(label[0], anchors_t[0])
    ih = mx.nd.minimum(label[3], anchors_t[3]) - mx.nd.maximum(label[1], anchors_t[1])
    I = mx.nd.maximum(iw, 0) * mx.nd.maximum(ih, 0)
    U = (label[3] - label[1]) * (label[2] - label[0]) + area_anchors_t

    iou = I / mx.nd.maximum((U - I), 0.000001)
    return iou.asnumpy() # (num_anchors, )

def _compute_overlap(anchors_t, area_anchors_t, img_shape):
    #
    iw = mx.nd.minimum(img_shape[2], anchors_t[2]) - mx.nd.maximum(img_shape[0], anchors_t[0])
    ih = mx.nd.minimum(img_shape[3], anchors_t[3]) - mx.nd.maximum(img_shape[1], anchors_t[1])
    I = mx.nd.maximum(iw, 0) * mx.nd.maximum(ih, 0)
    overlap = I / area_anchors_t
    return overlap.asnumpy()

def _compute_loc_target(gt_bb, bb, variances):
    loc_target = np.zeros_like(bb)
    aw = (bb[:, 2] - bb[:, 0])
    ah = (bb[:, 3] - bb[:, 1])
    loc_target[:, 0] = ((gt_bb[2] + gt_bb[0]) - (bb[:, 2] + bb[:, 0])) * 0.5 / aw
    loc_target[:, 1] = ((gt_bb[3] + gt_bb[1]) - (bb[:, 3] + bb[:, 1])) * 0.5 / ah
    loc_target[:, 2] = np.log2((gt_bb[2] - gt_bb[0]) / aw)
    loc_target[:, 3] = np.log2((gt_bb[3] - gt_bb[1]) / ah)
    return loc_target / variances, np.ones_like(loc_target)

def _expand_target(loc_target, cid, n_cls):
    n_target = loc_target.shape[0]
    loc_target_e = np.zeros((n_target, 4 * n_cls), dtype=np.float32)
    loc_mask_e = np.zeros_like(loc_target_e)
    sidx = cid * 4
    loc_target_e[:, sidx:sidx+4] = loc_target
    loc_mask_e[:, sidx:sidx+4] = 1
    # for i in range(n_target):
    #     loc_target_e[i, sidx:sidx+4] = loc_target
    #     loc_mask_e[i, sidx:sidx+4] = 1
    return loc_target_e, loc_mask_e

@mx.operator.register("multibox_target2")
class MultiBoxTargetProp2(mx.operator.CustomOpProp):
    def __init__(self, n_class, img_wh,
            th_iou=0.5, th_iou_neg=1.0/3.0, th_nms_neg=1.0/3.0, max_pos_sample=5120,
            reg_sample_ratio=1.0, hard_neg_ratio=3.0, ignore_label=-1, # box_ratios=(1.0,),
            variances=(0.1, 0.1, 0.2, 0.2), per_cls_reg=False, normalization=False):
        #
        super(MultiBoxTargetProp2, self).__init__(need_top_grad=True)
        self.n_class = int(n_class)
        if isinstance(img_wh, str):
            img_wh = make_tuple(img_wh)
        self.img_wh = img_wh
        self.th_iou = float(th_iou)
        self.th_iou_neg = float(th_iou_neg)
        self.th_nms_neg = float(th_nms_neg)
        self.max_pos_sample = int(max_pos_sample)
        self.reg_sample_ratio = int(reg_sample_ratio)
        self.hard_neg_ratio = int(hard_neg_ratio)
        self.ignore_label = float(ignore_label)
        # if isinstance(box_ratios, str):
        #     box_ratios = make_tuple(box_ratios)
        # self.box_ratios = np.array(box_ratios)
        assert self.ignore_label == -1
        if isinstance(variances, str):
            variances = make_tuple(variances)
        self.variances = np.reshape(np.array(variances), (1, -1))
        self.per_cls_reg = bool(make_tuple(str(per_cls_reg)))
        self.normalization = bool(make_tuple(str(normalization)))

        self.max_sample = int(self.max_pos_sample + \
                          self.reg_sample_ratio * self.max_pos_sample + \
                          self.hard_neg_ratio * self.max_pos_sample)

    def list_arguments(self):
        return ['preds_cls', 'preds_reg', 'anchors', 'label', 'probs_cls']

    def list_outputs(self):
        return ['sample_cls', 'sample_reg', 'target_cls', 'target_reg', 'mask_reg']

    def list_auxiliary_states(self):
        return ['target_loc_weight']

    def infer_shape(self, in_shape):
        n_class = in_shape[0][2]

        sample_cls_shape = (self.max_sample, n_class)
        if self.per_cls_reg:
            sample_reg_shape = (self.max_sample, 4 * n_class)
        else:
            sample_reg_shape = (self.max_sample, 4)
        target_cls_shape = (self.max_sample, )
        target_reg_shape = sample_reg_shape
        mask_reg_shape = sample_reg_shape

        out_shape = [sample_cls_shape, sample_reg_shape,
                target_cls_shape, target_reg_shape, mask_reg_shape]

        target_loc_shape = (self.max_sample, 2)
        return in_shape, out_shape, [target_loc_shape]

    def create_operator(self, ctx, shapes, dtypes):
        return MultiBoxTarget2( \
                self.n_class, self.img_wh, self.th_iou, self.th_iou_neg, self.th_nms_neg,
                self.max_pos_sample, self.reg_sample_ratio, self.hard_neg_ratio,
                self.ignore_label, # self.box_ratios,
                self.variances, self.per_cls_reg, self.normalization)


from decimal import Decimal
import time
from time import mktime
from datetime import datetime

import bottle
import pymongo
from bson.objectid import ObjectId
from bottle import static_file, Bottle

import entryDAO
import categoryDAO
import accountDAO

# import billDAO
import utils

__author__ = 'Rafael de Souza'
app = Bottle(4000)

@bottle.route('/resources/<folder>/<filepath:path>')
def server_static(folder, filepath):
    return static_file(filepath, root='./resources/%s' % folder)

@bottle.route('/')
def controle_financeiro_index():
    return bottle.template('main')

@bottle.get('/addentry/<entry_type>')
def present_add_entry(entry_type):
    list_of_categories = categories.get_by_category_type(entry_type)
    list_of_accounts = accounts.get_all()

    if list_of_categories.count() == 0:
        return "Erro: Nao ha categorias cadastradas!"

    if list_of_accounts.count() == 0:
        return "Erro: Nao ha contas cadastradas!"

    return bottle.template('add_entry',
                           dict(description="", value="", date="", account="", category="", description_error="",
                                value_error="", date_error="", account_error="", category_error="",
                                list_of_categories=list_of_categories, list_of_accounts=list_of_accounts,
                                entry_type=entry_type))

@bottle.post('/addentry/<entry_type>')
def process_add_entry(entry_type):
    description = bottle.request.forms.get("description")
    value = bottle.request.forms.get("value")
    date = bottle.request.forms.get("date")
    account = bottle.request.forms.get("account")
    category = bottle.request.forms.get("category")
    entry_type = entry_type

    real_date = time.strptime(date, "%d/%m/%Y")
    dt = datetime.fromtimestamp(mktime(real_date))
    insert_date = utils.now()

    entry = {'description': description, 'value': value, 'date': date, 'account': ObjectId(account),
             'category': ObjectId(category),
             'entry_type': entry_type, 'insert_date': insert_date, 'real_date': dt}

    entries.add(entry)

    bottle.redirect('/entries')

@bottle.get('/addcategory')
def present_add_category():
    return bottle.template('add_category', dict(description="", category_type="", description_error="",
                                                category_type_error=""))

@bottle.post('/addcategory')
def process_add_category():
    description = bottle.request.forms.get("description")
    category_type = bottle.request.forms.get("category_type")

    insert_date = utils.now()

    category = {'description': description, 'category_type': category_type, 'insert_date': insert_date}

    categories.add(category)

    bottle.redirect('/')

@bottle.get('/addaccount')
def present_add_account():
    return bottle.template('add_account', dict(description="", description_error="", initial_value="",
                                               initial_value_error=""))

@bottle.post('/addaccount')
def process_add_account():
    description = bottle.request.forms.get('description')
    initial_value = bottle.request.forms.get('initial_value')

    insert_date = utils.now()

    account = {'description': description, 'initial_value': initial_value, 'insert_date': insert_date}

    accounts.add(account)

    bottle.redirect('/')

@bottle.get('/entries')
def present_entries():
    all_entries = entries.get_all
    returned_entries = []

    for entry in all_entries:
        account = accounts.get(entry['account'])
        category = categories.get(entry['category'])

        all_entries_with_description = {'id': entry['_id'], 'description': entry['description'],
                                        'value': entry['value'],
                                        'date': entry['date'], 'account': account['description'],
                                        'entry_type': entry['entry_type'], 'category': category['description']}

        returned_entries.append(all_entries_with_description)

    return bottle.template('entries', dict(list_of_entries=returned_entries))

@bottle.get('/editentry/<id>')
def present_edit_entry(id):
    entry = entries.get_by_id(id)
    list_of_categories = categories.get_by_category_type(entry['entry_type'])
    list_of_accounts = accounts.get_all()

    return bottle.template('edit_entry', dict(id=entry['_id'], description=entry['description'], value=entry['value'],
                                              date=entry['date'],
                                              account=entry['account'], category=entry['category'],
                                              description_error="", value_error="",
                                              date_error="", account_error="", category_error="",
                                              list_of_categories=list_of_categories, list_of_accounts=list_of_accounts,
                                              entry_type=""))

@bottle.post('/editentry/<id>')
def process_edit_entry(id):
    _id = bottle.request.forms.get('id')
    description = bottle.request.forms.get('description')
    value = bottle.request.forms.get('value')
    date = bottle.request.forms.get('date')
    account = bottle.request.forms.get('account')
    category = bottle.request.forms.get('category')

    real_date = time.strptime(date, "%d/%m/%Y")
    dt = datetime.fromtimestamp(mktime(real_date))

    entry = {'_id': ObjectId(_id), 'description': description, 'value': value, 'date': date,
             'account': ObjectId(account), 'category': ObjectId(category), 'real_date': dt}

    entries.update(entry)

    bottle.redirect('/entries')

@bottle.get('/removeentry/<id>')
def process_remove_entry(id):
    entries.remove(id)

    bottle.redirect('/entries')

@bottle.get('/categories')
def presente_categories():
    all_categories = categories.get_all

    return bottle.template('categories', dict(list_of_categories=all_categories))

@bottle.get('/editcategory/<id>')
def present_edit_entry(id):
    category = categories.get_by_id(id)

    return bottle.template('edit_category', dict(id=category['_id'], description=category['description'],
                                                 category_type=category['category_type'], description_error="",
                                                 category_type_error=""))

@bottle.post('/editcategory/<id>')
def process_edit_category(id):
    _id = bottle.request.forms.get('id')
    description = bottle.request.forms.get('description')
    category_type = bottle.request.forms.get('category_type')

    category = {'_id': ObjectId(_id), 'description': description, 'category_type': category_type}

    categories.update(category)

    bottle.redirect('/categories')

@bottle.get('/removecategory/<id>')
def process_remove_category(id):
    categories.remove(id)

    bottle.redirect('/categories')

@bottle.get('/accounts')
def present_accounts():
    all_accounts = accounts.get_all()

    return bottle.template('accounts', dict(list_of_accounts=all_accounts))

@bottle.get('/editaccount/<id>')
def present_edit_account(id):
    account = accounts.get_by_id(id)
    account_entries = entries.get_by_account_id(account['_id'])

    returned_entries = []

    total_value_account = Decimal(str(account['initial_value']).replace(',', '.'))

    for entry in account_entries:
        account = accounts.get(entry['account'])
        category = categories.get(entry['category'])

        if (entry['entry_type']) == 'C':
            total_value_account += Decimal(str(entry['value']).replace(',', '.'))
        else:
            total_value_account -= Decimal(str(entry['value']).replace(',', '.'))

        all_entries_with_description = {'id': entry['_id'], 'description': entry['description'],
                                        'value': entry['value'],
                                        'date': entry['date'], 'account': account['description'],
                                        'entry_type': entry['entry_type'], 'category': category['description']}

        returned_entries.append(all_entries_with_description)

    return bottle.template('edit_account', dict(id=account['_id'], description=account['description'],
                                                initial_value=account['initial_value'], description_error="",
                                                total_value_account=str(total_value_account).replace('.', ','),
                                                initial_value_error="", list_of_entries=returned_entries))

@bottle.post('/editaccount/<id>')
def process_edit_account(id):
    _id = bottle.request.forms.get('id')
    description = bottle.request.forms.get('description')
    initial_value = bottle.request.forms.get('initial_value')

    account = {'_id': ObjectId(_id), 'description': description, 'initial_value': initial_value}

    accounts.update(account)

    bottle.redirect('/accounts')

@bottle.get('/removeaccount/<id>')
def process_remove_account(id):
    accounts.remove(id)

    bottle.redirect('/accounts')

@bottle.get('/reports')
def present_reports():
    return bottle.template('reports')

@bottle.get('/monthreport')
def present_month_report():
    list_of_categories = categories.get_all
    list_of_accounts = accounts.get_all()
    return bottle.template('monthreport', dict(filtered_entries="", list_of_categories=list_of_categories,
                                               list_of_accounts=list_of_accounts, selected_month="", selected_year="",
                                               selected_account="", selected_category=""))

@bottle.post('/monthreport')
def process_month_report():
    selected_month = bottle.request.forms.get('month_selection')
    selected_year = bottle.request.forms.get('year_selection')
    selected_account = bottle.request.forms.get('account_selection')
    selected_category = bottle.request.forms.get('category_selection')
    list_of_categories = categories.get_all
    list_of_accounts = accounts.get_all()

    filtered_entries = entries.get_report_data(selected_month, selected_year, selected_account, selected_category)
    returned_entries = []

    for entry in filtered_entries:
        account = accounts.get(entry['account'])
        category = categories.get(entry['category'])

        all_entries_with_description = {'id': entry['_id'], 'description': entry['description'],
                                        'value': entry['value'],
                                        'date': entry['date'], 'account': account['description'],
                                        'entry_type': entry['entry_type'], 'category': category['description']}

        returned_entries.append(all_entries_with_description)

    return bottle.template('monthreport', dict(filtered_entries=returned_entries, list_of_categories=list_of_categories,
                                               selected_month=selected_month, list_of_accounts=list_of_accounts,
                                               selected_year=selected_year, selected_account=selected_account,
                                               selected_category=selected_category))

# @bottle.get('/bills')
# def present_bills():
#    all_bills = bills.get_all()

#    return bottle.template('bills', dict(list_of_bills=all_bills))

# @bottle.get('/addbill')
# def present_add_bill():
#    return bottle.template('addbill')

connection_string = "mongodb://localhost"
connection = pymongo.MongoClient(connection_string)
database = connection.controlefinanceiro

entries = entryDAO.EntryDAO(database)
categories = categoryDAO.CategoryDAO(database)
accounts = accountDAO.AccountDAO(database)
#bills = billDAO.BillDAO(database)
utils = utils.Utils()

bottle.debug(True)
# if __name__ == '__main__':
bottle.run(host='', port=4000)
# bottle.run(reloader=True, debug=True)


# -*- coding: utf-8 -*-
'''
    Author    : Huseyin BIYIK <husenbiyik at hotmail>
    Year      : 2016
    License   : GPL

    This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
'''

import urllib2
import re
import time
import json
import os
import zipfile
import StringIO
import shutil
import sys

import xbmc
import xbmcgui

_dom = "http://www.github.com"
_ddir = xbmc.translatePath("special://userdata/addon_data/")

def _mkdir(*args):
    p = os.path.join(*args)
    if not os.path.exists(p):
        os.makedirs(p)
    return p

_ddir = _mkdir(_ddir, "script.module.ghub", "src")

def _page(u):
    return urllib2.urlopen(u).read()

def _getrels(uname, rname):
    page = _page("%s/%s/%s/tags" % (_dom, uname, rname))
    rels = re.findall('
(.*?
.*?)
', page, re.DOTALL)
    allrels = []
    for rel in rels:
        commit = re.findall('/commit/(.*?)"', rel)[0]
        zipu = re.findall('href="(.*?\.zip)"', rel)[0]
        zipu = _dom + zipu
        allrels.append([commit, zipu])
    return allrels

def _getcommits(uname, rname, branch, commit):
    if not commit:
        page = _page("%s/%s/%s/commits/%s" % (_dom, uname, rname, branch))
        comms = re.findall('data-url=".*?/commit/(.*?)/', page)
    else:
        comms = [commit]
    allcoms = []
    for commit in comms:
        zipu = "%s/%s/%s/archive/%s.zip" % (
                                            _dom,
                                            uname,
                                            rname,
                                            commit
                                            )
        allcoms.append([commit, zipu])
    return allcoms

def _updatezip(u, tdir):
    for d in os.listdir(tdir):
        p = os.path.join(tdir, d)
        if os.path.isdir(p):
            shutil.rmtree(p)
    zip = zipfile.ZipFile(StringIO.StringIO(_page(u)))
    zip.extractall(tdir)

def _makepack(tdir, path):
    sdir = None
    for d in os.listdir(tdir):
        sdir = os.path.join(tdir, d)
        if os.path.isdir(sdir):
            break
    if sdir and not sdir in sys.path:
        sys.path.append(os.path.join(sdir, *path))

def load(uname, rname, branch, commit=None, path = [], period=24):
    """
    Loads, caches, and arranges paths for a repo from github.
    This module, downloads a github repo to
    userdata/addon_data/scirpt.module.gub/uname/package/branch/commit
    It also updates the package automatically in a given period (24h default)
    Note that this is not a package manager and does not do any dependency check

    Example:
        import ghub

        ghub.load("hbiyik", "script.module.boogie", "master", None, ["lib"])

        #now you can safely import this module
        import boogie

    Params:
        uname: github username
        rname: repository name
        branch: github branch (ie master), if this parameter is None, latest tagged
            version is used.
        commit: [optional,None] commit of the specified branch, if None is specified
            latest commit is fetched, if brach is None, this parameter is dismissed
        path: [optional,[]] list of directories pointing to the root directory of source
            ie, if source is in lib/src folder path should be ["lib", "src"]
        period: [optional, 24] period in hours of how frequent the repo should be checked
            for an update

    Returns:
        None

    """
    mem = None
    bdir = _mkdir(_ddir, uname)
    if not branch:
        bdir = _mkdir(bdir, "release")
    else:
        bdir = _mkdir(bdir, "branch", branch)
    memfile = os.path.join(bdir, "memory.json")
    if os.path.exists(memfile):
        with open(memfile) as infile:
            mem = json.load(infile)
    if mem and time.time() - mem["ts"] < period * 60 * 60:
        return _makepack(bdir, path)
    else:
        if branch:
            ref = _getcommits(uname, rname, branch, commit)
        else:
            ref = _getrels(uname, rname)
        latest = ref[0]
        if not mem or not latest[0] == mem["latest"]:
            _updatezip(latest[1], bdir)
        data = {
                "ts": time.time(),
                "latest": latest[0]
                }
        with open(memfile, 'w') as outfile:
            json.dump(data, outfile)
        dialog = xbmcgui.Dialog()
        if not branch:
            branch = "release"
        dialog.notification("%s:%s:%s" % (rname, uname, branch), latest[0][:7])
        return _makepack(bdir, path)


import sys
from sklearn.cluster import KMeans
from sklearn.mixture import GaussianMixture
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from os import listdir
from glob import glob
import cmocean

'''
  - Read the wind data divided by positive and negative SHF values
  - Apply clustering analysis to each dataset to divide it between Unstable/Shear Driven and WSBL/VSBL

  to do: Weibull distribution, other things?
'''

def main():

  lats = []
  lons = []
  stnames = []

  stations = open('stations.txt', 'r')
  for line in stations:
    aa = line.replace("\n", '').split(';')
    if (aa[0] != "#"):
      lats.append(float(aa[3]))
      lons.append(float(aa[5]))
      stnames.append(aa[1].replace(',',"_"))

  datai = 1986
  dataf = 2015

  percentage = open('percentage_seasonal.txt', 'w')
  percentage.write("Station Neg Pos Neg1 Neg2 Pos1 Pos2\n")

  # looping throught all the stations
  for lat, lon, name in zip(lats, lons, stnames):

    for season, sname in zip([[12, 1, 2],[6, 7, 8],[3, 4, 5],[9, 10, 11]], ['DJF','JJA','MAM','SON']):

      filepaths_n = []
      filepaths_p = []

      for month in season:
        print(name, month)

        for year in range(datai, dataf+1):

          # Open the .csv
          #filepaths_n.extend(glob('CSV/*{1}*_windpress_neg.csv'.format(month, year)))
          filepaths_n.extend(glob('CSV/*{2}_{1}{0:02d}_windpress_neg.csv'.format(month, year, name)))
          filepaths_p.extend(glob('CSV/*{2}_{1}{0:02d}_windpress_pos.csv'.format(month, year, name)))
      #print(filepaths_n)
      #sys.exit()

      df_n = pd.concat((pd.read_csv(f, index_col=0) for f in filepaths_n), ignore_index=True)
      df_p = pd.concat((pd.read_csv(f, index_col=0) for f in filepaths_p), ignore_index=True)

      #[10.0, 15.0, 20.0, 30.0, 50.0, 70.0, 100.0, 150.0, 200.0, 250.0, 300.0, 400.0, 500.0, 600.0, 700.0, 800.0, 850.0, 900.0, 925.0, 950.0, 975.0, 1000.0]
      # Delete the upper levels of the atmosphere. I need only up to 700 hPa.
      df_n = df_n.drop(columns=['300.0', '400.0', '500.0', '600.0'])
      df_p = df_p.drop(columns=['300.0', '400.0', '500.0', '600.0'])

      p_neg = len(df_n.index)*100/(len(df_n.index) + len(df_p.index))
      p_pos = len(df_p.index)*100/(len(df_n.index) + len(df_p.index))

      centroids_n, histo_n, perc_n = kmeans_probability(df_n)

      cent = []
      histo = []
      perc = []
      shf = []

      if (perc_n[0] > perc_n[1]):
        k = 0
        j = 1
      else:
        k = 1
        j = 0

      cent.append(centroids_n[k])
      cent.append(centroids_n[j])

      histo.append(histo_n[k])
      histo.append(histo_n[j])

      perc.append(perc_n[k])
      perc.append(perc_n[j])

      shf.append('SHF-')
      shf.append('SHF-')

      #plot_wind(centroids_n[0], histo_n[0], perc_n[0], datai, dataf, name, "negative_type1", sname)
      #plot_wind(centroids_n[1], histo_n[1], perc_n[1], datai, dataf, name, "negative_type2", sname)

      centroids_p, histo_p, perc_p = kmeans_probability(df_p)

      if (perc_p[0] > perc_p[1]):
        k = 0
        j = 1
      else:
        k = 1
        j = 0

      cent.append(centroids_p[k])
      cent.append(centroids_p[j])

      histo.append(histo_p[k])
      histo.append(histo_p[j])

      perc.append(perc_p[k])
      perc.append(perc_p[j])

      shf.append('SHF+')
      shf.append('SHF+')

      plot_wind_seasonal(cent, histo, perc, shf, datai, dataf, name, sname)

      #plot_wind(centroids_p[0], histo_p[0], perc_p[0], datai, dataf, name, "positive_type1", sname)
      #plot_wind(centroids_p[1], histo_p[1], perc_p[1], datai, dataf, name, "positive_type2", sname)

      percentage.write("{0} {1:2.2f} {2:2.2f} {3:2.2f} {4:2.2f} {5:2.2f} {6:2.2f} {7}\n".format(name, p_neg, p_pos, perc_n[0], perc_n[1], perc_p[0], perc_p[1], sname))

  percentage.close()

def plot_wind(centroids, histo, perc, datai, dataf, name, ptype, month):

  y = [700.0, 800.0, 850.0, 900.0, 925.0, 950.0, 975.0, 1000.0]
  x = np.arange(0,40,1)
  X, Y= np.meshgrid(x, y)
  vmin=0
  vmax=15
  v = np.arange(vmin, vmax+1, 2)

  fig = plt.figure(figsize=[14,8])
  CS = plt.contourf(X, Y, histo, v, cmap='cmo.haline', extend='max')
  CS.set_clim(vmin, vmax)
  plt.gca().invert_yaxis()
  plt.plot(centroids, y, color='white', marker='o', lw=4, markersize=10, markeredgecolor='k')
  CB = plt.colorbar(CS, extend='both', ticks=v)
  CB.ax.tick_params(labelsize=20)
  plt.xlim(0,40)
  plt.ylim(1000,700)
  plt.xticks(np.arange(0,40,5), fontsize=20)
  plt.yticks(y, fontsize=20)
  plt.title('{0}:{1} - {2:2.2f}% - {3} - {4}'.format(datai, dataf, perc, name, month), fontsize='20')
  plt.tight_layout()
  plt.savefig('Images/{0}_{1}{2}_{3}_{4}.png'.format(name, datai, dataf, ptype, month), pad_inches=0.0, bbox_inches='tight')
  plt.close()

  return None

def plot_wind_seasonal(centroids, histo, perc, shf, datai, dataf, name, period):

  y = [700.0, 800.0, 850.0, 900.0, 925.0, 950.0, 975.0, 1000.0]
  x = np.arange(0,40,1)
  X, Y= np.meshgrid(x, y)
  vmin=0
  vmax=15
  v = np.arange(vmin, vmax+1, 2)

  fig = plt.figure(figsize=[28,16])

  for k, letter in zip(range(0,4), ['a', 'b', 'c', 'd']):
    subplt = '22{0}'.format(k+1)
    plt.subplot(subplt)

    CS = plt.contourf(X, Y, histo[k], v, cmap='cmo.haline', extend='max')
    CS.set_clim(vmin, vmax)
    plt.gca().invert_yaxis()
    plt.plot(centroids[k], y, color='white', marker='o', lw=4, markersize=10, markeredgecolor='k')
    CB = plt.colorbar(CS, extend='both', ticks=v)
    CB.ax.tick_params(labelsize=20)
    plt.xlim(0,39)
    plt.ylim(1000,700)
    plt.xticks(np.arange(0,40,5), fontsize=20)
    plt.yticks(y, fontsize=20)
    plt.title('({0}) {1:2.2f} % {2}'.format(letter, perc[k], shf[k]), fontsize='20')
  plt.tight_layout()
  plt.savefig('Images/{0}_{1}{2}_{3}.png'.format(name, datai, dataf, period), pad_inches=0.0, bbox_inches='tight')
  plt.close()

  return None

def kmeans_probability(df):
  '''
    For now fixed at 2 clusters

    returns: Array of the centroids, the two histograms and % of each group
  '''
  kmeans = KMeans(n_clusters=2, random_state=0).fit(df)

  # Getting the location of each group.
  pred = kmeans.predict(df)
  labels = np.equal(pred, 0)

  # Converting to numpy array
  df_a = np.array(df)

  # Dividing between the 2 clusters
  df_0 = df_a[labels,:]
  df_1 = df_a[~labels,:]

  # Getting the probability distribution. Bins of 0.5 m/s
  hist_0 = calc_histogram(df_0)
  hist_1 = calc_histogram(df_1)

  #print(np.mean(df_0, axis=0), np.mean(df_1, axis=0), kmeans.cluster_centers_)
  #sys.exit()

  return kmeans.cluster_centers_, [hist_0, hist_1], [df_0.shape[0]*100/df_a.shape[0], df_1.shape[0]*100/df_a.shape[0]]

def calc_histogram(df):

  hist_l = []
  bins = np.arange(0,40.25,1)
  for i in range(0, df.shape[1]):
    hist, bins = np.histogram(df[:,i], bins=bins)
    hist_l.append(hist*100/sum(hist))

  return np.array(hist_l)

if __name__ == "__main__":
  main()


from __future__ import print_function
from keras.preprocessing import sequence
from keras.models import Sequential
from keras.layers import AveragePooling1D,MaxPooling1D,LSTM,LeakyReLU,K,Layer,TimeDistributed, Multiply,Permute,Dense, Embedding,Dropout,Activation,Bidirectional,Convolution1D, MaxPooling1D, Flatten
from keras.callbacks import ModelCheckpoint
from keras.preprocessing.text import  Tokenizer
from nested_lstm import NestedLSTM
import  keras
from keras.layers.normalization import BatchNormalization
from ind_rnn import IndRNN
import numpy as np
from keras.optimizers import Adam
import pickle as cp
import  re
from  GroupNormalization import  GroupNormalization
from sru import SRU
from highway import Highway
import  os

num_classes=10
max_features = 100000
maxlen = 2000  #2000 cut texts after this number of words (among top max_features most common words)
batch_size =64

class LAttenLayer(Layer):
    def __init__(self,nb_filter=1,filter_length=5,**kwargs):
        self.nb_filter = nb_filter
        self.filter_length = filter_length
        super(LAttenLayer, self).__init__(**kwargs)

    def build(self, input_shape):
        assert  len(input_shape)==3, 'attention layer 输入是三维矩阵'
        self.input_dim = input_shape[2]

        self.atten_layer = Convolution1D(
            filters=self.nb_filter,
            kernel_size=self.filter_length,
            padding='same',
            activation='sigmoid'
        )
        self.trainable_weights = self.atten_layer.trainable_weights
        super(LAttenLayer, self).build(input_shape)

    def call(self, x, mask=None):
        x_padding = x
        word_score = self.atten_layer(x_padding)
        out = Multiply()([x, word_score])
        return out

    def compute_output_shape(self, input_shape):
        return tuple(input_shape)

class Sum(Layer):
    def call(self, inputs, **kwargs):
        return K.sum(inputs, axis=1)

    def compute_output_shape(self, input_shape):
        return (input_shape[0], input_shape[-1])

class GAttenLayer(Layer):
    def __init__(self, **kwargs):
        super(GAttenLayer, self).__init__(**kwargs)

    def build(self, input_shape):
        assert  len(input_shape)==3
        self.dense = Dense(1)
        self.trainable_weights = self.dense.trainable_weights
        super(GAttenLayer, self).build(input_shape)

    def call(self, x, mask=None):
        atten = self.dense(x)
        atten = Permute((2, 1))(atten)
        atten = Activation('softmax')(atten)
        atten = Permute((2, 1))(atten) #[n_sample, n_length, 1]
        conv_feature = Multiply()([x, atten]) #[n_sample, n_length, embedding]
        conv_feature = Sum()(conv_feature) #[n_sample, embedding]
        return conv_feature

    def compute_output_shape(self, input_shape):
        return (input_shape[0], input_shape[-1])

def predataset(file):
    train=[]
    label=[]
    max_len=0
    with open(file,encoding="utf8") as file:
        for line in file:
            temp=line.split("\t\t")
            train.append(temp[3])
            label.append(int(temp[2])-1)
            if max_len<len(temp[3]):
                max_len=len(temp[3])
    print("max",max_len)
    return  train ,label

# #将篇章进行切割
# def predataset(file):
#     train = []
#     label = []
#     with open(file, encoding="utf8") as f:
#         for line in f:
#             temp = line.split("\t\t")
#             lines=temp[3].split("<sssss>")
#             for l in lines:
#                 train.append(l)
#                 label.append(int(temp[2])-1)
#     return train, label

#训练数据预处理
x_train,y_train=predataset("../data/yelp13/train.txt")

# 对句子进行序列映射
tokenizer = Tokenizer()
tokenizer.fit_on_texts(x_train)
x_train = tokenizer.texts_to_sequences(x_train)
y_train = keras.utils.to_categorical(y_train,num_classes=num_classes)
word_index = tokenizer.word_index

#测试数据预处理
x_test,y_test=predataset("../data/yelp13/test.txt")
# 分词并对句子进行序列映射
x_test = tokenizer.texts_to_sequences(x_test)
y_test = keras.utils.to_categorical(y_test,num_classes=num_classes)

#测试数据预处理
x_val,y_val=predataset("../data/yelp13/dev.txt")
# 分词并对句子进行序列映射
x_val = tokenizer.texts_to_sequences(x_val)
y_val = keras.utils.to_categorical(y_val,num_classes=num_classes)

print('Pad sequences (samples x time)')

x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
x_val = sequence.pad_sequences(x_val, maxlen=maxlen)
print('x_train shape:', x_train.shape)
print('x_test shape:', x_test.shape)
print('x_val shape:', x_val.shape)

# 读取文件，转成向量
embeddings_index = {}
filename="wiki.en.vec"
with open(os.path.join("../data/", filename),encoding='utf8') as f:
    if filename=="wiki.en.vec":
        first=next(f)
        s=first.split()
        num=s[0]
        v_size=s[1]
    for line in f:
        values = line.rstrip().split(" ")
        word = values[0]
        coefs = np.asarray(values[1:], dtype='float32')
        embeddings_index[word] = coefs
print(len(embeddings_index))
print('Preparing embedding matrix.')

embedding_matrix = np.zeros((len(word_index)+1, 300))

for word, i in word_index.items():
    embedding_vector = embeddings_index.get(word)
    if embedding_vector is not None:
        # words not found in embedding index will be all-zeros.
        embedding_matrix[i] = embedding_vector
# configuration matches 4.47 Million parameters with `units=600` and `64 embedding dim`
print('Build model...')
#weights=[embedding_W]
model = Sequential()
model.add(Embedding(len(word_index)+1, 300, input_shape=(maxlen,),trainable=True,weights=[embedding_matrix]))
model.add(Dropout(0.5))
model.add(AveragePooling1D())
model.add(LAttenLayer())
model.add(Bidirectional(IndRNN(300, return_sequences=True)))

model.add(Convolution1D(filters=32,
                        kernel_size=5,
                        strides=1,
                        padding='same'))

model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=2,strides=2,padding='same'))
model.add(GAttenLayer())
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(num_classes, activation='softmax'))
# try using different optimizers and different optimizer configs

model.compile(
        loss='categorical_crossentropy',
        optimizer='adadelta',
        metrics=['accuracy']
    )

model.summary()

print('Train...')

model.fit(x_train, y_train,
              batch_size=batch_size,
              epochs=15,
              shuffle=True,
              validation_data=(x_val, y_val),
              callbacks=[ModelCheckpoint('weights/IMDB_indrnn.h5', monitor='val_acc',
                                         save_best_only=True, save_weights_only=False)])

model.load_weights('weights/IMDB_indrnn.h5')

score, acc = model.evaluate(x_test, y_test,batch_size = batch_size)
print('Test score:', score)
print('Test accuracy:', acc)


import os

from requests.exceptions import RequestException

from conans.client.graph.graph import (RECIPE_DOWNLOADED, RECIPE_INCACHE, RECIPE_NEWER,
                                       RECIPE_NOT_IN_REMOTE, RECIPE_NO_REMOTE, RECIPE_UPDATEABLE,
                                       RECIPE_UPDATED, RECIPE_EDITABLE)
from conans.client.output import ScopedOutput
from conans.client.recorder.action_recorder import INSTALL_ERROR_MISSING, INSTALL_ERROR_NETWORK
from conans.client.remover import DiskRemover
from conans.errors import ConanException, NotFoundException,\
    RecipeNotFoundException
from conans.model.manifest import FileTreeManifest
from conans.util.tracer import log_recipe_got_from_local_cache

class ConanProxy(object):
    def __init__(self, cache, output, remote_manager):
        # collaborators
        self._cache = cache
        self._out = output
        self._remote_manager = remote_manager

    def get_recipe(self, ref, check_updates, update, remotes, recorder):
        if self._cache.installed_as_editable(ref):
            conanfile_path = self._cache.package_layout(ref).conanfile()
            status = RECIPE_EDITABLE
            # TODO: log_recipe_got_from_editable(reference)
            # TODO: recorder.recipe_fetched_as_editable(reference)
            return conanfile_path, status, None, ref

        with self._cache.package_layout(ref).conanfile_write_lock(self._out):
            result = self._get_recipe(ref, check_updates, update, remotes, recorder)
            conanfile_path, status, remote, new_ref = result

            if status not in (RECIPE_DOWNLOADED, RECIPE_UPDATED):
                log_recipe_got_from_local_cache(new_ref)
                recorder.recipe_fetched_from_cache(new_ref)

        return conanfile_path, status, remote, new_ref

    def _get_recipe(self, ref, check_updates, update, remotes, recorder):
        output = ScopedOutput(str(ref), self._out)
        # check if it is in disk
        conanfile_path = self._cache.package_layout(ref).conanfile()

        # NOT in disk, must be retrieved from remotes
        if not os.path.exists(conanfile_path):
            remote, new_ref = self._download_recipe(ref, output, remotes, remotes.selected, recorder)
            status = RECIPE_DOWNLOADED
            return conanfile_path, status, remote, new_ref

        metadata = self._cache.package_layout(ref).load_metadata()
        cur_revision = metadata.recipe.revision
        cur_remote = metadata.recipe.remote
        cur_remote = remotes[cur_remote] if cur_remote else None
        selected_remote = remotes.selected or cur_remote

        check_updates = check_updates or update
        # Recipe exists in disk, but no need to check updates
        requested_different_revision = (ref.revision is not None) and cur_revision != ref.revision
        if requested_different_revision and not check_updates:
            raise NotFoundException("The recipe in the local cache doesn't match the specified "
                                    "revision. Use '--update' to check in the remote.")

        if not requested_different_revision:
            if not check_updates:
                status = RECIPE_INCACHE
                ref = ref.copy_with_rev(cur_revision)
                return conanfile_path, status, cur_remote, ref

            if not selected_remote:
                status = RECIPE_NO_REMOTE
                ref = ref.copy_with_rev(cur_revision)
                return conanfile_path, status, None, ref
        else:  # Requested different revision and with --update
            remote, new_ref = self._download_recipe(ref, output, remotes, selected_remote, recorder)
            status = RECIPE_DOWNLOADED
            return conanfile_path, status, remote, new_ref

        try:  # get_recipe_manifest can fail, not in server
            upstream_manifest, ref = self._remote_manager.get_recipe_manifest(ref, selected_remote)
        except NotFoundException:
            status = RECIPE_NOT_IN_REMOTE
            ref = ref.copy_with_rev(cur_revision)
            return conanfile_path, status, selected_remote, ref

        export = self._cache.package_layout(ref).export()
        read_manifest = FileTreeManifest.load(export)
        if upstream_manifest != read_manifest:
            if upstream_manifest.time > read_manifest.time:
                if update:
                    DiskRemover().remove_recipe(self._cache.package_layout(ref), output=output)
                    output.info("Retrieving from remote '%s'..." % selected_remote.name)
                    self._download_recipe(ref, output, remotes, selected_remote, recorder)
                    with self._cache.package_layout(ref).update_metadata() as metadata:
                        metadata.recipe.remote = selected_remote.name
                    status = RECIPE_UPDATED
                    return conanfile_path, status, selected_remote, ref
                else:
                    status = RECIPE_UPDATEABLE
            else:
                status = RECIPE_NEWER
        else:
            status = RECIPE_INCACHE

        ref = ref.copy_with_rev(cur_revision)
        return conanfile_path, status, selected_remote, ref

    def _download_recipe(self, ref, output, remotes, remote, recorder):

        def _retrieve_from_remote(the_remote):
            output.info("Trying with '%s'..." % the_remote.name)
            # If incomplete, resolve the latest in server
            _ref = self._remote_manager.get_recipe(ref, the_remote)
            output.info("Downloaded recipe revision %s" % _ref.revision)
            with self._cache.package_layout(ref).update_metadata() as metadata:
                metadata.recipe.remote = the_remote.name
            recorder.recipe_downloaded(ref, the_remote.url)
            return _ref

        if remote:
            output.info("Retrieving from server '%s' " % remote.name)
        else:
            try:
                remote_name = self._cache.package_layout(ref).load_metadata().recipe.remote
                if remote_name:
                    remote = remotes[remote_name]
            except (IOError, RecipeNotFoundException):
                pass
            else:
                if remote:
                    output.info("Retrieving from predefined remote '%s'" % remote.name)

        if remote:
            try:
                new_ref = _retrieve_from_remote(remote)
                return remote, new_ref
            except NotFoundException:
                msg = "%s was not found in remote '%s'" % (str(ref), remote.name)
                recorder.recipe_install_error(ref, INSTALL_ERROR_MISSING,
                                              msg, remote.url)
                raise NotFoundException(msg)
            except RequestException as exc:
                recorder.recipe_install_error(ref, INSTALL_ERROR_NETWORK,
                                              str(exc), remote.url)
                raise exc

        output.info("Not found in local cache, looking in remotes...")
        remotes = remotes.values()
        if not remotes:
            raise ConanException("No remote defined")
        for remote in remotes:
            try:
                new_ref = _retrieve_from_remote(remote)
                return remote, new_ref
            # If not found continue with the next, else raise
            except NotFoundException:
                pass
        else:
            msg = "Unable to find '%s' in remotes" % ref.full_repr()
            recorder.recipe_install_error(ref, INSTALL_ERROR_MISSING,
                                          msg, None)
            raise NotFoundException(msg)