import re
from os import path

import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

with open(path.join(path.dirname(__file__), "linkify_it", "__init__.py")) as f:
    match = re.search(r"__version__\s*=\s*[\'\"](.+?)[\'\"]", f.read())
    version = match.group(1)

setuptools.setup(
    name="linkify-it-py",
    version=version,
    license="MIT",
    author="tsutsu3",
    description="Links recognition library with FULL unicode support.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/tsutsu3/linkify-it-py",
    packages=setuptools.find_packages(exclude=["test", "benchmark"]),
    include_package_data=True,
    classifiers=[
        "Development Status :: 3 - Alpha",
        "Programming Language :: Python :: 3",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3.7",
        "Programming Language :: Python :: 3.8",
        "Programming Language :: Python :: 3.9",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
        "Intended Audience :: Developers",
        "Topic :: Software Development :: Libraries :: Python Modules",
    ],
    python_requires=">=3.6",
    keywords=["linkify", "linkifier", "autolink", "autolinker"],
    install_requires=["uc-micro-py"],
    extras_require={
        "dev": ["pre-commit", "isort", "flake8", "black"],
        "benchmark": ["pytest", "pytest-benchmark"],
        "test": ["coverage", "pytest", "pytest-cov"],
        "doc": ["sphinx", "sphinx_book_theme", "myst-parser"],
    },
)


from copy import deepcopy
import numpy as np
from scipy.sparse import csc_matrix
import time

from aux import lognormal_mu_sig, sgmd
from replay import cxn
from ntwk import LIFNtwk, join_w

cc = np.concatenate

def run(p, s_params):
    """
    Run smln and return rslt.

    :param p: dict of model params
    :param s_params: dict of smln params
    """
    # prepare smln
    prep_start = time.time()

    ## build trajectory
    trj = build_trj(s_params)
    schedule = s_params['schedule']

    ## adjust schedule
    t = np.arange(0, schedule['D_SMLN'], s_params['DT'])

    ## build ntwk
    ntwk = build_ntwk(p, s_params)

    ## get apx. real-valued mask ("veil") over trj nrns;
    ## values are >= 0 and correspond to apx. scale factors on
    ## corresponding G->PC weights minus 1
    trj_veil = get_trj_veil(trj, ntwk, p, s_params)
    ntwk.sgm = trj_veil + 1

    ## approximate G->PC weights if desired
    ntwk = apx_ws_up(ntwk, trj_veil)

    spks_up, i_ext = build_stim(t, trj, ntwk, p, s_params, schedule)

    prep_end = time.time()
    prep_time = prep_end - prep_start

    # run smln
    run_start = time.time()

    rslt = ntwk.run(spks_up=spks_up, dt=s_params['DT'], i_ext=i_ext)
    run_end = time.time()

    run_time = run_end - run_start

    # consolidate smln rslt
    rslt.ntwk = ntwk
    rslt.schedule = schedule
    rslt.trg = s_params['TRG']

    rslt.p = p
    rslt.s_params = s_params

    rslt.trj = trj
    rslt.trj_veil = trj_veil

    rslt.prep_time = prep_time
    rslt.run_time = run_time

    return rslt

def build_ntwk(p, s_params):
    """
    Construct a network object from the model and
    simulation params.
    """
    np.random.seed(s_params['RNG_SEED'])

    # set membrane properties
    n = p['N_PC'] + p['N_INH']

    t_m = cc(
        [np.repeat(p['T_M_PC'], p['N_PC']), np.repeat(p['T_M_INH'], p['N_INH'])])
    e_l = cc(
        [np.repeat(p['E_L_PC'], p['N_PC']), np.repeat(p['E_L_INH'], p['N_INH'])])
    v_th = cc(
        [np.repeat(p['V_TH_PC'], p['N_PC']), np.repeat(p['V_TH_INH'], p['N_INH'])])
    v_r = cc(
        [np.repeat(p['V_R_PC'], p['N_PC']), np.repeat(p['V_R_INH'], p['N_INH'])])
    t_rp = cc(
        [np.repeat(p['T_R_PC'], p['N_PC']), np.repeat(p['T_R_INH'], p['N_INH'])])

    # set latent nrn positions
    lb = [-s_params['BOX_W']/2, -s_params['BOX_H']/2]
    ub = [s_params['BOX_W']/2, s_params['BOX_H']/2]

    # sample evenly spaced place fields
    ## E cells
    pfxs_e, pfys_e = cxn.apx_lattice(lb, ub, p['N_PC'], randomize=True)
    ## I cells
    pfxs_i, pfys_i = cxn.apx_lattice(lb, ub, p['N_INH'], randomize=True)

    ## join E & I place fields
    pfxs = cc([pfxs_e, pfxs_i])
    pfys = cc([pfys_e, pfys_i])

    # make upstream ws
    if p['W_PC_PL'] > 0:
        w_pc_pl_flat = np.random.lognormal(
            *lognormal_mu_sig(p['W_PC_PL'], p['S_PC_PL']), p['N_PC'])
    else:
        w_pc_pl_flat = np.zeros(p['N_PC'])

    if p['W_PC_G'] > 0:
        w_pc_g_flat = np.random.lognormal(
            *lognormal_mu_sig(p['W_PC_G'], p['S_PC_G']), p['N_PC'])
    else:
        w_pc_g_flat = np.zeros(p['N_PC'])

    ws_up_temp = {
        'E': {
            ('PC', 'PL'): np.diag(w_pc_pl_flat),
            ('PC', 'G'): np.diag(w_pc_g_flat),
        },
    }

    targs_up = cc([np.repeat('PC', p['N_PC']), np.repeat('INH', p['N_INH'])])
    srcs_up = cc([np.repeat('PL', p['N_PC']), np.repeat('G', p['N_PC'])])

    ws_up = join_w(targs_up, srcs_up, ws_up_temp)

    # make rcr ws
    w_pc_pc = cxn.make_w_pc_pc(pfxs[:p['N_PC']], pfys[:p['N_PC']], p)

    w_inh_pc = cxn.make_w_inh_pc(
        pfxs_inh=pfxs[-p['N_INH']:],
        pfys_inh=pfys[-p['N_INH']:],
        pfxs_pc=pfxs[:p['N_PC']],
        pfys_pc=pfys[:p['N_PC']],
        p=p)

    w_pc_inh = cxn.make_w_pc_inh(
        pfxs_pc=pfxs[:p['N_PC']],
        pfys_pc=pfys[:p['N_PC']],
        pfxs_inh=pfxs[-p['N_INH']:],
        pfys_inh=pfys[-p['N_INH']:],
        p=p)

    ws_rcr_temp = {
        'E': {
            ('PC', 'PC'): w_pc_pc,
            ('INH', 'PC'): w_inh_pc,
        },
        'I': {
            ('PC', 'INH'): w_pc_inh,
        },
    }
    targs_rcr = cc([np.repeat('PC', p['N_PC']), np.repeat('INH', p['N_INH'])])

    ws_rcr = join_w(targs_rcr, targs_rcr, ws_rcr_temp)

    # make ntwk
    ntwk = LIFNtwk(
        t_m=t_m,
        e_l=e_l,
        v_th=v_th,
        v_r=v_r,
        t_r=t_rp,
        es_syn={'E': p['E_E'], 'I': p['E_I']},
        ts_syn={'E': p['T_E'], 'I': p['T_I']},
        ws_up=ws_up,
        ws_rcr=ws_rcr)

    ntwk.pfxs = pfxs
    ntwk.pfys = pfys

    ntwk.types_up = srcs_up
    ntwk.types_rcr = targs_rcr

    ntwk.n_pc = p['N_PC']
    ntwk.n_inh = p['N_INH']
    ntwk.n_g = p['N_PC']
    ntwk.n_inp = p['N_PC']
    ntwk.n_rcr = p['N_PC'] + p['N_INH']
    ntwk.n_up = 2 * p['N_PC']

    ntwk.types_up_slc = {
        'PL': slice(0, p['N_PC']),
        'G': slice(p['N_PC'], 2*p['N_PC'])
    }
    ntwk.types_rcr_slc = {
        'PC': slice(0, p['N_PC']),
        'INH': slice(p['N_PC'], p['N_PC'] + p['N_INH'])
    }

    return ntwk

def build_trj(s_params):
    """
    Build trajectory.
    """
    # calc trj segs
    segs_x = []
    segs_y = []

    for (x_0, y_0), (x_1, y_1) in zip(s_params['TRJ'][:-1], s_params['TRJ'][1:]):

        path_len = np.sqrt((x_1-x_0)**2 + (y_1-y_0)**2)
        dur = path_len / s_params['SPD']

        # num timesteps
        n_t = int(round(dur/s_params['DT'])) - 1

        # x and y segs
        segs_x.append(np.linspace(x_0, x_1, n_t, endpoint=False))
        segs_y.append(np.linspace(y_0, y_1, n_t, endpoint=False))

    # make full x, y seq from segs
    x = np.concatenate(segs_x)
    y = np.concatenate(segs_y)

    t = np.arange(len(x), dtype=float) * s_params['DT']
    spd = s_params['SPD'] * np.ones(len(t))

    return {'x': x, 'y': y, 't': t, 'spd': spd}

def get_trj_veil(trj, ntwk, p, s_params):
    """
    Return a "veil" (positive real-valued mask) over cells in the ntwk
    with place fields along the trajectory path.
    """
    # compute scale factor for all PCs
    ## get distance to trj
    d = dist_to_trj(ntwk.pfxs, ntwk.pfys, trj['x'], trj['y'])[0]

    # sensory-driven firing rates as function of d
    r = p['R_MAX'] * np.exp(-.5*(d**2)/(p['L_PL']**2))
    sgm = 1 + (p['SGM_MAX'] - 1) * sgmd(p['B_SGM']*(r - p['R_SGM']))
    veil = sgm - 1

    return veil

def dist_to_trj(pfxs, pfys, x, y):
    """
    Compute distance of static points (pfxs, pfys) to trajectory (x(t), y(t)).

    :return: dists to nearest pts, idxs of nearest pts
    """
    # get dists to all pts along trj
    dx = np.tile(pfxs[None, :], (len(x), 1)) - np.tile(x[:, None], (1, len(pfxs)))
    dy = np.tile(pfys[None, :], (len(y), 1)) - np.tile(y[:, None], (1, len(pfys)))

    d = np.sqrt(dx**2 + dy**2)

    # return dists of cells to nearest pts on trj
    return np.min(d, 0), np.argmin(d, 0)

def apx_ws_up(ntwk, trj_veil):
    """
    Replace G->PC weights with apxns expected following
    initial sensory input.
    """
    scale = trj_veil[ntwk.types_rcr == 'PC'] + 1

    ws_up_e_dense = np.array(ntwk.ws_up['E'].todense())

    n_pc = ntwk.n_pc

    mask = np.zeros((ntwk.n, 2*n_pc), dtype=bool)
    mask[:n_pc, n_pc:] = np.eye(n_pc, dtype=bool)

    ws_up_e_dense[mask] *= scale

    ntwk.ws_up['E'] = csc_matrix(ws_up_e_dense)

    return ntwk

def build_stim(t, trj, ntwk, p, s_params, schedule):
    """
    Put together upstream spk and external current inputs
    according to stimulation params and schedule.
    """
    np.random.seed(s_params['RNG_SEED'])

    # initialize upstream spks array
    spks_up = np.zeros((len(t), 2*p['N_PC']), int)

    # fill in replay epoch Gate inputs
    spks_up += spks_up_from_g(t, ntwk, p, s_params, schedule)

    # initialize external current array
    i_ext = np.zeros((len(t), p['N_PC'] + p['N_INH']))

    # add replay trigger
    i_ext += i_ext_trg(t, ntwk, p, s_params, schedule)

    return spks_up, i_ext

def spks_up_from_g(t, ntwk, p, s_params, schedule):
    """
    Add G --> PC spks to upstream spk array.
    """
    spks_up = np.zeros((len(t), 2*p['N_PC']), int)

    # replay epoch
    spks_up[:, p['N_PC']:] += np.random.poisson(
        p['R_G'] * s_params['DT'], (len(t), p['N_PC']))

    return spks_up

def i_ext_trg(t, ntwk, p, s_params, schedule):
    """
    Add replay trigger to external current stim.
    """
    i_ext = np.zeros((len(t), p['N_PC'] + p['N_INH']))

    # get mask over cells to trigger to induce replay
    for trg in s_params['TRG']:

        ## compute distances to trigger center
        trg_mask = get_trg_mask_pc(ntwk, p, trg)

        ## get time mask
        t_mask = (trg['T'] <= t) \
            & (t < (trg['T'] + trg['D']))

        ## add in external trigger
        i_ext[np.outer(t_mask, trg_mask)] = trg['A']

    return i_ext

def get_trg_mask_pc(ntwk, p, trg):
    """Get spatial mask for replay trigger."""
    dx = ntwk.pfxs - trg['X']
    dy = ntwk.pfys - trg['Y']
    d = np.sqrt(dx**2 + dy**2)

    ## get mask
    trg_mask = (d < trg['R']) & (ntwk.types_rcr == 'PC')

    return trg_mask


import numpy as np
import matplotlib.pyplot as plt
from lmfit import Parameters, minimize, fit_report
from ttim import *

observed = np.loadtxt('oudekorendijk_h30.dat')
to1 = observed[:,0] / 60.0 / 24.0
ho1 = -observed[:,1]
ro1 = 30.0
Qo = 788.0
ml = ModelMaq(kaq=60, z=(-18,-25), Saq=1e-4, tmin=1e-5, tmax=1)
w = Well(ml, xw=0, yw=0, rw=0.1, tsandQ=[(0, 788)], layers=0)
ml.solve(silent='.')

#################
import numpy as np
from lmfit import Parameters, minimize, fit_report

class Fit:
    def __init__(self, model):
        self.model = model
        self.lmfitparams = Parameters()
        self.parameterdict = {}
        self.seriesdict = {}
    def parameter(self, name, par=None, initial=0, vmin=None, vmax=None, vary=True):
        if par is not None:
            if type(par) != np.ndarray:
                p = np.array(par)
            else:
                p = par
        else:
            if name[:3] == 'kaq':
                layer = int(name[3:])
                p = self.model.aq.kaq[layer:layer + 1]
            elif name[:3] == 'Saq':
                layer = int(name[3:])
                p = self.model.aq.Saq[layer:layer + 1]
            else:
                print('parameter name not recognized or no par reference supplied')
                return
        self.lmfitparams.add(name, value=initial, min=vmin, max=vmax, vary=vary)
        self.parameterdict[name] = p
    def series(self, name, x, y, t, h):
        s = Series(x, y, t, h)
        self.seriesdict['name'] = s

class Series:
    def __init__(self, x, y, t, h):
        self.x = x
        self.y = y
        self.t = t
        self.h = h

#####################

fit = Fit(ml)
fit.parameter('kaq0', initial=10)
fit.parameter('Saq0', initial=1e-4)
fit.series('obs1', ro1, 0, to1, ho1)

#def residual(p, t=None, rdata=None, data=None):
#    vals = p.valuesdict()
#    k =  vals['k']
#    Ss =  vals['Ss']
#    ml.aq.set_kaq(k, 0)
#    ml.aq.set_Saq(Ss, 0)
#    ml.solve(silent='.')
#    hm = ml.head(rdata, 0.0, t, layers=[0])
#    if data is None:
#        return hm[0]
#    else:
#        return hm[0] - data  # head in layer 0
#
#p = Parameters()
#p.add('k', value=10.0, min=5)
#p.add('Ss', value=1e-4)
#
#print('Parameter estimation using data at observation well 30 m')
#p30 = minimize(residual, p, kws={'t':to1, 'rdata':ro1, 'data':ho1}, epsfcn=1e-4)
#print()
#print(fit_report(p30.params))


import math
import random
from copy import deepcopy
import numpy as np
import time

import torch

import torch.nn as nn
import torch.optim as optim
import torch.nn.functional as F
from torch.distributions import Normal
from torch.utils.tensorboard import SummaryWriter

from multiprocessing import Process
from multiprocessing.managers import BaseManager

from PZR_bubblegeneration.CNS_PZR import ENVCNS
from MONITORINGTOOL_PZR import MonitoringMEM, Monitoring

torch.multiprocessing.set_start_method('spawn', force=True)

class SharedAdam(optim.Optimizer):
    r"""Implements Adam algorithm.
    It has been proposed in `Adam: A Method for Stochastic Optimization`_.
    Arguments:
        params (iterable): iterable of parameters to optimize or dicts defining
            parameter groups
        lr (float, optional): learning rate (default: 1e-3)
        betas (Tuple[float, float], optional): coefficients used for computing
            running averages of gradient and its square (default: (0.9, 0.999))
        eps (float, optional): term added to the denominator to improve
            numerical stability (default: 1e-8)
        weight_decay (float, optional): weight decay (L2 penalty) (default: 0)
        amsgrad (boolean, optional): whether to use the AMSGrad variant of this
            algorithm from the paper `On the Convergence of Adam and Beyond`_
            (default: False)
    .. _Adam\: A Method for Stochastic Optimization:
        https://arxiv.org/abs/1412.6980
    .. _On the Convergence of Adam and Beyond:
        https://openreview.net/forum?id=ryQu7f-RZ
    """

    def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8,
                 weight_decay=0, amsgrad=False):
        if not 0.0 <= lr:
            raise ValueError("Invalid learning rate: {}".format(lr))
        if not 0.0 <= eps:
            raise ValueError("Invalid epsilon value: {}".format(eps))
        if not 0.0 <= betas[0] < 1.0:
            raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
        if not 0.0 <= betas[1] < 1.0:
            raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
        defaults = dict(lr=lr, betas=betas, eps=eps,
                        weight_decay=weight_decay, amsgrad=amsgrad)
        super(SharedAdam, self).__init__(params, defaults)

    def __setstate__(self, state):
        super(SharedAdam, self).__setstate__(state)
        for group in self.param_groups:
            group.setdefault('amsgrad', False)

    def step(self, closure=None):
        """Performs a single optimization step.
        Arguments:
            closure (callable, optional): A closure that reevaluates the model
                and returns the loss.
        """
        loss = None
        if closure is not None:
            loss = closure()

        for group in self.param_groups:
            for p in group['params']:
                if p.grad is None:
                    continue
                grad = p.grad.data
                if grad.is_sparse:
                    raise RuntimeError('Adam does not support sparse gradients, please consider SparseAdam instead')
                amsgrad = group['amsgrad']

                state = self.state[p]
                ### ADD
                device = p.device
                for k, v in state.items():
                    if isinstance(v, torch.Tensor):
                        state[k] = v.to(device)
                ### ADD

                # State initialization
                if len(state) == 0:
                    state['step'] = 0
                    # Exponential moving average of gradient values
                    state['exp_avg'] = torch.zeros_like(p.data)
                    # Exponential moving average of squared gradient values
                    state['exp_avg_sq'] = torch.zeros_like(p.data)
                    if amsgrad:
                        # Maintains max of all exp. moving avg. of sq. grad. values
                        state['max_exp_avg_sq'] = torch.zeros_like(p.data)

                exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
                if amsgrad:
                    max_exp_avg_sq = state['max_exp_avg_sq']
                beta1, beta2 = group['betas']

                state['step'] += 1

                if group['weight_decay'] != 0:
                    grad.add_(group['weight_decay'], p.data)

                # Decay the first and second moment running average coefficient
                #exp_avg.mul_(beta1).add_(1 - beta1, grad)
                #exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)

                exp_avg.mul_(beta1).add_(grad, alpha=1 - beta1)
                exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)

                if amsgrad:
                    # Maintains the maximum of all 2nd moment running avg. till now
                    torch.max(max_exp_avg_sq, exp_avg_sq, out=max_exp_avg_sq)
                    # Use the max. for normalizing running avg. of gradient
                    denom = max_exp_avg_sq.sqrt().add_(group['eps'])
                else:
                    denom = exp_avg_sq.sqrt().add_(group['eps'])

                bias_correction1 = 1 - beta1 ** state['step']
                bias_correction2 = 1 - beta2 ** state['step']
                step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1

                p.data.addcdiv_(-step_size, exp_avg, denom)

        return loss

class ReplayBuffer:
    def __init__(self, capacity):
        self.capacity = capacity
        self.buffer = []
        self.position = 0
        self.tot_ep = 0

    def push(self, state, action, reward, next_state, done):
        if len(self.buffer) < self.capacity:
            self.buffer.append(None)
        self.buffer[self.position] = (state, action, reward, next_state, done)
        self.position = int((self.position + 1) % self.capacity)  # as a ring buffer

    def sample(self, batch_size):
        batch = random.sample(self.buffer, batch_size)
        state, action, reward, next_state, done = map(np.stack,
                                                      zip(*batch))  # stack for each element
        '''
        the * serves as unpack: sum(a,b) <=> batch=(a,b), sum(*batch) ;
        zip: a=[1,2], b=[2,3], zip(a,b) => [(1, 2), (2, 3)] ;
        the map serves as mapping the function on each list element: map(square, [2,3]) => [4,9] ;
        np.stack((1,2)) => array([1, 2])
        '''
        return state, action, reward, next_state, done

    def __len__(
            self):  # cannot work in multiprocessing case, len(replay_buffer) is not available in proxy of manager!
        return len(self.buffer)

    def get_length(self):
        return len(self.buffer)

    def add_ep(self):
        self.tot_ep += 1

    def get_ep(self):
        return self.tot_ep

class SoftQNetwork(nn.Module):
    def __init__(self, num_inputs, num_actions, hidden_size, init_w=3e-3):
        super(SoftQNetwork, self).__init__()

        self.linear1 = nn.Linear(num_inputs + num_actions, hidden_size)
        # self.linear2 = nn.Linear(hidden_size, hidden_size)
        # self.linear3 = nn.Linear(hidden_size, hidden_size)
        self.linear4 = nn.Linear(hidden_size, 1)

        self.linear4.weight.data.uniform_(-init_w, init_w)
        self.linear4.bias.data.uniform_(-init_w, init_w)

    def forward(self, state, action):
        x = torch.cat([state, action], 1)  # the dim 0 is number of samples
        x = F.relu(self.linear1(x))
        # x = F.relu(self.linear2(x))
        # x = F.relu(self.linear3(x))
        x = self.linear4(x)
        return x

class PolicyNetwork(nn.Module):
    def __init__(self, num_inputs, num_actions, hidden_size, action_range=1., init_w=3e-3, log_std_min=1e-6,
                 log_std_max=1):
        super(PolicyNetwork, self).__init__()

        self.log_std_min = log_std_min
        self.log_std_max = log_std_max

        self.linear1 = nn.Linear(num_inputs, hidden_size)
        # self.linear2 = nn.Linear(hidden_size, hidden_size)
        # self.linear3 = nn.Linear(hidden_size, hidden_size)
        self.linear4 = nn.Linear(hidden_size, hidden_size)

        self.mean_linear = nn.Linear(hidden_size, num_actions)
        self.mean_linear.weight.data.uniform_(-init_w, init_w)
        self.mean_linear.bias.data.uniform_(-init_w, init_w)

        self.log_std_linear = nn.Linear(hidden_size, num_actions)
        self.log_std_linear.weight.data.uniform_(-init_w, init_w)
        self.log_std_linear.bias.data.uniform_(-init_w, init_w)

        self.action_range = action_range
        self.num_actions = num_actions

    def forward(self, state):
        x = F.relu(self.linear1(state))
        # x = F.relu(self.linear2(x))
        # x = F.relu(self.linear3(x))
        x = F.relu(self.linear4(x))

        # mean = (torch.clamp(self.mean_linear(x), -2, 2))
        mean = self.mean_linear(x)
        # mean    = F.leaky_relu(self.mean_linear(x))
        log_std = self.log_std_linear(x)
        log_std = torch.clamp(log_std, self.log_std_min, self.log_std_max)

        return mean, log_std

    def evaluate(self, state, epsilon=1e-6):
        '''
        generate sampled action with state as input wrt the policy network;
        '''
        mean, log_std = self.forward(state)
        std = log_std.exp()  # no clip in evaluation, clip affects gradients flow

        normal = Normal(0, 1)
        z = normal.sample(mean.shape)
        action_0 = torch.tanh(mean + std * z.cuda())  # TanhNormal distribution as actions; reparameterization trick

        action = self.action_range * action_0
        log_prob = Normal(mean, std).log_prob(mean + std * z.cuda()) - torch.log(
            1. - action_0.pow(2) + epsilon) - np.log(self.action_range)
        # both dims of normal.log_prob and -log(1-a**2) are (N,dim_of_action);
        # the Normal.log_prob outputs the same dim of input features instead of 1 dim probability,
        # needs sum up across the features dim to get 1 dim prob; or else use Multivariate Normal.
        log_prob = log_prob.sum(dim=1, keepdim=True)

        return action, log_prob, z, mean, log_std

    def get_action(self, state, deterministic):
        state = torch.FloatTensor(state).unsqueeze(0).cuda()
        # print(state)
        mean, log_std = self.forward(state)
        std = log_std.exp()

        normal = Normal(0, 1)
        z = normal.sample(mean.shape).cuda()
        action = self.action_range * torch.tanh(mean + std * z)

        action = self.action_range * torch.tanh(mean).detach().cpu().numpy()[0] if deterministic else \
            action.detach().cpu().numpy()[0]
        return action, mean.data.tolist()[0], std.data.tolist()[0]

    def sample_action(self, state):
        a = torch.FloatTensor(self.num_actions).uniform_(-1, 1)
        _, mean, std = self.get_action(state, deterministic=True)
        return self.action_range * a.numpy(), mean, std

class Alpha(nn.Module):
    ''' nn.Module class of alpha variable, for the usage of parallel on gpus '''

    def __init__(self):
        super(Alpha, self).__init__()
        self.log_alpha = torch.nn.Parameter(torch.zeros(1))  # initialized as [0.]: alpha->[1.]

    def forward(self):
        return self.log_alpha

class SAC_Trainer():
    def __init__(self, replay_buffer, hidden_dim, action_range):
        self.replay_buffer = replay_buffer
        self.action_dim = action_dim

        self.soft_q_net1 = SoftQNetwork(state_dim, action_dim, hidden_dim)
        self.soft_q_net2 = SoftQNetwork(state_dim, action_dim, hidden_dim)
        self.target_soft_q_net1 = SoftQNetwork(state_dim, action_dim, hidden_dim)
        self.target_soft_q_net2 = SoftQNetwork(state_dim, action_dim, hidden_dim)
        self.policy_net = PolicyNetwork(state_dim, action_dim, hidden_dim, action_range)
        self.log_alpha = Alpha()
        print('Soft Q Network (1,2): ', self.soft_q_net1)
        print('Policy Network: ', self.policy_net)

        for target_param, param in zip(self.target_soft_q_net1.parameters(),
                                       self.soft_q_net1.parameters()):
            target_param.data.copy_(param.data)
        for target_param, param in zip(self.target_soft_q_net2.parameters(),
                                       self.soft_q_net2.parameters()):
            target_param.data.copy_(param.data)

        self.soft_q_criterion1 = nn.MSELoss()
        self.soft_q_criterion2 = nn.MSELoss()

        soft_q_lr = 3e-4
        policy_lr = 3e-4
        alpha_lr = 3e-4

        self.soft_q_optimizer1 = SharedAdam(self.soft_q_net1.parameters(), lr=soft_q_lr)
        self.soft_q_optimizer2 = SharedAdam(self.soft_q_net2.parameters(), lr=soft_q_lr)
        self.policy_optimizer = SharedAdam(self.policy_net.parameters(), lr=policy_lr)
        self.alpha_optimizer = SharedAdam(self.log_alpha.parameters(), lr=alpha_lr)

    def to_cuda(self):  # copy to specified gpu
        self.soft_q_net1 = self.soft_q_net1.cuda()
        self.soft_q_net2 = self.soft_q_net2.cuda()
        self.target_soft_q_net1 = self.target_soft_q_net1.cuda()
        self.target_soft_q_net2 = self.target_soft_q_net2.cuda()
        self.policy_net = self.policy_net.cuda()
        self.log_alpha = self.log_alpha.cuda()

    def update(self, batch_size, reward_scale=5., auto_entropy=True, target_entropy=-2, gamma=0.99,
               soft_tau=0.005):
        # soft_tau=1e-2
        state, action, reward, next_state, done = self.replay_buffer.sample(batch_size)
        # print('sample:', state, action,  reward, done)

        state = torch.FloatTensor(state).cuda()
        next_state = torch.FloatTensor(next_state).cuda()
        action = torch.FloatTensor(action).cuda()
        # reward is single value, unsqueeze() to add one dim to be [reward] at the sample dim;
        reward = torch.FloatTensor(reward).unsqueeze(1).cuda()
        done = torch.FloatTensor(np.float32(done)).unsqueeze(1).cuda()

        predicted_q_value1 = self.soft_q_net1(state, action)
        predicted_q_value2 = self.soft_q_net2(state, action)
        new_action, log_prob, z, mean, log_std = self.policy_net.evaluate(state)
        new_next_action, next_log_prob, _, _, _ = self.policy_net.evaluate(next_state)
        reward = reward_scale * (reward - reward.mean(dim=0)) / (reward.std(
            dim=0) + 1e-6)  # normalize with batch mean and std; plus a small number to prevent numerical problem

        # Updating alpha wrt entropy
        # alpha = 0.0
        # trade-off between exploration (max entropy) and exploitation (max Q)
        if auto_entropy is True:
            # self.log_alpha as forward function to get value
            alpha_loss = -(self.log_alpha() * (log_prob - 1.0 * self.action_dim).detach()).mean()
            # print('alpha loss: ',alpha_loss)
            self.alpha_optimizer.zero_grad()
            alpha_loss.backward()
            self.alpha_optimizer.step()
            self.alpha = self.log_alpha().exp()
        else:
            self.alpha = 1.
            alpha_loss = 0

        # print(self.alpha)
        # Training Q Function
        target_q_min = torch.min(self.target_soft_q_net1(next_state, new_next_action),
                                 self.target_soft_q_net2(next_state,
                                                         new_next_action)) - self.alpha * next_log_prob
        target_q_value = reward + (1 - done) * gamma * target_q_min  # if done==1, only reward
        q_value_loss1 = self.soft_q_criterion1(predicted_q_value1,
                                               target_q_value.detach())  # detach: no gradients for the variable
        q_value_loss2 = self.soft_q_criterion2(predicted_q_value2, target_q_value.detach())

        self.soft_q_optimizer1.zero_grad()
        q_value_loss1.backward()
        self.soft_q_optimizer1.step()
        self.soft_q_optimizer2.zero_grad()
        q_value_loss2.backward()
        self.soft_q_optimizer2.step()

        # Training Policy Function
        predicted_new_q_value = torch.min(self.soft_q_net1(state, new_action),
                                          self.soft_q_net2(state, new_action))
        policy_loss = (self.alpha * log_prob - predicted_new_q_value).mean()

        self.policy_optimizer.zero_grad()
        policy_loss.backward()
        self.policy_optimizer.step()

        # print('q loss: ', q_value_loss1, q_value_loss2)
        # print('policy loss: ', policy_loss )

        # Soft update the target value net
        for target_param, param in zip(self.target_soft_q_net1.parameters(),
                                       self.soft_q_net1.parameters()):
            target_param.data.copy_(  # copy data value into target parameters
                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
            )
        for target_param, param in zip(self.target_soft_q_net2.parameters(),
                                       self.soft_q_net2.parameters()):
            target_param.data.copy_(  # copy data value into target parameters
                target_param.data * (1.0 - soft_tau) + param.data * soft_tau
            )
        # return predicted_new_q_value.mean()
        q1_, q2_, po_ = q_value_loss1.detach(), q_value_loss2.detach(), policy_loss.detach()

        return 0, q1_.item(), q2_.item(), po_.item()

    def save_model(self, path):
        torch.save(self.soft_q_net1.state_dict(),
                   path + '_q1')  # have to specify different path name here!
        torch.save(self.soft_q_net2.state_dict(), path + '_q2')
        torch.save(self.policy_net.state_dict(), path + '_policy')

    def load_model(self, path):
        self.soft_q_net1.load_state_dict(
            torch.load(path + '_q1', map_location='cuda:0'))  # map model on single gpu for testing
        self.soft_q_net2.load_state_dict(torch.load(path + '_q2', map_location='cuda:0'))
        self.policy_net.load_state_dict(torch.load(path + '_policy', map_location='cuda:0'))

        self.soft_q_net1.eval()
        self.soft_q_net2.eval()
        self.policy_net.eval()

def worker(id, sac_trainer, replay_buffer, monitoring_mem, max_episodes, max_steps, batch_size,
           explore_steps, update_itr, AUTO_ENTROPY, DETERMINISTIC):
    """
    the function for sampling with multi-processing

    :param id:
    :param sac_trainer:
    :param replay_buffer:
    :param monitoring_mem:
    :param max_episodes:
    :param max_steps:
    :param batch_size:
    :param explore_steps:
    :param update_itr:
    :param AUTO_ENTROPY:
    :param DETERMINISTIC:
    :return: 0
    """

    with torch.cuda.device(id % torch.cuda.device_count()):
        sac_trainer.to_cuda()
        # Divide agents according to below information
        # 'id' is 0, 1, 2, 3, ...

        _CNS_info = {
            0: ['192.168.0.4', 7101, False],
            1: ['192.168.0.4', 7102, False],
            2: ['192.168.0.4', 7103, False],
            3: ['192.168.0.4', 7104, False],
            4: ['192.168.0.4', 7105, False],

            5: ['192.168.0.9', 7201, False],
            6: ['192.168.0.9', 7202, False],
            7: ['192.168.0.9', 7203, False],
            8: ['192.168.0.9', 7204, False],
            9: ['192.168.0.9', 7205, False],
        }

        # Set CNS
        env = ENVCNS(Name=id, IP=_CNS_info[id][0], PORT=_CNS_info[id][1])
        env.PID_Mode = _CNS_info[id][2]
        # env.PID_Mode = True if id == 2 else False  # PID는 마지막 2번 에이전트가 담당함.
        action_dim = env.action_space

        # Agent info and get_shared mem
        print(f'Agent {id}|Trainer {sac_trainer}|'
              f'ReplayBuffer {replay_buffer}|MonitoringMem {monitoring_mem}|'
              f'ENV CNSIP{env.CNS_ip}-CNSPort{env.CNS_port}-ComIP{env.Remote_ip}-ComPort{env.Remote_port}')

        # Worker mem
        Wm = {'ep_acur': 0, 'ep_q1': 0, 'ep_q2': 0, 'ep_p': 0}

        frame_idx = 0
        # training loop
        for eps in range(max_episodes):
            # Buffer <-
            replay_buffer.add_ep()
            # CNS reset <-
            state = env.reset(file_name=f'{id}_{eps}')
            time.sleep(1)

            for step in range(max_steps):
                if frame_idx > explore_steps:
                    action, mean_, std_ = sac_trainer.policy_net.get_action(state, deterministic=DETERMINISTIC)
                else:
                    action, mean_, std_ = sac_trainer.policy_net.sample_action(state)
                old_action = deepcopy(action)

                # MonitoringMem <-
                monitoring_mem.push_ENV_val(id, CNSMem=env.mem)

                # CNS Step <-
                next_state, reward, done, action = env.step(action)

                # MonitoringMem <-
                monitoring_mem.push_ENV_ActDis(id, Dict_val={'Mean': mean_, 'Std': std_,
                                                             'OA0': old_action[0], 'A0': action[0],
                                                             'OA1': old_action[1], 'A1': action[1],
                                                             # 'OA2': old_action[2], 'A2': action[2],
                                                             })

                # MonitoringMem <- last order ...
                Wm['ep_acur'] += reward
                monitoring_mem.push_ENV_reward(id, Dict_val={'R': reward, 'AcuR': Wm['ep_acur']})

                # Buffer <-
                if env.PID_Mode == False:
                    replay_buffer.push(state, action, reward, next_state, done)

                # s <- next_s
                state = next_state
                frame_idx += 1

                # if len(replay_buffer) > batch_size:
                if replay_buffer.get_length() > batch_size and env.PID_Mode == False:
                    for i in range(update_itr):
                        _, episode_q1, episode_q2, episode_p = sac_trainer.update(batch_size,
                                                                                reward_scale=10.,
                                                                                auto_entropy=AUTO_ENTROPY,
                                                                                target_entropy=-1. * action_dim)
                        Wm['ep_q1'] += episode_q1
                        Wm['ep_q2'] += episode_q2
                        Wm['ep_p'] += episode_p

                # Done ep ??
                if done:
                    print(f"END_EP [{replay_buffer.get_ep()}]|"
                          f"Ep AccR [{Wm['ep_acur']}]|"
                          f"EP q1 [{Wm['ep_q1']}]|"
                          f"EP q2 [{Wm['ep_q2']}]|"
                          f"EP p [{Wm['ep_p']}]")

                    # MonitoringMem < - last order...
                    monitoring_mem.push_ENV_epinfo({'AcuR/Ep': Wm['ep_acur'], 'q1/Ep': Wm['ep_q1'],
                                                    'q2/Ep': Wm['ep_q2'], 'p/Ep': Wm['ep_p']})
                    monitoring_mem.init_ENV_val(id)

                    # Dump reward to txt
                    with open('DumpRewardLog.txt', 'a') as f:
                        f.write(f'{id},{Wm["ep_acur"]},{Wm["ep_q1"]},{Wm["ep_q2"]},{Wm["ep_p"]}\n')

                    # Dump Model
                    sac_trainer.save_model('Model/Agent')

                    for _ in Wm.keys():
                        Wm[_] = 0
                    break
                else:
                    # every step ...
                    pass

            # Keep ...
            # sac_trainer.save_model(model_path)

def ShareParameters(adamoptim):
    ''' share parameters of Adamoptimizers for multiprocessing '''
    for group in adamoptim.param_groups:
        for p in group['params']:
            state = adamoptim.state[p]
            # initialize: have to initialize here, or else cannot find
            state['step'] = 0
            state['exp_avg'] = torch.zeros_like(p.data)
            state['exp_avg_sq'] = torch.zeros_like(p.data)

            # share in memory
            state['exp_avg'].share_memory_()
            state['exp_avg_sq'].share_memory_()

if __name__ == '__main__':

    replay_buffer_size = 1e7 #1e6
    num_workers = 10  # or: mp.cpu_count()
    # hyper-parameters for RL training, no need for sharing across processes
    max_episodes = 1000
    max_steps = 2000
    explore_steps = 0  # for random action sampling in the beginning of training
    batch_size = 640
    update_itr = 1
    action_itr = 3
    AUTO_ENTROPY = True
    DETERMINISTIC = False
    hidden_dim = 512
    model_path = ''

    # the replay buffer is a class, have to use torch manager to make it a proxy for sharing across processes
    BaseManager.register('ReplayBuffer', ReplayBuffer)
    BaseManager.register('MonitoringMEM', MonitoringMEM)
    manager = BaseManager()
    manager.start()
    replay_buffer = manager.ReplayBuffer(replay_buffer_size)  # share the replay buffer through manager
    monitoring_mem = manager.MonitoringMEM(num_workers)

    # choose env
    env = ENVCNS(Name='GETINFO', IP='192.168.0.4', PORT=int(f'7100'))
    action_dim = env.action_space
    state_dim = env.observation_space
    action_range = 1.

    sac_trainer = SAC_Trainer(replay_buffer, hidden_dim=hidden_dim, action_range=action_range)

    # share the global parameters in multiprocessing
    sac_trainer.soft_q_net1.share_memory()
    sac_trainer.soft_q_net2.share_memory()
    sac_trainer.target_soft_q_net1.share_memory()
    sac_trainer.target_soft_q_net2.share_memory()
    sac_trainer.policy_net.share_memory()

    ShareParameters(sac_trainer.soft_q_optimizer1)
    ShareParameters(sac_trainer.soft_q_optimizer2)
    ShareParameters(sac_trainer.policy_optimizer)
    ShareParameters(sac_trainer.alpha_optimizer)

    # process ------------------------------------------------------------------------------------------
    processes = []
    # Worker process
    for i in range(num_workers):
        process = Process(target=worker, args=(
            i, sac_trainer, replay_buffer, monitoring_mem, max_episodes, max_steps, \
            batch_size, explore_steps, update_itr, AUTO_ENTROPY, DETERMINISTIC
        ))  # the args contain shared and not shared
        process.daemon = True  # all processes closed when the main stops
        processes.append(process)

    # Monitoring process
    m_process = Monitoring(monitoring_mem)
    processes.append(m_process)

    [p.start() for p in processes]
    [p.join() for p in processes]  # finished at the same time


# Given a string s consists of upper/lower-case alphabets and empty space characters ' ', return the length of last word in the string.
# If the last word does not exist, return 0.
# Note: A word is defined as a character sequence consists of non-space characters only.
# For example, Given s = "Hello World", return 5.
class Solution(object):
    def lengthOfLastWord(self, s):
        """
        :type s: str
        :rtype: int
        """
        n = len(s) - 1
        if n == -1 :
            return 0
        c_space = ord(' ')
        while n >= 0 and ord(s[n]) == c_space :
            n -= 1
        size = n + 1
        while n >= 0 :
            if ord(s[n]) == c_space :
                return size - n - 1
            n -= 1
        return size

py/058 Length of Last Word/LengthofLastWord.py


import numpy as np
import cv2 as cv
import collections
from myUtils import *
import ransac
import math

font = cv2.FONT_HERSHEY_SIMPLEX

def left90degree(frame):
    frame = cv.transpose(frame)
    frame = cv.flip(frame, 0)
    return frame

# 背景建模法
if __name__ == "__main__":

    # 测试视频
    videoUrl = "D:\develop\python3\jupyter\water-spot-detection\\pink-red-drop.mp4"
    cap = cv.VideoCapture(videoUrl)

    fps = cap.get(cv.CAP_PROP_FPS)
    # 形态学操作需要内核
    kernel = cv.getStructuringElement(cv.MORPH_ELLIPSE, (10, 10))
    # 基于KNN的背景/前景分割算法
    history = 5
    fgbg = cv.createBackgroundSubtractorKNN()
    fgbg.setHistory(history)
    ret, orignal = cap.read()
    # 左转90度
    orignal = left90degree(orignal)
    # orignal = cv.resize(orignal, (0, 0), None, fx=0.5, fy=0.5)
    # orignal = cv.blur(orignal, (5, 5))
    # 记录滴液质心位置
    points_list = []

    # 拟合数据
    n_inputs = 1
    n_outputs = 1
    input_columns = range(n_inputs)  # the first columns of the array
    output_columns = [n_inputs + i for i in range(n_outputs)]  # the last columns of the array
    model = ransac.LinearLeastSquaresModel(input_columns, output_columns)
    while (True):
        ret, frame = cap.read()
        # 左转90度
        frame = left90degree(frame)

        if ret == False:
            break

        # 算法比对
        fgmask = fgbg.apply(frame)
        # 建立一下历史
        if history + 10 > 0:
            history -= 1
            continue

        frame_copy = frame.copy()

        # frame = cv.blur(frame, (5, 5))
        # 形态学开运算加腐蚀去噪点
        fgmask = cv.morphologyEx(fgmask, cv.MORPH_OPEN, kernel)
        fgmask = cv.dilate(fgmask, None, iterations=3)
        # 寻找轮廓
        binary, contours, hierarchy = cv.findContours(image=fgmask, mode=cv.RETR_EXTERNAL, method=cv.CHAIN_APPROX_NONE)
        for c in contours:
            # 计算各轮廓面积
            area = cv.contourArea(c)

            if area > 500:
                x, y, w, h = cv.boundingRect(c)
                cv.rectangle(frame, (x, y), (x + h, y + w), (255, 255, 0), 2)
                gravity = center([[x, y], [x + w, y], [x, y + h], [x + w, y + h]])
                # gravity = center(x, y, w, h)
                # points_list.append([int(gravity[0]), int(gravity[1])])
                points_list.append((gravity[0], gravity[1]))
                # 轮廓切割roi
                frame_roi = frame[y:y + h, x:x + w]
                roi = orignal[y:y + h, x:x + w]
                # 查看与原来模块roi区域的相似度
                similarity = hist_similarity(cv.cvtColor(frame_roi, cv.COLOR_BGR2GRAY),
                                             cv.cvtColor(roi, cv.COLOR_BGR2GRAY))
                # similarity = classify_hist_with_split(frame_roi, roi)
                # print(similarity)
                if similarity < 0.5:
                    # 画出这个矩形
                    cv.rectangle(frame_copy, (x, y), (x + w, y + h), (0, 0, 255), 2)
                    cv.putText(frame_copy, "water spot", (x, y - 50), cv.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 2)

        #
        for i in range(len(points_list)):
            if i == 0:
                continue
            cv.line(frame_copy, points_list[i - 1], points_list[i], (255, 255, 0), 2)
        #
        np_array = []
        if len(points_list) > 50:
            for i in range(len(points_list)):
                np_array.append(list(points_list[i]))
            np_array = np.array(np_array, dtype=np.float64)
            lstsq_array = np_array[:, input_columns]
            lstsq_array = lstsq_array.reshape((1, -1))
            A = np.vstack([lstsq_array**0,lstsq_array**1])

            lstsq_array = np_array[:, output_columns]
            lstsq_array = lstsq_array.ravel()

            print("A:",A)
            print("B:",lstsq_array)
            np_array2  = np_array
            print(np_array)
            # print(np_array[:, input_columns])
            # print(np_array[:, output_columns])
            try:
                linear_fit, resids, rank, s = scipy.linalg.lstsq(A.T,
                                                                 lstsq_array )
                b = linear_fit[0]
                a = linear_fit[1]
                linear_fit = np.array(linear_fit)
                linear_fit.reshape((2,1))
                print("b:",b)
                print("a:",a)
                # run RANSAC algorithm
                ransac_fit, ransac_data= ransac.ransac(np_array, model,
                                           5, 500, 3e5, 30,  # misc. parameters
                                           debug=False, return_all=True)
                tmp = ransac_fit[0][0]
                ransac_fit[0][0] = ransac_fit[1][0]
                ransac_fit[1][0] = tmp

            except ValueError as e:
                print("not find", e)
                pass
            else :
                print("ransac_fit:", ransac_fit)
                # print(type(ransac_fit))
                np_array = np_array[:, 0]
                np_array = np_array.reshape(-1, 1)
                # print("np_array:",np_array)
                sort_idxs = np.argsort(np_array[:, 0])
                # print("sort_idxs:",sort_idxs)

                k = np.max(ransac_fit[0])
                if k<0:
                    k = -k
                print("K:", k)
                angle = math.atan2(k, 1)
                theta = angle * (180 / math.pi)
                if theta > 90:
                    print(theta)
                    theta = 180 - theta
                if (theta >= 0 and theta <= 22) or (theta >= 70 and theta <= 90):
                    cv2.putText(frame_copy, "water line appear", (20, 100)
                                , font, 1,
                                (0, 255, 0),
                                1,
                                cv2.LINE_AA)
                print("theta:", theta, "度")

                # angle = Math.atan2((p2.x - p1.x), (p2.y - p1.y)) // 弧度
                # 0.9272952180016122
                # theta = angle * (180 / Math.PI); // 角度
                # 53.13010235415598
                A_col0_sorted = np_array[sort_idxs]  # maintain as rank-2 array
                import pylab
                pylab.plot(A_col0_sorted[:, 0],
                           np.dot(A_col0_sorted, ransac_fit.T)[:, 0],
                           label='RANSAC fit')
                # pylab.plot(A_col0_sorted[:, 0],
                #            np.dot(A_col0_sorted, linear_fit.T)[:, 0],
                #            label='linear_fit')
                # pylab.scatter(ransac_data[:, 0], ransac_data[:, 1], color="b")
                pylab.scatter(np_array2[:, 0], np_array2[:, 1], color="r")
                # x y 互换
                # pylab.scatter(np_array2[:, 1], np_array2[:, 0], color="r")
                pylab.show()
                print(points_list)
                points_list.clear()

        # TODO 根据points_list的点拟合线性函数
        cv2.namedWindow("frame", 0)
        cv2.resizeWindow("frame", 600, 800)
        cv2.namedWindow("fgmask", 0)
        cv2.resizeWindow("fgmask", 600, 800)
        cv.imshow("frame", frame_copy)
        cv.imshow("fgmask", fgmask)
        k = cv.waitKey(int(1000.0 / fps)) & 0xff
        # k = cv.waitKey(70) & 0xff
        if k == 27:
            break

    cv.destroyAllWindows()
    cap.release()


import os
import re
import shutil
import subprocess
import pkg_resources
import pandas
# import toytree
# import toyplot
import phylovega
import phylopandas
import pyasr
import dendropy
from functools import wraps
from time import localtime, strftime

import Bio.Phylo.Applications

def track_in_history(method):
    """Track this call in the history DataFrame"""
    @wraps(method)
    def wrapper(self, *args, **kwargs):
        """"""
        # Now run method
        output = method(self, *args, **kwargs)

        # Store method in history.
        # Prepare data for history dataframe
        time =strftime("%Y-%m-%d %H:%M:%S", localtime())
        args_as_str = ",".join([str(a) for a in args])
        kwargs_as_str = ",".join([str((key, val)) for key, val in kwargs.items()])

        # Create row
        history = pandas.DataFrame({'time':[time],
            'method':[method.__name__],
            'args':[args_as_str],
            'kwargs':[kwargs_as_str]}, dtype=str)

        # Append to main history dataframe
        self.history = self.history.append(history, ignore_index=True)

        # Write history to file.
        history_file = os.path.join(self.project_dir, 'history.csv')
        self.history.to_csv(history_file)
        self.save()
        return output
    return wrapper

class PhylogeneticsProject(object):
    """A lightweight python object that manages phylogenetic data.

    There are three datatypes stored in a PhylogeneticsProject object:

        * 'tips' : A DataFrame with an alignment and information about the tips of the tree.
        * 'ancs' : A DataFrame with information about the ancestors of the tree.
        * 'tree' : A dendropy.Tree object containing the topology of the phylogenetic tree.

    Parameters
    ----------
    project_dir : str
        the directory to store the phylogenetic data.
    overwrite : bool (default: False)
        allow overwriting a project that already exists in project_dir location.
    """
    def __init__(self, project_dir, overwrite=False):
        # Get current time for history.
        time = strftime("%Y-%m-%d %H:%M:%S", localtime())

        # Set up a project directory
        if os.path.exists(project_dir) and overwrite is False:
            raise Exception("Project already exists! Use `PhylogeneticsProject.load` or delete the project.")
        elif not os.path.exists(project_dir):
            os.makedirs(project_dir)

        self.project_dir = project_dir

        # DataFrame for storing data.
        self.data = {'tips':None, 'ancs':None, 'tree':None}

        # Create a history database.
        self.history = pandas.DataFrame({'time':[time],
            'method':['__init__'],
            'args':[project_dir],
            'kwargs':[None]}, dtype=str)

        # Write history to file.
        history_file = os.path.join(self.project_dir, 'history.csv')
        self.history.to_csv(history_file)

    def __str__(self):
        # Get history
        history = self.history.iloc[-1]
        tips_bool = self.data['tips'] is not None
        ancs_bool = self.data['ancs'] is not None
        tree_bool = self.data['tree'] is not None

        # Start building history.
        info = ["PhylogeneticsProject(project_dir={})\n".format(self.project_dir),
                "    last modified\t{}\n".format(history['time']),
                "    last edit\t\t{}\n".format(history['method'])]

        # If leafs, add stats
        info += ["    tips\t\t{}\n".format(tips_bool)]
        if tips_bool:
            leafs = self.data['tips']
            info += ["        - num of tips\t{}\n".format(len(leafs))]

        # If ancestors, add stats.
        info += ["    ancs\t\t{}\n".format(ancs_bool)]
        if ancs_bool:
            ancs = self.data['ancs']
            info += ["        - num of ancs\t{}\n".format(len(ancs))]

        # If tree, add stats.
        info += ["    tree\t\t{}\n".format(tree_bool)]
        return "".join(info).strip()

    def __repr__(self):
        return self.__str__()

    def save(self):
        """Save this project to file."""
        # Save history
        history_path = os.path.join(self.project_dir, 'history.csv')
        self.history.to_csv(history_path)

        # Save tips data.
        if self.data['tips'] is not None:
            tips_path = os.path.join(self.project_dir, 'tips.csv')
            self.data['tips'].to_csv(tips_path)

        # Save ancs data.
        if self.data['ancs'] is not None:
            ancs_path = os.path.join(self.project_dir, 'ancs.csv')
            self.data['ancs'].to_csv(ancs_path)

        # Save tree data.
        if self.data['tree'] is not None:
            tree_path = os.path.join(self.project_dir, 'tree.newick')
            self.data['tree'].write(path=tree_path, schema='newick')
        return self

    @classmethod
    def load(cls, project_dir):
        """Load a project from a project directory."""
        if not os.path.exists(project_dir):
            raise Exception("project_dir does not exist")

        # load history
        history_path = os.path.join(project_dir, 'history.csv')
        history = pandas.read_csv(history_path, index_col=0)

        # Initialize a TreeProject class.
        self = cls(project_dir, overwrite=True)

        # Append old history.
        self.history = history

        # Try reading tips
        try:
            tips_path = os.path.join(self.project_dir, 'tips.csv')
            tips_df = pandas.read_csv(tips_path, index_col=0)
            self._add_tips(tips_df)
        except: pass

        # Try reading ancestors
        try:
            ancs_path = os.path.join(self.project_dir, 'ancs.csv')
            ancs_df = pandas.read_csv(ancs_path, index_col=0)
            self._add_ancs(ancs_df)
        except: pass

        # Try reading tree
        try:
            tree_path = os.path.join(self.project_dir, 'tree.newick')
            tree = dendropy.Tree.get(path=tree_path, schema='newick')
            self._add_tree(tree)
        except: pass

        return self

    def _add_tips(self, data):
        """Add data about the tips (must be a DataFrame) to the project class."""
        if isinstance(data, pandas.DataFrame) is False:
            raise Exception('Bad datatype.')

        # Add unique ids
        if 'unique_id' not in data:
            unique_ids = ["tip{:07d}".format(i) for i in range(len(data))]
            col = pandas.Series(unique_ids, index=data.index)
            data['unique_id'] = col

        self.data['tips'] = data
        self.tips = data

    def _add_ancs(self, data):
        """Add data about the ancestors (must be a DataFrame) to the project class."""
        if isinstance(data, pandas.DataFrame) is False:
            raise Exception('Bad datatype.')

        # Add unique ids
        if 'unique_id' not in data:
            unique_ids = ["anc{:07d}".format(i) for i in range(len(data))]
            col = pandas.Series(unique_ids, index=data.index)
            data['unique_id'] = col

        self.data['ancs'] = data
        self.ancestors = data

    def _add_tree(self, data):
        """Add tree object (must be dendropy.Tree object) to the project class."""
        if isinstance(data, dendropy.Tree) == False:
            raise Exception('Bad datatype.')
        self.data['tree'] = data
        self.tree = data

    @track_in_history
    def add_data(self, dtype, data):
        """Add data to this phylogenetic project.

        Three types of data can be passed to this object.
            'tips' : A DataFrame with an alignment and information about the tips of
                    the tree.
            'ancs' : A DataFrame with information about the ancestors of the tree.
            'tree' : A dendropy.Tree object containing the topology of the phylogenetic tree.

        Parameters
        ----------
        dtype : str
            the type of data you are giving the project: 'tips', 'ancs', or 'tree'.
        data :
            the data to store.
        """
        if dtype not in ['tips', 'ancs', 'tree']:
            raise Exception('dtype is not valid.')

        # Call method on data.
        method = getattr(self, '_add_{}'.format(dtype))
        method(data)
        return self

    @track_in_history
    def read_data(self, dtype, path, schema='fasta', **kwargs):
        """Read data from file.

        Three types of data can be passed to this object.
            'tips' : A DataFrame with an alignment and information about the tips of
                    the tree.
            'ancs' : A DataFrame with information about the ancestors of the tree.
            'tree' : A dendropy.Tree object containing the topology of the phylogenetic tree.

        Parameters
        ----------
        dtype : str
            the type of data you are giving the project: 'tips', 'ancs', or 'tree'.
        path : str
            file path to the data.
        schema : str
            the format of the datafile. Must be one of the formats supported by
            PhyloPandas.
        """
        try:
            method_read = getattr(phylopandas, 'read_{}'.format(schema))
        except AttributeError:
            method_read = getattr(pandas, 'read_{}'.format(schema))

        df = method_read(path, **kwargs)
        method_add = getattr(self, '_add_{}'.format(dtype))
        method_add(df)
        return self

    def write_data(self, dtype, path, schema='fasta', **kwargs):
        """Write a piece of the project data to file.

        Parameters
        ----------
        dtype : str
            the type of data you are giving the project: 'tips', 'ancs', or 'tree'.
        path : str
            file path to the data.
        schema : str
            the format of the datafile. Must be one of the formats supported by
            PhyloPandas.
        """
        df = self.data[dtype]

        # Get write method.
        try:
            method_write = getattr(df, 'to_{}'.format(schema))
        except AttributeError:
            method_write = getattr(df.phylo, 'to_{}'.format(schema))

        method_write(filename=path)
        return self

    @track_in_history
    def run_tree(self, id_col='unique_id', sequence_col='sequence',
        datatype='aa',
        bootstrap='-1',
        model='LG',
        frequencies='e',
        **kwargs):
        """Use PhyML to build a phylogenetic tree.

        Parameters
        ----------
        id_col : str (default: 'unique_id')
            column in `tips` DataFrame to use as the labels in the tree. Don't change
            this unless you know what you are doing.
        sequence_col : str (default: 'sequence')
            Column in the `tips` DataFrame that contains the alignment sequences
            for constructing the phylogenetic tree.
        bootstrap : str (default : -1 [aLRT values])
            Bootstrap values to use as support for the nodes of the tree. See PhyML's
            docs for more information. The default is to calculate aLRT supports.
        model : str (default : 'lg')
            the evolutionary transition matrix to use to model substitutions.
        frequencies :  str (default : 'e')
            Nucleotide or amino-acid frequencies.

        Note
        ----
        keyword arguments from what is mentioned above are translated to
        commandline arguments for PhyML. Read PhyML's docs to see a list of all
        arguments that can passed to PhyML.
        """
        df = self.data['tips']

        # Write file to disk
        fasta_file = os.path.join(self.project_dir, 'alignment.phy')
        df.phylo.to_phylip(fasta_file, sequence_col=sequence_col, id_col=id_col)

        # Prepare options
        options = {
            'input':fasta_file,
            'datatype':datatype,
            'bootstrap':bootstrap,
            'model':model,
            'frequencies':frequencies
        }
        # Update with any kwargs manually set by users.
        options.update(**kwargs)

        # Build command line arguments for PhyML.
        cml = Bio.Phylo.Applications.PhymlCommandline(**options)
        cml_args = str(cml).split()
        output = subprocess.run(cml_args)

        # # Get path (catch variations in file extension generated by phyml)
        try:
            tree_file = os.path.join(self.project_dir, 'alignment.phy_phyml_tree')
        except FileNotFoundError:
            tree_file = os.path.join(self.project_dir, 'alignment.phy_phyml_tree.txt')

        tree = dendropy.Tree.get(path=tree_file, schema='newick')
        self._add_tree(tree)
        return self

    @track_in_history
    def run_reconstruction(self, id_col='unique_id', sequence_col='sequence', **kwargs):
        """Use PAML (via PyASR) to build a phylogenetic tree.

        Parameters
        ----------
        id_col : str (default: 'unique_id')
            column in `tips` DataFrame to use as the labels in the tree. Don't change
            this unless you know what you are doing.
        sequence_col : str (default: 'sequence')
            Column in the `tips` DataFrame that contains the alignment sequences
            for constructing the phylogenetic tree.

        Note
        ----
        Keyword arguments from what is mentioned above are translated to
        commandline arguments for PAML. Read PAML's docs to see a list of all
        arguments that can passed to PAML.
        """
        df_seqs = self.data['tips']
        tree = self.data['tree']

        ###### BIT OF A HACK  ########################

        # Parse PhyML stats for alpha (catch variations in phyml output file extensions)
        data = {}
        try:
            phyml_file = os.path.join(self.project_dir, 'alignment.phy_phyml_stats')
        except FileNotFoundError:
            phyml_file = os.path.join(self.project_dir, 'alignment.phy_phyml_stats.txt')

        with open(phyml_file, 'r') as f:
            data_string = f.read()

        option_regex = re.compile("\.[\w\t ]+:.+\n|\- [\w\t ]+:.+\n")
        for pair in option_regex.findall(data_string):
            # Split the pair
            parse = pair.split(":")
            # Each pair has either a `. ` or `- ` in front. Remove those
            key = parse[0][2:]
            # Remove whitespace
            value = parse[1].lstrip().rstrip()
            # add to data dict
            data[key] = value

        # Get alpha
        alpha = data['Gamma shape parameter']
        alpha = alpha[:5]

        ###### END HACK #################################

        # Write file to disk
        df_seqs, df_ancs, tree_ancs = pyasr.reconstruct(df_seqs, tree,
            working_dir=self.project_dir,
            id_col=id_col, sequence_col=sequence_col,
            alpha=alpha,
            save_ancestors=True,
            **kwargs)

        # Add data to TreeProject.
        self._add_ancs(df_ancs)
        self._add_tree(tree_ancs)
        return self

    @track_in_history
    def run_pipeline(self, id_col='unique_id', sequence_col='sequence', tree_kwargs={}, reconstruction_kwargs={}):
        """Construct a phylogenetic tree from an alignment and reconstruct
        the ancestoral sequences in the resulting tree.

        Parameters
        ----------
        id_col : str (default: 'unique_id')
            column in `tips` DataFrame to use as the labels in the tree. Don't change
            this unless you know what you are doing.
        sequence_col : str (default: 'sequence')
            Column in the `tips` DataFrame that contains the alignment sequences
            for constructing the phylogenetic tree.
        tree_kwargs : dict
            keyword arguments to pass to PhyML
        reconstruction_kwargs: dict
            keyword arguments to pass to PAML
        """
        self.run_tree(id_col=id_col, sequence_col=sequence_col, **tree_kwargs)
        self.run_reconstruction(id_col=id_col, sequence_col=sequence_col, **reconstruct_kwargs)
        return self

    # def draw_tree(self, width=200, height=500,
    #     tip_labels=None,
    #     tip_labels_color=None,
    #     tip_labels_align=False,
    #     use_edge_lengths=False,
    #     edge_style=None,
    #     edge_align_style=None,
    #     node_labels=None,
    #     node_size=None,
    #     node_labels_style=None,
    #     node_color=None,
    #     **kwargs):
    #     """Draw tree (using the toytree package.)
    #
    #     Note
    #     ----
    #     Keyword arguments are passed directly to ToyTree's draw method. However,
    #     if you give the keyword arguments a column number in the tips/ancs DataFrames,
    #     this method will map those values onto the paired tree attribute.
    #     """
    #     if hasattr(self, 'tree') is False:
    #         raise Exception("TreeProject isn't away of any tree. Have you added it?")
    #
    #     newick_str = self.tree.as_string(schema='newick')
    #     tree = toytree.tree(newick_str)
    #
    #     # Tip settings
    #     tree_tips_labels = tree.get_tip_labels()
    #     tree_ancs_labels = tree.get_node_values('support', show_root=False, show_tips=False)
    #
    #     # Get tips
    #     tips = self.data['tips']
    #     ancs = self.data['ancs']
    #
    #     # set tip labels to column in tip dataframe
    #     if type(tip_labels) is str:
    #         mapping = dict(zip(tree_tips_labels, tips[tip_labels]))
    #         tip_labels = [mapping[label] for label in tree_tips_labels]
    #
    #     # set tip label colors to column in tips dataframe
    #     if type(tip_labels_color) is str:
    #         mapping = dict(zip(tree_tips_labels, tips[tip_label_colors]))
    #         tip_labels_color = [mapping[label] for label in tree_tips_labels]
    #
    #     # # set anc labels to column in anc dataframe
    #     # if type(node_labels) is str:
    #     #     mapping = dict(zip(ancs['id'], ancs[node_labels]))
    #     #     node_labels = [mapping[tree_ancs_labels[i]] for i in range(len(tree_ancs_labels))]
    #     #
    #     # # set anc labels to column in anc dataframe
    #     # if type(node_size) is str:
    #     #     mapping = dict(zip(ancs['id'], ancs[node_size]))
    #     #     node_size = [mapping[tree_ancs_labels[i]] for i in range(len(tree_ancs_labels))]
    #     #
    #     # # set anc labels to column in anc dataframe
    #     # if type(node_color) is str:
    #     #     mapping = dict(zip(ancs['id'], ancs[node_color]))
    #     #     node_color = [mapping[tree_ancs_labels[i]] for i in range(len(tree_ancs_labels))]
    #
    #     options = dict(
    #         width=width,
    #         height=height,
    #         tip_labels=tip_labels,
    #         tip_labels_color=tip_labels_color,
    #         tip_labels_align=tip_labels_align,
    #         use_edge_lengths=use_edge_lengths,
    #         edge_style=edge_style,
    #         edge_align_style=edge_align_style,
    #         node_labels=tree_ancs_labels, #### FIX LATER
    #         node_size=node_size,
    #         node_labels_style=node_labels_style,
    #         node_color=node_color,
    #         **kwargs)
    #
    #     return tree.draw(**options)

    def show(self):
        """Show rough interactive tree.
        """
        # Write a dummy tree
        tree_path = os.path.join(self.project_dir, 'vega-tree.newick')
        self.data['tree'].write(path=tree_path, schema='newick')
        tree_df = phylopandas.read_newick(tree_path)

        # Create vega tree
        vegatree = phylovega.VegaTree(tree_df)

        # Write vega tree.
        vega_path = os.path.join(self.project_dir, 'vega-tree.vg.json')
        vegatree.to_json(vega_path)

        # Display vega tree
        vegatree.display()

    def decorate_tree(self, tips_attrs, anc_attrs):
        """Map taxon attributes from dataframe to a dendropy tree.
        """
        ###### Map Taxon info to tree
        # Iterate over index
        for uid in self.tips['unique_id']:
            # Get row
            row = self.tips[self.tips['unique_id'] == uid]

            # Get node in tree
            node = self.tree.find_node_with_taxon_label(uid)

            # Set attributes
            for label in tips_attrs:
                # add to annotations
                value = row[label].values[0]
                node.taxon.annotations.add_new(label, value)

        # Map ancestors if info is given
        for uid in self.ancestors['id']:
            # Get row
            row = self.ancestors[self.ancestors['id'] == uid]

            # Get node in tree
            node = self.tree.find_node_with_label(str(uid))

            # Set attributes
            if node is not None:
                for label in anc_attrs:
                    # add to annotations
                    value = row[label].values[0]
                    node.annotations.add_new(label, value)


from typing import List, Any

import rocketbot.bots as b
import rocketbot.commands as c
import rocketbot.models as m

class Prefix(c.BaseCommand):
    def __init__(self, bot: b.BaseBot):
        self.bot = bot
        self.enable_public_channel = False
        self.enable_private_group = False
        self.enable_direct_message = False
        self.prefixes: List[Any] = []

    def is_applicable(self, room: m.RoomRef2) -> bool:
        if room.roomType == m.RoomType.PUBLIC and self.enable_public_channel:
            return True
        if room.roomType == m.RoomType.PRIVATE and self.enable_private_group:
            return True
        if room.roomType == m.RoomType.DIRECT and self.enable_direct_message:
            return True
        return False

    async def handle(self, message: m.Message) -> bool:
        if not message.msg:
            return True
        command = message.msg.split()[0].lower()
        args = message.msg[len(command):].lstrip()

        for commands, callback in self.prefixes:
            if command in commands:
                result = await callback(args, message)
                if result:
                    return True
        return False


from datetime import datetime, timedelta
from random import randint
from typing import Optional

from discord import Member, Embed
from discord.ext.commands import Cog
from discord.ext.commands import command, has_permissions
from discord.ext.menus import MenuPages, ListPageSource
from lib.bot import bot # pylint: disable=no-name-in-module, import-error
from ..db import db # pylint: disable=relative-beyond-top-level

class Menu(ListPageSource):
    def __init__(self, ctx, data):
        self.ctx = ctx
        self.bot = bot

        super().__init__(data, per_page=10)

    async def write_page(self, menu, offset, fields=[]):
        offset = (menu.current_page*self.per_page) + 1
        len_data = len(self.entries)

        embed = Embed(title="Global XP Leaderboard", description="See who is on top!", colour=self.ctx.author.colour)
        embed.set_author(name="Warning ⚠: Global XP leaderboard is super slow at the moment.")
        embed.set_thumbnail(url=self.ctx.guild.me.avatar_url)
        embed.set_footer(text=f"{offset:,} - {min(len_data, offset+self.per_page-1):,} of {len_data:,} members.")

        for name, value in fields:
            embed.add_field(name=name, value=value, inline=False)

        return embed

    async def format_page(self, menu, entries):
        offset = (menu.current_page*self.per_page) + 1
        fields = []
        table = ("\n".join(f"{idx+offset}. {bot.get_user(entry[0]).name} (XP: {entry[1]} | Level: {entry[2]})"
                   for idx, entry in enumerate(entries)))

        fields.append(("Ranks", table))

        return await self.write_page(menu, offset, fields)

class ServerMenu(ListPageSource):
    def __init__(self, ctx, data):
        self.ctx = ctx
        self.bot = bot

        super().__init__(data, per_page=10)

    async def write_page(self, menu, offset, fields=[]):
        offset = (menu.current_page*self.per_page) + 1
        len_data = len(self.entries)

        embed = Embed(title="Server XP Leaderboard", description="See who is on top!", colour=self.ctx.author.colour)
        embed.set_thumbnail(url=self.ctx.guild.icon_url)
        embed.set_footer(text=f"{offset:,} - {min(len_data, offset+self.per_page-1):,} of {len_data:,} members.")

        for name, value in fields:
            embed.add_field(name=name, value=value, inline=False)

        return embed

    async def format_page(self, menu, entries):
        offset = (menu.current_page*self.per_page) + 1
        fields = []
        table = ("\n".join(f"{idx+offset}. {bot.get_user(entry[0]).display_name} (XP: {entry[1]} | Level: {entry[2]})"
                   for idx, entry in enumerate(entries)))

        fields.append(("Ranks", table))

        return await self.write_page(menu, offset, fields)

class Exp(Cog):
    def __init__(self, bot):
        self.bot = bot

    async def process_xp(self, message):
        xp, lvl, xplock = db.record("SELECT XP, Level, XPLock FROM exp WHERE UserID = ?", message.author.id)
        xp_g, lvl_g, xplock_g = db.record(f"SELECT XP, Level, XPLock FROM guildexp WHERE UserID = {message.author.id} AND GuildID = {message.guild.id}")

        if datetime.utcnow() > datetime.fromisoformat(xplock):
            await self.add_xp(message, xp, lvl)

        if datetime.utcnow() > datetime.fromisoformat(xplock_g):
            await self.add_gxp(message, xp_g, lvl_g)

    async def add_xp(self, message, xp, lvl):
        xp_to_add = randint(10, 20)
        level_up_messages = db.record("SELECT LevelMessages FROM guilds WHERE GuildID = ?", message.guild.id)

        new_lvl = int(((xp + xp_to_add)//42) ** 0.55)

        db.execute("UPDATE exp SET XP = XP + ?, Level = ?, XPLock = ? WHERE UserID = ?",
                   xp_to_add, new_lvl, (datetime.utcnow()+timedelta(seconds=50)).isoformat(), message.author.id)

        if new_lvl > lvl:
            if level_up_messages == "('yes',)" or level_up_messages == "('Yes',)":
                await message.channel.send(f"{message.author.mention} leveled up to {new_lvl:,}!", delete_after = 10)

    async def add_gxp(self, message, xp, lvl):
        xp_to_add = randint(10, 20)
        level_up_messages = db.record("SELECT LevelMessages FROM guilds WHERE GuildID = ?", message.guild.id)

        new_lvl = int(((xp + xp_to_add)//42) ** 0.55)

        db.execute(f"UPDATE guildexp SET XP = XP + ?, Level = ?, XPLock = ? WHERE UserID = {message.author.id} AND GuildID = {message.guild.id}",
                   xp_to_add, new_lvl, (datetime.utcnow()+timedelta(seconds=50)).isoformat())
        db.commit()

        if new_lvl > lvl:
            if level_up_messages == "('yes',)" or level_up_messages == "('Yes',)":
                await message.channel.send(f"{message.author.mention} leveled up to server level {new_lvl:,}!", delete_after = 10)

    @command(name="level", aliases=["rank", "lvl"], brief="Shows your level, and rank.")
    async def display_level(self, ctx, target: Optional[Member]):
        """Shows your Global+Server Doob level, rank and XP!"""
        target = target or ctx.author

        ids = db.column("SELECT UserID FROM exp ORDER BY XP DESC")
        # ids_g = db.column("SELECT UserID FROM exp ORDER BY XP DESC WHERE GuildID = ?", ctx.guild.id)
        xp, lvl = db.record("SELECT XP, Level FROM exp WHERE UserID = ?", target.id) or (None, None)
        xp_g, lvl_g = db.record("SELECT XP, Level FROM guildexp WHERE (UserID, GuildID) = (?, ?)", target.id, ctx.guild.id) or (None, None)

        if lvl is not None:
            await ctx.send(f"`Global Rank:`\n{target.display_name} is level {lvl:,} with {xp:,} xp and is rank {ids.index(target.id)+1} of {len(ids):,} users globally.\n`Server Rank:`\n{target.display_name} is server level {lvl_g:,} with {xp_g:,} server xp.")

        else:
            ctx.send("That member is not in the XP Database.")

    @command(name="levelmessages", aliases=["slm", "lm", "setlevelmessages"], brief="Set the server's level messages")
    @has_permissions(manage_guild=True)
    async def set_level_messages(self, ctx, *, yes_or_no: Optional[str]):
        """PLEASE, put 'yes' if you DO want level messages\n`Manage Server` permission required."""
        levelmessages = db.records("SELECT LevelMessages FROM guilds WHERE GuildID = ?", ctx.guild.id) or (None)
        prefix = db.records("SELECT Prefix FROM guilds WHERE GuildID = ?", ctx.guild.id)

        if yes_or_no == "Yes" or yes_or_no == "yes" or yes_or_no == "no" or yes_or_no == "No":
            db.execute("UPDATE guilds SET LevelMessages = ? WHERE GuildID = ?", yes_or_no, ctx.guild.id)
            db.commit()
            await ctx.send(f"Level messages set to `{yes_or_no}`.")

        else:
            await ctx.send(f"The current setting for Level Messages is: `{levelmessages[0][0]}`\nTo change it, type `{prefix[0][0]}levelmessages (yes or no)`")

    @command(name="leaderboard", aliases=["lb", "xplb"], brief="Show who's on top of the Doob GlobalXP Leaderboard!")
    async def display_leaderboard(self, ctx):
        """Displays the Global XP Leaderboard for Doob."""
        records = db.records("SELECT UserID, XP, Level FROM exp ORDER BY XP DESC")

        menu = MenuPages(source=Menu(ctx, records), clear_reactions_after=True, timeout=100.0)
        await menu.start(ctx)

    @command(name="serverleaderboard", aliases=["serverlb", "serverxplb", "svxp", "svxplb"], brief="Show who's on top of the Doob ServerXP Leaderboard!")
    async def display_serverxp_leaderboard(self, ctx):
        """Displays the Server XP Leaderboard for Doob."""
        records = db.records("SELECT UserID, XP, Level FROM guildexp WHERE GuildID = ? ORDER BY XP DESC", ctx.guild.id)

        menu = MenuPages(source=ServerMenu(ctx, records), clear_reactions_after=True, timeout=100.0)
        await menu.start(ctx)

    @Cog.listener()
    async def on_ready(self):
        if not self.bot.ready:
            self.bot.cogs_ready.ready_up("exp")

    @Cog.listener()
    async def on_message(self, message):
        if not message.author.bot:
            await self.process_xp(message)

def setup(bot):
    bot.add_cog(Exp(bot))


#!/home/kmtopel/mysite/env/bin python3
from flask import Flask, render_template, redirect, flash, url_for, request, Markup, jsonify
from flask_nav import Nav
from flask_sqlalchemy import SQLAlchemy
from flask_wtf.csrf import CSRFProtect
from forms import SubmitForm, ContactAdmin
from datetime import datetime as dt
import smtplib
import os

app = Flask(__name__)

basedir = os.path.abspath(os.path.dirname(__file__))
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///' + os.path.join(basedir, 'data.sqlite')
app.config['SQLALCHEMY_COMMIT_ON_TEARDOWN'] = True

csrf = CSRFProtect(app)
nav = Nav(app)
db = SQLAlchemy(app)

class Posts(db.Model):
    __tablename__ = "posts"
    id = db.Column(db.Integer, primary_key=True)
    timestamp = db.Column(db.DateTime,nullable=True)
    post_title = db.Column(db.String(80),nullable=False)
    email = db.Column(db.String(120),nullable=False)
    link = db.Column(db.String(120))
    post_type = db.Column(db.String(80),nullable=False)
    description = db.Column(db.Text)
    active = db.Column(db.Boolean, default=0)
    ph_num = db.Column(db.String(80),nullable=True)
    sql_autoincrement=True

class Messages(db.Model):
    __tablename__ = "messages"
    id = db.Column(db.Integer, primary_key=True)
    timestamp = db.Column(db.DateTime)
    email = db.Column(db.String(80))
    msg = db.Column(db.Text)

key = os.urandom(24)
app.config['SECRET_KEY'] = key

choices = ['Show All','Information','Financial','Food','Health Care','Employment/Vocational','Utilities','Housing/Rent Assistance','Fitness/Recreation','Mental Health/Substance Abuse','Social Service','Education','Transportation','Personal Care','Child Care','Disability Services','Other']

@app.route('/', methods=["GET","POST"])
def main():
    form = SubmitForm(request.form)
    if form.validate_on_submit():
        flash("Success! Thank you for submitting a resource. Your post is being reviewed.")
        post = Posts(
            post_title=form.post_title.data,
            email=form.email.data,
            description=form.description.data,
            post_type=form.post_type_submit.data,
            link = form.link.data,
            ph_num = form.ph_num.data,
            timestamp=dt.utcnow()
                )
        db.session.add_all([post])
        db.session.commit()
        return redirect(url_for('main'))
    data=Posts.query.order_by(Posts.timestamp.desc()).all()
    return render_template('main.html',form=form, data=data, choices=choices)

@app.route('/contact', methods=["GET","POST"])
def contact():
    form = SubmitForm(request.form)
    contact = ContactAdmin(request.form)
    if contact.validate_on_submit():
        flash(Markup("Thank you! Your message has been sent."))
        msg = Messages(
                timestamp=dt.utcnow(),
                email = contact.email.data,
                msg = contact.msg.data
                )
        db.session.add_all([msg])
        db.session.commit()
        return redirect(url_for('contact'))

    if form.validate_on_submit():
        flash(Markup("Success! Thank you for submitting a resource. Your post is being reviewed."))
        post = Posts(
            post_title=form.post_title.data,
            email=form.email.data,
            description=form.description.data,
            post_type=form.post_type_submit.data,
            link = form.link.data,
            ph_num = form.ph_num.data,
            timestamp=dt.utcnow()
                )
        db.session.add_all([post])
        db.session.commit()
        return redirect(url_for('contact'))
    return render_template('contact.html', contact=contact, form=form)

# @app.route('/api',methods=['GET'])
# def api():
#     return jsonify(Posts.query.order_by(Posts.timestamp.desc()).all().to_dict())


# To add a new cell, type '# %%'
# To add a new markdown cell, type '# %% [markdown]'
# %% [markdown]
# The following code is used to obtain the size and power under six different cases:

# %%
import numpy as np
from numpy.linalg import inv
import pickle
import multiprocessing as mp

path = '/home/ubuntu/python/2020/Project_LRL'
# %%
with open(path+'/GSW.pickle', 'rb') as f:
    cvalues = pickle.load(f)['GSW']
cvl0 = np.zeros((12,3))
cvl1 = np.zeros((12,3))
for jj in range(12):
    cvl0[jj,:] = np.quantile(cvalues[:,0,jj],(0.90,0.95,0.99))
    cvl1[jj,:] = np.quantile(cvalues[:,1,jj],(0.90,0.95,0.99))

# %%
def lag_v(x,lag,max_lag):
    n = x.shape[0]
    y = x[max_lag-lag:n-lag,:]
    return y

# %%
def sim_process(case_no,T0,TT,tau0,d,rw,seeds,n):
    # Data Generating Process
    np.random.seed(seeds)
    Z0 = np.zeros((n,T0+TT+tau0))
    eps = np.random.multivariate_normal(np.zeros(n),np.eye(n),T0+TT+tau0)
    pi = np.zeros((n,n))
    if case_no == 1:
        pi[0,0] = 0
        pi[1,1] = 0
        pi[0,1] = 0
    elif case_no == 2:
        pi[0,0] = 0
        pi[1,1] = -1*d
        pi[0,1] = 0
    elif case_no == 3:
        pi[0,0] = -1*d
        pi[1,1] = -1*d
        pi[0,1] = 0
    elif case_no == 4:
        pi[0,0] = 0
        pi[1,1] = -1*d
        pi[0,1] = -1*d
    elif case_no == 5:
        pi[0,0] = -1*d
        pi[1,1] = 0
        pi[0,1] = -1*d
    else:
        pi[0,0] = -1*d
        pi[1,1] = -1*d
        pi[0,1] = -1*d
    for tt in range(2,T0+TT+tau0):
        Z0[:,tt] =  Z0[:,tt-1]+pi@Z0[:,tt-1]+eps[tt,:].T
    Z  = Z0[:,50:].T
    T1 = tau0
    T2 = T0+tau0
    y = Z[T1-tau0:T2-tau0,:1]
    x = Z[T1:T2,1:2]
    # The Estimation Procedure
    dy0 = y[1:,:1] - y[:-1,:1]
    Ly0 = lag_v(y,1,1)
    Lx0 = lag_v(x,1,1)
    T = len(Ly0)
    ss = int(T- np.ceil(T*rw)+1) # the toal shift of moving the x,  the reason +1 is because tau could be zero
    Ty0 = ss
    Ty1 = T
    W = np.zeros((ss,1))
    for tau in range(ss): #
        Tx0 = Ty0-tau
        Tx1 = Ty1-tau
        Y = dy0[Ty0:Ty1,:1]
        X = np.concatenate((Ly0[Ty0:Ty1,:1],Lx0[Tx0:Tx1,:1]),1)
        bhat = inv(X.T@X)@X.T@Y
        sigu2 = np.mean((Y-X@bhat)**2)
        W[tau,0] = bhat.T@(X.T@X)@bhat/sigu2
    return [np.max(W),int(np.where(W[:,0]==np.max(W[:,0]))[0])]

# %%
TT0 =  [200,500,1000,2000]
N_T = len(TT0)
TT = 50
n = 2
R = 1000
d0 = [0.2,0.5,1]
N_d = len(d0)
rw0 =  [0.05,0.10,0.15,0.20,0.25,0.30,0.35,0.40,0.45,0.50,0.55,0.60]
tau0 = 30
N_rw = len(rw0)
size = np.zeros((N_rw,3,N_T,N_d,2))
power0 = np.zeros((N_rw,3,N_T,N_d,4))
power1 = np.zeros((N_rw,3,N_T,N_d,4))
datemean =  np.zeros((N_rw,N_T,N_d,4))
case_no = [1,2,3,4,5,6]
for ii in case_no:
    for n_d in range(N_d):
        d = d0[n_d]
        for n_t in range(N_T):
            T0 = TT0[n_t]
            for rw1 in range(N_rw):
                rw = rw0[rw1]
                GSW = np.zeros((R,1))
                date = np.zeros((R,1))
                p = mp.Pool(processes = 16)
                seeds = np.random.randint(low=1,high=100000,size=R)
                multi_res = [p.apply_async(sim_process,(ii,T0,TT,tau0,d,rw,seeds[rr],2)) for rr in range(R)]
                p.close()
                p.join()
                GSW[:,0]=np.array(([res.get()[0] for res in multi_res]))
                date[:,0]=np.array(([res.get()[1] for res in multi_res]))
                if ii == 1:
                    size[rw1,:,n_t,n_d,ii-1] = np.sum(GSW>cvl1[rw1,:],0)/R
                elif ii ==2:
                    size[rw1,:,n_t,n_d,ii-1] = np.sum(GSW>cvl0[rw1,:],0)/R
                elif ii ==3:
                    power0[rw1,:,n_t,n_d,ii-3] = np.sum(GSW>cvl0[rw1,:],0)/R
                    power1[rw1,:,n_t,n_d,ii-3] = np.sum(GSW>cvl1[rw1,:],0)/R
                    datemean[rw1,n_t,n_d,ii-3] =np.mean(date)
                elif ii ==4:
                    power0[rw1,:,n_t,n_d,ii-3] = np.sum(GSW>cvl0[rw1,:],0)/R
                    power1[rw1,:,n_t,n_d,ii-3] = np.sum(GSW>cvl1[rw1,:],0)/R
                    datemean[rw1,n_t,n_d,ii-3] =np.mean(date)
                elif ii ==5:
                    power0[rw1,:,n_t,n_d,ii-3] = np.sum(GSW>cvl0[rw1,:],0)/R
                    power1[rw1,:,n_t,n_d,ii-3] = np.sum(GSW>cvl1[rw1,:],0)/R
                    datemean[rw1,n_t,n_d,ii-3] =np.mean(date)
                else:
                    power0[rw1,:,n_t,n_d,ii-3] = np.sum(GSW>cvl0[rw1,:],0)/R
                    power1[rw1,:,n_t,n_d,ii-3] = np.sum(GSW>cvl1[rw1,:],0)/R
                    datemean[rw1,n_t,n_d,ii-3] =np.mean(date)

res_dict = {'power0': power0,'power1':power1,'size':size, 'datemean':datemean}

file = open(path+'/sim_res.pickle', 'wb')
pickle.dump(res_dict, file)
file.close()

from sendmail import sendemail
sendemail()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Mar  4 15:32:38 2018

@author: Rogelio Carrillo

Solución del Problema 4.3 del Libro "An Introduction to Computational Fluid Dynamics"
de H K Versteeg and W Malalasekera

----------------------------------
En el ejemplo final trabajado de este capítulo discutimos el enfriamiento de una
aleta circular por medio de transferencia de calor convectiva a lo largo de su longitud.
La convección da lugar a un término de pérdida de calor dependiente de la temperatura
o sumidero en la ecuación gobernante. En la Figura 4.9 se muestra una aleta cilíndrica
con un área transversal uniforme A. La base está a una temperatura de 100 ° C (T_A) y
el extremo está aislado. La aleta está expuesta a una temperatura ambiente de 20 ° C.
La transferencia de calor dimensional en esta situación se rige por

$
\dfrac{d}{dx} \left( k A \dfrac{d T}{dx} \right) - hP(T - T∞) = 0
$

donde h es el coeficiente de transferencia de calor por convección, P el perímetro,
k la conductividad térmica del material y $ T_ \ infty $ la temperatura ambiente.
Calcule la distribución de temperatura a lo largo de la aleta y compare los
resultados con la solución analítica dada por

$\frac{T - T∞}{T_A - T∞} = \frac{\cosh[n(L-x)]}{\cosh(nL)}$

Donde $n^2 = hP/(kA)$, $L$ es la longitud de la aleta y $x$ la distancia a lo largo de la aleta.
Datos: $L = 1$ m, $hP/(kA) = 25/m^2$ (Note que $kA$ es constante).

La ecuación que rige en el ejemplo contiene un término sumidero, -hP (T - T∞),
la pérdida de calor por convección, que es una función de la temperatura local T.

___________________________________________________________

 |<-------------- 1.0 m ------------->|
 A
 |
 |
 |------------------------------------|
 |------------------------------------| Flujo igual a zero
 |        Temp. Ambiente (T∞)
 |                                    |
                                      |
T_A = 100                  \frac{\partial T}{\partial dx} = q = 0
___________________________________________________________
Figure 4.9

Cuando $kA$ = es constante, la ecuación gobernante se puede escribir como

$
\dfrac{d}{dx} \left( \dfrac{d T}{dx} \right) - n^2(T - T∞) = 0
$

"""

#Importar Script donde se encuentra el Metodo de Volumen Finito
import FiniteVolumeMethod as fvm
#Librerias de Python
import numpy as np
import matplotlib.pyplot as plt

### Calculo Mediante la Solución Analitica
def analyticSol(x):
    return (TA - Tambiente) * np.cosh(n * (longitud - x)) / np.cosh(n * longitud) + Tambiente

# Datos del Problema (Variables)
longitud = 1.0 # metros
Tambiente = 20  # °C
TA = 100  # °C
n2 = 25 # /m^2 = hP/(kA)
n = 5
fluxB = 0 # Flujo igual a cero
N = 6 # Número de nodos

#
# Creamos la malla y obtenemos datos importantes
#
malla = fvm.Mesh(nodes = N, length = longitud)
nx    = malla.nodes()     # Número de nodos
nvx   = malla.volumes()   # Número de volúmenes
delta = malla.delta()     # Tamaño de los volúmenes
#
# Imprimimos los datos del problema (nicely)
#
fvm.printData(Longitud = longitud,
              Temperatura_A = TA,
              Flujo_B = fluxB,
              n2 = n2,
              Nodos = nx,
              Volúmenes = nvx,
              Delta = delta)
#
#  Creamos los coeficientes de FVM
#
df1 = fvm.Diffusion1D(nvx, Gamma = 1, dx = delta)
df1.alloc(nvx) # Se aloja memoria para los coeficientes
df1.calcCoef() # Se calculan los coeficientes
df1.setSu(n2 * Tambiente)  # Se agrega la fuente uniforme
df1.setSp(-n2)
#print(df1.aP(),df1.aW(), df1.aE(), df1.Su(), sep = '\n')
#
# Se construye el arreglo donde se guardará la solución
#
T = np.zeros(nvx) # El arreglo contiene ceros
T[0]  = TA        # Condición de frontera izquierda
df1.bcDirichlet('LEFT_WALL', T[0])   # Se actualizan los coeficientes
df1.bcNeumman('RIGHT_WALL', fluxB) # de acuerdo a las cond. de frontera
#
# Se construye el sistema lineal de ecuaciones a partir de los coef. de FVM
#
Su = df1.Su()  # Vector del lado derecho
A = fvm.Matrix(malla.volumes())  # Matriz del sistema
A.build(df1) # Construcción de la matriz en la memoria
print('A = ', A.mat(),
      'b = {}'.format(Su[1:-1]), sep='\n')
print('.'+'-'*70+'.')
#
# Se resuelve el sistema usando un algoritmo del módulo linalg
#
T[1:-1] = np.linalg.solve(A.mat(),Su[1:-1])
T[-1] = T[-2] # Condición de frontera tipo Neumman
print('Solución = {}'.format(T))
print('.'+'-'*70+'.')
#
# Se construye un vector de coordenadas del dominio
#
x = malla.createMesh()
#
# Calculamos la solución exacta y el error
#
Ta = analyticSol(x)
error = fvm.calcError(T, Ta)
datos = {'x(m)': x,
         'T(x)': T,
         'Analytic': Ta,
         'Error': error}
fvm.printFrame(datos)
print('||Error|| = ', np.linalg.norm(error))
print('.'+ '-'*70 + '.')
#
# Calculamos la solución exacta en una malla más fina para graficar
#
x1 = np.linspace(0,longitud,100)
n = np.sqrt(n2)
Ta = analyticSol(x1)
#
#  Se grafica la solución
#
plt.plot(x1,Ta, '-', label = 'Sol. analítica')
plt.plot(x,T,'o', label = 'Sol. FVM')
plt.title('Solución de $\partial^2 T/\partial x^2 - hP(T-T_\infty) = 0$ con FVM')
plt.ylim(10,110)
plt.xlabel('$x$ [m]')
plt.ylabel('Temperatura [$^o$C]')
plt.grid()
plt.legend()
plt.savefig('Tarea_ejemplo4.3.pdf')
plt.show()


# SOLVED BY __proRajukians__
# LANGUAGE --> python 3.7.5
# COMPILER --> VISUAL STUDIO CODE

# ICPC 2018 <1>
name=input()
print(name[-1::-1])

# ICPC 2018 <2>
line=int(input())
box=[]
for i in range(line):
    sides=input()
    box.append(sides)

def area(i):
    a,b,c=i.split(" ")
    a=int(a)
    b=int(b)
    c=int(c)
    if (a+b)<=c or (b+c)<=a or (a+c)<=b:
        return "Oh, No!"
    else:
        import math
        s=(a+b+c)/2
        value=math.sqrt(s*(s-a)*(s-b)*(s-c))
        return "%.2f"% round(value,2)
a=list(map(area,box))
for i in a:
    print(i)

# ICPC 2018 <3>
for i in range (10,0,-1):
    print(i)

# ICPC 2018 <4>
dx=input()
dy =input()
for i in range(dx):
    while len(input())<=dy:
        maze=input()


#!/usr/bin/env python
# -*- coding:UTF-8 -*-

__author__ = 'taloo'
__date__ = '15-1-10'
__time__ = '10:08'
__description__ = 'BaseHandler'

import tornado.web
from lib.session import Session

class BaseHandler(tornado.web.RequestHandler):
    @property
    def db(self):
        return self.application.db

    @property
    def session(self):
        if hasattr(self,"_session"):
            return self._session
        else:
            session_id = self.get_secure_cookie("sid")
            self._session = Session(self.application.session_store,session_id,expires_days=1)
            if not session_id:
                self.set_secure_cookie("sid",self._session.id,expires_days=1)
            return self._session

    def get_current_user(self):
        '''
        session : return self.session['user'] if 'user' in self.session else None
        cookie : return self.get_secure_cookie("user")
        '''
        #return self.get_secure_cookie("user")
        return self.session['user'] if 'user' in self.session else None