# -*- coding: utf-8 -*-
from django.conf import settings
from django.views.generic import CreateView
from levangersundet.forms import DeltagerForm

from post_office import mail

class TestCreateView(CreateView):
    form_class = DeltagerForm
    template_name = 'test.html'

    def get_success_url(self):
        return '/nb/%s/' % self.testtype

    def form_valid(self, form):
        response = super(TestCreateView, self).form_valid(form)
        mail.send(
            [self.object.epost],
            settings.SERVER_EMAIL,
            template=self.testtype,
            context={'deltager': self.object},
            priority='now'
        )
        mail.send(
            [settings.TEST_NOTIFY_EMAIL],
            settings.SERVER_EMAIL,
            template='%s_notify' % self.testtype,
            context={'deltager': self.object},
            priority='now'
        )
        return response

    def dispatch(self, *args, **kwargs):
        self.testtype = kwargs.get('testtype', False)
        return super(TestCreateView, self).dispatch(*args, **kwargs)


import sys

input = sys.stdin.readline

data1 = input().rstrip()
data2 = input().rstrip()

n1 = len(data1)
n2 = len(data2)

ans = 0
i = 0

while i <= n1-n2:
    if data1[i:i+n2] == data2:
        ans += 1
        i += n2
    else:
        i += 1

print(ans)


from typing import *

# Definition for a Node.
class Node:
    def __init__(self, val=None, children=None):
        self.val = val
        self.children = children

class Solution:
    def levelOrder(self, root: 'Node') -> List[List[int]]:
        res = []
        if root is None:
            return res
        que = [root]
        while len(que) > 0:
            cur_res = []
            q_size = len(que)
            for _ in range(q_size):
                node = que.pop(0)
                if node.children is not None:
                    for child in node.children:
                        que.append(child)
                cur_res.append(node.val)
            res.append(cur_res)
        return res

if __name__ == "__main__":
    s = Solution()

    node5 = Node(5)
    node6 = Node(6)
    node3 = Node(3, [node5, node6])
    node2 = Node(2)
    node4 = Node(4)
    node1 = Node(1, [node3, node2, node4])

    assert s.levelOrder(node1) == [[1], [3, 2, 4], [5, 6]]


from src.models.caja_moneda import CajaMoneda
from src.models.moneda_digital import MonedaDigital
from src.models.tipo_operacion import TipoOperacion
from src.models.transaccion import Transaccion

class Usuario(object):
    def __init__(self, nombre, codigo):
        self._nombre = nombre
        self._codigo = codigo
        self._caja_monedas = []
        self._historial_transacciones = []

        self._caja_monedas = self.iniciar_lista_monedas(100)

    def iniciar_lista_monedas(self, cantidad_monedas_inicial):
        cajas = []
        for moneda in MonedaDigital:
            caja = CajaMoneda(moneda)
            caja.sumar_cantidad_monedas(cantidad_monedas_inicial)
            cajas.append(caja)
        return cajas

    def agregar_transaccion(self, transaccion):
        self._historial_transacciones.append(transaccion)

    def restar_monedas(self, moneda_digital, cantidad):
        position = self.buscar_pocicion(moneda_digital)
        accion_realizada = False

        if position is not None:
            if cantidad <= self._caja_monedas[position].cantidad:
                self._caja_monedas[position].restar_cantidad_monedas(cantidad)
                accion_realizada = True
        return accion_realizada

    def sumar_monedas(self, moneda_digital, cantidad):
        posicion = self.buscar_pocicion(moneda_digital)
        accion_realizada = False
        if posicion is not None:
            self._caja_monedas[posicion].sumar_cantidad_monedas(cantidad)
            accion_realizada = True
        return accion_realizada

    # busca en la lista de caja_monedas y devuelve
    # si encontro la monedaDigital devuelve su posicion
    # si no encotro una monedaDIgital devuelve None
    def buscar_pocicion(self, moneda_digital: MonedaDigital):
        position = None
        for caja in self._caja_monedas:
            if caja.moneda is moneda_digital.value:
                position = self._caja_monedas.index(caja)
        return position

    def __str__(self):
        mensaje = "Usuario \n nombre: " + self._nombre + "\n" + "Cajas:\n"
        for caja in self._caja_monedas:
            mensaje += "* " + caja.__str__()
        mensaje += "Transacciones\n"
        if len(self._historial_transacciones) == 0:
            mensaje += "No hay Transacciones monetarias"
        else:
            for transaccion in self._historial_transacciones:
                mensaje += "* " + transaccion.__str__()
        return mensaje

    @property
    def caja_monedas(self):
        return self._caja_monedas

    @property
    def nombre(self):
        return self._nombre

    @property
    def historial_transacciones(self):
        if len(self._historial_transacciones) == 0:
            return "Sin transacciones"
        else:
            transacciones = "Transacciones de " + self.nombre + "\n"
            for transaccion in self._historial_transacciones:
                transacciones += transaccion.__str__()
            return transacciones

    @property
    def codigo(self):
        return self._codigo


import logging
import os
import sys

def configLogger():
    root = logging.getLogger()
    root.setLevel(logging.DEBUG)

    file_handler = logging.FileHandler(_get_logfile_name())
    file_handler.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    file_handler.setFormatter(formatter)
    root.addHandler(file_handler)

def _get_logfile_name():
    executable_name = os.path.basename(sys.argv[0]).split('.')[0]
    return '/tmp/logs/%s.log' % executable_name


import sys
import datetime
import csv
from os import path

""""
This section describes all functions called when the program is started.
"""
def startup():
    create_date_file()
    create_bought_file()
    create_sold_file()

def create_date_file():
    """Check if there is already a file present containing the date set as current date. If not: create one."""
    if path.exists('date.txt') == False:
        date = str(datetime.date.today())
        file = open('date.txt', 'w')
        file.write(date)
        file.close()

def create_bought_file():
    """Check if there is already a bought.csv file present. If not: create one"""
    if path.exists('bought.csv') == False:
        with open('bought.csv', 'w', newline='') as csvfile:
            bought_creation = csv.writer(csvfile)
            bought_creation.writerow(['id', 'product_name', 'buy_date', 'buy_price', 'expiration_date'])

def create_sold_file():
    """Check if there is already a sold.csv file present. If not: create one"""
    if path.exists('sold.csv') == False:
        with open('sold.csv', 'w', newline='') as csvfile:
            sold_creation = csv.writer(csvfile)
            sold_creation.writerow(['id', 'bought_id', 'sell_date', 'sell_price'])


import os
import gc
import re
import sys
import copy
import time
import random
import tempfile
import logging
import cPickle as cp
import multiprocessing
import subprocess
import deepity
import numpy as np
import numpy.random as npr
import smat as sm
import scipy
import scipy.stats
from . import std
from .        import hpsearch as hp
from .        import _io_
from .        import util
from .data    import datasource
from .        import globals
from .report  import training_report, calc_auc, bootstrap_auc
import node    as _node
import trainer as _trainer

class _object_factory_from_file(object):
    def __init__(self,filename,fieldname=None):
        self.filename = filename
        self.fieldname = fieldname

    def __call__(self):
        obj = _io_.load(self.filename)
        if self.fieldname and isinstance(obj,dict):
            return obj[self.fieldname]
        return obj

def _create_model(model_proto, hparams):
    model = copy.deepcopy(model_proto)
    for key,val in hparams.iteritems():
        prefix,path = key.split(":")  # look for hparams named "model:..."
        if prefix == "model":
            nodepath,attrname = path.rsplit(".",1)
            node = model.find(nodepath)
            if hasattr(node,"set_"+attrname):
                getattr(node,"set_"+attrname)(model,val)   # call model.set_xxx(val)
            else:
                setattr(node,attrname,val)
    return model

def _create_trainer(trainer_proto, hparams):
    trainer = copy.deepcopy(trainer_proto)
    for key,val in hparams.iteritems():
        prefix,attrname = key.split(":")  # look for hparams named "trainer:..."
        if prefix == "trainer":
            if hasattr(trainer,"set_"+attrname):
                getattr(trainer,"set_"+attrname)(model,val)   # call trainer.set_xxx(val)
            else:
                setattr(trainer,attrname,val)
    return trainer

def _slice_hparams(hparams, inst):
    h = copy.deepcopy(hparams)
    for key in h.keys():
        h[key] = h[key][inst]
    return h

def load_hparams_result(filename):
    with open(filename,'r') as f:
        lines = f.readlines()
    params = {}
    result = 0.0
    for line in lines:
        # Look for validation performance
        matches = re.findall("# metric = (\S+)", line)
        if len(matches) > 0:
            result = float(matches[0])
            continue

        # Add hparam
        name, value = re.findall(" *(\S+) += (\S+)", line)[0]
        if name in [":cfgname"]:
            params[name] = value
        else:
            params[name] = float(value)

    return hp.sample(params, result)

def save_hparams_result(filename, hparams_result, metric_key):
    util.makepath(os.path.dirname(filename))
    with open(filename,'w') as f:
        if metric_key:
            f.write("# metric = %f (%s)\n" % (hparams_result.metrics[metric_key], metric_key))
        f.write(hparams2str(hparams_result.params))

def _save_model_inst(filename, inst, model, hparams):
    m = copy.deepcopy(model)
    sm.sync()

    # Slice the trainable weights
    m.slice_inst(inst)

    # Also slice the hyperparams, and replace corresponding 'arrayed'
    # attributes in the model with their scalar (sliced element) counterpart.
    h = _slice_hparams(hparams,inst)
    for key,val in h.iteritems():
        prefix,path = key.split(":")  # look for hparams named "model:..."
        if prefix != "model":
            continue
        nodepath,attrname = path.rsplit(".",1)
        node = m.find(nodepath)
        if hasattr(node,"set_"+attrname):
            getattr(node,"set_"+attrname)(model,val)   # call model.set_xxx(val)
        else:
            setattr(node,attrname,val)

    # Dump the model
    util.makepath(os.path.dirname(filename))
    with open(filename,'wb') as f:
        cp.dump(m,f)

    sm.sync() # Make sure we wait until the sarrays are all dumped

def gen_predictions(model, data):
    # We must feed each sequence through the model several times
    # by applying the model repeatedly on sliding a window along the sequence.
    # That generates a prediction map, from which we can take max, sum, etc.
    predictions = []
    gmaps = {}
    batches = data.asbatches(batchsize=128, reshuffle=False)
    for batch in batches:
        args = batch.input_data()
        args["want_bprop_inputs"] = False
        if isinstance(model.Z.origin().node,std.softmaxnode):
            args["bprop_inputs_loss"] = std.nll()
        else:
            args["bprop_inputs_loss"] = std.mse()
        outputs = model.eval(**args)
        Z = outputs['Z'].asnumpy()
        Zmask = outputs.get('Zmask',None)
        if Zmask is not None:
            Zmask = Zmask.asnumpy()
            Z = Z[Zmask.ravel()]
        predictions.append(Z)

    # Concatenate all numpy arrays if they're the same size
    predictions = np.vstack(predictions)

    return predictions

def getinstdir(outdir, targetname, trialnum, foldid):
    if isinstance(outdir,str):
        return outdir
    outdir = [_ for _ in outdir] # Make a copy that we can substitute elements in
    args = {"target" : targetname,
            "trial"  : trialnum,
            "fold"   : foldid}
    for i,item in enumerate(outdir):
        if isinstance(item, tuple):
            name, patt = item
            if args[name] is None:
                outdir[i] = None
            else:
                outdir[i] = patt % args[name]
    instdir = "/".join([part for part in outdir if part is not None])
    return instdir

def load_metrics(filename):
    metrics = {}
    with open(filename) as f:
        groupnames = f.readline().rstrip().split()
        for line in f:
            line = line.rstrip().split()
            for i,val in enumerate(line[1:]):
                metrics.setdefault(groupnames[i],{})[line[0]] = val
    return metrics

def save_metrics(outfile, metrics):
    with open(outfile,"w") as f:
        groupnames = sorted(metrics.keys())
        fieldnames = set()
        for groupname in groupnames:
            for fieldname in metrics[groupname].keys():
                fieldnames.add(fieldname)
        fieldnames = sorted(list(fieldnames))

        f.write(" "*14+"\t".join(groupnames) + "\n")
        rows = {}
        for groupname in groupnames:
            for fieldname in fieldnames:
                fieldval = metrics[groupname].setdefault(fieldname, np.nan)
                if not isinstance(fieldval,np.ndarray):
                    if isinstance(fieldval, float):
                        fmt = "%.2e" if fieldname.endswith(".p") else "%.6f"
                        fieldval = fmt%fieldval
                    rows.setdefault(fieldname,[]).append(str(fieldval))
        f.writelines([fieldname + " "*max(0,14-len(fieldname)) + "\t".join(rows[fieldname]) +"\n" for fieldname in fieldnames])

def call_dumpviz(dumpdir):
    subprocess.Popen(["python", os.path.dirname(__file__)+"/dumpviz.py", dumpdir])

##########################################

class hypertrain_worker(object):
    """
    Given a dataset and specific hyperparameters, this object will
    simply train a model (an array of models) and return the
    validation error (array of validation errors).
    """
    def __init__(self, worker_id, model_proto, trainer_proto, datasrc,
                                  nfold, allfolds, outdir, report_class, devices, verbose,
                                  default_dtype, global_flags, auxfilter, mode, dumpviz):

        self.worker_id = worker_id
        # All the data subsets in 'trainset' will be merged into a single fold.
        self.model_proto   = model_proto
        self.trainer_proto = trainer_proto
        self.datasrc = datasrc    # Load a copy of the dataset into this worker process.
        self.nfold  = nfold
        self.allfolds = allfolds
        self.outdir  = outdir
        self.mode = mode
        self.aucrange = (0.5,0.5)  # threshold for making AUCs out of non-binary targets, presumed to be in range [0,1]
        self.report_class = report_class
        self.auxfilter = auxfilter
        self.dumpviz = dumpviz

        globals.flags.copy_from(global_flags)

        # If we've been called from a new process, create a separate log file.
        # Otherwise everything is logged into the original log file.
        if multiprocessing.current_process().name != "MainProcess":
            logdir = getinstdir(outdir,None,None,None)
            worker_logfile = os.path.join(logdir,"hypertrain_worker%d.log" % worker_id)
            globals.set_logging(worker_logfile,level=verbose,echo=False)
            logging.info("\n----------------------------- %s -----------------------------" % time.strftime("%y-%m-%d %H-%M-%S",time.localtime()))

        # Configure deepity to use this worker's GPU device.
        logging.info("worker %d starting on device %d using %s" % (worker_id,devices[worker_id],sm.get_default_dtype().__name__))
        rseed = int((time.time()*100000 + worker_id)  % 2000)
        globals.reset_backend(device=devices[worker_id], seed=rseed)
        random.seed(rseed)
        sm.set_default_dtype(default_dtype)
        npr.seed(rseed)

        # Seed this process's random number generator, for reproducibility
        sm.sync()

        # Prepare the datasource to serve data.
        self.datasrc.open()

    def __del__(self):
        self.datasrc.close()
        self.datasrc = None
        gc.collect()  # Clear out the cruft and make sure the backend can be destroyed
        sm.sync()
        sm.destroy_backend()

    def __call__(self, hparams, task_ids, sample_ids):

        # Determine what kind of targets we want to train on
        data = self.datasrc.astargets(task_ids) # Copies of arbitrary targets
        data = data[:] # Copy so that when we normalize etc we don't affect the original data

        # Normalize the targets. For logisitic-output models this means
        # scaling targets to [0,1]. For other models this means scaling
        # targets to have mean=0, variance=1.
        data.requirements = self.model_proto.data_requirements()
        #print np.percentile(data.Y[data.Ymask].ravel(), [99, 99.99, 99.995, 99.999])
        #print data.Y.size, int(data.Y.size*(100-99.95)/100)
        if "clamp_targets" in globals.flags:
            data.clamp_extremes(0.0,99.95)
        if "normalize_targets" in globals.flags:
            data.normalize_targets()
            #data.arcsinhtransform_targets()

        if self.mode != 'calib':
            # If we're not in calibration mode, then there's no need for multiple checkpoints
            #  -- just keep the last checkpoint so that it can be dumped to disk
            #del hparams["trainer:checkpoints"]
            self.trainer_proto.checkpoints = 1

        # Shuffle the individual rows of data, always the same random shuffle
        # and therefore always the same random split each time the code is run.
        data.shuffle()

        # Create a callback handler to collect predictions and evaluate final performance
        checkpoints = self.report_class()

        # Perform k-fold cross validation (k=nfold), training with one fold held out at a time.
        for foldid in range(self.nfold):
            checkpoints.setfold(foldid)  # Tell the checkpoint

            # Create a new model and trainer with the given hyperparams
            model   = _create_model(self.model_proto,     hparams)
            trainer = _create_trainer(self.trainer_proto, hparams)

            # Split the data into training and validation sets
            trdata, vadata = data.split(foldid, self.nfold-1)
            trdata = trdata.augmented(trdata)
            datasets = { "train" : trdata }
            if vadata:
                vadata = vadata.augmented(vadata)
                datasets["validate"] = vadata
                if self.auxfilter:
                    datasets["validate_aux"] = vadata[[i for i in range(len(vadata)) if vadata.foldids[i] in self.auxfilter]]
            for dset in datasets.values():
                dset.requirements = model.data_requirements()

            #if not checkpoint_callback:
            #    trainer.viz_steps = False # Disable periodic updates if no reports

            # Train the model and remember how well it performed.
            trainer.train(model, datasets, checkpoints)

            if self.mode == 'train' and self.nfold > 1:
                entries = checkpoints.curr()
                metrics = self.calc_metrics(entries)
                self.save_model(model, hparams, task_ids, sample_ids, foldid)
                self.save_metrics(metrics, task_ids, sample_ids, foldid)
                self.save_predictions(entries, task_ids, sample_ids, foldid)
                self.call_dumpviz(task_ids, sample_ids, foldid)

            # If we`re only supposed to try one fold, then don`t bother looping over the other splits
            if not self.allfolds:
                break

        # Consolidate the separate folds, and dump them if need be
        entries = checkpoints.combined()

        # Calculate the performance stats associated with each target
        metrics = self.calc_metrics(entries)

        # Save the current model and predictions
        if self.mode == 'train':
            self.save_predictions(entries, task_ids, sample_ids, None)
            self.save_metrics(metrics, task_ids, sample_ids, None)
            if self.nfold == 1:
                self.save_model(model, hparams, task_ids, sample_ids, None)
                self.save_preprocessors(data, task_ids, sample_ids, None)
            #self.call_dumpviz(task_ids, sample_ids, None)

        # Return a new hparams object with the performance incorporated
        hpsearch_result = self.add_hparam_metrics(hparams, metrics)
        return hpsearch_result

    def save_model(self, model, hparams, task_ids, sample_ids, foldid):
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            util.makepath(dumpdir)

            # Slice out model i and save it to disk
            _save_model_inst(dumpdir+"/model.pkl", i, model, hparams)

    def save_predictions(self, entries, task_ids, sample_ids, foldid):
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            util.makepath(dumpdir)

            # Save out the predictions for model i
            assert len(entries[i]) == 1, "Bug. Expected only a single unique 'step' in the list of entries"
            groups = entries[i].values()[0]
            np.savez_compressed(dumpdir+"/predict.npz",
                                targetname=np.asarray(taskid, dtype=object),
                                groups=np.asarray(groups, dtype=object))

    def save_metrics(self, metrics, task_ids, sample_ids, foldid):
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            util.makepath(dumpdir)

            # Save out the predictions for model i
            assert len(metrics[i]) == 1, "Bug. Expected only a single unique 'step' in the list of entries"
            groups = metrics[i].values()[0]
            save_metrics(dumpdir+"/metrics.txt", groups)

    def call_dumpviz(self, task_ids, sample_ids, foldid):
        if not self.dumpviz:
            return
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            call_dumpviz(dumpdir)

    def save_preprocessors(self, data, task_ids, sample_ids, foldid):
        for i, taskid in enumerate(task_ids):
            dumpdir = getinstdir(self.outdir, taskid, sample_ids[i], foldid)
            data.dump_preprocessors(dumpdir, slice(i,i+1))

    def add_hparam_metrics(self, hparams, metrics):
        groupkey = "validate" if "validate" in metrics[0].values()[0] else "train"
        hpsearch_result = {}
        for i in metrics:
            for step in metrics[i]:
                hparams_i = { key : val[i] for key,val in hparams.iteritems() }
                hparams_i["trainer:max_steps"] = step
                metrics_i = metrics[i][step][groupkey]
                hpsearch_result.setdefault(i,[]).append((hparams_i, metrics_i)) # Thus tuple is returned to hpsearch
        return hpsearch_result

        """
            if "vloss" in stats and stats["vloss"] is not None:
                loss.append(stats["vloss"])
                auc.append(stats["vauc"])
            else:
                loss.append(stats["tloss"])
                auc.append(stats["tauc"])

            if self.testfilter is not None:
                tidx = [i for i in range(len(vdata)) if vdata.foldids[i] in self.testfilter]
                tdata = vdata[tidx]
                tpred = gen_predictions(model, tdata)
                testauc,teststd = bootstrap_auc(tpred.ravel(), tdata.Y.ravel(), ntrial=20)
                flogfile = self.outdir + "/%s_%04d/fold%d.log" % (task_ids[0], sample_ids[0], foldid)
                with open(flogfile) as fh:
                    flog = fh.readlines()
                flog[-1] = flog[-1].rstrip() + "\ttestAUC=%.3f (%f)\n" % (testauc,teststd)
                with open(flogfile,"w") as fh:
                    fh.writelines(flog)
                testaucs.append((testauc, teststd))

            if report:
                reports.append(report)
                report.dump(want_html=True)
                #report.dump(want_html=self.want_html)

                # Dump each model to a separate file
                for inst in range(len(sample_ids)):
                    filename = self.outdir + ("/%s_%04d/fold%d.model.pkl" % (task_ids[inst], sample_ids[inst], foldid))
                    _save_model_inst(filename, inst, model, hparams)

            """
            #break
        """"
        if reports != []:
            # Dump the separate (individual) hyperparams that were used for each instance trained
            for inst in range(len(sample_ids)):
                dumpdir = self.outdir + ("/%s_%04d/" % (task_ids[inst], sample_ids[inst]))
                vloss = self.validation_performance[task_ids[inst]] if self.validation_performance else None
                _dump_hparams(dumpdir, _slice_hparams(hparams,inst), vloss)
                tdata.dump_preprocessors(dumpdir, slice(inst,inst+1))
            merged = self.report_class.merge_reports(self.outdir + "/%(task_id)s_%(sample_id)04d/final.log", task_ids, sample_ids, reports)
            #merged.dump(want_html=self.want_html)
            merged.dump()
            if testaucs:
                flogfile = self.outdir + "/%s_%04d/final.log" % (task_ids[0], sample_ids[0])
                with open(flogfile) as fh:
                    flog = fh.readlines()
                testauc = sum([_auc for _auc, _std in testaucs]) / len(testaucs)
                teststd = sum([_std for _auc, _std in testaucs]) / len(testaucs)
                flog[-1] = flog[-1].rstrip() + "\ttestAUC=%.3f (%f)\n" % (testauc,teststd)
                with open(flogfile,"w") as fh:
                    fh.writelines(flog)

        # Average the loss over each fold
        loss = np.mean(np.asarray(loss),axis=0)
        auc  = np.mean(np.asarray(auc),axis=0)

        # Dump each average loss and corresponding hyperparameters into a log file
        for inst in range(len(sample_ids)):
            util.makepath(self.outdir+"/hpsearch")
            with open(self.outdir+"/hpsearch/%s.log"%task_ids[inst],"a") as f:
                f.write("%.6f\t%.4f\t%s\n"%(loss[inst], auc[inst], hparams2str( _slice_hparams(hparams,inst) ).replace("\n",";")) )
                """
        sm.sync()

        # Return a list of objective values, one per search_id
        values = [float(x) for x in loss]
        return values

    def calc_metrics(self, entries):
        metrics = {}
        for taskidx in entries:
            for step in entries[taskidx]:
                for group in entries[taskidx][step]:
                    entry = entries[taskidx][step][group]
                    Z = entry["Z"]
                    Y = entry["Y"]

                    # Start computing stats
                    metric = metrics.setdefault(taskidx,{}).setdefault(step,{}).setdefault(group,{})
                    metric["loss"] = entry["L"]
                    if Z.shape[1] == 1:
                        metric.update(deepity.calc_metrics(Z.ravel(), Y.ravel(), self.aucrange))
        return metrics

def hparams2str(params):
    txt = ""
    for key in sorted(params.keys()):
        value = params[key]
        if isinstance(value, np.ndarray) and value.size > 10:
            value = "ndarray"
        txt += "  %s = %s\n" % (key + " "*max(0,20-len(key)),value)
    return txt

#######################################

def hypertrain(model, trainer, data,
               nfold=2, allfolds=True, outdir=None, nsample=20,
               devices=None, verbose=None, report_class=None,
               auxfilter=None):

    if report_class is None: report_class = training_report

    # Create the output directory if it doesn't already exist.
    if outdir is None:
        outdir = join(tempfile.gettempdir(),"hypertrain")

    # Define the search space
    space = _get_hypertrain_searchspace(model, trainer)

    # Perform the search, returning the best parameters in the search space.
    logging.info("calibrating...")
    samples = hp.search(space,
                        objective  = hypertrain_worker,
                        objective_initargs = (model,trainer,data,nfold,allfolds,outdir,report_class,devices,False,sm.get_default_dtype(),globals.flags,auxfilter,"calib",False),
                        task_ids = data.targetnames,
                        nsample  = nsample,
                        nprocess = len(devices),
                        nsample_per_process = 15,
                        print_progress = True)
    logging.info("...calibrating done")

    return samples

###########################################

def train(model, trainer, data, hparams=None, hparams_metric=None,
          nfold=1, outdir=None, nsample=1,
          devices=None, verbose=None, report_class=None,
          auxfilter=None, dumpviz=True):

    if report_class is None: report_class = training_report

    if hparams:
        for targetname in data.targetnames:
            for sample in range(nsample):
                for fold in range(nfold):
                    save_hparams_result(getinstdir(outdir, targetname, sample, fold)+"/calib.txt", hparams[targetname], hparams_metric)

    space = _get_fixed_searchspace(model, trainer, data.targetnames, hparams)

    #space = _get_hypertrain_searchspace(model, trainer)

    #if space and not hparams:
    #    raise ValueError("The given model has undetermined hyperparamters. Must call hypertrain first.")

    # Replace the randomly sampled hparams with fixed values specified by 'hparams'
    #for pname in space._pdefs.iterkeys():
    #    pbest = np.asarray([hparams[task_id].params[pname] for task_id in data.targetnames])
    #    space._pdefs[pname] = hp.fixed(pbest, pname)
        #print "assigning hparam",pname,"<-",pbest

    final_outdir = outdir
    logging.info("train...")
    hp.search(space,
              objective  = hypertrain_worker,
              objective_initargs = (model,trainer,data,nfold,True,final_outdir,report_class,devices,verbose,sm.get_default_dtype(),globals.flags,auxfilter,"train",dumpviz),
              task_ids = data.targetnames,
              nsample  = nsample,
              nsample_per_process = 2,#len(data.targetnames),  # Hack: only train numtargets models at a time, to ensure that when nsample>1 the next sample gets a different minibatch order
              nprocess = len(devices))
    logging.info("...train done")

#######################################################

def _get_fixed_searchspace(model, trainer, targetnames, hparams):
    pdefs = []
    if hparams:
        # Convert the hparams list-of-dictionaries (all dictionaries having same key)
        # into a single dictionary-of-lists
        hpvec = {}
        for targetname in targetnames:
            sample = hparams[targetname]
            for pkey in sample.params:
                hpvec.setdefault(pkey,[]).append(sample.params[pkey])
        for key in hpvec:
            pdefs.append(hp.fixed(np.array(hpvec[key]), key))
    space = hp.space(pdefs)
    return space

def _get_hypertrain_searchspace(model, trainer):
    # First, collect all hparams by visiting the model's dependency graph
    model_hparams = []
    def collect_hparam(path,attr):
        if isinstance(attr,hp.paramdef):
            attr.name = "model:" + path  # model:...path
            model_hparams.append(attr)
    model.visit(collect_hparam)

    # Next, ask the trainer for its hyperparams, and put a "trainer." prefix on the name of each one
    # so that they don't conflict with model_hparams
    trainer_hparams = []
    for name,attr in trainer.__dict__.iteritems():
        if isinstance(attr,hp.paramdef):
            attr.name = "trainer:" + name  # trainer:...path
            trainer_hparams.append(attr)

    # Return a search space built from model and trainer hyperparams
    return hp.space(trainer_hparams + model_hparams)


from exercises import *
from graders import *
from training_util import *

from rlbottraining.common_exercises.kickoff_exercise import *

def make_default_playlist():
    exercises = [
        # KickoffExercise('Both Corners', blue_spawns=[Spawns.CORNER_R, Spawns.CORNER_L], orange_spawns = []),
        #KickoffExercise('Right Corner 50/50', blue_spawns=[Spawns.CORNER_R], orange_spawns = [Spawns.CORNER_R]),
        KickoffExercise('Right Corner', blue_spawns=[Spawns.CORNER_R], orange_spawns = []),
        KickoffExercise('Left Corner', blue_spawns=[Spawns.CORNER_L], orange_spawns = []),
        KickoffExercise('Back Right', blue_spawns=[Spawns.BACK_R], orange_spawns = []),
        KickoffExercise('Back Left', blue_spawns=[Spawns.BACK_L], orange_spawns = []),
        KickoffExercise('Straight', blue_spawns=[Spawns.STRAIGHT], orange_spawns = []),
    ]
    for exercise in exercises:
        exercise.match_config = make_match_config_with_cfg()

    return exercises


import json
import os
import pygame
from pprint import pformat

from pyggui.core import TRANSPARENT

pygame.font.init()

_DEFAULT_THEME_PATH = 'assets/themes/default_theme.json'
_DEFAULT_THEME_PATH = os.path.join(os.path.dirname(__file__), _DEFAULT_THEME_PATH)

NULL_THEME_DEFAULTS = {
    "col": TRANSPARENT,
    "width": 0,
    "padding": [0, 0],
    "radius": 0,
    "font": {
      "name": "calibri",
      "size": 32
    }
}

class Theme:
    def __init__(self, file=None):
        self.file = file

        if self.file is None:
            self.file = _DEFAULT_THEME_PATH

        self.changed = False
        self._all_styles = self._load_theme_json(self.file)
        self._top_level_theme = self._all_styles.get("*")
        self._styles = self._top_level_theme.copy()

    @staticmethod
    def _load_theme_json(file):
        with open(file, mode='r') as theme_file:
            themes = json.load(theme_file)

        return themes

    def get_widget_theme(self, widget=None):
        widget_theme = self._top_level_theme.copy()

        if widget is None:
            return widget_theme

        widget_type = widget.__class__.__name__.lower()

        # Update the widget theme with the widget type level attributes
        widget_type_theme = self._all_styles.get(widget_type, {})
        widget_theme |= widget_type_theme

        new_theme = Theme()
        new_theme._styles = widget_theme.copy()

        return new_theme

    def __getitem__(self, item):
        value = self._styles.get(item, None)
        if value is None and any(key in item for key in NULL_THEME_DEFAULTS.keys()):
            value = [v for k, v in NULL_THEME_DEFAULTS.items() if k == item or k in item][0]

        return value

    def __setitem__(self, item, value):
        old_styles = self._styles.copy()
        if item in self._styles:
            self._styles["bg-col"] = value

        self.changed = self.changed or old_styles != self._styles

    def __repr__(self):
        return pformat(self._styles)

    @property
    def font(self):
        font_dict = self['font']
        name = font_dict.get('name')
        size = font_dict.get('size')

        try:
            font_ = pygame.font.Font(name, size)
        except FileNotFoundError:
            font_ = pygame.font.SysFont(name, size)

        font_.set_bold(font_dict.get('bold', False))
        font_.set_italic(font_dict.get('italic', False))
        font_.set_underline(font_dict.get('underline', False))

        return font_

    def copy(self):
        copy = Theme()
        copy._styles = self._styles.copy()

        return copy

if __name__ == '__main__':
    theme = Theme()
    print(theme)


import sys, dpkt, socket
from dpkt.compat import compat_ord

class Statistics:               #Statistic Class: Used just to store global stats of the following info
    connCount = 0
    rstCount = 0
    openCount = 0
    closeCount = 0
    duration = 0
    minDuration = 0
    meanDuration = 0
    maxDuration = 0
    RTTCount = 0
    RTT = []
    minRTT = -1
    meanRTT = -1
    maxRTT = -1
    pktCount = 0
    minPacket = 0
    meanPacket = 0
    maxPacket = 0
    window = []
    minWindow = -1
    meanWindow = -1
    maxWindow = -1

class Packet(object):       #Packet Class: used to store packet info. A packet class is created for each packet, and destroyed after being analyzed
    srcMac = ""
    dstMac = ""
    srcIP = ""
    dstIP = ""
    IPLen = -1
    id = -1
    seq = -1
    ack = -1
    windowSize = -1
    flagsBin = -1
    flags = []
    srcPort = -1
    dstPort = -1
    time = -1

class Connection:                           #Connection Info. Used to store
    def __init__(self, packet):
        self.srcAdd = packet.srcIP
        self.dstAdd = packet.dstIP
        self.srcPort = packet.srcPort
        self.dstPort = packet.dstPort
        self.status = [0, 0, 0]             #SYN Count, FIN Count, RST Count
        self.startTime = packet.time
        self.endTime = packet.time
        self.srcDstPacketCount = 0
        self.dstSrcPacketCount = 0
        self.packetCount = 0
        self.srcDstByteCount = 0
        self.dstSrcByteCount = 0
        self.byteCount = 0
        self.initialClientSeq = packet.seq + 1
        self.initialServerSeq = 0
        self.pastClientSeq = -50
        self.pastServerSeq = 0
        self.pastClientPacketTime = packet.time
        self.pastServerPacketTime = 0
        self.RTTCount = 0
        self.calRTT = 0

        self.duration = 0
        self.RTT = []
        self.window = []

class Connections:
    def __init__(self):
        self.links = []
        self.size = 0

    def add(self, connection):
        self.links.append(connection)
        self.size = self.size + 1

    def printConnections(self):
        count = 1
        for link in self.links:
            print("Connection " + str(count) + ":")
            print("Source Address: " + link.srcAdd)
            print("Destination Address: " + link.dstAdd)
            print("Source Port: " + str(link.srcPort))
            print("Destination Port: " + str(link.dstPort))
            print("Status: " + "S" + str(link.status[0]) + "F" + str(link.status[1]) + "R" + str(link.status[2]))
            if link.status[0] >= 1:
                if link.status[1] >= 1:
                    print("Start Time: " + str(link.startTime) + "ms")
                    print("End Time: " + str(link.endTime) + "ms")
                    print("Duration: " + str(link.duration) + "ms")
                    print("Number of packets send from Source to Destination: " + str(link.srcDstPacketCount))
                    print("Number of packets send from Destination to Source: " + str(link.dstSrcPacketCount))
                    print("Total number of packets: " + str(link.packetCount))
                    print("Number of data bytes send from Source to Destination: " + str(link.srcDstByteCount))
                    print("Number of data bytes send from Destination to Source: " + str(link.dstSrcByteCount))
                    print("Total number of data bytes: " + str(link.byteCount))
                    print("END")
            count = count + 1
            if count <= (self.size): print("++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++")

def printPacket(packet):
    print("Source MAC: " + packet.srcMac)
    print("Destination MAC: " + packet.dstMac)
    print("Source IP: " + packet.srcIP)
    print("Destination IP: " + packet.dstIP)
    print("IP Header Length: " + str(packet.IPLen))
    print("Packet ID: " + str(packet.id))
    print("Sequence: " + str(packet.seq))
    print("Acknowledgement: " + str(packet.ack))
    print("Window Size: " + str(packet.windowSize))
    print("Flag Binary: " + bin(packet.flagsBin))
    print("Flags: " + str(packet.flags))
    print("Source Port: " + str(packet.srcPort))
    print("Destination Port: " + str(packet.dstPort))
    print("Time: " + str(packet.time))

def mac_addr(address):      #Refer to Reference                     #Used to convert binary to mac addresses
    return ":".join("%02x" % compat_ord(b) for b in address)

def inet_to_str(inet):      #Refer to Reference                     #Used to convert binary to Ip Addresses
    return socket.inet_ntop(socket.AF_INET, inet)

def binToFlags(packet):                                             #Binary Flag parsing
    packet.flags = []
    if packet.flagsBin & 0b1: packet.flags.append("FIN")
    if packet.flagsBin & 0b10: packet.flags.append("SYN")
    if packet.flagsBin & 0b100: packet.flags.append("RST")
    if packet.flagsBin & 0b1000: packet.flags.append("PSH")
    if packet.flagsBin & 0b10000: packet.flags.append("ACK")
    if packet.flagsBin & 0b100000: packet.flags.append("URG")
    return packet

def clientInitialRTT(stats, connection, packet):                    #The initial time for RTT
    connection.pastClientSeq = packet.seq                           #Initial sequence number sent
    connection.pastClientPacketTime = packet.time                   #Initial packet time
    return 0

def clientFinalRTT(stats, connection, packet):                      #Client final RTT
    if connection.pastClientSeq <= packet.ack:                      #Ensure that the ack receieved corresponds to an ack
        RTT = packet.time - connection.pastClientPacketTime         #Calculate RTT time between the matching seq to ack
        connection.RTT.append(RTT)                                  #Append RTT calculation to connection for mean and other purposes
    return 0

def updateDstSrcCount(connection, packet):                          #Calculation to update the byte and packet count from the destunation to source
    connection.packetCount = connection.packetCount + 1
    connection.srcDstPacketCount = connection.srcDstPacketCount + 1
    connection.dstSrcByteCount = packet.ack - connection.initialServerSeq - 1
    connection.byteCount = connection.srcDstByteCount + connection.dstSrcByteCount
    return packet.ack - connection.initialServerSeq - 1

def updateSrcDstCount(connection, packet):                          #Method to update the byte and packet count from the source to destionation
    connection.packetCount = connection.packetCount + 1
    connection.dstSrcPacketCount = connection.dstSrcPacketCount + 1
    if connection.initialServerSeq == 0:                            #Initial server / client 3 way hand shake scenario handling
        connection.initialServerSeq = packet.seq + 1
    connection.srcDstByteCount = packet.ack - connection.initialClientSeq
    connection.byteCount = connection.srcDstByteCount + connection.dstSrcByteCount
    return packet.ack - connection.initialClientSeq

def printFinal(stats, connections):
    print("A) Total number of connections: " + str(connections.size))
    print("___________________________________________________________________________________")
    print("")
    print("B) Connection's details:")
    print("")
    connections.printConnections()
    print("___________________________________________________________________________________")
    print("")
    print("C) General:")
    print("")
    print("Total number of complete TCP connections: " + str(stats.closeCount))
    print("Number of reset TCP connections: " + str(stats.rstCount))
    print("Number of TCP connections that were still open when the trace capture ended: " + str(stats.openCount))
    print("___________________________________________________________________________________")
    print("")
    print("D) Complete TCP connections:")
    print("")
    print("Minimum time durations: " + str(stats.minDuration) + "ms")
    print("Mean time durations: " + str(stats.meanDuration) + "ms")
    print("Maximum time duration: " + str(stats.maxDuration) + "ms")
    print("")
    print("Minimum RTT values: " + str(stats.minRTT) + "ms")
    print("Mean RTT values: " + str(stats.meanRTT) + "ms")
    print("Maximum RTT values: " + str(stats.maxRTT) + "ms")
    print("")
    print("Minimum number of packets including both send/received: " + str(stats.minPacket))
    print("Mean number of packets including both send/received: " + str(stats.meanPacket))
    print("Maximum number of packets including both send/received: " + str(stats.maxPacket))
    print("")
    print("Minimum receive window sizes including both send/received: " + str(stats.minWindow))
    print("Mean receive window sizes including both send/received: " + str(stats.meanWindow))
    print("Maximum receive window sizes including both send/receive: " + str(stats.maxWindow))
    print("___________________________________________________________________________________")

def analyzePacket(stats, connections, packet):              #Series of function calls that analyzes all the packets
    for connection in connections.links:                    #Checks whether a connection exists in file for the packet being analyzed
        if (connection.srcAdd == packet.srcIP) and (connection.dstAdd == packet.dstIP) and (connection.srcPort == packet.srcPort) and (connection.dstPort == packet.dstPort):
            if "SYN" in packet.flags:
                connection.status[0] = connection.status[0] + 1         #Update SYN Count
            if "FIN" in packet.flags:
                connection.status[1] = connection.status[1] + 1         #Update FIN Count
                connection.endTime = packet.time                        #Update END TIME
            if "RST" in packet.flags:
                connection.status[2] = connection.status[2] + 1         #Update RST Count

            connection.window.append(packet.windowSize)                 #Store Window Size
            byteTransfered = updateDstSrcCount(connection, packet)      #Calculate if any bytes were sent /received, and store

            if "SYN" in packet.flags or "FIN" in packet.flags:          #Calculate the RTT if it is SYN or FIN
                connection.calRTT = 1
                clientInitialRTT(stats, connection, packet)

            return 1

                                        #Serires of function calls that analyzes all the packets
                                        #Similar as to above, but for server -> destionation packets
        if (connection.dstAdd == packet.srcIP) and (connection.srcAdd == packet.dstIP) and (connection.dstPort == packet.srcPort) and (connection.srcPort == packet.dstPort):
            if "SYN" in packet.flags:
                connection.status[0] = connection.status[0] + 1
            if "FIN" in packet.flags:
                connection.status[1] = connection.status[1] + 1
                connection.endTime = packet.time
            if "RST" in packet.flags:
                connection.status[2] = connection.status[2] + 1

            connection.window.append(packet.windowSize)
            byteTransfered = updateSrcDstCount(connection, packet)

                                            #Only calculate RTT is, it is SYN, has data transfered
            if ((byteTransfered > 0 and "ACK" in packet.flags) or "SYN" in packet.flags) and connection.calRTT == 1:
                connection.calRTT = 0
                clientFinalRTT(stats, connection, packet)

            return 1

    connection = Connection(packet)                                                 #If there isn't any existing connection, create a connection
    connection.srcDstPacketCount = connection.srcDstPacketCount + 1                 #Update packet count
    connection.packetCount = connection.packetCount + 1                             #
    connection.status[0] = connection.status[0] + 1                                 #Add syn count
    stats.openCount = stats.openCount + 1                                           #Add open count
    stats.connCount = stats.connCount + 1                                           #Add connection cvount
    connection.window.append(packet.windowSize)                                     #Store c
    connections.add(connection)

    if "SYN" in packet.flags:                                                       #If SYN, prepare values for RTT
        connection.calRTT = 1
        clientInitialRTT(stats, connection, packet)
    return 0

def finalStatCheck(stats, connections):                                             #After analyzing all the packets
    for connection in connections.links:                                            #For all connections
        if connection.status[0] >= 1:                                               #If SYN
            if connection.status[1] >= 1:                                           #If FIN
                stats.openCount = stats.openCount - 1                               #Complete connection
                stats.closeCount = stats.closeCount + 1

                connection.duration = connection.endTime - connection.startTime     #min mean max duration for all compelete connections
                stats.duration = stats.duration + connection.duration
                if stats.minDuration == 0:
                    stats.minDuration = connection.duration
                if stats.maxDuration == 0:
                    stats.maxDuration = connection.duration
                if connection.duration <= stats.minDuration:
                    stats.minDuration = connection.duration
                if connection.duration >= stats.maxDuration:
                    stats.maxDuration = connection.duration

                stats.pktCount = stats.pktCount + connection.packetCount        #min mean max packet count for all complete connections
                if stats.minPacket == 0:
                    stats.minPacket = connection.packetCount
                if stats.maxPacket == 0:
                    stats.maxPacket = connection.packetCount
                if connection.packetCount <= stats.minPacket:
                    stats.minPacket = connection.packetCount
                if connection.packetCount >= stats.maxPacket:
                    stats.maxPacket = connection.packetCount

                stats.window.extend(connection.window)                          #update connection window for min, mean max calculations

                stats.RTT.extend(connection.RTT)                                #add rtt for min mean max valculations

            if connection.status[2] >= 1:
                stats.rstCount = stats.rstCount + 1

    stats.meanDuration = stats.duration / stats.closeCount                      #mean duration calculation
    stats.meanPacket = stats.pktCount / stats.closeCount                        #mean packet count duration

    stats.minWindow = min(stats.window)                                         #min mean max window size calculation
    stats.maxWindow = max(stats.window)
    stats.meanWindow = sum(stats.window)/stats.pktCount

    stats.minRTT = min(stats.RTT)                                               #min mean max RTT calculation
    stats.maxRTT = max(stats.RTT)
    stats.meanRTT = sum(stats.RTT) / len(stats.RTT)

    return 1

def main():
    traceFileName = sys.argv[1]                         #name of file to read from

    traceFile = open(traceFileName, "rb")               #open the file to read in binary
    tracePcap = dpkt.pcap.Reader(traceFile)             #use a reader to parse

    stats = Statistics()
    connections = Connections()
    count = 0

    for timeStamp, buf in tracePcap:                    #Refer to reference. Parts of the referenced code has been deleted or modified.
        eth = dpkt.ethernet.Ethernet(buf)
        ip = eth.data                                   #IP Header
        tcp = ip.data                                   #TCP Header

        packet = Packet()                               #Storing various values into a packet class
        packet.srcMac = mac_addr(eth.src)
        packet.dstMac = mac_addr(eth.dst)

        packet.srcIP = inet_to_str(ip.src)
        packet.dstIP = inet_to_str(ip.dst)
        packet.IPLen = ip.len
        packet.id = ip.id

        packet.seq = tcp.seq
        packet.ack = tcp.ack
        packet.windowSize = tcp.win
        packet.flagsBin = tcp.flags
        packet.srcPort = tcp.sport
        packet.dstPort = tcp.dport
        packet.time = timeStamp
        packet = binToFlags(packet)

        analyzePacket(stats, connections, packet)       #For each packet, analyze
        del packet

    finalStatCheck(stats, connections)
    printFinal(stats, connections)

main()

# Parsing is taken from the link below, in particluar the mac_addr, and inet_to_str.
# Opening the file, and obtaining the buffer and timestamp is from
# http://dpkt.readthedocs.io/en/latest/_modules/examples/print_packets.html?highlight=print%20ip


mylist = ["eat", "tea", "tan", "ate", "nat", "bat"]

emptylist = []

for i in mylist:

    tempSet = set(i)

    templist = []

    for j in range(len(mylist)):

        if tempSet == tempSet.intersection(set(mylist[j])):

            templist.append(mylist[j])

    if templist not in emptylist:

            emptylist.append(templist)

    templist = []

print(emptylist)


import numpy as np

def calibrate(input_path, output_path, filename):
    X = np.load(input_path + filename + '.npy')

    y_gt = np.zeros(X.shape[0], dtype=int);
    m = 0
    # for i in range(0,1250):
    #     y_gt[i] = 1
    #     m += 1
    for i in range(9346,13577):
        y_gt[i] = 1
        m += 1
    # for i in range(11512,11712):
    #     y_gt[i] = 1
    #     m += 1
    for i in range(14069,18355):
        y_gt[i] = 1
        m += 1

    print("保存标定的真实值:" + output_path + filename + '.npy', "测试集异常点数量: ", int(m / 2))
    np.save(output_path + filename + '.npy', y_gt)
    return


import time
import json
import math
import csv
import serial # conda install pyserial
import sys
import glob
# pygame needs python 3.6, not available for 3.7
import pygame # conda install -c cogsci pygame; maybe because it only is supplied for earlier python, might need conda install -c evindunn pygame ; sudo apt-get install libsdl-ttf2.0-0
import pygame.joystick as joystick # https://www.pygame.org/docs/ref/joystick.html
from datetime import datetime
# our imports
import kbhit
from pendulum import Pendulum

POLOLU_MOTOR            = False # set true to set options for this motor, which has opposite sign for set_motor TODO needs fixing in firmware or wiring of motor

SERIAL_PORT             = "COM4" #"/dev/ttyUSB0" # might move if other devices plugged in
SERIAL_BAUD             = 230400  # default 230400, in firmware.   Alternatives if compiled and supported by USB serial intervace are are 115200, 128000, 153600, 230400, 460800, 921600, 1500000, 2000000
PRINT_PERIOD_MS         = 100  # shows state every this many ms
CONTROL_PERIOD_MS       = 5
CALIBRATE               = False #False # important to calibrate if running standalone to avoid motor burnout because limits are determined during this calibration
MOTOR_FULL_SCALE        =  7199 # 7199 # with pololu motor and scaling in firmware #7199 # with original motor
MOTOR_MAX_PWM           = int(round(0.95 * MOTOR_FULL_SCALE))

JOYSTICK_SCALING        = MOTOR_MAX_PWM # how much joystick value -1:1 should be scaled to motor command
JOYSTICK_DEADZONE       = 0.05 # deadzone around joystick neutral position that stick is ignored

ANGLE_TARGET            = 3129 # 3383  # adjust to exactly vertical angle value, read by inspecting angle output
ANGLE_CTRL_PERIOD_MS    = 5         # Must be a multiple of CONTROL_PERIOD_MS
ANGLE_AVG_LENGTH        = 4 # adc routine in firmware reads ADC this many times quickly in succession to reduce noise
ANGLE_SMOOTHING         = 1      # 1.0 turns off smoothing
ANGLE_KP                = 400
ANGLE_KD                = 400

POSITION_TARGET         = 0 # 1200
POSITION_CTRL_PERIOD_MS = 25        # Must be a multiple of CONTROL_PERIOD_MS
POSITION_SMOOTHING      = 1       # 1.0 turns off smoothing
POSITION_KP             = 20
POSITION_KD             = 300

def serial_ports(): # from https://stackoverflow.com/questions/12090503/listing-available-com-ports-with-python
    """ Lists serial port names

        :raises EnvironmentError:
            On unsupported or unknown platforms
        :returns:
            A list of the serial ports available on the system
    """
    if sys.platform.startswith('win'):
        ports = ['COM%s' % (i + 1) for i in range(256)]
    elif sys.platform.startswith('linux') or sys.platform.startswith('cygwin'):
        # this excludes your current terminal "/dev/tty"
        # if cannot open, check permissions
        ports = glob.glob('/dev/ttyUSB[0-9]*')
    elif sys.platform.startswith('darwin'):
        ports = glob.glob('/dev/tty.*')
    else:
        raise EnvironmentError('Unsupported platform')

    result = []
    for port in ports:
        try:
            s = serial.Serial(port)
            s.close()
            result.append(port)
        except (OSError, serial.SerialException):
            pass
    return result

def saveparams():
    print("\nSaving parameters")
    p={}
    p['ANGLE_TARGET']=ANGLE_TARGET
    p['ANGLE_KP']=ANGLE_KP
    p['ANGLE_KD']=ANGLE_KD
    p['POSITION_TARGET']=POSITION_TARGET
    p['POSITION_KP']=POSITION_KP
    p['POSITION_KD']=POSITION_KD
    p['ANGLE_SMOOTHING']=ANGLE_SMOOTHING
    p['POSITION_SMOOTHING']=POSITION_SMOOTHING
    with open('control.json','w') as f:
        json.dump(p,f)

def loadparams():
    print("\nLoading parameters")
    f=open('control.json')
    try:
        p=json.load(f)
        global ANGLE_TARGET, ANGLE_KP,ANGLE_KD,POSITION_TARGET,POSITION_KP,POSITION_KD,ANGLE_SMOOTHING,POSITION_SMOOTHING
        ANGLE_TARGET=p['ANGLE_TARGET']
        ANGLE_KP=p['ANGLE_KP']
        ANGLE_KD=p['ANGLE_KD']
        POSITION_TARGET=p['POSITION_TARGET']
        POSITION_KP=p['POSITION_KP']
        POSITION_KD=p['POSITION_KD']
        ANGLE_SMOOTHING=p['ANGLE_SMOOTHING']
        POSITION_SMOOTHING=p['POSITION_SMOOTHING']
    except:
        print("something went wrong loading parameters")
    printparams()

def help():
    print("\n***********************************")
    print("keystroke commands")
    print("ESC quit")
    print("k toggle control on/off (initially off)")
    print("K trigger motor position calibration")
    print("=/- increase/decrease angle target")
    print("[/] increase/decrease position target")
    print("w/q angle proportional gain")
    print("s/a angle derivative gain")
    print("z/x angle smoothing")
    print("r/e position proportional gain")
    print("f/d position derivative gain")
    print("c/v position smoothing")
    print("l toggle logging data")
    print("S/L Save/Load param values from disk")
    print("D Toggle dance mode")
    print("***********************************")

def printparams():
    print("\nAngle PD Control Parameters")
    print("    Set point       {0}".format(ANGLE_TARGET))
    print("    Average Length  {0}".format(ANGLE_AVG_LENGTH))
    print("    Smoothing       {0:.2f}".format(ANGLE_SMOOTHING))
    print("    P Gain          {0:.2f}".format(ANGLE_KP))
    print("    D Gain          {0:.2f}".format(ANGLE_KD))

    print("Position PD Control Parameters")
    print("    Set point       {0}".format(POSITION_TARGET))
    print("    Control Period  {0} ms".format(POSITION_CTRL_PERIOD_MS))
    print("    Smoothing       {0:.2f}".format(POSITION_SMOOTHING))
    print("    P Gain          {0:.2f}".format(POSITION_KP))
    print("    D Gain          {0:.2f}".format(POSITION_KD))

ratio=1.05

def inc(param):
    if param < 2:
        param = round(param + 0.1, 1)
    else:
        old=param
        param = round(param*ratio)
        if param==old:
            param+=1
    return param

def dec(param):
    if param < 2:
        param = max(0,round(param - 0.1, 1))
    else:
        old=param
        param = round(param/ratio)
        if param==old:
            param-=1
    return param

if ANGLE_CTRL_PERIOD_MS < CONTROL_PERIOD_MS or POSITION_CTRL_PERIOD_MS <CONTROL_PERIOD_MS:
    raise Exception("angle or position control periods too short compared to CONTROL_PERIOD_MS")

# check that we are running from terminal, otherwise we cannot control it
if sys.stdin.isatty():
    # running interactively
    print('running interactively from an interactive terminal, ok')
else:
   print('run from an interactive terminal to allow keyboard input')
   quit()

################################################################################
# OPEN SERIAL PORT
################################################################################
p = Pendulum()
serialPorts=serial_ports()
print('Available serial ports: '+str(serialPorts))
if len(serialPorts)==0:
    print('no serial ports available, or cannot open it; check linux permissions\n Under linux, sudo chmod a+rw [port] transiently, or add user to dialout or tty group')
    quit()

if len(serialPorts)>1:
    print(str(len(serialPorts))+' serial ports, taking first one which is '+str(serialPorts[0]))
SERIAL_PORT=str(serialPorts[0])
try:
    p.open(SERIAL_PORT, SERIAL_BAUD)
except:
    print('cannot open port '+str(SERIAL_PORT)+': available ports are '+str(serial_ports()))
    quit()

print('opened '+str(SERIAL_PORT)+' successfully')
p.control_mode(False)
p.stream_output(False)

joystickExists=False
pygame.init()
joystick.init()
if joystick.get_count()==1:
    stick = joystick.Joystick(0)
    stick.init()
    axisNum = stick.get_numaxes()
    buttonNum = stick.get_numbuttons()
    joystickExists=True
    print('joystick found with '+str(axisNum)+' axes and '+str(buttonNum)+' buttons')
else:
    print('no joystick found, only PD control or no control possible')

if CALIBRATE:
    print("Calibrating motor position....")
    if not p.calibrate():
        print("Failed to connect to device")
        p.close()
        exit()
    print("Done calibrating")

loadparams()
time.sleep(1)

################################################################################
# SET PARAMETERS
################################################################################
p.set_angle_config(     ANGLE_TARGET,
                        ANGLE_AVG_LENGTH,
                        ANGLE_SMOOTHING,
                        ANGLE_KP,
                        ANGLE_KD)

p.set_position_config(  POSITION_TARGET,
                        POSITION_CTRL_PERIOD_MS,
                        POSITION_SMOOTHING,
                        POSITION_KP,
                        POSITION_KD)

################################################################################
# GET PARAMETERS
################################################################################
(   ANGLE_TARGET,
    ANGLE_AVG_LENGTH,
    ANGLE_SMOOTHING,
    ANGLE_KP,
    ANGLE_KD)           = p.get_angle_config()

(   POSITION_TARGET,
    POSITION_CTRL_PERIOD_MS,
    POSITION_SMOOTHING,
    POSITION_KP,
    POSITION_KD)        = p.get_position_config()

################################################################################
# CONTROL LOOP (PC BASED)
################################################################################
printCount          = 0

angleErrPrev        = 0
angleCmd            = 0
positionErrPrev     = 0
positionCmd         = 0

controlEnabled=False

danceEnabled=False
danceAmpl=500
dancePeriodS=8

loggingEnabled=False

kbAvailable=True
try:
    kb = kbhit.KBHit() # can only use in posix terminal; cannot use from spyder ipython console for example
except:
    kbAvailable=False

printparams()
help()
startTime=time.time()
lastTime=startTime
lastAngleControlTime=lastTime
lastPositionControlTime=lastTime
angleErr=0
positionErr=0 # for printing even if not controlling
p.stream_output(True)  # now start streaming state
while True:

    # Adjust Parameters
    if kbAvailable & kb.kbhit():
        c = kb.getch()

        if c=='D':
            danceEnabled=~danceEnabled
            print("\ndanceEnabled= {0}".format(danceEnabled))
        elif c == 'l':
            loggingEnabled=~loggingEnabled
            print("\nloggingEnabled= {0}".format(loggingEnabled))
            if loggingEnabled:
                try:
                    csvfilename=datetime.now().strftime("cartpole-%Y-%m-%d-%H-%M-%S.csv")
                    csvfile=open(csvfilename, 'w', newline='')
                    csvwriter = csv.writer(csvfile, delimiter=',')
                    csvwriter.writerow(['time'] + ['deltaTimeMs']+['angle'] + ['position'] + ['angleTarget']  + ['angleErr'] + ['positionTarget'] + ['positionErr'] + ['angleCmd'] + ['positionCmd'] + ['motorCmd']+['actualMotorCmd'])
                    print("\n Started logging data to "+csvfilename)
                except Exception as e:
                    loggingEnabled=False
                    print("\n" + str(e) + ": Exception opening csvfile; logging disabled")
            else:
                csvfile.close()
                print("\n Stopped logging data to "+csvfilename)

        elif c == 'k':
            controlEnabled=~controlEnabled
            print("\ncontrolEnabled= {0}".format(controlEnabled))
        elif c == 'K':
            controlEnabled=False
            print("\nCalibration triggered")
            p.calibrate()
            print("\nCalibration finished")
        elif c == 'h' or c=='?':
            help()
        elif c == 'p' :
            printparams()
        # Increase Target Angle
        elif c == '=':
            ANGLE_TARGET += 1
            print("\nIncreased target angle to {0}".format(ANGLE_TARGET))
        # Decrease Target Angle
        elif c == '-':
            ANGLE_TARGET -= 1
            print("\nDecreased target angle to {0}".format(ANGLE_TARGET))

        # Increase Target Position
        elif c == ']':
            POSITION_TARGET += 200
            print("\nIncreased target position to {0}".format(POSITION_TARGET))
        # Decrease Target Position
        elif c == '[':
            POSITION_TARGET -= 200
            print("\nDecreased target position to {0}".format(POSITION_TARGET))

        # Angle Gains
        elif c == 'w':
            ANGLE_KP=inc(ANGLE_KP)
            print("\nIncreased angle KP {0}".format(ANGLE_KP))
        elif c == 'q':
            ANGLE_KP=dec(ANGLE_KP)
            print("\nDecreased angle KP {0}".format(ANGLE_KP))
        elif c == 's':
            ANGLE_KD=inc(ANGLE_KD)
            print("\nIncreased angle KD {0}".format(ANGLE_KD))
        elif c == 'a':
            ANGLE_KD=dec(ANGLE_KD)
            print("\nDecreased angle KD {0}".format(ANGLE_KD))
        elif c == 'x':
            ANGLE_SMOOTHING=dec(ANGLE_SMOOTHING)
            if ANGLE_SMOOTHING>1:
                ANGLE_SMOOTHING=1
            print("\nIncreased ANGLE_SMOOTHING {0}".format(ANGLE_SMOOTHING))
        elif c == 'z':
            ANGLE_SMOOTHING=inc(ANGLE_SMOOTHING)
            if ANGLE_SMOOTHING>1:
                ANGLE_SMOOTHING=1
            print("\nDecreased ANGLE_SMOOTHING {0}".format(ANGLE_SMOOTHING))

        # Position Gains
        elif c == 'r':
            POSITION_KP=inc(POSITION_KP)
            print("\nIncreased position KP {0}".format(POSITION_KP))
        elif c == 'e':
            POSITION_KP=dec(POSITION_KP)
            print("\nDecreased position KP {0}".format(POSITION_KP))
        elif c == 'f':
            POSITION_KD=inc(POSITION_KD)
            print("\nIncreased position KD {0}".format(POSITION_KD))
        elif c == 'd':
            POSITION_KD=dec(POSITION_KD)
            print("\nDecreased position KD {0}".format(POSITION_KD))
        elif c == 'v':
            POSITION_SMOOTHING=dec(POSITION_SMOOTHING)
            if POSITION_SMOOTHING>1:
                POSITION_SMOOTHING=1
            print("\nIncreased POSITION_SMOOTHING {0}".format(POSITION_SMOOTHING))
        elif c == 'c':
            POSITION_SMOOTHING=inc(POSITION_SMOOTHING)
            if POSITION_SMOOTHING>1:
                POSITION_SMOOTHING=1
            print("\nDecreased POSITION_SMOOTHING {0}".format(POSITION_SMOOTHING))
        elif c=='S':
            saveparams()
        elif c=='L':
            loadparams()

        # Exit
        elif ord(c) == 27 : # ESC
            print("\nquitting....")
            break

    # This function will block at the rate of the control loop
#    p.clear_read_buffer() # if we don't clear read buffer, state output piles up in serial buffer
    (angle, position, command) = p.read_state()
    # angle count is more positive CCW facing cart, position encoder count is more positive to right facing cart

    timeNow=time.time()
    deltaTime=timeNow-lastTime
    if deltaTime==0:
        deltaTime=1e-6
    lastTime=timeNow
    elapsedTime=timeNow-startTime
    diffFactor=(CONTROL_PERIOD_MS/(deltaTime*1000))

    positionTargetNow=POSITION_TARGET
    if controlEnabled and danceEnabled:
        positionTargetNow=POSITION_TARGET+danceAmpl*math.sin(2*math.pi*(elapsedTime/dancePeriodS))

    # Balance PD Control
    # Position PD Control
    if timeNow -lastPositionControlTime >= POSITION_CTRL_PERIOD_MS*.001:
        lastPositionControlTime=timeNow
        positionErr         = POSITION_SMOOTHING*(position - positionTargetNow) + (1.0 - POSITION_SMOOTHING)*positionErrPrev # First order low-P=pass filter
        positionErrDiff     = (positionErr - positionErrPrev)*diffFactor
        positionErrPrev     = positionErr
        # Naive solution: if too positive (too right), move left (minus on positionCmd),
        # but this does not produce correct control.
        # The correct strategy is that if cart is too positive (too right),
        # produce lean to the left by introducing a positive set point angle leaning slightly to left,
        # i.e. more positve positionErr makes more positive effective ANGLE_TARGET
        # End result is that sign of positionCmd is flipped
        # Also, if positionErr is increasing more, then we want even more lean, so D sign is also positive
        positionCmd         = +(POSITION_KP*positionErr + POSITION_KD*positionErrDiff)

    if timeNow-lastAngleControlTime >= ANGLE_CTRL_PERIOD_MS*.001:
        lastAngleControlTime=timeNow
        angleErr            = ANGLE_SMOOTHING*(angle - ANGLE_TARGET) + (1.0 - ANGLE_SMOOTHING)*angleErrPrev # First order low-pass filter
        angleErrDiff        = (angleErr - angleErrPrev)*diffFactor # correct for actual sample interval; if interval is too long, reduce diff error
        angleErrPrev        = angleErr
        angleCmd            = -(ANGLE_KP*angleErr + ANGLE_KD*angleErrDiff) # if too CCW (pos error), move cart left

    motorCmd = int(round(angleCmd + positionCmd)) # change to plus for original, check that when cart is displayed, the KP term for cart position leans cart the correct direction
    motorCmd =  MOTOR_MAX_PWM if motorCmd >  MOTOR_MAX_PWM else motorCmd
    motorCmd = -MOTOR_MAX_PWM if motorCmd < -MOTOR_MAX_PWM else motorCmd

    stickPos=0.0
    if joystickExists:
        # for event in pygame.event.get(): # User did something.
        #     if event.type == pygame.QUIT: # If user clicked close.
        #         done = True # Flag that we are done so we exit this loop.
        #     elif event.type == pygame.JOYBUTTONDOWN:
        #         print("Joystick button pressed.")
        #     elif event.type == pygame.JOYBUTTONUP:
        #         print("Joystick button released.")
        pygame.event.get() # must call get() to handle internal queue
        stickPos=stick.get_axis(0) # 0 left right, 1 front back 2 rotate

    if abs(stickPos)>JOYSTICK_DEADZONE:
        actualMotorCmd=int(round(stickPos*JOYSTICK_SCALING))
    elif controlEnabled:
        actualMotorCmd=motorCmd
    else:
        actualMotorCmd=0

    if POLOLU_MOTOR==False:
        p.set_motor(-actualMotorCmd) # positive motor cmd moves cart right
    else:
        p.set_motor(actualMotorCmd)  # positive motor cmd moves cart right

    if loggingEnabled:
#      csvwriter.writerow(['time'] + ['deltaTimeMs']+['angle'] + ['position']  + ['angleErr'] + ['positionErr'] + ['angleCmd'] + ['positionCmd'] + ['motorCmd'])
       csvwriter.writerow([elapsedTime,deltaTime*1000,angle, position, ANGLE_TARGET, angleErr, positionTargetNow, positionErr, angleCmd,positionCmd,motorCmd,actualMotorCmd])

    # Print output
    printCount += 1
    if printCount >= (PRINT_PERIOD_MS/CONTROL_PERIOD_MS):
        printCount = 0
        print("\r angle {:+4d} angleErr {:+6.1f} position {:+6d} positionErr {:+6.1f} angleCmd {:+6d} positionCmd {:+6d} motorCmd {:+6d} dt {:.3f}ms  stick {:.3f}         \r".format(int(angle), angleErr, int(position),  positionErr, int(round(angleCmd)), int(round(positionCmd)), actualMotorCmd, deltaTime*1000, stickPos), end = '')
# if we pause like below, state info piles up in serial input buffer
        # instead loop at max possible rate to get latest state info
#    time.sleep(CONTROL_PERIOD_MS*.001)  # not quite correct since there will be time for execution below

# when x hit during loop or other loop exit
p.set_motor(0) # turn off motor
p.close()
joystick.quit()

if loggingEnabled:
    csvfile.close()


import numpy as np
import netCDF4
from datetime import datetime, timedelta
from glob import glob
import os, sys
"""
    This program is used to read input data.
"""
#******************************************
# Edit here (input file directories)
#------------------------------------------
slpbasedir = "/mnt/nas02/data/CMIP6"
tabasedir  = "/mnt/nas02/data/CMIP6"
uabasedir  = "/mnt/nas02/data/CMIP6"
vabasedir  = "/mnt/nas02/data/CMIP6"
tsbasedir  = "/mnt/nas02/data/CMIP6"
topobasedir= "/mnt/nas02/data/CMIP6"
landbasedir= "/mnt/nas02/data/CMIP6"
prbasedir  = "/mnt/nas02/data/CMIP6"   # not used for detection

#******************************************

dbasedir = {
    "slp":slpbasedir,
    "ta" :tabasedir,
    "ua" :uabasedir,
    "va" :vabasedir,
    "sst":tsbasedir,
    "topo":topobasedir,
    "land":landbasedir,
    "pr" :prbasedir,     # not used for detection
    }

dvar = {
    "slp":"psl",
    "ta" :"ta",
    "ua" :"ua",
    "va" :"va",
    "sst":"ts",
    "topo":"orog",
    "land":"sftlf",
    "pr" :"pr",    # not used for detection
    }

def ret_lats(model):
    return np.load(slpbasedir + "/%s/lat.npy"%(model))
    # MIROC6: -88.92773535 ~ 88.92773535, d=~1.4007664

def ret_lons(model):
    return np.load(slpbasedir + "/%s/lon.npy"%(model))
    # MIROC6: 0 ~ 358.59375, d=1.40625

def ret_ny(model):
    return len(ret_lats(model))

def ret_nx(model):
    return len(ret_lons(model))
    # MIROC6:   (128, 256)
def ret_miss(model):
    modelname = model.split(".")[0]
    if   modelname=="MIROC6":       miss_in= 9.969209968386869e+36
    elif modelname=="MRI-ESM2-0":   miss_in= 9.969209968386869e+36
    elif modelname=="MPI-ESM1-2-HR":miss_in= 9.969209968386869e+36

    return miss_in

def Load_6hrPlev(model, var, DTime, plev):
    modelname, expr, ens = model.split(".")
    vname = dvar[var]
    iplev = [850, 500, 250].index(plev)
    # Search file
    srcdir  = dbasedir[var] + "/%s"%(model)
    ssearch = srcdir + "/%s_6hrPlev*.nc"%(vname)
    lsrcpath = glob(ssearch)
    for srcpath in lsrcpath:
        stime = os.path.basename(srcpath).split("_")[6].split(".")[0]
        stime0, stime1 = stime.split("-")

        dtime0 = datetime.strptime(stime0, "%Y%m%d%H%M")
        dtime1 = datetime.strptime(stime1, "%Y%m%d%H%M")
        if (dtime0<=DTime)&(DTime<=dtime1):
            break
    nc = netCDF4.Dataset(srcpath)

    #print(nc.variables)
    #print(srcpath)
    # Find time index
    basetime = {
        ("MIROC6","piControl"):         datetime(3200,1,1),

        ("MRI-ESM2-0","piControl"):     datetime(1850,1,1),
        ("MRI-ESM2-0","historical"):    datetime(1850,1,1),

        ("MPI-ESM1-2-HR","piControl"):  datetime(1850,1,1),
        }[modelname,expr]

    dtime0 = basetime + timedelta(days=float(nc.variables["time"][0]))
    idxtime = int((DTime - dtime0).total_seconds()/21600)    # 6-hour = 21600 sec
    #print(DTime, dtime0)
    #print(idxtime)
    return nc.variables[vname][idxtime, iplev]

def Load_6hrSfc(model, var, DTime):
    modelname, expr, ens = model.split(".")
    vname = dvar[var]

    # Search file
    srcdir  = dbasedir[var] + "/%s"%(model)
    ssearch = srcdir + "/%s_6hrPlev*.nc"%(vname)
    lsrcpath = np.sort(glob(ssearch))
    for srcpath in lsrcpath:
        stime = os.path.basename(srcpath).split("_")[6].split(".")[0]
        stime0, stime1 = stime.split("-")

        dtime0 = datetime.strptime(stime0, "%Y%m%d%H%M")
        dtime1 = datetime.strptime(stime1, "%Y%m%d%H%M")
        if (dtime0<=DTime)&(DTime<=dtime1):
            break
    nc = netCDF4.Dataset(srcpath)
    #print(nc.variables)
    #print(srcpath)

    # Find time index
    basetime = {
        ("MIROC6","piControl"):         datetime(3200,1,1),
        ("MRI-ESM2-0","piControl"):     datetime(1850,1,1),
        ("MRI-ESM2-0","historical"):    datetime(1850,1,1),
        }[modelname,expr]

    dtime0 = basetime + timedelta(days=float(nc.variables["time"][0]))
    idxtime = int((DTime - dtime0).total_seconds()/21600)    # 6-hour = 21600 sec

    return nc.variables[vname][idxtime]
    #return nc.variables[vname].shape

def Load_monSfc(model, var, Year, Mon):
    modelname, expr, ens = model.split(".")
    vname = dvar[var]
    DTime = datetime(Year,Mon,1)
    # Search file
    srcdir  = dbasedir[var] + "/%s"%(model)
    ssearch = srcdir + "/%s_Amon*.nc"%(vname)
    lsrcpath = np.sort(glob(ssearch))
    for srcpath in lsrcpath:
        stime = os.path.basename(srcpath).split("_")[6].split(".")[0]
        stime0, stime1 = stime.split("-")

        dtime0 = datetime.strptime(stime0, "%Y%m")
        dtime1 = datetime.strptime(stime1, "%Y%m")
        if (dtime0<=DTime)&(DTime<=dtime1):
            break
    nc = netCDF4.Dataset(srcpath)
    #print(nc.variables)
    #print(srcpath)

    #print(nc.variables["time"][:])
    #print(len(nc.variables["time"][:]))

    # Find time index
    Year0,Mon0 = dtime0.timetuple()[:2]
    Year1,Mon1 = dtime1.timetuple()[:2]
    idxtime = int(Year-Year0)*12 -Mon0 + Mon
    #print(idxtime)
    return nc.variables[vname][idxtime]

def Load_const(model, var):
    vname = dvar[var]
    srcdir  = dbasedir[var] + "/%s"%(model)
    ssearch = srcdir + "/%s_*.nc"%(vname)
    lsrcpath= glob(ssearch)
    srcpath = lsrcpath[0]
    nc = netCDF4.Dataset(srcpath)
    #print(nc.variables)
    return nc.variables[vname][:]


import boto3
import sys
import time
# input value 'ansible-controller' while running the instance
#import json
ec2_client = boto3.client('ec2', region_name = "us-east-1")
instances = ec2_client.describe_instances()
for reservation in instances['Reservations']:
    for instance in reservation["Instances"]:
        if instance["Tags"][0]["Value"] == sys.argv[1]:
            response = ec2_client.start_instances(InstanceIds=[instance["InstanceId"]])
            istance_id = instance["InstanceId"]

while True:
    print('checking the status........')
    time.sleep(3)
    status_value = ec2_client.describe_instance_status(InstanceIds=[istance_id])
    if len(status_value['InstanceStatuses']) != 0:
        if status_value['InstanceStatuses'][0]['InstanceState']['Name'] == 'running':
            print(f"{sys.argv[1]} EC2 Instance got:- {status_value['InstanceStatuses'][0]['InstanceState']['Name']}")
            break

print ("Press Enter to continue ..." )
input()


import numpy as np
from sklearn.datasets import make_classification
import pytest
from pygbm.binning import BinMapper
from pygbm.grower import TreeGrower
from pygbm import GradientBoostingRegressor
from pygbm import GradientBoostingClassifier

X, y = make_classification(n_samples=150, n_classes=2, n_features=5,
                           n_informative=3, n_redundant=0,
                           random_state=0)

def test_plot_grower(tmpdir):
    pytest.importorskip('graphviz')
    from pygbm.plotting import plot_tree

    X_binned = BinMapper().fit_transform(X)
    gradients = np.asarray(y, dtype=np.float32).copy()
    hessians = np.ones(1, dtype=np.float32)
    grower = TreeGrower(X_binned, gradients, hessians, max_leaf_nodes=5)
    grower.grow()
    filename = tmpdir.join('plot_grower.pdf')
    plot_tree(grower, view=False, filename=filename)
    assert filename.exists()

def test_plot_estimator(tmpdir):
    pytest.importorskip('graphviz')
    from pygbm.plotting import plot_tree

    n_trees = 3
    est = GradientBoostingRegressor(max_iter=n_trees)
    est.fit(X, y)
    for i in range(n_trees):
        filename = tmpdir.join('plot_predictor.pdf')
        plot_tree(est, tree_index=i, view=False, filename=filename)
        assert filename.exists()

def test_plot_estimator_and_lightgbm(tmpdir):
    pytest.importorskip('graphviz')
    lightgbm = pytest.importorskip('lightgbm')
    from pygbm.plotting import plot_tree

    n_classes = 3
    X, y = make_classification(n_samples=150, n_classes=n_classes,
                               n_features=5, n_informative=3, n_redundant=0,
                               random_state=0)

    n_trees = 3
    est_pygbm = GradientBoostingClassifier(max_iter=n_trees,
                                           n_iter_no_change=None)
    est_pygbm.fit(X, y)
    est_lightgbm = lightgbm.LGBMClassifier(n_estimators=n_trees)
    est_lightgbm.fit(X, y)

    n_total_trees = n_trees * n_classes
    for i in range(n_total_trees):
        filename = tmpdir.join('plot_mixed_predictors.pdf')
        plot_tree(est_pygbm, est_lightgbm=est_lightgbm, tree_index=i,
                  view=False, filename=filename)
        assert filename.exists()


__doc__ = """
=====================
Plugs Introduction
=====================

:Author: Limodou <limodou@gmail.com>

.. contents::

About Plugs
----------------

Plugs is an apps collection project for uliweb. So you can use any app of it
to compose your project.

License
------------

Plugs is released under BSD license. Of cause if there are some third party
apps not written by me(limodou), it'll under the license of itself.

"""

from uliweb.utils.setup import setup
import plugs

setup(name='plugs',
    version=plugs.__version__,
    description="Apps collection for uliweb",
    long_description=__doc__,
    classifiers=[
        "Development Status :: 2 - Pre-Alpha",
        "Intended Audience :: Developers",
        "License :: OSI Approved :: BSD License",
        "Topic :: Internet :: WWW/HTTP :: WSGI",
        "Programming Language :: Python",
        "Operating System :: OS Independent",
        "Programming Language :: Python :: 2.6",
        "Programming Language :: Python :: 2.7",
    ],
    packages = ['plugs'],
    platforms = 'any',
    keywords='wsgi web framework',
    author=plugs.__author__,
    author_email=plugs.__author_email__,
    url=plugs.__url__,
    license=plugs.__license__,
    include_package_data=True,
    zip_safe=False,
    entry_points = {
        'uliweb_apps': [
          'helpers = plugs',
        ],
    },
)


from unittest import mock

from django.test import TestCase
from django.urls import resolve, reverse
from nltk import word_tokenize

from .models import ScrapeHistory
from .views import WordCountView

class ScrapeTest(TestCase):

    def _mock_response(
            self,
            status=200,
            content="CONTENT",
            json_data=None,
            raise_for_status=None):

        mock_resp = mock.Mock()

        # mock raise_for_status call w/optional error
        mock_resp.raise_for_status = mock.Mock()

        if raise_for_status:
            mock_resp.raise_for_status.side_effect = raise_for_status
        # set status code and content
        mock_resp.status_code = status
        mock_resp.content = content

        # add json data if provided
        if json_data:
            mock_resp.json = mock.Mock(
                return_value=json_data
            )
        return mock_resp

    def setUp(self):
        self.url = reverse("wordcount")
        self.post_params = {
            "word": "fit",
            "url": "https://www.virtusize.jp/"
        }

    def test_wordcount_url_resolves(self):
        self.assertEqual(resolve(self.url).func.view_class, WordCountView)

    def test_missing_param(self):
        params = {
            "word": "ABC"
        }

        response = self.client.post(
            self.url, params, content_type='application/json')

        self.assertEqual(response.status_code, 400)

    def test_invalid_link_value(self):
        params = {
            "word": "WORd",
            "url": "https://virtuse"
        }

        response = self.client.post(
            self.url, params, content_type='application/json')

        self.assertEqual(response.status_code, 500)

    @mock.patch("scrape.views.requests.get")
    def test_wordcount_request_to_url(self, m):
        mock_resp = self._mock_response(
            content="this is fit")

        m.return_value = mock_resp

        response = WordCountView.scrap_url_and_word_count(self,
                                                          "fit", "https://www.virtusize.jp/")

        self.assertEqual(response, 1)

    @mock.patch("scrape.views.WordCountView.scrap_url_and_word_count", return_value=1)
    def test_wordcount_success_result(self, m):
        response = self.client.post(
            self.url, self.post_params, content_type='application/json')

        result = response.json()

        test_content = {
            "status": "ok",
            "count": 1
        }

        self.assertEqual(result, test_content)

    @mock.patch("scrape.views.WordCountView.scrap_url_and_word_count", return_value=1)
    def test_save_to_db(self, m):
        _ = ScrapeHistory.objects.create(
            url="https://www.virtusize.jp/",
            word="fit",
            word_count=1
        )

        _ = self.client.post(
            self.url, self.post_params, content_type='application/json')

        self.assertEqual(ScrapeHistory.objects.last().word_count, 1)

        self.assertNotEqual(ScrapeHistory.objects.last().word_count, 13)

    def test_word_tokenize(self):
        txt = "Virtusize works for innovative idea. <'new idea'> idea-ly Although there are lot of new ideas but it focuses e-commerce"

        words = word_tokenize(txt)

        self.assertEqual(words.count("idea"), 2)
        self.assertNotEqual(words.count("idea"), 1)


import sys
from PyQt5.QtWidgets import *

class Main(QDialog):
    def __init__(self):
        super().__init__()
        self.initUI()

    def initUI(self):
        main_layout=QVBoxLayout()

        #버튼 만들기
        btn = QPushButton("Click me")
        main_layout.addWidget(btn)

        self.setLayout(main_layout)
        self.resize(500,500)
        self.show()

if __name__ == '__main__':
    app = QApplication(sys.argv)
    main = Main()
    sys.exit(app.exec_())


import time

def test1():
    now = 1543786615000
    time_array = time.localtime(now / 1000)
    other_style_time = time.strftime("%Y--%m--%d %H:%M:%S", time_array)
    print(other_style_time)

def test2():
    k = None
    if isinstance(k, dict):
        print(111)
    else:
        print(222)

def test3():
    condition = False
    # if not condition:
    #     raise AssertionError()
    # assert condition
    lists = ["", "", ""]
    assert len(lists) >= 3, '列表元素个数小于3'

if __name__ == '__main__':
    test3()


class Persona():

    def __init__(self, nombre, edad, lugar):

        self.nombre = nombre
        self.edad = edad
        self.lugar = lugar

    def descripcion(self):

        print("Nombre: ", self.nombre, " Edad: ", self.edad, " Lugar: ", self.lugar)

class Empleado(Persona):

    def __init__(self, nombre_empleado, edad_empleado, lugar_empleado, sueldo, antiguedad):

        super().__init__(nombre_empleado, edad_empleado, lugar_empleado)

        self.sueldo = sueldo
        self.antiguedad = antiguedad

    def descripcion(self):

        super().descripcion()
        print("Sueldo: ", self.sueldo, "Antigüedad: ", self.antiguedad)

Antonio = Persona('Antonio', '54', 'Canarias')
Antonio.descripcion()

Miguel = Empleado('Miguel', '37', 'Valencia', '35.000', '12')
Miguel.descripcion()

print(isinstance(Miguel, Empleado))