#!/usr/bin/env python

import argparse
import pathlib
import random

import numpy as np
import torch
import torch.nn as nn
import torch.utils.data
import torch.distributed as dist
from tensorboardX import SummaryWriter

import yolov3.models
from yolov3.config import get_default_config
from yolov3.utils.checkpoint import CheckPointer
from yolov3.utils.data.dataloader import create_dataloader
from yolov3.utils.logger import create_logger
from yolov3.utils.scheduler import create_scheduler

def load_config():
    parser = argparse.ArgumentParser()
    parser.add_argument('--config', type=str)
    parser.add_argument('--resume', type=str, default='')
    parser.add_argument('--local_rank', type=int, default=0)
    parser.add_argument('options', default=None, nargs=argparse.REMAINDER)
    args = parser.parse_args()

    config = get_default_config()
    if args.config is not None:
        config.merge_from_file(args.config)
    config.merge_from_list(args.options)
    if not torch.cuda.is_available():
        config.train.device = 'cpu'
        config.train.dataloader.pin_memory = False
    if args.resume != '':
        config_path = pathlib.Path(args.resume) / 'config.yaml'
        config.merge_from_file(config_path.as_posix())
        config.merge_from_list(['train.resume', True])
    config.merge_from_list(['train.dist.local_rank', args.local_rank])
    config.freeze()
    return config

def set_seed(seed):
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)

def create_model(config):
    local_rank = config.train.dist.local_rank
    device = torch.device(config.train.device, local_rank)

    model = yolov3.models.YOLOv3(config)
    model.to(device)
    if dist.is_available() and dist.is_initialized():
        model = nn.parallel.DistributedDataParallel(
            model, device_ids=[local_rank], output_device=local_rank)
    else:
        if config.train.device == 'cuda':
            model = nn.DataParallel(model)
    return model

def create_optimizer(model, config):
    param_list = []
    for name, params in model.named_parameters():
        if 'conv.weight' in name:
            param_list.append({
                'params': params,
                'weight_decay': config.train.weight_decay,
            })
        else:
            param_list.append({
                'params': params,
                'weight_decay': 0,
            })
    optimizer = torch.optim.SGD(
        param_list,
        lr=config.train.base_lr,
        momentum=config.train.momentum,
        nesterov=config.train.nesterov)
    return optimizer

def get_rank():
    if not (dist.is_available() and dist.is_initialized()):
        return 0
    else:
        return dist.get_rank()

def save_config(config, outdir):
    with open(outdir / 'config.yaml', 'w') as fout:
        fout.write(str(config))

def main():
    config = load_config()

    torch.backends.cudnn.benchmark = (config.train.augmentation.min_size ==
                                      config.train.augmentation.max_size)

    set_seed(config.train.seed)

    if config.train.distributed:
        dist.init_process_group(
            backend=config.train.dist.backend,
            init_method=config.train.dist.init_method,
            rank=config.train.dist.node_rank,
            world_size=config.train.dist.world_size)

    outdir = pathlib.Path(config.train.outdir)
    if get_rank() == 0:
        if not config.train.resume and outdir.exists():
            raise RuntimeError(
                f'Output directory `{outdir.as_posix()}` already exists')
        outdir.mkdir(exist_ok=True, parents=True)
        if not config.train.resume:
            save_config(config, outdir)

    logger = create_logger(
        name=__name__,
        outdir=outdir,
        distributed_rank=get_rank(),
        filename='log.txt')
    logger.info(config)

    model = create_model(config)
    optimizer = create_optimizer(model, config)
    scheduler = create_scheduler(optimizer, config)
    checkpointer = CheckPointer(
        model,
        optimizer=optimizer,
        scheduler=scheduler,
        checkpoint_dir=outdir,
        logger=logger,
        distributed_rank=get_rank())

    start_iter = config.train.start_iter
    scheduler.last_epoch = start_iter
    if config.train.resume:
        ckpt_config, start_iter = checkpointer.load()
        config.defrost()
        config.merge_from_other_cfg(ckpt_config)
        config.train.start_iter = start_iter
        config.freeze()
    elif config.train.ckpt_path != '':
        _, start_iter = checkpointer.load(
            config.train.ckpt_path, backbone=False)
        config.defrost()
        config.train.start_iter = start_iter
        config.freeze()
        for index in range(len(scheduler.base_lrs)):
            scheduler.base_lrs[index] = config.train.base_lr
        save_config(config, outdir)
    elif config.train.backbone_weight != '':
        checkpointer.load(config.train.backbone_weight, backbone=True)

    if get_rank() == 0 and config.train.tensorboard:
        if start_iter > 0:
            writer = SummaryWriter(
                outdir.as_posix(), purge_step=start_iter + 1)
        else:
            writer = SummaryWriter(outdir.as_posix())
    else:
        writer = None

    train_loader = create_dataloader(config, is_train=True)

    for step, (data, targets) in enumerate(train_loader, start_iter):
        step += 1

        scheduler.step()
        data_chunks = data.chunk(config.train.subdivision)
        target_chunks = targets.chunk(config.train.subdivision)

        optimizer.zero_grad()
        for data_chunk, target_chunk in zip(data_chunks, target_chunks):
            losses = model(data_chunk, target_chunk)
            losses[0].backward()
        if config.train.gradient_clip > 0:
            torch.nn.utils.clip_grad_norm_(model.parameters(),
                                           config.train.gradient_clip)
        for param in model.parameters():
            if param.grad is not None:
                param.grad.data.div_(data.size(0))
        optimizer.step()

        losses *= config.train.subdivision / config.train.batch_size
        if step == 1 or step % config.train.log_period == 0:
            logger.info(f'iter {step}/'
                        f'{config.train.scheduler.max_iterations}, '
                        f'lr {scheduler.get_lr()[0]:.7f}, '
                        f'image size: {data.shape[2]}, '
                        f'losses: [total {losses[0].item():.3f}, '
                        f'xy {losses[1].item():.3f}, '
                        f'wh {losses[2].item():.3f}, '
                        f'object {losses[3].item():.3f}, '
                        f'class {losses[4].item():.3f}]')

        if writer is not None:
            writer.add_scalar('train/loss', losses[0], step)
            writer.add_scalar('train/loss_xy', losses[1], step)
            writer.add_scalar('train/loss_wh', losses[2], step)
            writer.add_scalar('train/loss_object', losses[3], step)
            writer.add_scalar('train/loss_class', losses[4], step)
            writer.add_scalar('train/lr', scheduler.get_lr()[0], step)
            writer.add_scalar('train/image_size', data.shape[2], step)

        if (step % config.train.ckpt_period == 0) or (
                step == config.train.scheduler.max_iterations):
            ckpt_config = {'iteration': step, 'config': config}
            checkpointer.save(f'model_{step:07d}', **ckpt_config)

    if writer is not None:
        writer.close()

if __name__ == '__main__':
    main()


#! /usr/bin/env python

# This script converts the fits files from the NIRCam CRYO runs
# into ssb-conform fits files.

# Before running it, make sure to set environment variables:
#
# export UAZCONVDIR='/grp/jwst/wit/nircam/nircam-tools/pythonmodules/'
# export JWSTTOOLS_PYTHONMODULES='$JWSTTOOLS_ROOTDIR/pythonmodules'
# export JWSTTOOLS_ROOTDIR='/grp/jwst/wit/nircam/nircam-tools'
# export JWSTTOOLS_INSTRUMENT='NIRCAM'

import sys, os,re,math
import optparse,scipy
from astropy.io import fits as pyfits
from jwst import datamodels as models
from astropy.io import ascii
import numpy as np
from nircam2ssb import nircam2ssbclass
sys.path.append(os.environ['UAZCONVDIR'])

# Bryan's subarray list for the simulator
#subarrays = ascii.read("NIRCam_subarray_definitions.list")

class sci2ssbclass(nircam2ssbclass):
    def __init__(self):
        nircam2ssbclass.__init__(self)

    def image2ssb(self,inputfilename, outfilebasename='auto',outdir=None,outsuffix=None,outsubdir=None):
        outfilebasename = self.mkoutfilebasename(inputfilename, outfilebasename=outfilebasename,outdir=outdir,outsuffix=outsuffix,outsubdir=outsubdir)
        (self.data,self.hdr)=pyfits.getdata(inputfilename, 0, header=True)
        print('input shape:',self.data.shape)

        self.runID = self.getRunID(filename=inputfilename)
        self.hdr['SUBARRAY'] = 'FULL'

        # How many groups and integrations?
        if self.runID=='TUCSONNEW':
            Nint=1
        else:
            Nint = int(self.hdr['NINT'])
        Ngroup =int(self.hdr['NGROUP'])
        Nframe = int(self.hdr['NFRAME'])
        print('NGROUP:',Ngroup)
        print('NINT:',Nint)
        if (Ngroup*Nint)!=self.data.shape[0]:
            raise RuntimeError('something is wrong! NGROUP=%d, NINT=%d, sum is not shape[0]=%d' % (Ngroup,Nint,self.data.shape[0]))

        # rearrange the data: 4th coordinate is integration
        scinew=scipy.zeros((Nint,Ngroup,self.data.shape[1],self.data.shape[2]), dtype=float)
        for i in range(Nint):
            scinew[i,:,:,:]=self.data[i*Ngroup:(i+1)*Ngroup,:,:]
        print('output shape:',scinew.shape)

        self.outputmodel = models.RampModel(data=scinew)

        #Mask dead pixels
        #mask = scipy.where(np.isnan(scinew))
        # mask = np.where(np.isnan(a))
        #mask = np.isnan(scinew)
        # a=scinew[mask]
        #print mask
        #sys.exit(0)

        #updates the date string
        self.updatemetadata(inputfilename,reffileflag=False)
        print('meta data updated')
        #update detector
        self.cryo_update_meta_detector(reffileflag=False)
        print('cryo meta data updated')
        #flip the data around to place it in the science orientation expected by SSB
        if self.runID in ['CV2','CV3', 'OTIS']:
            self.native_to_science_image_flip()

        if self.runID in ['CV3']:
            self.outputmodel.meta.exposure.readpatt = self.hdr['READOUT']

        if self.runID in ['OTIS']:
            self.outputmodel.meta.exposure.readpatt = self.hdr['READOUT']

        self.outputmodel.meta.exposure.nints = Nint
        self.outputmodel.meta.exposure.nframes = Nframe
        #self.outputmodel.meta.exposure.ngroups = self.hdr['NAXIS3']
        self.outputmodel.meta.exposure.ngroups = Ngroup

        # put the version of nircam2ssb into header
        # self.outputmodel.meta.channel = self.version

        outfilename = outfilebasename
        if not re.search('fits$',outfilename): outfilename += '_uncal.fits'

        print('Saving %s' % outfilename)
        self.outputmodel.save(outfilename)

        return(outfilename)

    def native_to_science_image_flip(self):
        #flip the data to match the orientation expected by SSB
        data = self.outputmodel.data
        print('trying to flip the image')

        # NIRCAM A2, A4, B1, B3, BLONG
        # Flip vertically
        if self.hdr['DETECTOR'] in ['NRCA2','NRCA4','NRCB1','NRCB3','NRCBLONG']:
            flip = data[:,:,::-1]
            self.outputmodel.data = flip
            self.outputmodel.meta.subarray.fastaxis = 1
            self.outputmodel.meta.subarray.slowaxis = -2
            try:
                detector_row_start = self.hdr['ROWCORNR']
            except KeyError:
                print('Unable to get subarray ROWCORNR, using 1')
                detector_row_start = '0.000000'
                try:
                    detector_row_start = int(float(detector_row_start)) + 1
                except ValueError:
                    print('Unable to convert ROWCORNR to a valid integer, using 1')
                    detector_row_start = 1
            try:
                detector_column_start = self.hdr['COLCORNR']
            except KeyError:
                print('Unable to get subarray COLCORNR, using 1')
                detector_column_start = '0.00000'
                try:
                    detector_column_start = int(float(detector_column_start)) + 1
                except ValueError:
                    print('Unable to convert COLCORNR to a valid integer, using 1')
                    detector_column_start = 1

            ncols = self.hdr['NAXIS1']
            nrows = self.hdr['NAXIS2']
            #
            # Since we're flipping these data in the Y-direction only, COLCORNR and COLSTOP
            # will be unchanged.
            # ROWCORNR and ROWSTOP will swap and subtract from 2049
            # FASTAXIS is 1, as the detector is still read from left to right
            colstart = int(detector_column_start)
            colstop = int(colstart + ncols - 1)
            rowstop = int(2049 - detector_row_start)
            rowstart = int(rowstop - nrows + 1)
            fastaxis = 1
            slowaxis = -2

        # NIRCAM A1, A3, ALONG, B2, B4
        # Flip horizontally
        elif self.hdr['DETECTOR'] in ['NRCA1','NRCA3','NRCB2','NRCB4','NRCALONG']:
            flip = data[:,:,:,::-1]
            self.outputmodel.data = flip
            self.outputmodel.meta.subarray.fastaxis = -1
            self.outputmodel.meta.subarray.slowaxis = 2

            try:
                detector_row_start = self.hdr['ROWCORNR']
            except KeyError:
                print('Unable to get subarray ROWCORNR, using 1')
                detector_row_start = '0.000000'
                try:
                    detector_row_start = int(float(detector_row_start)) + 1
                except ValueError:
                    print('Unable to translate ROWCORNR to a valid integer, using 1')
                    detector_row_start = 1
            try:
                detector_column_start = self.hdr['COLCORNR']
            except KeyError:
                print('Unable to get subarray COLCORNR, using 1')
                detector_column_start = '0.00000'
                try:
                    detector_column_start = int(float(detector_column_start)) + 1
                except ValueError:
                    print('Unable to translate COLCORNR to a valid integer, using 1')
                    detector_column_start = 1

            ncols = self.hdr['NAXIS1']
            nrows = self.hdr['NAXIS2']
            #
            # Since we're flipping these data in the X-direction only, ROWCORNR and ROWSTOP
            # will be unchanged.
            # COLCORNR and COLSTOP will swap and subtract from 2049
            # FASTAXIS is -1, as the detector is now read from right to left
            rowstart = int(detector_row_start)
            rowstop = int(rowstart + nrows - 1)
            colstop = int(2049 - detector_column_start)
            colstart = int(colstop - ncols + 1)
            fastaxis = -1
            slowaxis = 2
        else:
            print("WARNING! I don't recognize {} as a valid detector!".format(self.detector))
            sys.exit(0)

        self.outputmodel.meta.subarray.xstart = rowstart
        self.outputmodel.meta.subarray.xsize = ncols
        self.outputmodel.meta.subarray.ystart = colstart
        self.outputmodel.meta.subarray.ysize = nrows
        self.outputmodel.meta.subarray.fastaxis = fastaxis
        self.outputmodel.meta.subarray.slowaxis = slowaxis

        print('trying to get the subarray name')

        #  Update the subarray parameters
        #subarray_name = self.get_subarray_name(subarrays, colstart-1, colstop-1, rowstart-1, rowstop-1)
        subarray_name = 'FULL'
        self.outputmodel.meta.subarray.name = subarray_name
        print('subarray name is: '+subarray_name)

if __name__=='__main__':

    usagestring='USAGE: sci3ssb.py infile1 infile2 ...'

    sci2ssb=sci2ssbclass()
    parser = sci2ssb.add_options(usage=usagestring)
    options,  args = parser.parse_args()

    if len(args)<1:
        parser.parse_args(['--help'])
        sys.exit(0)

    sci2ssb.verbose=options.verbose

    for infile in args:
        sci2ssb.image2ssb(infile,outfilebasename=options.outfilebasename,outdir=options.outdir,outsuffix=options.outsuffix,outsubdir=options.outsubdir)


#!/usr/bin/env python
# -*- coding: UTF-8 -*-
__author__ = "noxoomo"
__email__ = "noxoomo@yandex-team.ru"

import json
import subprocess
import os
import sys
from subprocess import Popen
import subprocess
import argparse
import os
import os.path
import pandas as pd
import numpy as np

lightgbm_path = "./lightgbm"

fit_template = " sparse_threshold=1.0 num_threads=16 device=gpu early_stopping_rounds=300"

if __name__ == '__main__':

    parser = argparse.ArgumentParser()

    parser.add_argument('--base-step',
                        type=float,
                        required=True)
    parser.add_argument('--base-iter',
                        type=float,
                        required=True)
    parser.add_argument('--mult',
                        type=float,
                        required=True)
    parser.add_argument('--count',
                        type=int,
                        required=True)
    parser.add_argument('--device',
                        required=True)
    parser.add_argument('--target',
                        required=True)
    parser.add_argument('--test-metric',
                        required=True)
    parser.add_argument('--pool-dir',
                        required=True)
    parser.add_argument('--read_pool_args')
    parser.add_argument('--other-args')

    parser.add_argument('--bins',type=int,default=15)
    parser.add_argument('--leaves',type=int,default=64)

    parser.add_argument('--skip-up-tasks', action='store_true')
    parser.add_argument('--skip-down-tasks', action='store_true')

    args = parser.parse_args(sys.argv[1:])
    pool_dir = args.pool_dir

    def get_lightgbm_times(log_path):
        elapsed_times = []
        with open(log_path, 'r') as input:
            for line in input.readlines():
                if 'seconds elapsed, finished' in line:
                    elapsed_times.append(float(line.split('seconds')[0].strip().split(' ')[-1]) * 1000)
        return np.array(elapsed_times)

    def estimate_fit_time(log_path, iter):
        times = get_lightgbm_times(log_path)
        return times[iter]

    def get_metric_from_lightgbm(log_path, metric):
        scores = []
        metric_name = "l2" if metric == "mse" else metric
        with open(log_path, 'r') as input:
            for line in input.readlines():
                if "Info" in line and 'valid_1' in line and metric_name in line:
                    score=  float(line.split("valid_1 "+ metric_name+' :')[-1])
                    scores.append(score)
        scores = np.array(scores)
        if metric == "mse":
            return scores ** 0.5 #we need rmse, lightgbm computes l2
        else:
            return scores

    def get_features_abs_path():
        return os.path.join(pool_dir, "features.train")

    def get_test_abs_path():
        return os.path.join(pool_dir, "features.test")

    def set_pool_path(cmd, metric=None):
        result = cmd + " data={__features}".format(__features=get_features_abs_path())
        if metric is not None:
            result = result + " valid={__test} metric={__metric}".format(__test=get_test_abs_path(), __metric=metric)
        if args.read_pool_args:
            result = result + " " + args.read_pool_args.sp
        return result

    def get_algo_name():
        return "lightgbm_{__bins}_{__leaves}".format(__bins=args.bins, __leaves=args.leaves)

    def set_tree_params(cmd, leaves, bins):
        return cmd + " max_bin={__bins} num_leaves={__leaves}".format(__bins=bins, __leaves=leaves)

    def set_params(cmd, step, iters, target, leaves, bins):
        result = cmd + " learning_rate={__step} num_iterations={__iters} objective={__target}".format(__iters=iters, __step=step, __target=target)
        if args.other_args is not None:
            result = result + " " + args.other_args

        return set_tree_params(result, leaves, bins)

    def is_max_optimal(metric):
        return "mse" not in metric

    def get_best_score(score_values, metric):
        if is_max_optimal(metric):
            best = max(score_values)
        else:
            best = min(score_values)
        return best

    def get_best_iter(scores, metric):
        if is_max_optimal(metric):
            return np.argmax(scores)
        else:
            return np.argmin(scores)

    def get_best_iter_from_log(dir, metric, suffix):
        scores = get_metric_from_lightgbm(os.path.join(dir, "lightgbm_stdout" + suffix), metric)
        return get_best_iter(scores, metric)

    def get_best_score_from_log(dir, metric, suffix):
        scores = get_metric_from_lightgbm(os.path.join(dir, "lightgbm_stdout" + suffix), metric)
        return get_best_score(scores, metric)

    def get_fit_time_from_log(dir, iter, suffix):
        return estimate_fit_time(os.path.join(dir, "lightgbm_stdout" + suffix), iter)

    def update_plot_log(step,  iter, best_score, best_iter, time, output):
        with open(output, 'a+') as outfile:
            outfile.write("{__iter}\t{__score}\t{__step}\t{__time}\t{__total}\t{__name}\n".format(__iter=best_iter,
                                                                                __score=best_score,
                                                                                __step=step,
                                                                                __time=time,
                                                                                __name=get_algo_name(),
                                                                                __total=iter))

    def run_on_device(args, device_id, task_dir, log_suffix):
        ensure_dir_exists(task_dir)
        cmd = args.split(' ')
        print("Running command: {}\n".format(" ".join(cmd)))
        env = os.environ.copy()
        env['CUDA_VISIBLE_DEVICES'] = str(device_id)
        full_path = os.path.join(os.getcwd(), task_dir)

        stderr = open(os.path.join(task_dir, "lightgbm_stderr" + log_suffix), "w")
        stdout = open(os.path.join(task_dir, "lightgbm_stdout" + log_suffix), "w")
        return subprocess.call(cmd, stdout=stdout, stderr=stderr, env=env, cwd=full_path)

    def run_on_device_relative(args, device_id, task_dir, log_suffix):
        return run_on_device(args, device_id, os.path.join(os.getcwd(), task_dir), log_suffix)

    def ensure_dir_exists(dir):
        if not os.path.exists(dir):
            os.mkdir(dir)

    def get_steps(str):
        return str.split(",")

    def get_dir_for_step(step):
        return "step_{__step}".format(__step=step)

    def generate_tasks(base_step, base_iter, mult, count, add_up, add_down):
        tasks = []
        for i in range(0, count):
            if add_up:
                tasks.append({"step": base_step * (mult ** i), "iter": int(base_iter / (mult ** i))})
            if add_down:
                tasks.append({"step": base_step / (mult ** i), "iter": int(base_iter * (mult ** i))})
        return tasks

    add_up = not args.skip_up_tasks
    add_down = not args.skip_down_tasks
    tasks = generate_tasks(args.base_step, args.base_iter, args.mult, args.count, add_up, add_down)
    #fit
    for task in tasks:
        step_ = task["step"]
        iter_ = task["iter"]
        task_dir = get_dir_for_step(step_)
        base_cmd = lightgbm_path + fit_template
        if args.other_args:
            base_cmd = base_cmd + " " + args.other_args
        run_with_quality_calc = set_pool_path(base_cmd, args.test_metric)
        run_with_quality_calc = set_params(run_with_quality_calc, step_, iter_, args.target, args.leaves, args.bins)
        quality_log_suffix = ".quality.log"
        speed_log_suffix = ".speed.log"
        run_on_device_relative(run_with_quality_calc, args.device, task_dir, quality_log_suffix)
        best_score = get_best_score_from_log(task_dir, args.test_metric, quality_log_suffix)
        best_iter = int(get_best_iter_from_log(task_dir, args.test_metric, quality_log_suffix))
        run_for_speed = set_pool_path(base_cmd)
        run_for_speed = set_params(run_for_speed, step_, best_iter + 1, args.target, args.leaves, args.bins)
        run_on_device_relative(run_for_speed, args.device, task_dir, speed_log_suffix)
        best_time = get_fit_time_from_log(task_dir, best_iter, speed_log_suffix)
        update_plot_log(step_, iter_, best_score, best_iter, best_time, "scores_plot.tsv")


import socket
import threading
import numpy as np
import webrtcvad
import sys
import logging
import wave

from collections import deque
from sonosco.inputs.audio import SonoscoAudioInput

logging.basicConfig(level=logging.INFO)

class MicrophoneClient(SonoscoAudioInput):
    """
    Connects to mic_server.py, aggregates chunked audio in to wave format and returns it to the caller in
    request_audio method.
    """

    def __init__(self, port=10002, host='192.168.1.138', sample_rate=16000, chunk_size=1024):
        # self.format = pyaudio.paInt16

        self.SAMPLE_WIDTH = 2  # pyaudio.get_sample_size(self.format)  # size of each sample
        self.SAMPLE_RATE = sample_rate  # sampling rate in Hertz
        self.CHUNK = chunk_size
        self.CHANNELS = 1
        self.record = False

        self.RATE = 16000
        self.CHUNK_DURATION_MS = 30  # supports 10, 20 and 30 (ms)
        self.PADDING_DURATION_MS = 1000
        self.CHUNK_SIZE = int(self.RATE * self.CHUNK_DURATION_MS / 1000)
        self.CHUNK_BYTES = self.CHUNK_SIZE * 2
        self.NUM_PADDING_CHUNKS = int(self.PADDING_DURATION_MS / self.CHUNK_DURATION_MS)
        self.NUM_WINDOW_CHUNKS = int(240 / self.CHUNK_DURATION_MS)

        self.stream = MicrophoneClient.BytesLoop()
        self.vad = webrtcvad.Vad(2)

        self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.s.connect((host, port))
        logging.info("Microphone client connected")

        d = threading.Thread(target=self.write_to_streams)
        self.lock = threading.RLock()
        d.setDaemon(True)
        d.start()

    def write_to_streams(self):
        logging.info("Started mic client deamon")
        while True:
            data = self.s.recv(self.CHUNK_BYTES)
            if self.record:
                data = np.frombuffer(data, dtype=np.int16)
                self.stream.write(data.tobytes())

    def request_audio(self, *args, **kwargs):
        try:
            got_a_sentence = False

            ring_buffer = deque(maxlen=self.NUM_PADDING_CHUNKS)
            triggered = False
            voiced_frames = []
            ring_buffer_flags = [0] * self.NUM_WINDOW_CHUNKS

            ring_buffer_index = 0
            buffer_in = ''

            logging.info("* recording")

            while not got_a_sentence:  # and not leave:
                chunk = self.stream.read(self.CHUNK_BYTES)
                active = self.vad.is_speech(chunk, self.RATE)
                # logging.info('1' if active else '0')
                ring_buffer_flags[ring_buffer_index] = 1 if active else 0
                ring_buffer_index += 1
                ring_buffer_index %= self.NUM_WINDOW_CHUNKS
                if not triggered:
                    ring_buffer.append(chunk)
                    num_voiced = sum(ring_buffer_flags)
                    if num_voiced > 0.5 * self.NUM_WINDOW_CHUNKS:
                        logging.info('+')
                        triggered = True
                        voiced_frames.extend(ring_buffer)
                        ring_buffer.clear()
                else:
                    voiced_frames.append(chunk)
                    ring_buffer.append(chunk)
                    num_unvoiced = self.NUM_WINDOW_CHUNKS - sum(ring_buffer_flags)
                    if num_unvoiced > 0.9 * self.NUM_WINDOW_CHUNKS:
                        sys.stdout.write('-')
                        triggered = False
                        got_a_sentence = True

            data = b''.join(voiced_frames)
            wf = wave.open("audio2.wave", "wb")

            wf.setframerate(16000)
            wf.setsampwidth(2)
            wf.setnchannels(1)
            wf.writeframes(data)
            with open("audio2.wave", "rb") as data_wav:
                logging.info("* done recording")
                return data_wav.read()

        except Exception as e:
            logging.exception(f"Mic client exception {e}")
            raise e

    def __enter__(self):
        # assert self.stream is None, "This audio source is already inside a context manager"
        self.record = True
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        # self.stream = None
        self.record = False

    class BytesLoop:
        def __init__(self, s=b''):
            self.buffer = s
            self.lock = threading.RLock()

        def read(self, n=-1):
            # print("read %i"%n)
            while len(self.buffer) < n:
                pass
            # print("got enough data")
            # self.lock.acquire()

            chunk = self.buffer[:n]
            self.buffer = self.buffer[n:]
            # self.lock.release()
            return chunk

        def write(self, s):
            # print("write %i"%len(s))
            # self.lock.acquire()
            self.buffer += s
            # self.lock.release()


import numpy as np
from keras.models import Model, load_model
from keras.layers import Dropout, Flatten, Dense, Input
from keras.layers.convolutional import Conv2D, MaxPooling2D
from keras.utils import plot_model, to_categorical
from keras import optimizers
import os
import cv2
from keras.applications import VGG16
from keras.applications.vgg16 import preprocess_input
from keras.callbacks import TensorBoard

class vgg():
    def __init__(self, shape, num_classes, data_path, label_path, model_path):
        self.shape = shape
        self.num_classes = num_classes
        self.data_path = data_path
        self.label_path = label_path
        self.model_path = model_path
        self.log_path = "./logs"
        self.classes = self.classname()

    def classname(self, prepath="./tea_category/"):
        # 数据集的类别序号和对应名称，注意的是序号是从1开始，而label中编码实际是从0开始
        classes = os.listdir(prepath)  # 类别序号和名称
        class_dict = {int(Class.split(".")[0]): Class.split(".")[1] for Class in classes[0:5]}
        return class_dict

    def generate_data(self, prepath="tea_category/"):
        classes = os.listdir(prepath)  # 类别序号和名称

        data_path = self.data_path
        label_path = self.label_path
        datas = []
        labels = []
        for i, abspath in enumerate(classes):  # prepath的每一个文件目录
            img_names = os.listdir(prepath + abspath)
            for img_name in img_names:  # 子目录中的每一张图片
                img = cv2.imread(os.path.join(prepath + abspath, img_name))  # cv2读取
                if not isinstance(img, np.ndarray):
                    print("read img error")
                    continue
                img = cv2.resize(img, (224, 224))  # 尺寸变换224*224
                # img = img.astype(np.float32)  # 类型转换为float32
                img = preprocess_input(img)
                label = to_categorical(i, self.num_classes)
                labels.append(label)
                datas.append(img)
        datas = np.array(datas)
        labels = np.array(labels)
        np.save(data_path, datas)
        np.save(label_path, labels)
        return True

    def vgg_model(self):  # 自行构建VGG16
        input_1 = Input(shape=self.shape)  # 输入224*224*3
        # 第一部分
        # 卷积 64深度，大小是3*3 步长为1 使用零填充 激活函数relu
        # 2次卷积 一次池化 池化尺寸2*2 步长2*2
        x = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(input_1)
        x = Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)  # 64 224*224
        x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="SAME")(x)  # 64 112*112

        # 第二部分 2次卷积 一次池化
        # 卷积 128深度 大小是3*3 步长1 零填充
        x = Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)
        x = Conv2D(filters=128, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)  # 128 112*112
        x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="SAME")(x)  # 128 56*56

        # 第三部分 3次卷积 一次池化 卷积256 3*3
        x = Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)
        x = Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)
        x = Conv2D(filters=256, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)  # 256 56*56
        x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="SAME")(x)  # 256 28*28

        # 第四部分 3次卷积 一次池化 卷积 512 3*3
        x = Conv2D(filters=512, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)
        x = Conv2D(filters=512, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)
        x = Conv2D(filters=512, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)  # 512 28*28
        x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="SAME")(x)  # 512 14*14

        # 第五部分 3次卷积 一次池化 卷积 512 3*3
        x = Conv2D(filters=512, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)
        x = Conv2D(filters=512, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)
        x = Conv2D(filters=512, kernel_size=(3, 3), strides=(1, 1), padding="SAME", activation="relu")(x)  # 512 14*14
        x = MaxPooling2D(pool_size=(2, 2), strides=(2, 2), padding="SAME")(x)  # 512 7*7

        x = Flatten()(x)  # 扁平化，用在全连接过渡
        # 第六部分 三个全连接
        # 第一个和第二个全连接相同 输出4096 激活relu 使用dropout，随机丢弃一半
        x = Dense(4096, activation="relu")(x)
        Dropout(0.5)(x)
        x = Dense(4096, activation="relu")(x)
        Dropout(0.5)(x)  # 输出 4096 1*1
        # 第三个全连接层 输出 softmax分类
        out_ = Dense(self.num_classes, activation="softmax")(x)
        model = Model(inputs=input_1, outputs=out_)
        # print(model.summary())
        sgd = optimizers.sgd(lr=0.001, momentum=0.9, nesterov=True)
        model.compile(sgd, loss="categorical_crossentropy", metrics=["accuracy"])
        # plot_model(model,"model.png")
        return model

    def pretrain_vgg(self):  # 采用预训练的VGG16,修改最后一层
        model_vgg = VGG16(include_top=False, weights='imagenet', input_shape=(224, 224, 3))  # 不包含最后一层
        model = Flatten(name='Flatten')(model_vgg.output)
        model = Dense(self.num_classes, activation='softmax')(model)  # 最后一层自定义

        model_vgg = Model(inputs=model_vgg.input, outputs=model, name='vgg16')
        sgd = optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True)
        # sgd效果比Adam更好
        # adam = optimizers.Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)
        model_vgg.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])
        # model_vgg.summary()
        return model_vgg

    def train(self, load_pretrain=False, batch_size=32, epoch=50):
        # if load_pretrain:
        #     model = self.pretrain_vgg()
        # else:
        model = self.vgg_model()
        # TensorBoard查看日志
        logs = TensorBoard(log_dir=self.log_path, write_graph=True, write_images=True)

        data_path = self.data_path
        label_path = self.label_path
        save_path = self.model_path
        x = np.load(data_path)
        y = np.load(label_path)
        # 必须打乱 否则验证集loss和acc会出问题
        np.random.seed(200)
        np.random.shuffle(x)
        np.random.seed(200)
        np.random.shuffle(y)
        model.fit(x, y, batch_size=batch_size, epochs=epoch, verbose=1, validation_split=0.3, callbacks=[logs])
        model.save(save_path)

    def predict(self, img_path="test.jpg"):
        # model = vgg_model((224,224,3),5)
        model_path = self.model_path
        model = load_model(model_path)
        test_img = cv2.imread(img_path)
        test_img = cv2.resize(test_img, (224, 224))
        test_img = preprocess_input(test_img)
        ans = model.predict(test_img.reshape(1, 224, 224, 3))
        max_index = np.argmax(ans, axis=1)  # 预测结果是值范围0-4的行向量，因此对应的类别序号要+1
        print("预测结果是%s" % (self.classes[max_index[0] + 1]))

data = "datas_tea/train_data.npy"
label = "label_tea/labels.npy"
mode_path = "tea_category/tea_category.h5"

vgg16 = vgg((224, 224, 3), 4, data, label, mode_path)
vgg16.generate_data()
vgg16.train(batch_size=32,epoch=50)
# vgg16.predict(imgpath)


import os
import numpy as np
import pickle
import re

def load_dict(dict_path):
    """Loads a dictionary in pickle format."""
    with open(dict_path,'rb') as fh:
        d = pickle.load(fh)
    return d

def parse_search(workspace,corpus_name,node,pos,case_sensitive,regexp):
    """Parses the input search and returns a tuple with encoded word and pos node.
    """
    # load dictionaries for encoding words to numbers
    if case_sensitive == False:
        dict_words = load_dict(workspace+corpus_name+'/data/idx/wordslc.pickle') # lowercase
    else:
        dict_words = load_dict(workspace+corpus_name+'/data/idx/words.pickle')
        dict_words = {v: k for k, v in dict_words.items()}
    dict_tags = load_dict(workspace+corpus_name+'/data/idx/tags.pickle')
    dict_tags = {v: k for k, v in dict_tags.items()}
    # parse node
    n = []
    if case_sensitive == False:
        node = node.lower()
    for w in node.strip().split(' '):
        if len(w.strip()) != 0:
            n.append(w.strip())
    # parse pos
    if pos != None:
        p = []
        if type(pos) == list:
            for t in pos:
                if len(t.strip())!= 0:
                    p.append(t.strip())
        else:
            for t in pos.strip().split(' '):
                if len(t.strip())!=0:
                    p.append(t.strip())
    # encode node
    encoded_node = []
    if regexp == False:
        for w in n:
            if w in dict_words:
                arr = np.array(dict_words[w],dtype=int)
                encoded_node.append(arr)
            else: # if any word has no match, return None
                encoded_node = None
                break
    else: # use regexp:
        patterns = []
        for ptrn in n:
            patterns.append(re.compile(ptrn))
        for ptrn in patterns:
            a = []
            for k in dict_words:
                if ptrn.match(k) != None:
                    a+=dict_words[k]
            if len(a)!=0:
                arr = np.array(a,dtype=int)
                encoded_node.append(arr)
            else: # if any pattern has no match, return None
                encoded_node = None
                break
    # encode pos
    if pos != None:
        encoded_pos = []
        for t in p:
            if t in dict_tags:
                arr = np.array(dict_tags[t],dtype=int)
                encoded_pos.append(arr)
            else: # if any tag has no match, return None
                encoded_pos = None
                break
    else:
        encoded_pos = None
    return (encoded_node, encoded_pos)

def search_node(npy_path,encoded_node,encoded_pos):
    """Searches the node in every text file and gets positions."""
    files = os.listdir(npy_path)
    data = []
    for filename in files:
        # load arr
        arr = np.load(npy_path + filename)
        # get matching indexes
        ix = arr_search_indexes(arr,encoded_node,encoded_pos)
        positions = []
        for i in ix.tolist():
            p = round((i / arr.shape[0])*100,2)
            positions.append(p)
        # get positions
        data.append((filename.replace('.npy',''),arr.shape[0],len(ix),positions))
    return data

def arr_search_indexes(arr,search_w,search_t):
    """Searches the node in arr and returns the matching indexes. (Max. 3 words)"""
    size = len(search_w)
    if search_t == None:
        # search only words
        if size == 1:
            ix = np.isin(arr[:,[0]],search_w)
            ix = np.where(ix)[0]
        elif size == 2:
            # w1
            x1 = np.isin(arr[:,[0]],search_w[0])
            if np.any(x1)==False:
                return np.where(x1)[0]
            # w2
            x2 = np.isin(arr[:,[0]],search_w[1])
            if np.any(x2)==False:
                return np.where(x2)[0]
            # match indexes
            x = np.arange(0,x1.shape[0]-1)
            y = np.arange(1,x1.shape[0])
            # return indexes
            ix = np.where(x1[x] & x2[y])[0]
            return ix
        elif size == 3:
            # w1
            x1 = np.isin(arr[:,[0]],search_w[0])
            if np.any(x1)==False:
                return np.where(x1)[0]
            # w2
            x2 = np.isin(arr[:,[0]],search_w[1])
            if np.any(x2)==False:
                return np.where(x2)[0]
            # w3
            x3 = np.isin(arr[:,[0]],search_w[2])
            if np.any(x3)==False:
                return np.where(x3)[0]
            # match indexes
            x = np.arange(0,x1.shape[0]-2)
            y = np.arange(1,x1.shape[0]-1)
            z = np.arange(2,x1.shape[0])
            # return indexes
            ix = np.where(x1[x] & x2[y] & x3[z])[0]
    else:
        # search words and POS
        if size == 1:
            iw = np.isin(arr[:,[0]],search_w)
            it = np.isin(arr[:,[1]],search_t)
            ix = np.where(iw & it)[0]
        elif size == 2:
            # w1 t1
            x1 = np.isin(arr[:,[0]],search_w[0])
            t1 = np.isin(arr[:,[1]],search_t[0])
            if np.any(x1)==False:
                return np.where(x1)[0]
            if np.any(t1)==False:
                return np.where(t1)[0]
            # w2 t2
            x2 = np.isin(arr[:,[0]],search_w[1])
            t2 = np.isin(arr[:,[1]],search_t[1])
            if np.any(x2)==False:
                return np.where(x2)[0]
            if np.any(t2)==False:
                return np.where(t2)[0]
            # match indexes
            x = np.arange(0,x1.shape[0]-1)
            y = np.arange(1,x1.shape[0])
            # return indexes
            ix = np.where(x1[x] & t1[x] & x2[y] & t2[y])[0]
        elif size == 3:
            # w1 t1
            x1 = np.isin(arr[:,[0]],search_w[0])
            t1 = np.isin(arr[:,[1]],search_t[0])
            if np.any(x1)==False:
                return np.where(x1)[0]
            if np.any(t1)==False:
                return np.where(t1)[0]
            # w2 t2
            x2 = np.isin(arr[:,[0]],search_w[1])
            t2 = np.isin(arr[:,[1]],search_t[1])
            if np.any(x2)==False:
                return np.where(x2)[0]
            if np.any(t2)==False:
                return np.where(t2)[0]
            # w3 t3
            x3 = np.isin(arr[:,[0]],search_w[2])
            t3 = np.isin(arr[:,[1]],search_t[2])
            if np.any(x3)==False:
                return np.where(x3)[0]
            if np.any(t3)==False:
                return np.where(t3)[0]
            # match indexes
            x = np.arange(0,x1.shape[0]-2)
            y = np.arange(1,x1.shape[0]-1)
            z = np.arange(2,x1.shape[0])
            # return indexes
            ix = np.where(x1[x] & t1[x] & x2[y] & t2[y] & x3[z] & t3[z])[0]
    return ix

def translate(positions):
    """Translate numbers back to words in a list format for kitconc KIWC object."""
    dispersion = []
    dpts = dict()
    i = 0
    total_s1 = 0
    total_s2 = 0
    total_s3 = 0
    total_s4 = 0
    total_s5 = 0

    for d in positions:
        i+=1
        s1 = 0
        s2 = 0
        s3 = 0
        s4 = 0
        s5 = 0
        dpts[d[0]] = d[3]
        for point in d[3]:
            p = round(point)
            if p >= 0 and p <= 19:
                s1+=1
            elif p >= 20 and p <= 39:
                s2+=1
            elif p >= 40 and p <= 59:
                s3+=1
            elif p >= 60 and p <= 79:
                s4+=1
            elif p >= 80 and p <= 100:
                s5+=1
        total_s1 += s1
        total_s2 += s2
        total_s3 += s3
        total_s4 += s4
        total_s5 += s5
        dispersion.append((i ,d[0],d[1],d[2],s1,s2,s3,s4,s5))
    return ((total_s1,total_s2,total_s3,total_s4,total_s5),dpts,dispersion)

def make_dispersion(workspace,corpus_name,node,pos,case_sensitive,regexp):
    dispersion=[]
    dpts = dict()
    totals = tuple()
    encoded_node,encoded_pos = parse_search(workspace,corpus_name,node,pos,case_sensitive,regexp)
    if encoded_node != None: # search node is OK
        positions = search_node(workspace+corpus_name + '/data/npy/',encoded_node,encoded_pos)
        if len(positions)!=0:
            totals,dpts,dispersion = translate(positions)
    return (totals,dpts,dispersion)


#!/usr/bin/env python
# coding: utf-8

#
# # Investigation of the Dataset: No-show Medical Appointments
#
# ## Table of Contents
#

#
Introduction

#
Data Wrangling

#
Exploratory Data Analysis

#
Conclusions

#

#
# ## Introduction
#
# I am goint to investigate the **No-show appointments** dataset, which collects information from 100k medical appointments in Brazil and is focused on the question of whether or not patients show up for their appointment. A number of characteristics about the patient are included in each row.
#
# This dataset does not have many variables, so it will be necessary to make good use of all of them in order to answer as many questions as we can. Let's take a look at out Data Dictionary:
#
# **Independent variables:**
# >
 PatientId: Identification of a patient
# >
 AppointmentID: Identification of each appointment
# >
 Gender: Male or Female . Female is the greater proportion
# >
 ScheduledDay: The day of the actuall appointment, when they have to visit the doctor.
# >
 AppointmentDay: The day someone called/registered the appointment.
# >
 Age: How old is the patient.
# >
 Neighborhood: here the appointment takes place.
#
# >**Categorical variables** *(True or False)*:
#
# >
 Scholarship: indicates if the patient is enrolled in Brasilian welfare Bolsa Familia.
# >
 Hypertension: indicates if the patient has hypertension
# >
 Diabetes: indicates if the patient has diabetes
# >
 Alcoholism: indicates if the patient is an alcoholic
# >
 Handcap: indicates if the patient is handicapped
# >
 SMS_received: 1 or more messages sent to the patient
#
# **Dependent variable:**
#
# >
 No-show: it says ‘No’ if the patient showed up to their appointment, and ‘Yes’ if they did not show up.
#
#
# ### Questions:
#
# Every hospital would love to predict whether a patient will show up or not, as this would optimize appointments with the doctor allowing more patients to be attended during the day and reducing the waiting time in the hospital.
#
#   - **Question 1:** What is the proportion of patients who showed up and did not showed up?
#   - **Question 2:** What is the patient age distribution of shows versus no_shows?
#
#   - **Question 3:** Is there a relationship between the gender and the Appointment status?
#   - **Question 4:** Is there a relationship between being involved in a scholarship and the status of the appointment?
#   - **Question 5:** Is there a relationship between the patient health designation and the status of the appointment?
#   - **Question 6:** Top 10 neighborhoods with the highest number of no-show ups?
#   - **Question 7:** Is there a specific day of the week in which more not show ups occur?
#   - **Question 8:** Does the waiting days period of time affect the status of the appointment?
#
#
#

#
# ## Data Wrangling & Data Cleaning
#
# **I am going to clean the data as I wrangle it**
#
# Data loading, observations, check for cleanliness and trim / clean of the dataset.
#
# ### General Properties

# In[110]:

# Import statements for all of the packages that I am going to use

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
get_ipython().run_line_magic('matplotlib', 'inline')

# In[111]:

# Load data to a dataframe and visualize the first lines
df=pd.read_csv('noshowappointments-kagglev2-may-2016.csv')
df.head()

# In[112]:

# Get information about the shape of the dataframe
df.shape

# - There are 110527 records and 14 columns in the dataset.

# ### Observations & Data Cleaning Process

# **Step 1:** Check the column names to see if we have to rename them:

# In[113]:

df.columns

# In[114]:

# Let's rename some columns to make it easier to read and nicer
df.rename(columns={'PatientId':'Patient_Id',
                   'AppointmentID':'Appointment_ID',
                   'ScheduledDay':'Scheduled_Day',
                   'AppointmentDay':'Appointment_Day',
                   'No-show':'No_Show'},inplace=True)

# **Step 2:** Get information about the variables in the dataframe and make changes if necessary

# In[115]:

df.info()

# As we can see in the previous query, there are **no missing values** in the dataset.
#
# Although, some variable type changes are necessary:
#
# - Patient_Id to **int**
# - Scheduled_Day and Appointment_ID to **datetime**

# In[116]:

# We want to get rid of the float type and convert the Patient_ID to int64, also in
# order to operate with the '...Day' columns, we need to change its type to datetime.

df['Patient_Id']=df['Patient_Id'].astype('int64')
df['Scheduled_Day']=pd.to_datetime(df['Scheduled_Day'])
df['Appointment_Day']=pd.to_datetime(df['Appointment_Day'])

df.head(6)

# >**Create new columns**

# - We won't be able to use the time of the appointment day, since it looks like they are marked as 00:00:00
# - It'll be helpfull for further studies if we create a new column showing how long the patient has been waiting for the medical appointment
#

# In[117]:

# Create the new column Waiting_Days

df['Waiting_Days'] = (df.Appointment_Day-df.Scheduled_Day).dt.days
df.Waiting_Days

# In[118]:

df.head(2)

# Since the time in the Appointment_Day column doesn't show, for those appointments that took time on the same day as they weere scheduled, it shows -1 days of waiting. Therefore, we must replace the -1 with 0 days of waiting.
#

# In[119]:

df.Waiting_Days.replace(-1,0,inplace=True)
df.head(2)

# - It will also be useful to show the weekday in which the appointment takes place, so let's create a new column for it:

# In[120]:

# Create the new column Appointment_weekday
df['Appointment_Weekday'] = (df.Appointment_Day).dt.weekday_name

#Check shows up and no shows up per weekday
df.groupby('Appointment_Weekday')['No_Show'].value_counts()

#
#
# **Step 3:** Check if there are any duplicated rows:

# In[121]:

sum(df.duplicated())

# - There are **no duplicated rows.**

# **Step 4:** Let's get some statistical data for each column:

# In[122]:

df.describe()

# - The average patients age is 37 years old
# - 75% of the patients received an SMS
# - The average of waiting days for the appointment can be rounded to 10 days (75% of the patients waited 14 days for the Appointment and 50% waited up to 3 days).
# - We can already see **some errors in the data**:
# > 1. **The minimum age is -1**, therefore we will have to get rid of this row or change replace it by the mean age which we can see is 37 years. But before we do that, lets take a look at that particular patient:
# > 2. The are still **negative waiting days**, as this is impossible, we will have to fix this ploblem
#

# **Fix the errors:**
#
# 1. Age = -1:

# In[123]:

df.query('Age == "-1"')

# As no particular characteristic gives you any hint on how old the patient could be, **let's drop that row** and make sure there aren't any more errors on the age:

# In[124]:

# drop the row with Age = -1
df.drop(df[df['Age'] == -1].index, inplace =True)

# In[125]:

# Check if there are more Ages with error values
df.query('Age < 0')

# - There are no more rows with invalid age values.

# 2. Negative waiting days:

# In[126]:

# Check the data related to negative waiting days:

neg_waiting_days = df.query('Waiting_Days < 0')
neg_waiting_days

# In[127]:

neg_waiting_days.describe()

# All the errors (with a mean of -3 in the waiting days) happened on May 2016 and at different Neighbourhoods (except for 2 cases at the Santo Antonio Neighbourhood), 2 of the patients are handicapped and their average age is 33 years old.
#
# - As we cannot relate the error to any of this data, we will drop these rows, since they don't represent a high percentage (0.0045%) of the total data volume.

# In[128]:

# drop the rows with negative waiting days:
df.drop(df[df['Waiting_Days'] < 0].index, inplace =True)

# Check they are dropped
df.query('Waiting_Days < 0')

# **Step 5:** Check if there are several rows for the same patient and appointment:

# In[129]:

sum(df.Patient_Id.duplicated()), sum(df.Appointment_ID.duplicated())

# - There are no dupplicated Appointment_ID values (which would be an error)
# - There are 48228 patients for whom we have data from different appointments, so let's check these out:

# In[130]:

#Create a table for all the duplicated patients
duplc_patients=df[df.Patient_Id.duplicated()==True]

# In[131]:

# select top 10 patients who scheduled appointments more than once
dup=duplc_patients['Patient_Id'].value_counts().iloc[0:10]
print('Top 10 patients who scheduled appointments more than once:\n\n{}'.format(dup))

# In[132]:

# Get the first and last date in out dataset for which an appointment was scheduled
first_date_sch=df.sort_values(by='Scheduled_Day', ascending=True, inplace=False).Scheduled_Day.head(1)
last_date_sch=df.sort_values(by='Scheduled_Day', ascending=True, inplace=False).Scheduled_Day.tail(1)
print('First day an appointment was scheduled:{}'.format(first_date_sch))
print('Last day an appointment was scheduled:{}'.format(last_date_sch))

# In[133]:

# Get the firstand last date in out dataset for the appointments
first_date_app=df.sort_values(by='Appointment_Day', ascending=True, inplace=False).Appointment_Day.head(1)
last_date_app=df.sort_values(by='Appointment_Day', ascending=True, inplace=False).Appointment_Day.tail(1)
print('First appointment day:{}'.format(first_date_app))
print('Last appointment day:{}'.format(last_date_app))

# Therefore, there are patients that scheduled more than 60 appointments in less than a month and a half. As there are too many of these to just take them as an error in the data, I will assume that a patient who must attend different specialists appointments, attends all of them the same day and the system creates different records for each specialist visit.

#
# ## Exploratory Data Analysis

# First of all, I would like to get a quick histogram view of all of the different variables and try to get some observations.

# In[134]:

df.hist(figsize=(15,15));

# **Observations:**
#
# - The sample of young-adult patients is more representative than that of patients older than 60, reaching the peak of representation in infants.
# - A very small sample of patients is represented by alcoholic, diabetics or handicapped patients.
# - There are fewer patients who did not receive a reminder SMS for their appointment than patients who did receive one.

# Secondly, I am going to create a mask for our dependent variable so that I can see the correlations with the different independent variables.

# In[135]:

#Create a mask
shows = df['No_Show'] == 'No'
no_shows = df['No_Show'] == 'Yes'

# From now on, in order to get better insights on the data, I am going to create **contingency tables** (used in statistics to summarize the relationship between several categorical variables) using  the crosstab function to see the results in proportion.
#
# Also, to make sure if a relationship exist or not between variables and answer the question: "is it statistically significant?" I am going to conduct a **Chi-Square Test** on each contingency tables. Therefore, if I obtain a small Chi-Square value, that means the correlation between my two variables is very little, while if I obtain a larger value, it'll mean that there is a correlation between my two variables.
#
# >I stablish my significance level at 95%:
#
# >- If p ≤ 0.05 --> Statistically significant - indicates strong evidence against the null hypothesis, so I reject the null hypothesis.
# >- If p > 0.05 --> Not statistically significant - indicates weak evidence against the null hypothesis, so I fail to reject the null hypothesis.

# ### Question 1:
# >### What is the proportion of patients who showed up and did not showed up?

# In[136]:

# Count the number of patients who showed up and did not showed up
# as well as the total number of patients
No_Show_counts=df.groupby('No_Show').count()['Patient_Id']
No_Show_total=df.count()['Patient_Id']

# In[137]:

# Get proportion of appointment shows versus no shows
showsup=No_Show_counts.iloc[0]/No_Show_total
noshowsup=No_Show_counts.iloc[1]/No_Show_total
No_Show_prop=[showsup,noshowsup]
No_Show_prop

# In[138]:

# Plot a Pie chart with Appointment status (showed up - not showed up)

labels = ['Not Showed up', 'Showed up']
sizes = [noshowsup, showsup]
colors = ['pink', 'palegreen']
explode = (0.1, 0) # only "explode" the Not Showed up slice

plt.pie(sizes, explode=explode, labels=labels, colors=colors,
        autopct='%1.1f%%', shadow=True, startangle=90)
plt.title('Show status of the Appointments')
plt.axis('equal'); # Equal aspect ratio ensures that pie is drawn as a circle.

# The majority of appointment status are **show up (79.8%)**, while those marked as **not show up** represent only the **20.2%** of the total number of records.
#

# ### Question 2:
# >### What is the patient age distribution of shows versus no_shows?
#

# In[139]:

#Create a contingency table: age vs. No-show to check the difference for all ages
age_vs_no_show = pd.crosstab(index = df['Age'], columns = df['No_Show'], normalize = 'index') #With normalize='index’ we normalize over each row.
age_vs_no_show.head()

# In[140]:

# Plot a histogram chart Age vs. Appointment status (showed up - not showed up)

colors = ['palegreen','pink']

age_vs_no_show.plot(kind='bar', stacked=True, color=colors, figsize=(10,5))
plt.xlabel('Age')
plt.ylabel('Show vs. No Show proportion')
plt.title('Age vs Appointment status')
L=plt.legend()
L.get_texts()[0].set_text('Showed up')
L.get_texts()[1].set_text('No Showed up')

# - We can see in the chart above that **patients whose age is above 60 take appointment more seriously** than patients under that age.
# - Patients **between 9 and 45 years old** are those that represents a **greater proportion of no showed up appointmens.**
#

# In[141]:

# Average ages for different show status
df.groupby('No_Show')['Age'].mean()

# The mean for patients who showed up can be rounded to 38 and for the patients that did not showed up can be rounded to 34.

# ### T-Test
# #### Age vs No shows up

# In this case, as we have calculated the means, we are goint to conduct the T-Test, which is used to compare means between two groups of a small sample size and tell you if they are different from each other and let you know if those differences could have happened by chance.

# In[142]:

# Import the package needed
import scipy

# In[143]:

# Create columns for shows and no_whows ages
shows_age = df[df.No_Show == 'No'].Age
no_shows_age = df[df.No_Show == 'Yes'].Age

# In[144]:

# Perform the T-Test
ttest_results = scipy.stats.ttest_ind(shows_age, no_shows_age)
ttest_results

# A statistical hypothesis test determines whether there is a statistically significant relationship between the data. The null hypothesis (considered the default in the test experiment) states that there is no meaningful relationship while the alternative hypothesis states the opposite.
#
# As we can see in our output, the p-value is notoriously small. This shows evidences against the null hypothesis, therefore, I am going to reject it and assume that **there is a meaningful relationship between the age and the appointment status.**

# ### Question 3:
# >### Is there a relationship between the gender and the Appointment status?

# In[145]:

#Create a contingency table: Gender vs. No-show
gender_vs_no_show = pd.crosstab(index = df['Gender'], columns = df['No_Show'], normalize = 'index') #With normalize='index’ we normalize over each row.
gender_vs_no_show.head()

# In[146]:

# Plot a histogram chart Gender vs. Appointment status (showed up - not showed up)

colors = ['palegreen','pink']

gender_vs_no_show.plot(kind='bar', stacked=True, color=colors, figsize=(7,5))
plt.xlabel('Gender')
plt.ylabel('Number of Appointments')
plt.title('Gender vs Appointment status')
L=plt.legend(fancybox=True, framealpha=0.5)
L.get_texts()[0].set_text('Showed up')
L.get_texts()[1].set_text('No Showed up')

# As we can see from the plot, there is not a lot of difference on the appointment status based on the patient gender.

# ### Chi-Square Test
# #### Gender vs No shows up

# In[147]:

from scipy.stats import chi2_contingency

# In[148]:

#Conduct the Chi-Square Test(chi2), showing the p (p-value), dof(degrees of freedom) and ex(expected frequencies)
chi2, p, dof,ex = chi2_contingency(gender_vs_no_show, correction=False)
chi2, p

# The p-value is greater than 0.05% (0.99%), therefore, I am going not going to assume that **there is NOT a meaningful relationship between the gender and the appointment status.**

# ### Questions 4 & 5:
# >### Is there a relationship between being involved in a scholarship and the status of the appointment?
# >### Is there a relationship between the patient health designation and the status of the appointment?

# Our **categorical variables** are: scholarship, Hypertension, Diabetes,  Alcoholism, Handcap, SMS_received and No_Show.

# In[149]:

#Create a list with all the categorical variables
cat_var = ['Scholarship','Hipertension','Diabetes','Alcoholism','Handcap','SMS_received']

# multiple line plot
fig=plt.figure(figsize=(14,15))
for i, var in enumerate(cat_var):
    ax = fig.add_subplot(3, 3, i+1)
    # Add title and labels
    ax.set_title(var + ' vs No_Show')
    ax.set_xlabel(var)
    ax.set_ylabel('Number of Appointments')
    # Plot the proportions
    pd.crosstab(index = df[var], columns = df['No_Show'], normalize = 'index').plot(ax=ax, kind='bar', stacked=True, color=['palegreen','pink'])
    # Modify legends
    L=plt.legend(fancybox=True, framealpha=0.7, loc=3)
    L.get_texts()[0].set_text('Showed up')
    L.get_texts()[1].set_text('No Showed up');

# In general, there is not a big difference between both appointment status (show / no-show) due to the health designation. What is very remarkable, in my opinion, is that in proportion, there are **more patients who did not attend the appointment and got a reminder SMS than those who did not attend it without having received it.**

# ### Chi-Square Test
# #### Categorical variables vs No shows up

# In[150]:

#Create a list with all the categorical variables
cat_var_chi2_test = ['Scholarship','Hipertension','Diabetes','Alcoholism','Handcap','SMS_received']

#Conduct the Chi-Square Test(chi2), showing the p (p-value), dof(degrees of freedom) and ex(expected frequencies)
# multiple chi-square test
for i, result in enumerate(cat_var_chi2_test):
    chi2, p, dof,ex = chi2_contingency(pd.crosstab(index = df[result], columns = df['No_Show']), correction=False)

    print(p)

# The Chi-Square Test p values are:
#
#     - p value for Scholarship: 3.13149499541e-22
#     - p value for Hipertension: 2.00724179106e-32
#     - p value for Diabetes: 4.6773921245e-07
#     - p value for Alcoholism: 0.952033520537
#     - p value for Handcap: 0.112251011129
#     - p value for SMS_received: 0.0
#
# Therefore:
#
# - There is a **meaningful relationship** between the independent variables **Scholarship**, **Hipertension**, **Diabetes** and **SMS_received** with our dependent variable Appointment status (No_Show)
#
# - There is **NOT a meaningful relationship** between the independent variables **Alcoholism** and **Handcap** with our dependent variable Appointment status (No_Show)

# ### Question 6:
# >### Top 10 neighborhoods with the highest number of no-show ups?

# In[151]:

# Top 10 neighbourhood with NO shows up
hoods_no_show=df.Neighbourhood[no_shows].value_counts().head(10)
hoods_no_show

# In[152]:

# Plot a Pie chart with Appointment status (showed up - not showed up)

labels = ['JARDIM CAMBURI','MARIA ORTIZ','ITARARÉ','RESISTÊNCIA',
          'CENTRO','JESUS DE NAZARETH','JARDIM DA PENHA','CARATOÍRA',
          'TABUAZEIRO','BONFIM']

colors = ['darkred', 'firebrick','indianred','palevioletred','lightcoral',
          'salmon','darksalmon','lightsalmon','darkorange','orange']

plt.pie(hoods_no_show, labels=labels,
        autopct='%1.1f%%', shadow=True, startangle=90, colors=colors)
plt.title('\n\nTop 10 "No-Show Appointments" neighbourhoods\n\n',fontsize=18)
plt.axis('equal'); # Equal aspect ratio ensures that pie is drawn as a circle.

# ### Question 7:
# >### Is there a specific day of the week in which more not show ups occur?

# In[153]:

#Number of show ups and no show ups by weekdays
weekdays_vs_NoShow = pd.crosstab(index = df['Appointment_Weekday'], columns = df['No_Show'])
weekdays_vs_NoShow

# In[154]:

#Plot days vs Show ups and No show ups
days_shows = df['Appointment_Weekday'][shows].value_counts()
days_noshows= df['Appointment_Weekday'][no_shows].value_counts()

Appointment_Weekday=['Wednesday','Tuesday','Monday','Friday','Thursday','Saturday']

days_shows.plot(color='palegreen',linewidth=3.0,grid=.5)
days_noshows.plot(color='pink',linewidth=3.0,grid=.5)
plt.xlabel(Appointment_Weekday)
plt.ylabel('Number of Appointments')
plt.title('\nDays vs Appointment status\n')
L=plt.legend(fancybox=True, framealpha=0.9)
L.get_texts()[0].set_text('Showed up')
L.get_texts()[1].set_text('No Showed up');

# We can see from the graph how the curves of show and no-show appointments follow a very similar dynamic throughout the weekdays. **Wednesday is the day with the highest number of no show ups** (also with the highest number of show ups) while **saturday is the day with the lowest number of no show ups** (as well as for the show ups). However, we need to look at the data in proportion.

# In[155]:

#Create a table: Appointment_Weekday vs. No-show (to check proportionally)
weekdays_vs_no_show = pd.crosstab(index = df['Appointment_Weekday'], columns = df['No_Show'], normalize = 'index')
weekdays_vs_no_show

# In[156]:

# Plot a histogram chart Appointment_Weekday vs. Appointment status (showed up - not showed up)

colors = ['palegreen','pink']

weekdays_vs_no_show.plot(kind='bar', stacked=True, color=colors, figsize=(7,5))
plt.xlabel('Weekdays')
plt.ylabel('Number of Appointments')
plt.title('Appointment Weekday vs Appointment Status')
L=plt.legend(fancybox=True, framealpha=0.5)
L.get_texts()[0].set_text('Showed up')
L.get_texts()[1].set_text('No Showed up')

# From this graph we can conclude that the difference on Saturday might be due to the low number of records for this day, therefore, any Saturday  record represents a greater variation in proportion to the other days.

# ### Chi-Square Test
# #### Appointment Weekday vs No shows up

# In[157]:

#Conduct the Chi-Square Test(chi2), showing the p (p-value), dof(degrees of freedom) and ex(expected frequencies)
chi2, p, dof,ex = chi2_contingency(weekdays_vs_no_show, correction=False)
chi2, p

# The p-value is greater than 0.05% (0.99%), therefore, I am going not going to assume that **there is NOT a meaningful relationship between the appointment weekday and the appointment status.**

# ### Question 8:
# >### Does the waiting days period of time affect the status of the appointment?

# In[158]:

df.columns

# In[159]:

#Create a contingency table: Appointment_Weekday vs. No-show (to check proportionally)
waiting_days_vs_no_show = pd.crosstab(index = df['Waiting_Days'], columns = df['No_Show'])
waiting_days_vs_no_show

# In[160]:

#Plot Waiting Days vs Show ups and No Show ups
waitingdays_shows = df['Waiting_Days'][shows].value_counts()
waitingdays_noshows= df['Waiting_Days'][no_shows].value_counts()

waitingdays_shows.plot(kind='bar', color='palegreen',linewidth=3.0, figsize=(20,10))
waitingdays_noshows.plot(kind='bar', color='pink',linewidth=3.0, figsize=(20,8))
plt.xlabel('Waiting Days',fontsize=20)
plt.ylabel('Number of Appointments',fontsize=20)
plt.title('\nWaiting Days vs Appointment Status\n',fontsize=24)
L=plt.legend(fancybox=True, framealpha=0.9, fontsize=20)
L.get_texts()[0].set_text('Showed up')
L.get_texts()[1].set_text('No Showed up');

# The most relevant insight of this plot is that a large majority scheduled the appointment the same days they attended, causing a very positive trend in attendance. **The shorten awaiting period the more patients show up.**

#
# ## Conclusions
#
# 1. The majority of appointment status are **show up (79.8%)**, while those marked as **not show up** represent only the **20.2%** of the total number of records.
#
#
# 2. We can see in the chart above that **patients whose age is above 60 take appointment more seriously** than patients under that age.
#
#
# 3. Patients **between 9 and 45 years old** are those that represents a **greater proportion of no showed up appointmens,** while **patients above 60 take
# appointment more seriously**
#
#
# 4. There are **more patients who did not attend the appointment and got a reminder SMS than those who did not attend it without having received it.**
#
#
# 5. The **Top 10 neighborhoods** with the highest number of **no-show ups** are, in order: Jardim Camburi, Maria Ortiz, Itararé, Resistência, Centro, Jesus De Nazareth, Jardim Da Penha, Caratoíra, Tabuazeiro Y Bonfim.
#
#
# 6. **Wednesdays and Tuesdays** are the days with the **highest number of appointments**, while **Thursday and Saturday** are the days with the **least number of appointments**, although there is no significant relationship between the date of the appointment and its final status.
#
#
# 7. **The shorten awaiting period the more patients show up.**
#
#
# >Variables that show a significant relationship with the status of the appointment according to statistical tests:
# >-	Age
# >-	Scholarship
# >-	Hipertension
# >-	Diabetes
# >-	SMS_received
#
#
#
# ## Limitations
#
# • **The time period over the data is collected is relatively small** (6 months), if we had data referring to a longer period of time we would have drawn conclusions regarding seasonality and the rate of variation over the years.
#
# • **The sample is very unbalanced** since we only have the 20% of records for non-show, which results in a strong limitation of the study of their causes.
#
# • **Limitations of chi-square**: if the test result comes out as a meaningfult relathionship between the variables, that means there is some association but it does not provide further information.
#
#

# In[163]:

from subprocess import call
call(['python', '-m', 'nbconvert', 'Investigate_a_Dataset.ipynb'])


import json, random
import pandas as pd
import numpy as np
import sys

fileName = sys.argv[-1]

annots = []

AllData = pd.read_table("./VisFiles/ToJson_"+str(fileName)+".txt",  sep=" ")
chrs = [
	 "1", "2", "3", "4", "5","6","7","8","9",
	 "10", "11", "12", "13", "14", "15", "16", "17", "18", "19", "20",
	 "21", "22","X","Y"
]
ch1 = 1
ch2 = 1
ch3 = 1
ch4 = 1
ch5 = 1
ch6 = 1
ch7 = 1
ch8 = 1
ch9 = 1
ch10 = 1
ch11 = 1
ch12 = 1
ch13 = 1
ch14 = 1
ch15 = 1
ch16 = 1
ch17 = 1
ch18 = 1
ch19 = 1
ch20 = 1
ch21 = 1
ch22 = 1
chX = 1
chY = 1

lengths_GRCh38 = {
	"1": 248956422, "2": 242193529, "3": 198295559,
	"4": 190214555, "5": 181538259, "6": 170805979,
	"7": 159345973, "8": 145138636, "9": 138394717,
	"10": 133797422, "11": 135086622, "12": 133275309,
	"13": 114364328, "14": 107043718, "15": 101991189,
	"16": 90338345,	"17": 83257441, "18": 80373285,
	"19": 58617616, "20": 64444167, "21": 46709983,
	"22": 50818468, "X": 156040895, "Y": 57227415
}
#
lengths_GRCh37 = {
	"1": 249250621, "2": 243199373, "3": 198022430,
	"4": 191154276, "5": 180915260, "6": 171115067,
	"7": 159138663, "8": 146364022, "9": 141213431,
	"10": 135534747, "11": 135006516, "12": 133851895,
	"13": 115169878, "14": 107349540, "15": 102531392,
	"16": 90354753, "17": 81195210, "18": 78077248,
	"19": 59128983, "20": 63025520, "21": 48129895,
	"22": 51304566, "X": 155270560, "Y": 59373566
}

for chr in chrs:
	annots.append({"chr": chr, "annots":[]})

i = 0

while i < len(AllData):
	# j = str(i + 1)
	# print(j)
	#chr = i % 24
	# print(chr)

	# print(chr)
    #
	# start = int((i * lengths_GRCh38[chrs[chr]])/1000 + 1)
	# length = 0
	#
	#
	if (AllData['chr'][i] == 1):
		#chrlength1 = len(AllData[AllData['chr'] == 1])
		#if (ch1 != chrlength1):
			#a = int((ch1 * lengths_GRCh37[chrs[0]])/chrlength1 + 1)#100
			annot = [
				AllData['info'][i]+'\n'+
				''+'More Info'+'' ,
			    AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[0]["annots"].append(annot)
			ch1+=1

	if (AllData['chr'][i] == '2'):
		chrlength2 = len(AllData[AllData['chr'] == '2'])
		if (ch2 != chrlength2):
			b = int((ch2 * lengths_GRCh37[chrs[1]])/chrlength2 + 1)#100
			annot = [
				AllData['info'][i]+'\n'+
				''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[1]["annots"].append(annot)
			ch2+=1

	if (AllData['chr'][i] == '3'):
		chrlength3 = len(AllData[AllData['chr'] == '3'])
		if (ch3 != chrlength3):
			c = int((ch3 * lengths_GRCh37[chrs[2]])/chrlength3 + 1)#100
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[2]["annots"].append(annot)
			ch3+=1

	if (AllData['chr'][i] == '4'):
		chrlength4 = len(AllData[AllData['chr'] == '4'])
		if (ch4 != chrlength4):
			d = int((ch4 * lengths_GRCh37[chrs[3]])/chrlength4 + 1)
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[3]["annots"].append(annot)
			ch4+=1

	if (AllData['chr'][i] == '5'):
		chrlength5 = len(AllData[AllData['chr'] == '5'])
		if (ch5 != chrlength5):
			e = int((ch5 * lengths_GRCh37[chrs[4]])/chrlength5 + 1)#100
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[4]["annots"].append(annot)
			ch5+=1

	if (AllData['chr'][i] == '6'):
		chrlength6 = len(AllData[AllData['chr'] == '6'])
		if (ch6 != chrlength6):
			f = int((ch6 * lengths_GRCh37[chrs[5]])/chrlength6 + 1) #100
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[5]["annots"].append(annot)
			ch6+=1

	if (AllData['chr'][i] == '7'):
		chrlength7 = len(AllData[AllData['chr'] == '7'])
		if (ch7 != chrlength7 ):
			g = int((ch7 * lengths_GRCh37[chrs[6]])/chrlength7 + 1 ) #200
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[6]["annots"].append(annot)
			ch7+=1

	if (AllData['chr'][i] == '8'):
		chrlength8 = len(AllData[AllData['chr'] == '8'])
		if (ch8 != chrlength8 ):
			h = int((ch8 * lengths_GRCh37[chrs[7]])/chrlength8 + 1 ) #200
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[7]["annots"].append(annot)
			ch8+=1

	if (AllData['chr'][i] == '9'):
		chrlength9 = len(AllData[AllData['chr'] == '9'])
		if (ch9 != chrlength9 ):
			z = int((ch9 * lengths_GRCh37[chrs[8]])/chrlength9 + 1)
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[8]["annots"].append(annot)
			ch9+=1

	if (AllData['chr'][i] == '10'):
		chrlength10 = len(AllData[AllData['chr'] == '10'])
		if( ch10 != chrlength10 ):
			j = int((ch10 * lengths_GRCh37[chrs[9]])/chrlength10 +1 )#60
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[9]["annots"].append(annot)
			ch10+=1

	if (AllData['chr'][i] == '11'):
		chrlength11 = len(AllData[AllData['chr'] == '11'])
		if (ch11 != chrlength11):
			k = int((ch11 * lengths_GRCh37[chrs[10]])/chrlength11 + 1 )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[10]["annots"].append(annot)
			ch11+=1

	if (AllData['chr'][i] == '12'):
		chrlength12 = len(AllData[AllData['chr'] =='12'])
		if (ch12 != chrlength12):
			l = int((ch12 * lengths_GRCh37[chrs[11]])/chrlength12 + 1 )
		annot = [
			AllData['info'][i]+'\n'+''+'More Info'+'' ,
			AllData['pos'][i],
			AllData['len'][i],
			AllData['trackIndex'][i]
		]
		annots[11]["annots"].append(annot)
		ch12+=1

	if (AllData['chr'][i] == '13'):
		chrlength13 = len(AllData[AllData['chr'] == '13'])
		if (ch13 != chrlength13 ):
			m = int((ch13 * lengths_GRCh37[chrs[12]])/chrlength13 + 1  )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[12]["annots"].append(annot)
			ch13+=1

	if (AllData['chr'][i] == '14'):
		chrlength14 = len(AllData[AllData['chr'] == '14'])
		if (ch14 != chrlength14):
			n = int((ch14 * lengths_GRCh37[chrs[13]])/chrlength14 + 1  )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[13]["annots"].append(annot)
			ch14+=1

	if (AllData['chr'][i] == '15'):
		chrlength15 = len(AllData[AllData['chr'] == '15'])
		if (ch15 != chrlength15):
			o = int((ch15 * lengths_GRCh37[chrs[14]])/chrlength15 + 1  )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[14]["annots"].append(annot)
			ch15+=1

	if (AllData['chr'][i] == '16'):
		chrlength16 = len(AllData[AllData['chr'] == '16'])
		if (ch16 != chrlength16):
			p = int((ch16 * lengths_GRCh37[chrs[15]])/chrlength16 + 1  )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[15]["annots"].append(annot)
			ch16+=1

	if (AllData['chr'][i] == '17'):
		chrlength17 = len(AllData[AllData['chr'] == '17'])
		if (ch17 != chrlength17 ):
			q = int((ch17 * lengths_GRCh37[chrs[16]])/chrlength17 + 1  )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[16]["annots"].append(annot)
			ch17+=1

	if (AllData['chr'][i] == '18'):
		chrlength18 = len(AllData[AllData['chr'] == '18'])
		if (ch18 !=chrlength18 ):
			r = int((ch18 * lengths_GRCh37[chrs[17]])/chrlength18 + 1  )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[17]["annots"].append(annot)
			ch18+=1

	if (AllData['chr'][i] == '19'):
		chrlength19 = len(AllData[AllData['chr'] == '19'])
		if (ch19 != chrlength19):
			s = int((ch19 * lengths_GRCh37[chrs[18]])/chrlength19 + 1 )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[18]["annots"].append(annot)
			ch19+=1

	if (AllData['chr'][i] == '20'):
		chrlength20 = len(AllData[AllData['chr'] == '20'])
		if (ch20 != chrlength20):
			t = int((ch20 * lengths_GRCh37[chrs[19]])/chrlength20 + 1 )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[19]["annots"].append(annot)
			ch20+=1

	if (AllData['chr'][i] =='21'):
		chrlength21 = len(AllData[AllData['chr'] == '21'])
		if (ch21 != chrlength21):
			u = int((ch21 * lengths_GRCh37[chrs[20]])/chrlength21 + 1 )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				u,
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[20]["annots"].append(annot)
			ch21+=1

	if (AllData['chr'][i] == '22'):
		chrlength22 = len(AllData[AllData['chr'] == '22'])
		if (ch22 != chrlength22):
			v = int((ch22 * lengths_GRCh37[chrs[21]])/chrlength22 + 1  )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[21]["annots"].append(annot)
			ch22+=1

	if (AllData['chr'][i] == 'X'):
		chrlength23 = len(AllData[AllData['chr'] == 'X'])
		if (chX != chrlength23):
			w = int((chX * lengths_GRCh37[chrs[22]])/chrlength23 + 1 )
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[22]["annots"].append(annot)
			chX+=1

	if (AllData['chr'][i] == 'Y'):
		chrlength24 = len(AllData[AllData['chr'] == 'X'])
		if (chY != chrlength24):
			x = int((chY * lengths_GRCh37[chrs[23]])/chrlength24 + 1)
			annot = [
				AllData['info'][i]+'\n'+''+'More Info'+'' ,
				AllData['pos'][i],
				AllData['len'][i],
				AllData['trackIndex'][i]
			]
			annots[23]["annots"].append(annot)
			chY+=1

	i += 1

top_annots = {}
top_annots["keys"] = ["name", "start", "length", "trackIndex"]
top_annots["annots"] = annots
#annots=json.dumps(top_annots.serialize())

def default(o):
    if isinstance(o, np.int64): return int(o)
    raise TypeError

annots = json.dumps(top_annots, default=default,indent=4)

#annots = json.dumps(top_annots, indent=4)
print(annots)


'''Midi velocity to dynamics notation dictionary'''
dynamicsDict = dict([
    (8, 'pppp'), (20, 'ppp'), (31, 'pp'), (42, 'p'), (53, 'mp'), (64, 'mf'), (80, 'f'), (96, 'ff'), (112, 'fff'), (127, 'ffff')
])
#

'''Midi key number to piano key number dictionary'''
midikeyDict = dict([
    (108, 88), (107, 87), (106, 86), (105, 85), (104, 84), (103, 83), (102, 82), (101, 81), (100, 80), (99, 79), (98, 78), (97, 77), (96, 76), (95, 75),
    (94, 74), (93, 73), (92, 72), (91, 71), (90, 70), (89, 69), (88, 68), (87, 67), (86, 66), (85, 65), (84, 64), (83, 63), (82, 62), (81, 61), (80, 60),
    (79, 59), (78, 58), (77, 57), (76, 56), (75, 55), (74, 54), (73, 53), (72, 52), (71, 51), (70, 50), (69, 49), (68, 48), (67, 47), (66, 46), (65, 45),
    (64, 44), (63, 43), (62, 42), (61, 41), (60, 40), (59, 39), (58, 38), (57, 37), (56, 36), (55, 35), (54, 34), (53, 33), (52, 32), (51, 31), (50, 30),
    (49, 29), (48, 28), (47, 27), (46, 26), (45, 25), (44, 24), (43, 23), (42, 22), (41, 21), (40, 20), (39, 19), (38, 18), (37, 17), (36, 16), (35, 15),
    (34, 14), (33, 13), (32, 12), (31, 11), (30, 10), (29, 9), (28, 8), (27, 7), (26, 6), (25, 5), (24, 4), (23, 3), (22, 2), (21, 1)
])
#

def closest_dynamics(velocity):
    import numpy as np
    # Calculating closest dynamics variable from velocity integer
    lst = [8, 20, 31, 42, 53, 64, 80, 96, 112, 127]
    lst = np.asarray(lst)
    idx = (np.abs(lst - velocity)).argmin()
    return lst[idx]

def midi_dynamics(velocity):
    '''
    Convert the midi velocity to standard English notation, either short or long.
    params      velocity:   int     between 1 and 127
    returns     [dynamics_name, dynamics_description]
    '''
    dynamics_name = dict([(8, 'pppp'),
                     (20, 'ppp'),
                     (31, 'pp'),
                     (42, 'p'),
                     (53, 'mp'),
                     (64, 'mf'),
                     (80, 'f'),
                     (96, 'ff'),
                     (112, 'fff'),
                     (127, 'ffff')])

    dynamics_desc = {'ppp':  'extremely soft',
                         'pp':   'very soft',
                         'p':    'soft',
                        'mp':   'moderately soft',
                        'mf':   'moderately loud',
                        'f':    'loud',
                        'ff':   'very loud',
                        'fff':  'extremely loud'}

    closest_x = closest_dynamics(velocity)
    dname = dynamics_name[closest_x]
    # ddesc = dynamics_desc[dname]
    # return [str(dname), str(ddesc)]
    return dname

def dynamics_color(velocity):
    '''
    Color codes the dynamics
    parameters:
        velocity:    int    midi velocity
    '''
    if 0   <= velocity < 8:    color = '#0E0050'
    if 8   <= velocity < 11:  color = '#0E0066'
    if 11  <= velocity < 17:   color = '#14056B'
    if 17  <= velocity < 20:   color = '#1B0B70'
    if 20  <= velocity < 22:   color = '#221175'
    if 22  <= velocity < 28:   color = '#29167B'
    if 28  <= velocity < 31:   color = '#2F1C80'
    if 31  <= velocity < 33:   color = '#362285'
    if 33  <= velocity < 39:   color = '#3D278A'
    if 39  <= velocity < 42:   color = '#442D90'
    if 42  <= velocity < 44:   color = '#4A3395'
    if 44  <= velocity < 50:   color = '#51389A'
    if 50  <= velocity < 53:   color = '#583E9F'
    if 53  <= velocity < 55:   color = '#5F44A5'
    if 55  <= velocity < 61:   color = '#6E44A2'
    if 61  <= velocity < 64:   color = '#7D449F'
    if 64  <= velocity < 68:   color = '#8D449D'
    if 68  <= velocity < 76:   color = '#B34F7D'
    if 76  <= velocity < 80:   color = '#D95A5E'
    if 80  <= velocity < 84:   color = '#FF663F'
    if 84  <= velocity < 92:   color = '#FF5336'
    if 92  <= velocity < 96:   color = '#FF402D'
    if 96  <= velocity < 100:  color = '#FF2D25'
    if 100 <= velocity < 108:  color = '#FF4544'
    if 108 <= velocity < 112:  color = '#FF5D63'
    if 112 <= velocity < 115:  color = '#FF7582'
    if 115 <= velocity < 123:  color = '#FF939A'
    if 123 <= velocity < 127:  color = '#FFB1B3'
    if 127 <= velocity:        color = '#FFD0CC'

    return color


import sys
import os
import time
import argparse
from pathlib import Path

import torch
import torch.nn as nn
import torch.backends.cudnn as cudnn
from torch.autograd import Variable

from PIL import Image

import cv2
from skimage import io
import numpy as np
import craft_utils
import imgproc
import file_utils
import json
import zipfile

from collections import OrderedDict

from torch.autograd import Variable

import pycuda.driver as cuda
import pycuda.autoinit
import tensorrt as trt

from icecream import ic
TRT_LOGGER = trt.Logger()

class RTLayer():
    """

    """

    def __init__(self, config=None, model_path=None, data_path='./weights',
                 engine_path=None, cuda_ctx=None, input_shape=None):
        super().__init__()
        data_path = Path(data_path)
        model_path = sorted(data_path.glob('*.engine'))

        self.engine_path=model_path[0]

        self.cuda_ctx = cuda_ctx
        if self.cuda_ctx:
            self.cuda_ctx.push()

        self.trt_logger = trt.Logger(trt.Logger.INFO)
        self._load_plugins()
        self.engine = self._load_engine()
        self.input_shape = input_shape

    def __call__(self, args, image, text_threshold, link_threshold, low_text, poly):
        t0 = time.time()

        # resize
        img_resized, target_ratio, size_heatmap = imgproc.resize_aspect_ratio(image, args.canvas_size, interpolation=cv2.INTER_LINEAR, mag_ratio=args.mag_ratio)
        ratio_h = ratio_w = 1 / target_ratio

        # preprocessing
        img_resized = imgproc.normalizeMeanVariance(img_resized)
        img_resized = torch.from_numpy(img_resized).permute(2, 0, 1)    # [h, w, c] to [c, h, w]

        img_resized = Variable(img_resized.unsqueeze(0))                # [c, h, w] to [b, c, h, w]
        if cuda:
            img_resized = img_resized.cuda()
        # ic(img_resized.shape)

        # feed to engine and process output
        height, width = img_resized.shape[2:4]
        self.input_shape = (height,width)
        img_resized = img_resized.cpu().detach().numpy()

        segment_inputs, segment_outputs, segment_bindings = self._allocate_buffers()

        stream = cuda.Stream()

        with self.engine.create_execution_context() as context:
            context.active_optimization_profile = 0
            origin_inputshape=context.get_binding_shape(0)

            if (origin_inputshape[-1]==-1):
                origin_inputshape[-2],origin_inputshape[-1]=(self.input_shape)
                context.set_binding_shape(0,(origin_inputshape))

            input_img_array = np.array([img_resized] * self.engine.max_batch_size)
            img = torch.from_numpy(input_img_array).float().numpy()
            segment_inputs[0].host = img
            [cuda.memcpy_htod_async(inp.device, inp.host, stream) for inp in segment_inputs]#Copy from the Python buffer src to the device pointer dest (an int or a DeviceAllocation) asynchronously,
            stream.synchronize()#Wait for all activity on this stream to cease, then return.

            context.execute_async(bindings=segment_bindings, stream_handle=stream.handle)#Asynchronously execute inference on a batch.
            stream.synchronize()
            [cuda.memcpy_dtoh_async(out.host, out.device, stream) for out in segment_outputs]#Copy from the device pointer src (an int or a DeviceAllocation) to the Python buffer dest asynchronously
            stream.synchronize()
            bs = context.get_binding_shape(2)

            y_out = segment_outputs[0].host

            # ic(context.get_binding_shape(1))
            ic(bs)
        y1 =  y_out[0:np.array(bs).prod()].reshape(bs)
        ic('head: ',y_out[0:np.array(bs).prod()])
        ic('tail: ',y_out[np.array(bs).prod():])

        y = torch.from_numpy(y1)
        ic(5,'y2: ',y.shape)
        ic('value: ',y)

        # make score and link map
        score_text = y[0,:,:,0].cpu().data.numpy()
        score_link = y[0,:,:,1].cpu().data.numpy()

        t0 = time.time() - t0
        t1 = time.time()

        # Post-processing
        boxes, polys = craft_utils.getDetBoxes(score_text, score_link, text_threshold, link_threshold, low_text, poly)

        # coordinate adjustment
        boxes = craft_utils.adjustResultCoordinates(boxes, ratio_w, ratio_h)
        polys = craft_utils.adjustResultCoordinates(polys, ratio_w, ratio_h)
        for k in range(len(polys)):
            if polys[k] is None: polys[k] = boxes[k]

        t1 = time.time() - t1

        # render results (optional)
        render_img = score_text.copy()
        render_img = np.hstack((render_img, score_link))
        ret_score_text = imgproc.cvt2HeatmapImg(render_img)

        if args.show_time : print("\ninfer/postproc time : {:.3f}/{:.3f}".format(t0, t1))

        return boxes, polys, ret_score_text

    def _load_plugins(self):
        if trt.__version__[0] < '7':
            ctypes.CDLL("./libflattenconcat.so")
        trt.init_libnvinfer_plugins(self.trt_logger, '')

    def _load_engine(self):
        assert os.path.exists(self.engine_path)
        print("Reading engine from file {}".format(self.engine_path))
        with open(self.engine_path, "rb") as f, trt.Runtime(TRT_LOGGER) as runtime:
            return runtime.deserialize_cuda_engine(f.read())

    def _allocate_buffers(self):
        inputs = []
        outputs = []
        bindings = []
        class HostDeviceMem(object):
            def __init__(self, host_mem, device_mem):
                self.host = host_mem
                self.device = device_mem

            def __str__(self):
                return "Host:\n" + str(self.host) + "\nDevice:\n" + str(self.device)

            def __repr__(self):
                return self.__str__()
        for binding in self.engine:

            dims = self.engine.get_binding_shape(binding)
            # print(dims)
            if dims[-1] == -1:
                assert(self.input_shape is not None)
                dims[-2],dims[-1] = self.input_shape
            size = trt.volume(dims) * self.engine.max_batch_size#The maximum batch size which can be used for inference.
            dtype = trt.nptype(self.engine.get_binding_dtype(binding))
            # Allocate host and device buffers
            host_mem = cuda.pagelocked_empty(size, dtype)
            device_mem = cuda.mem_alloc(host_mem.nbytes)
            # Append the device buffer to device bindings.
            bindings.append(int(device_mem))
            if self.engine.binding_is_input(binding):#Determine whether a binding is an input binding.
                inputs.append(HostDeviceMem(host_mem, device_mem))
            else:
                outputs.append(HostDeviceMem(host_mem, device_mem))
        return inputs, outputs, bindings

    # def __del__(self):
    #     """Free CUDA memories and context."""
    #     del self.cuda_outputs
    #     del self.cuda_inputs
    #     del self.stream