import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import DataLoader
import torchvision
import pytorch_lightning as pl

from argparse import ArgumentParser, Namespace
from collections import OrderedDict

from dataloaders.dataloader_self import Multiview_Dataset
from utils.radam import RAdam

import numpy as np
eps = torch.tensor(np.finfo(float).eps).float()

class Discriminator(nn.Module):

    def __init__(self, tile_size=(64, 64, 64), in_channels=1, **kwargs):
        super(Discriminator, self).__init__()

        self.tile_size = tile_size
        self.in_channels = in_channels
        self.out_size = tuple([int(p/2**4) for p in tile_size])

        # Define layer instances
        self.conv1 = nn.Conv3d(in_channels=in_channels, out_channels=64, kernel_size=3, stride=2, padding=1)
        self.norm2 = nn.InstanceNorm3d(num_features=64)
        self.leaky1 = nn.LeakyReLU(negative_slope=0.2)

        self.conv2 = nn.Conv3d(in_channels=64, out_channels=128, kernel_size=3, stride=2, padding=1)
        self.norm2 = nn.InstanceNorm3d(num_features=128)
        self.leaky2 = nn.LeakyReLU(negative_slope=0.2)

        self.conv3 = nn.Conv3d(in_channels=128, out_channels=256, kernel_size=3, stride=2, padding=1)
        self.norm3 = nn.InstanceNorm3d(num_features=256)
        self.leaky3 = nn.LeakyReLU(negative_slope=0.2)

        self.conv4 = nn.Conv3d(in_channels=256, out_channels=512, kernel_size=3, stride=2, padding=1)
        self.norm4 = nn.InstanceNorm3d(num_features=512)
        self.leaky4 = nn.LeakyReLU(negative_slope=0.2)

        self.conv6 = nn.Conv3d(in_channels=512, out_channels=1, kernel_size=3, stride=1, padding=1)
        self.sig6 = nn.Sigmoid()

    def forward(self, img):

        out = self.conv1(img)
        out = self.leaky1(out)

        out = self.conv2(out)
        out = self.norm2(out)
        out = self.leaky2(out)

        out = self.conv3(out)
        out = self.norm3(out)
        out = self.leaky3(out)

        out = self.conv4(out)
        out = self.norm4(out)
        out = self.leaky4(out)

        out = self.conv6(out)
        out = self.sig6(out)

        return out

class Generator(nn.Module):
    """
    Implementation of the 3D U-Net architecture.
    """

    def __init__(self, in_channels=4, out_channels=1, feat_channels=64, num_views=4,
        out_activation='sigmoid', norm_method='instance', data_augmentation=True, **kwargs):
        super(Generator, self).__init__()

        self.in_channels = in_channels
        self.out_channels = out_channels
        self.feat_channels = feat_channels
        self.num_views = num_views
        self.out_activation = out_activation # relu | sigmoid | tanh | hardtanh | none
        self.norm_method = norm_method # instance | batch | none

        if self.norm_method == 'instance':
            self.norm = nn.InstanceNorm3d
        elif self.norm_method == 'batch':
            self.norm = nn.BatchNorm3d
        elif self.norm_method == 'none':
            self.norm = nn.Identity
        else:
            raise ValueError('Unknown normalization method "{0}". Choose from "instance|batch|none".'.format(self.norm_method))

        # The forward generator (point spread functions)
        self.PSF_conv = []
        self.data_augmentation = data_augmentation  # If true, the forward generator has to be re-assigned for a new batch
        self.kernels_initilized = False  # Whether the forward generator has been initialized

        # Define layer instances of the inverse generator (deconvolution network)
        self.c1 = nn.Sequential(
            nn.Conv3d(in_channels=in_channels, out_channels=feat_channels, kernel_size=3, stride=1, padding=1),
            self.norm(num_features=feat_channels),
            nn.ReLU(inplace=True),
            nn.Conv3d(in_channels=feat_channels, out_channels=feat_channels, kernel_size=3, stride=1, padding=1),
            self.norm(num_features=feat_channels),
            nn.ReLU(inplace=True)
            )
        self.d1 = nn.Sequential(nn.MaxPool3d(kernel_size=2, stride=2, padding=0))

        self.c2 = nn.Sequential(
            nn.Conv3d(in_channels=feat_channels, out_channels=feat_channels*2, kernel_size=3, stride=1, padding=1),
            self.norm(num_features=feat_channels*2),
            nn.ReLU(inplace=True),
            nn.Conv3d(in_channels=feat_channels*2, out_channels=feat_channels*2, kernel_size=3, padding=1),
            self.norm(num_features=feat_channels*2),
            nn.ReLU(inplace=True)
            )
        self.d2 = nn.Sequential(nn.MaxPool3d(kernel_size=2, stride=2, padding=0))

        self.c3 = nn.Sequential(
            nn.Conv3d(in_channels=feat_channels*2, out_channels=feat_channels*4, kernel_size=3, stride=1, padding=1),
            self.norm(num_features=feat_channels*4),
            nn.ReLU(inplace=True),
            nn.Conv3d(in_channels=feat_channels*4, out_channels=feat_channels*4, kernel_size=3, stride=1, padding=1),
            self.norm(num_features=feat_channels*4),
            nn.ReLU(inplace=True)
            )

        self.u1 = nn.Sequential(
            nn.ConvTranspose3d(in_channels=feat_channels*4, out_channels=feat_channels*2, kernel_size=4, stride=2, padding=1, output_padding=0),
            self.norm(num_features=feat_channels*4),
            nn.ReLU(inplace=True)
            )
        self.c4 = nn.Sequential(
            nn.Conv3d(in_channels=feat_channels*4, out_channels=feat_channels*2, kernel_size=3, stride=1, padding=1),
            self.norm(num_features=feat_channels*2),
            nn.ReLU(inplace=True),
            nn.Conv3d(in_channels=feat_channels*2, out_channels=feat_channels*2, kernel_size=3, stride=1, padding=1),
            self.norm(num_features=feat_channels*2),
            nn.ReLU(inplace=True)
            )

        self.u2 = nn.Sequential(
            nn.ConvTranspose3d(in_channels=feat_channels*2, out_channels=feat_channels, kernel_size=4, stride=2, padding=1, output_padding=0),
            self.norm(num_features=feat_channels*2),
            nn.ReLU(inplace=True)
            )
        self.c5 = nn.Sequential(
            nn.Conv3d(in_channels=feat_channels*2, out_channels=feat_channels, kernel_size=3, stride=1, padding=1),
            self.norm(num_features=feat_channels),
            nn.ReLU(inplace=True),
            nn.Conv3d(in_channels=feat_channels, out_channels=feat_channels, kernel_size=3, stride=1, padding=1),
            self.norm(num_features=feat_channels),
            nn.ReLU(inplace=True)
            )

        self.out = nn.Sequential(
            nn.Conv3d(in_channels=feat_channels, out_channels=out_channels, kernel_size=1, stride=1)
            )

        if self.out_activation == 'relu':
            self.out_fcn = nn.ReLU()
        elif self.out_activation == 'sigmoid':
            self.out_fcn = nn.Sigmoid()
        elif self.out_activation == 'tanh':
            self.out_fcn = nn.Tanh()
        elif self.out_activation == 'leakyrelu':
            self.out_fcn = nn.LeakyReLU(negative_slope=0.1)
        elif self.out_activation == 'none':
            self.out_fcn = None
        else:
            raise ValueError('Unknown output activation "{0}". Choose from "relu|sigmoid|tanh|hardtanh|none".'.format(self.out_activation))

    def set_kernel_weights(self, PSF):
        '''
        Initialize convolutional kernels with the given PSF
        '''

        for i in range(self.num_views):
            psf_size = tuple(PSF[0, i, ...].size())
            len_padding = tuple([(s - 1) // 2 for s in psf_size])

            if not self.kernels_initilized:
                self.PSF_conv.append(nn.Conv3d(in_channels=self.out_channels, out_channels=self.out_channels, \
                    kernel_size=psf_size, stride=1, padding=len_padding).cuda())

            normalized = (PSF[0, i, ...] / PSF[0, i, ...].sum()).unsqueeze(0).unsqueeze(0)

            self.PSF_conv[i].weight = nn.Parameter(normalized, requires_grad=False)
            self.PSF_conv[i].bias = nn.Parameter(torch.zeros_like(self.PSF_conv[i].bias), requires_grad=False)

        self.kernels_initilized = True

    def forward(self, img, PSF, direction):

        # Blurry-->sharp-->blurry
        if direction == 'BSB':

            c1 = self.c1(img)
            d1 = self.d1(c1)
            # print('c1 ', c1.size())
            # print('d1 ', d1.size())

            c2 = self.c2(d1)
            d2 = self.d2(c2)
            # print('c2 ', c2.size())
            # print('d2 ', d2.size())

            c3 = self.c3(d2)
            # print('c3 ', c3.size())

            u1 = self.u1(c3)
            c4 = self.c4(torch.cat((u1, c2), dim=1))
            # print('u1 ', u1.size())
            # print('c4 ', c4.size())

            u2 = self.u2(c4)
            c5 = self.c5(torch.cat((u2, c1), dim=1))
            # print('u2 ', u2.size())
            # print('c5 ', c5.size())

            if self.out_fcn is None:
                sharp_pred = self.out(c5)
            else:
                sharp_pred = self.out_fcn(self.out(c5))

            if PSF is None:
                blurred_with_PSF = None
            else:
                if self.data_augmentation or not self.kernels_initilized:
                    self.set_kernel_weights(PSF)

                blurred_with_PSF = []
                for i in range(self.num_views):
                    blurred_with_PSF.append(self.PSF_conv[i](sharp_pred))
                blurred_with_PSF = torch.cat(blurred_with_PSF, dim=1)

        # Sharp-->blurry-->sharp
        elif direction == 'SBS':

            if PSF is None:
                blurred_with_PSF = None
            else:
                if self.data_augmentation or not self.kernels_initilized:
                    self.set_kernel_weights(PSF)

                blurred_with_PSF = []
                for i in range(self.num_views):
                    blurred_with_PSF.append(self.PSF_conv[i](img))
                blurred_with_PSF = torch.cat(blurred_with_PSF, dim=1)

            c1 = self.c1(blurred_with_PSF)
            d1 = self.d1(c1)
            # print('c1 ', c1.size())
            # print('d1 ', d1.size())

            c2 = self.c2(d1)
            d2 = self.d2(c2)
            # print('c2 ', c2.size())
            # print('d2 ', d2.size())

            c3 = self.c3(d2)
            # print('c3 ', c3.size())

            u1 = self.u1(c3)
            c4 = self.c4(torch.cat((u1, c2), dim=1))
            # print('u1 ', u1.size())
            # print('c4 ', c4.size())

            u2 = self.u2(c4)
            c5 = self.c5(torch.cat((u2, c1), dim=1))
            # print('u2 ', u2.size())
            # print('c5 ', c5.size())

            if self.out_fcn is None:
                sharp_pred = self.out(c5)
            else:
                sharp_pred = self.out_fcn(self.out(c5))

        return sharp_pred, blurred_with_PSF

class cycleGAN(pl.LightningModule):

    def __init__(self, hparams):
        super().__init__()
        if type(hparams) is dict:
            hparams = Namespace(**hparams)
        self.hparams = hparams

        # Generator
        self.generator = Generator(in_channels=hparams.in_channels, out_channels=hparams.out_channels,
            feat_channels=hparams.feat_channels, num_views=hparams.num_views, out_activation=hparams.out_activation,
            norm_method=hparams.norm_method, data_augmentation=hparams.data_augmentation)

        # Discriminator with full-size tile
        self.discriminator_full = Discriminator(tile_size=hparams.tile_size, in_channels=hparams.out_channels)

        # Discriminator with half-size tile
        half_tile_size = (dim // 2 for dim in hparams.tile_size)
        self.discriminator_half = Discriminator(tile_size=half_tile_size, in_channels=hparams.out_channels)

        # Cache for generated images
        self.last_views = None
        self.last_BSB_sharp_pred = None
        self.last_SBS_sharp_pred = None
        self.last_gt = None
        self.slicing_half = None

        # Cycle loss function
        if hparams.cycle_loss_function == 'l1':
            self.cycle_loss_func = F.l1_loss
        else:
            self.cycle_loss_func = F.mse_loss

    def forward(self, img, PSF=None, direction='BSB'):
        return self.generator(img, PSF, direction)

    def compute_cycle_loss(self, img_before_cycle, img_after_cycle):
        return self.cycle_loss_func(img_before_cycle, img_after_cycle, reduction='mean')

    def compute_LSGAN_loss(self, pred, mask):
        return F.mse_loss(pred, mask, reduction='mean')

    def compute_gradient_loss(self, img):
        """
        3D adaption of numpy.gradient()
        """

        dz = torch.zeros_like(img)
        dy = torch.zeros_like(img)
        dx = torch.zeros_like(img)
        out = [dz, dy, dx]
        N = img.ndimension()
        s_dim = N - 2  # Number of spatial dimensions

        # Initialize slice objects with [:, ..., :]
        slice1 = [slice(None)]*N
        slice2 = [slice(None)]*N
        slice3 = [slice(None)]*N
        slice4 = [slice(None)]*N

        for i, axis in enumerate(range(-s_dim, 0)):

            # 2nd order difference in the interior
            slice1[axis] = slice(1, -1)
            slice2[axis] = slice(None, -2)
            slice3[axis] = slice(1, -1)
            slice4[axis] = slice(2, None)

            out[i][tuple(slice1)] = (img[tuple(slice4)] - img[tuple(slice2)]) / 2

            # 1st order difference on edges
            slice1[axis] = 0
            slice2[axis] = 1
            slice3[axis] = 0

            # 1D equivalent -- out[0] = (f[1] - f[0]) / (x[1] - x[0])
            out[i][tuple(slice1)] = img[tuple(slice2)] - img[tuple(slice3)]

            slice1[axis] = -1
            slice2[axis] = -1
            slice3[axis] = -2

            # 1D equivalent -- out[-1] = (f[-1] - f[-2]) / (x[-1] - x[-2])
            out[i][tuple(slice1)] = img[tuple(slice2)] - img[tuple(slice3)]

            # Reset the slice object in this dimension to ":"
            slice1[axis] = slice(None)
            slice2[axis] = slice(None)
            slice3[axis] = slice(None)
            slice4[axis] = slice(None)

            zero_map = (out[i]==0)
            out[i][zero_map] = eps

        return torch.mean(torch.cat(out, dim=1)**2, dim=(1, 2, 3, 4))

    def training_step(self, batch, batch_idx, optimizer_idx):

        if self.current_epoch==0 and batch_idx==0 and optimizer_idx==0:
            print('Training started...')

        # Get input view, PSF and unmatched ground truth of current batch
        self.last_views, last_PSF, last_slicing = batch['view'], batch['psf'], batch['slicing']
        self.slicing_half = last_slicing[0]

        # Train the generator
        if optimizer_idx == 0:

            # Generate images
            self.last_BSB_sharp_pred, self.last_BSB_blurred_with_PSF = self.forward(self.last_views, last_PSF, 'BSB')

            # Cycle loss
            cycle_loss = self.compute_cycle_loss(self.last_views, self.last_BSB_blurred_with_PSF)

            # WGAN loss
            # Full size
            D_full = self.discriminator_full(self.last_BSB_sharp_pred.detach())
            validity_full = torch.ones_like(D_full).cuda(self.last_BSB_sharp_pred.device.index)
            LSGAN_loss_full = self.compute_LSGAN_loss(D_full, validity_full)

            # Half size
            D_half = self.discriminator_half(self.last_BSB_sharp_pred.detach()[:, :, self.slicing_half[0][0]:self.slicing_half[1][0],
                self.slicing_half[0][1]:self.slicing_half[1][1], self.slicing_half[0][2]:self.slicing_half[1][2]])
            validity_half = torch.ones_like(D_half).cuda(self.last_BSB_sharp_pred.device.index)
            LSGAN_loss_half = self.compute_LSGAN_loss(D_half, validity_half)

            LSGAN_loss = 0.5 * (LSGAN_loss_full + LSGAN_loss_half)

            # Gradient loss
            grad_loss = self.compute_gradient_loss(self.last_BSB_sharp_pred)

            total_loss = self.hparams.cycle_loss_weight * cycle_loss + LSGAN_loss + grad_loss
            tensorboard_logs = {'total_loss': total_loss, 'cycle_loss': cycle_loss, 'LSGAN_loss_G': LSGAN_loss, 'grad_loss': grad_loss}

            return OrderedDict({'loss': total_loss, 'log': tensorboard_logs})

        # Train discriminator with full-size tile
        if optimizer_idx == 1:

            # Sharp prediction
            D_sharp_pred = self.discriminator_full(self.last_BSB_sharp_pred.detach())
            validity_sharp_pred = torch.zeros_like(D_sharp_pred).cuda(self.last_BSB_sharp_pred.device.index)
            LSGAN_loss = self.compute_LSGAN_loss(D_sharp_pred, validity_sharp_pred)

            tensorboard_logs = {'LSGAN_loss_full': LSGAN_loss}

            return OrderedDict({'loss': LSGAN_loss, 'log': tensorboard_logs})

        # Train discriminator with half-size tile
        if optimizer_idx == 2:

            # Sharp prediction
            D_sharp_pred = self.discriminator_half(self.last_BSB_sharp_pred.detach()[:, :, self.slicing_half[0][0]:self.slicing_half[1][0],
                self.slicing_half[0][1]:self.slicing_half[1][1], self.slicing_half[0][2]:self.slicing_half[1][2]])
            validity_sharp_pred = torch.zeros_like(D_sharp_pred).cuda(self.last_BSB_sharp_pred.device.index)
            LSGAN_loss = self.compute_LSGAN_loss(D_sharp_pred, validity_sharp_pred)

            tensorboard_logs = {'LSGAN_loss_half': LSGAN_loss}

            return OrderedDict({'loss': LSGAN_loss, 'log': tensorboard_logs})

    def validation_step(self, batch, batch_idx):
        view, PSF, slicing = batch['view'], batch['psf'], batch['slicing']
        slicing_half = slicing[0]
        slicing_quarter = slicing[1]

        # Generate images
        BSB_sharp_pred, BSB_blurred_with_PSF = self.forward(view, PSF, 'BSB')

        # Cycle loss
        cycle_loss = self.compute_cycle_loss(view, BSB_blurred_with_PSF)

        # WGAN loss
        # Full size
        D_full = self.discriminator_full(BSB_sharp_pred)
        validity_full_1 = torch.ones_like(D_full).cuda(BSB_sharp_pred.device.index)
        validity_full_0 = torch.zeros_like(D_full).cuda(BSB_sharp_pred.device.index)
        LSGAN_loss_full_G = self.compute_LSGAN_loss(D_full, validity_full_1)
        LSGAN_loss_full_D = self.compute_LSGAN_loss(D_full, validity_full_0)

        # Half size
        D_half = self.discriminator_half(BSB_sharp_pred[:, :, slicing_half[0][0]:slicing_half[1][0],
            slicing_half[0][1]:slicing_half[1][1], slicing_half[0][2]:slicing_half[1][2]])
        validity_half_1 = torch.ones_like(D_half).cuda(BSB_sharp_pred.device.index)
        validity_half_0 = torch.zeros_like(D_half).cuda(BSB_sharp_pred.device.index)
        LSGAN_loss_half_G = self.compute_LSGAN_loss(D_half, validity_half_1)
        LSGAN_loss_half_D = self.compute_LSGAN_loss(D_half, validity_half_0)

        LSGAN_loss_G = 0.5 * (LSGAN_loss_full_G + LSGAN_loss_half_G)

        # Gradient loss
        grad_loss = self.compute_gradient_loss(BSB_sharp_pred)

        return {'cycle_loss': cycle_loss, 'LSGAN_loss_G': LSGAN_loss_G, 'LSGAN_loss_full_D': LSGAN_loss_full_D, 'LSGAN_loss_half_D': LSGAN_loss_half_D, 'grad_loss': grad_loss}

    def validation_end(self, outputs):
        avg_cycle_loss = torch.stack([x['cycle_loss'] for x in outputs]).mean()
        avg_LSGAN_loss_G = torch.stack([x['LSGAN_loss_G'] for x in outputs]).mean()
        avg_LSGAN_loss_full_D = torch.stack([x['LSGAN_loss_full_D'] for x in outputs]).mean()
        avg_LSGAN_loss_half_D = torch.stack([x['LSGAN_loss_half_D'] for x in outputs]).mean()
        avg_grad_loss = torch.stack([x['grad_loss'] for x in outputs]).mean()

        tensorboard_logs = {'val_cycle_loss': avg_cycle_loss, 'val_LSGAN_loss_G': avg_LSGAN_loss_G, 'val_LSGAN_loss_full_D': avg_LSGAN_loss_full_D, 'val_LSGAN_loss_half_D': avg_LSGAN_loss_half_D, 'val_grad_loss': avg_grad_loss}

        return {'avg_val_loss': avg_cycle_loss, 'log': tensorboard_logs}

    def configure_optimizers(self):
        opt_g = RAdam(self.generator.parameters(), lr=self.hparams.learning_rate)
        opt_d_full = RAdam(self.discriminator_full.parameters(), lr=self.hparams.learning_rate)
        opt_d_half = RAdam(self.discriminator_half.parameters(), lr=self.hparams.learning_rate)
        return [opt_g, opt_d_full, opt_d_half], []

    @pl.data_loader
    def train_dataloader(self):
         if self.hparams.train_dir is None:
            return None
         else:
            dataset = Multiview_Dataset(data_dir=self.hparams.train_dir, num_views=self.hparams.num_views, tile_size=self.hparams.tile_size, shuffle=True, data_augmentation=True)
            return DataLoader(dataset, batch_size=self.hparams.batch_size, shuffle=True)

    @pl.data_loader
    def val_dataloader(self):
        if self.hparams.val_dir is None:
            return None
        else:
            dataset = Multiview_Dataset(data_dir=self.hparams.val_dir, num_views=self.hparams.num_views, tile_size=self.hparams.tile_size, shuffle=False, data_augmentation=False)
            return DataLoader(dataset, batch_size=self.hparams.batch_size)

    def on_epoch_end(self):

        slice_idx = self.last_views.size()[2] // 2

        # Log BSB sharp prediction
        BSB_prediction_grid = torchvision.utils.make_grid(torch.cat([self.last_BSB_sharp_pred[:, :, slice_idx, :, :] for i in range(3)], dim=1))
        self.logger.experiment.add_image('BSB_sharp_pred', BSB_prediction_grid, self.current_epoch)

        # Log the input view
        img_grid = torchvision.utils.make_grid(torch.cat([self.last_views[:, 0, slice_idx, :, :].unsqueeze(1) for i in range(3)], dim=1))
        self.logger.experiment.add_image('Raw_input_view', img_grid, self.current_epoch)

    @staticmethod
    def add_model_specific_args(parent_parser):
        """
        Parameters you define here will be available to your model through self.hparams
        """

        parser = ArgumentParser(parents=[parent_parser])

        # Network params
        parser.add_argument('--in_channels', default=4, type=int)
        parser.add_argument('--out_channels', default=1, type=int)
        parser.add_argument('--feat_channels', default=64, type=int)
        parser.add_argument('--norm_method', default='instance', type=str)
        parser.add_argument('--out_activation', default='sigmoid', type=str)
        parser.add_argument('--cycle_loss_function', default='l1', type=str)
        parser.add_argument('--cycle_loss_weight', default=10.0, type=float)

        # Data
        parser.add_argument('--train_dir', default='/datasets/quad_view/train/', type=str)
        parser.add_argument('--val_dir', default='/datasets/quad_view/val/', type=str)
        parser.add_argument('--data_augmentation', default=True, type=bool)
        parser.add_argument('--num_views', default=4, type=int)

        # Training params (opt)
        parser.add_argument('--batch_size', default=1, type=int)
        parser.add_argument('--learning_rate', default=0.0001, type=float)

        return parser


# SQLite_Database.py
import sqlite3

# Step 1: create 'connection object' representing the database:
conn = sqlite3.connect('example.db')        #Do the same for accessing the database later on

# Step 2: create Cursor object to be able to perform SQL commands on the database:
c = conn.cursor()

# Step 3: adding data to the database:
    # Create table
#c.execute('''CREATE TABLE stocks
#             (date text, trans text, symbol text, qty real, price real)''')
    # Insert a row of data
#c.execute("INSERT INTO stocks VALUES ('2006-01-05','BUY','RHAT',100,35.14)")
    # Always have to save (commit) the changes
#conn.commit()

# Step 4: querying the database:
    # Put the query in an execute statement
    # fetchall() returns a list [] of tuples (rows)
a = c.execute("SELECT * FROM stocks WHERE symbol = 'RHAT'").fetchall()     #Use "" (double) so you can use '' (single) in query
    # To use variables from Python in the query, use ? as placeholder
    # and provide tuple as second argument to placeholder:
symbol = ('RHAT',) #Tuple with only one value here
print(c.execute('SELECT * FROM stocks WHERE symbol=?', symbol).fetchall())

# Step 5: inserting multiple rows at once: c.executemany:
purchases = [('2006-03-28', 'BUY', 'IBM', 1000, 45.00),
             ('2006-04-05', 'BUY', 'MSFT', 1000, 72.00),
             ('2006-04-06', 'SELL', 'IBM', 500, 53.00),
            ]
#c.executemany("INSERT INTO stocks VALUES (?,?,?,?,?)", purchases) #Executemany takes list of tuples as second argument
#conn.commit()      #Don't forget commit!
try:  # Deal with last part of the data: general data without publishIndex
    publish_index = sensor['publishIndex']
except:
    publish_index = 17  # Last element on the device list is the 'general element'
# Step 5: close the connection (changes are lost if not commited before closing)
#conn.close()


from flask import Flask, jsonify, request
app = Flask(__name__)
contacts = [{'Name':'Ram', 'Number':'2123'}, {'Name':'Geeta','Number':'9034'},{'Name':'Mukesh','Number':'3139'}]

@app.route('/get-contact')

def getContacts():
    return jsonify({'contact':contacts})

@app.route('/add-contact', methods = ['POST'])

def addContacts():
    if not request.json:
        return jsonify({'status':'error', 'message':'Please provide the data'}, 400)
    contact = {'Name':request.json['Name'], 'Number':request.json['Number']}
    contacts.append(contact)
    return jsonify({'status':'Success', 'message':'Contact Added Sucessfully'})
if __name__ == '__main__':
    app.run(debug = True)
print(contacts)


import socket
import time
import numpy as np
import cv2

class CameraClient(object):

    def __init__(self):
        self.initSocketClient()

    def initSocketClient(self):
        host = "raspberrypi"
        port = 2222
        clientSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        clientSocket.connect((host, port))
        streamSocket = clientSocket.makefile('rb')
        print("Connected to camera server")
        try:
            print("Streaming...")
            stream_bytes = " "

            while True:
                stream_bytes += streamSocket.read(1024)
                first = stream_bytes.find('\xff\xd8')
                last  = stream_bytes.find('\xff\xd9')
                if first != -1 and last != -1:
                    jpg = stream_bytes[first:last + 2]
                    stream_bytes = stream_bytes[last + 2:]
                    image = cv2.imdecode(np.fromstring(jpg, dtype=np.uint8), cv2.CV_LOAD_IMAGE_UNCHANGED)

                    #cv2.namedWindow("window", cv2.WND_PROP_FULLSCREEN)
                    #cv2.setWindowProperty("window",cv2.WND_PROP_FULLSCREEN,cv2.WINDOW_FULLSCREEN)
                    cv2.imshow('Camera', image)

                    if cv2.waitKey(1) & 0xFF == ord('q'):
                        break
        finally:
            clientSocket.close()
            streamSocket.close()

#Main Function
if __name__ == '__main__':
    CameraClient()


import numpy as np
import math, sys, time
import numpy as np
import scipy.linalg, scipy.constants
import matplotlib.pyplot as plt
from scipy import signal
import pymatgen as pmg
import itertools
from pymatgen.core import Lattice
from pymatgen.core.tensors import Tensor
from pymatgen.analysis.elasticity.elastic import ElasticTensor
from pymatgen.analysis.elasticity.elastic import ElasticTensorExpansion
import pandas as pd

data = np.genfromtxt("./pcu_2.log", skip_footer=29, skip_header=17)

a_mean = []
b_mean = []
c_mean = []
alpha_mean = []
beta_mean = []
gamma_mean = []
vol_mean = []
mechanical_eig_max = []
mechanical_eig_min = []
temp = []

h = []
for frame in data:
    lattice = Lattice.from_lengths_and_angles([frame[14], frame[15], frame[16]], [frame[17], frame[18], frame[19]])
    hmatrix = lattice.matrix
    h.append(hmatrix)

a_mean.append(np.mean(data[:,14]))
b_mean.append(np.mean(data[:,15]))
c_mean.append(np.mean(data[:,16]))
alpha_mean.append(np.mean(data[:,17]))
beta_mean.append(np.mean(data[:,18]))
gamma_mean.append(np.mean(data[:,19]))
vol_mean.append(np.mean(data[:,13]))
temp.append(np.mean(data[:,11]))

h = np.array(h)
h0 = np.mean(h, axis=0)
h0_std = np.std(h, axis=0)
h0m1  = np.linalg.inv(h0)
h0m1t = np.linalg.inv(np.transpose(h0))
eps_mat = np.array(list(map(lambda hi : (np.dot(h0m1t, np.dot(hi.transpose(), np.dot(hi, h0m1))) - np.identity(3))/2.0, h)))
eps_voigt = np.empty([6,eps_mat.shape[0]], "d")
eps_voigt[0] = eps_mat[:,0,0]
eps_voigt[1] = eps_mat[:,1,1]
eps_voigt[2] = eps_mat[:,2,2]
eps_voigt[3] = eps_mat[:,2,1]
eps_voigt[4] = eps_mat[:,2,0]
eps_voigt[5] = eps_mat[:,1,0]
eps_voigt_mean = np.mean(eps_voigt, axis=1)
Smat = np.zeros([6,6], "d")
for i in range(6):
    for j in range(i,6):
        Smat[i,j] = np.mean(eps_voigt[i]*eps_voigt[j])-eps_voigt_mean[i]*eps_voigt_mean[j]
        if i != j: Smat[j,i] = Smat[i,j]
T=300
V = np.linalg.det(h0)
Cmat = np.linalg.inv(Smat)*((1.3806488E-23*T)/(V*1E-30))*1E-9
Cmat = np.around(Cmat, decimals=2)
print(Cmat)
Cmat_eig = np.linalg.eigvals(Cmat)
mechanical_eig_max.append(max(Cmat_eig))
mechanical_eig_min.append(min(Cmat_eig))
print(mechanical_eig_max)
print(mechanical_eig_min)

mat = Tensor.from_voigt(Cmat)
elastic = ElasticTensor(mat)
print(elastic.k_vrh)
#np.savetxt("pcu_1.cij", Cmat)


#Barbaric approach at /r/place timelapese program using pygame for visualization

import pygame, sys

#I could've put everything into a single list, but idk, this feels cleaner
time = []
coord = []
color = []
color_list = [
    (255, 255, 255),
    (228, 228, 228),
    (136, 136, 136),
    (34, 34, 34),
    (255, 167, 209),
    (229, 0, 0),
    (229, 149, 0),
    (160, 106, 66),
    (229, 217, 0),
    (148, 224, 68),
    (2, 190, 1),
    (0, 229, 240),
    (0, 131, 199),
    (0, 0, 234),
    (224, 74, 255),
    (130, 0, 128),
    ]

running = True
length = 0
speed = 10000
current_time = 0

#Loading the file and seperating it into different lists
path = "data/place.csv"
with open(path) as f:
    content = f.readlines()
    for x in content:
        temp = x.strip() #remove unnecessary things from string
        temp = temp.split(",") #seperate the string by ","
        #Adding the seperated string into the lists
        time.append(int(temp[0]))
        coord.append((int(temp[1]), int(temp[2])))
        color.append(int(temp[3]))
        length += 1

#Pygame initialisation stuff
pygame.init()
window = pygame.display.set_mode((1200,1000))
pygame.display.set_caption("/r/place")
font = pygame.font.SysFont('Arial', 30)
pygame.key.set_repeat(1,1)

#Function for writing text on screen
def text(string, coords):
    string = str(string) + "          "
    temp = font.render(string, True, (0,0,0), (255,255,255))
    window.blit(temp, coords)

#Fill the screen with white once at start
window.fill((255, 255, 255))

#Recalculates frame, insanely slow, probably going to add some key frames
def recalculate():
    window.fill((255, 255, 255))
    for i in range(0, current_time):
        window.set_at(coord[i], color_list[color[i]])

#Main loop
while running:
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        if event.type == pygame.KEYDOWN:
            if event.key == pygame.K_KP_PLUS: speed += 2
            if event.key == pygame.K_KP_MINUS: speed -= 2
            if event.key == pygame.K_KP1: speed = 1000
            if event.key == pygame.K_KP2: speed = 20000
            if event.key == pygame.K_KP0: #Resets to zero
                current_time = 0
                recalculate()
            if event.key == pygame.K_LEFT: #If you go backwards it has to recalculate everything
                current_time -= speed
                if current_time < 0: current_time = 0
                recalculate()
            if event.key == pygame.K_RIGHT: #Going forwards
                current_time += speed
                if current_time > length: current_time = length
                for i in range(current_time - speed, current_time):
                    window.set_at(coord[i], color_list[color[i]])

    text("Time:", (1020,20))
    text(current_time, (1020,60))
    text("Speed:", (1020, 120))
    text(speed, (1020, 160))

    pygame.display.update()


import os
import time
import argparse
import numpy as np
import pandas as pd
from tqdm import tqdm
from sklearn.metrics import roc_auc_score
import torch
from torch.utils.data import DataLoader
import torch.nn as nn
from dataset import get_dataframe, get_transforms, resamplingDataset
from models import Effnet_MMC, Resnest_MMC, Seresnext_MMC
from utils.util import *
from utils.torchsummary import summary

Precautions_msg = '(주의사항) Stone dataset의 경우 사람당 4장의 이미지기때문에 batch사이즈를 4의 배수로 해야 제대로 평가 된다.'

'''
- evaluate.py

학습한 모델을 평가하는 코드
Test셋이 아니라 학습때 살펴본 validation셋을 활용한다.
grad-cam 실행한다.

#### 실행법 ####
Terminal을 이용하는 경우 경로 설정 후 아래 코드를 직접 실행
python evaluate.py --kernel-type 5fold_b3_256_30ep --data-folder original_stone/ --enet-type tf_efficientnet_b3 --n-epochs 30

pycharm의 경우:
Run -> Edit Configuration -> evaluate.py 가 선택되었는지 확인
-> parameters 이동 후 아래를 입력 -> 적용하기 후 실행/디버깅
--kernel-type 5fold_b3_256_30ep --data-folder original_stone/ --enet-type tf_efficientnet_b3_ns --n-epochs 30 --k-fold 5

edited by MMCLab, 허종욱, 2020
'''

def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('--kernel-type', type=str, required=True)
    parser.add_argument('--data-dir', type=str, default='./data/')
    parser.add_argument('--data-folder', type=str, required=True)
    parser.add_argument('--image-size', type=int, required=True)
    parser.add_argument('--enet-type', type=str, required=True)
    parser.add_argument('--batch-size', type=int, default=4)
    parser.add_argument('--num-workers', type=int, default=32)
    parser.add_argument('--out-dim', type=int, default=2)
    parser.add_argument('--use-amp', action='store_true')
    parser.add_argument('--use-meta', action='store_true')
    parser.add_argument('--use-ext', action='store_true')
    parser.add_argument('--DEBUG', action='store_true')
    parser.add_argument('--model-dir', type=str, default='./weights')
    parser.add_argument('--log-dir', type=str, default='./logs')
    parser.add_argument('--oof-dir', type=str, default='./oofs')

    parser.add_argument('--k-fold', type=int, default=5)
    parser.add_argument('--eval', type=str, choices=['best', 'best_no_ext', 'final'], default="best")
    parser.add_argument('--CUDA_VISIBLE_DEVICES', type=str, default='0')
    parser.add_argument('--n-meta-dim', type=str, default='512,128')

    args, _ = parser.parse_known_args()
    return args

def val_epoch(model, loader, target_idx, is_ext=None, n_test=1, get_output=False):

    model.eval()
    val_loss = []
    LOGITS = []
    PROBS = []
    TARGETS = []
    with torch.no_grad():
        for (data, target) in tqdm(loader):

            if args.use_meta:
                data, meta = data
                data, meta, target = data.to(device), meta.to(device), target.to(device)
                logits = torch.zeros((data.shape[0], args.out_dim)).to(device)
                probs = torch.zeros((data.shape[0], args.out_dim)).to(device)
                for I in range(n_test):
                    l = model(get_trans(data, I), meta)
                    logits += l
                    probs += l.softmax(1)
            else:
                data, target = data.to(device), target.to(device)
                logits = torch.zeros((data.shape[0], args.out_dim)).to(device)
                probs = torch.zeros((data.shape[0], args.out_dim)).to(device)
                for I in range(n_test):
                    l = model(get_trans(data, I))
                    logits += l
                    probs += l.softmax(1)
            logits /= n_test
            probs /= n_test

            LOGITS.append(logits.detach().cpu())
            PROBS.append(probs.detach().cpu())
            TARGETS.append(target.detach().cpu())

            loss = criterion(logits, target)
            val_loss.append(loss.detach().cpu().numpy())

    val_loss = np.mean(val_loss)
    LOGITS = torch.cat(LOGITS).numpy()
    PROBS = torch.cat(PROBS).numpy()
    TARGETS = torch.cat(TARGETS).numpy()

    if get_output:
        return LOGITS, PROBS
    else:
        acc = (PROBS.argmax(1) == TARGETS).mean() * 100.
        auc = roc_auc_score((TARGETS == target_idx).astype(float), PROBS[:, target_idx])
        auc_no_ext = roc_auc_score((TARGETS[is_ext == 0] == target_idx).astype(float), PROBS[is_ext == 0, target_idx])
        return val_loss, acc, auc, auc_no_ext

def val_epoch_stonedata(model, loader, target_idx, is_ext=None, n_test=1, get_output=False):

    model.eval()
    val_loss = []
    LOGITS = []
    PROBS = []
    TARGETS = []

    with torch.no_grad(): # grad-cam 할때는 꺼야함
        for (data, target) in tqdm(loader):

            if args.use_meta:
                data, meta = data
                data, meta, target = data.to(device), meta.to(device), target.to(device)
                logits = torch.zeros((data.shape[0], args.out_dim)).to(device)
                probs = torch.zeros((data.shape[0], args.out_dim)).to(device)
                for I in range(n_test):
                    l = model(get_trans(data, I), meta)
                    logits += l
                    probs += l.softmax(1)
            else:
                data, target = data.to(device), target.to(device)
                logits = torch.zeros((data.shape[0], args.out_dim)).to(device)
                probs = torch.zeros((data.shape[0], args.out_dim)).to(device)
                for I in range(n_test):
                    l = model(get_trans(data, I))
                    logits += l
                    probs += l.softmax(1)
            logits /= n_test
            probs /= n_test

            # 4장 의료 데이터를 묶음으로 확률계산. 평균이용
            for b_i in range(int(data.shape[0] / 4)):
                b_i4 = b_i*4
                logits[0+b_i4:4+b_i4, 0] = torch.mean(logits[0+b_i4:4+b_i4, 0])
                logits[0+b_i4:4+b_i4, 1] = torch.mean(logits[0+b_i4:4+b_i4, 1])
                probs[0+b_i4:4+b_i4, 0] = torch.mean(probs[0+b_i4:4+b_i4, 0])
                probs[0+b_i4:4+b_i4, 1] = torch.mean(probs[0+b_i4:4+b_i4, 1])

            LOGITS.append(logits.detach().cpu())
            PROBS.append(probs.detach().cpu())
            TARGETS.append(target.detach().cpu())

            loss = criterion(logits, target)
            val_loss.append(loss.detach().cpu().numpy())

    val_loss = np.mean(val_loss)
    LOGITS = torch.cat(LOGITS).numpy()
    PROBS = torch.cat(PROBS).numpy()
    TARGETS = torch.cat(TARGETS).numpy()

    if get_output:
        return LOGITS, PROBS, TARGETS
    else:
        acc = (PROBS.argmax(1) == TARGETS).mean() * 100.
        auc = roc_auc_score((TARGETS == target_idx).astype(float), PROBS[:, target_idx])
        auc_no_ext = roc_auc_score((TARGETS[is_ext == 0] == target_idx).astype(float), PROBS[is_ext == 0, target_idx])
        return val_loss, acc, auc, auc_no_ext

def main():

    '''
    ####################################################
    # stone data 데이터셋 : dataset.get_df_stone
    ####################################################
    '''
    df_train, df_test, meta_features, n_meta_features, target_idx = get_dataframe(
        k_fold = args.k_fold,
        out_dim = args.out_dim,
        data_dir = args.data_dir,
        data_folder = args.data_folder,
        use_meta = args.use_meta,
        use_ext = args.use_ext
    )

    transforms_train, transforms_val = get_transforms(args.image_size)

    LOGITS = []
    PROBS = []
    TARGETS = []
    dfs = []

    folds = range(args.k_fold)
    for fold in folds:
        print(f'Evaluate data fold{str(fold)}')
        df_valid = df_train[df_train['fold'] == fold]

        # batch_normalization에서 배치 사이즈 1인 경우 에러 발생할 수 있으므로, 데이터 한개 버림
        if len(df_valid) % args.batch_size == 1:
            df_valid = df_valid.sample(len(df_valid)-1)

        if args.DEBUG:
            df_valid = pd.concat([
                df_valid[df_valid['target'] == target_idx].sample(args.batch_size * 3),
                df_valid[df_valid['target'] != target_idx].sample(args.batch_size * 3)
            ])

        dataset_valid = resamplingDataset(df_valid, 'valid', meta_features, transform=transforms_val)
        valid_loader = torch.utils.data.DataLoader(dataset_valid, batch_size=args.batch_size, num_workers=args.num_workers)

        if args.eval == 'best':
            model_file = os.path.join(args.model_dir, f'{args.kernel_type}_best_fold{fold}.pth')
        elif args.eval == 'best_no_ext':
            model_file = os.path.join(args.model_dir, f'{args.kernel_type}_best_no_ext_fold{fold}.pth')
        if args.eval == 'final':
            model_file = os.path.join(args.model_dir, f'{args.kernel_type}_final_fold{fold}.pth')

        model = ModelClass(
            args.enet_type,
            n_meta_features=n_meta_features,
            n_meta_dim=[int(nd) for nd in args.n_meta_dim.split(',')],
            out_dim=args.out_dim
        )
        model = model.to(device)

        # model summary
        if args.use_meta:
            pass
            # 코드 확인이 필요함
            # summary(model, [(3, args.image_size, args.image_size), n_meta_features])
        else:
            if fold == 0: # 한번만
                summary(model, (3, args.image_size, args.image_size))

        try:  # single GPU model_file
            model.load_state_dict(torch.load(model_file), strict=True)
        except:  # multi GPU model_file
            state_dict = torch.load(model_file)
            state_dict = {k[7:] if k.startswith('module.') else k: state_dict[k] for k in state_dict.keys()}
            model.load_state_dict(state_dict, strict=True)

        if len(os.environ['CUDA_VISIBLE_DEVICES']) > 1:
            model = torch.nn.DataParallel(model)

        model.eval()

        '''
        ####################################################
        # stone data를 위한 평가함수 : val_epoch_stonedata
        ####################################################
        '''
        this_LOGITS, this_PROBS, this_TARGETS = val_epoch_stonedata(model, valid_loader, target_idx, is_ext=df_valid['is_ext'].values, n_test=8, get_output=True)
        LOGITS.append(this_LOGITS)
        PROBS.append(this_PROBS)
        TARGETS.append(this_TARGETS)
        dfs.append(df_valid)

    dfs = pd.concat(dfs).reset_index(drop=True)
    dfs['pred'] = np.concatenate(PROBS).squeeze()[:, target_idx]

    Accuracy = (round(dfs['pred'])== dfs['target']).mean() * 100.
    auc_all_raw = roc_auc_score(dfs['target'] == target_idx, dfs['pred'])

    dfs2 = dfs.copy()
    for i in folds:
        dfs2.loc[dfs2['fold'] == i, 'pred'] = dfs2.loc[dfs2['fold'] == i, 'pred'].rank(pct=True)
    auc_all_rank = roc_auc_score(dfs2['target'] == target_idx, dfs2['pred'])

    if args.use_ext:
        # 외부데이터를 사용할 경우, 외부데이터를 제외하고 모델을 따로 평가해본다.
        dfs3 = dfs[dfs.is_ext == 0].copy().reset_index(drop=True)
        auc_no_ext_raw = roc_auc_score(dfs3['target'] == target_idx, dfs3['pred'])

        for i in folds:
            dfs3.loc[dfs3['fold'] == i, 'pred'] = dfs3.loc[dfs3['fold'] == i, 'pred'].rank(pct=True)
        auc_no_ext_rank = roc_auc_score(dfs3['target'] == target_idx, dfs3['pred'])

        content = time.ctime() + ' ' + f'Eval {args.eval}:\nAccuracy : {Accuracy:.5f}\n' \
                                       f'auc_all_raw : {auc_all_raw:.5f}\nauc_all_rank : {auc_all_rank:.5f}\n' \
                                       f'auc_no_ext_raw : {auc_no_ext_raw:.5f}\nauc_no_ext_rank : {auc_no_ext_rank:.5f}\n'
    else:
        content = time.ctime() + ' ' + f'Eval {args.eval}:\nAccuracy : {Accuracy:.5f}\n' \
                  f'AUC_all_raw : {auc_all_raw:.5f}\nAUC_all_rank : {auc_all_rank:.5f}\n'

    # 로그파일 맨 뒤에 결과 추가해줌
    print(content)
    with open(os.path.join(args.log_dir, f'log_{args.kernel_type}.txt'), 'a') as appender:
        appender.write(content + '\n')

    np.save(os.path.join(args.oof_dir, f'{args.kernel_type}_{args.eval}_oof.npy'), dfs['pred'].values)

    # 결과 csv 저장
    dfs[['filepath', 'patient_id', 'target', 'pred']].to_csv(os.path.join(args.oof_dir, f'{args.kernel_type}_{args.eval}_oof.csv'), index=True)

if __name__ == '__main__':

    print('----------------------------')
    print(Precautions_msg)
    print('----------------------------')
    args = parse_args()
    os.makedirs(args.oof_dir, exist_ok=True)
    if args.use_gradcam:
        os.makedirs(os.path.join(args.gradcam_dir, args.kernel_type), exist_ok=True)
    os.environ['CUDA_VISIBLE_DEVICES'] = args.CUDA_VISIBLE_DEVICES

    if args.enet_type == 'resnest101':
        ModelClass = Resnest_MMC
    elif args.enet_type == 'seresnext101':
        ModelClass = Seresnext_MMC
    elif 'efficientnet' in args.enet_type:
        ModelClass = Effnet_MMC
    else:
        raise NotImplementedError()

    DP = len(os.environ['CUDA_VISIBLE_DEVICES']) > 1

    device = torch.device('cuda')
    criterion = nn.CrossEntropyLoss()

    main()


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import csv
import re
import time
import os

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from seleniumwire import webdriver as wiredriver
from selenium.webdriver.chrome.service import Service as ChromeService

TIMER_SLEEP = 2
# Configurações do Chrome
chrome_options = Options()
# Descomente a linha abaixo se você quiser executar o Chrome em modo headless (sem interface gráfica)
# chrome_options.add_argument("--headless")

# Configurações do Selenium Wire
# proxy_address = 'https://179.83.82.10'  # Substitua pelo endereço do seu proxy
# seleniumwire_options = {
#     'backend': 'seleniumwire',
#     'proxy': {
#         'http': proxy_address,
#         'https': proxy_address,
#         'no_proxy': 'localhost,127.0.0.1',  # Domínios que devem ser acessados diretamente sem usar o proxy
#     }
# }

# # # Configuração do serviço do ChromeDriver
# # service = webdriver.chrome.service.Service('chromedriver.exe')

# # # Inicialize o driver do Chrome com as opções do Chrome e do Selenium Wire
# driver = wiredriver.Chrome(service=service, chrome_options=chrome_options, seleniumwire_options=seleniumwire_options)

# # Inicialize o servidor proxy do Selenium Wire
# proxy = driver.request_interceptor

# # Abra a página de login
# url = 'https://sales.ticketsforfun.com.br/#/authentication/login'
# driver.get(url)
# # Defina a função JavaScript a ser executada no navegador
# js_script = """
# function interceptXHR() {
#     console.log("Executando script");
#     try {
#         // Interceptar as requisições XHR
#         var originalOpen = window.XMLHttpRequest.prototype.open;
#         var originalSend = window.XMLHttpRequest.prototype.send;

#         window.XMLHttpRequest.prototype.open = function(method, url) {
#             // Salvar a URL da requisição para uso posterior
#             this._url = url;
#             originalOpen.apply(this, arguments);
#         };

#         window.XMLHttpRequest.prototype.send = function(data) {
#             // Verificar se a URL da requisição corresponde ao padrão desejado
#             if (this._url && this._url.startsWith('https://services.tix.byinti.com/neofront-v3/authentication/send-pin-code?')) {

#                 // Modificar a URL da requisição
#                 var modifiedUrl = this._url.replace('email=jalmirrosa6%40gmail.com', 'email=igorbzreis%40gmail.com');

#                 // Chamar o método originalOpen para aplicar a modificação
#                 originalOpen.call(this, 'GET', modifiedUrl, true);

#                 // Agora, chamar o método originalSend com os dados modificados
#                 originalSend.call(this, data);

#                 console.log("Requisição modificada e enviada:", modifiedUrl);
#             } else {
#                 // Se não for a requisição desejada, chamar o método originalSend sem modificação
#                 originalSend.call(this, data);
#             }
#         };
#     } catch (error) {
#         console.error('Erro na interceptação XHR:', error.message);
#     }
# }

# function addClickListenersToButtons() {
#     try {
#         // Adicionar um ouvinte de evento de clique a todos os botões
#         var buttons = document.querySelectorAll('button');
#         buttons.forEach(function(button) {
#             button.addEventListener('click', function(event) {
#                 console.log("Botão clicado! Texto do botão: " + event.target.innerText);
#                 // Chamar a função de interceptação de requisições ao clicar em um botão
#                 interceptXHR();
#             });
#         });
#     } catch (error) {
#         console.error('Erro ao adicionar ouvinte de evento de clique:', error.message);
#     }
# }

# // Executar a função de interceptação de requisições ao carregar a página
# interceptXHR();

# // Executar a função de adicionar ouvinte de evento de clique ao carregar a página
# addClickListenersToButtons();

# """

# # Execute o script JavaScript no navegador
# driver.execute_script(js_script)

# # Caminho do seu driver do Selenium (exemplo usando o Chrome)
# # Certifique-se de baixar o driver correspondente à sua versão do navegador

# # Aguarde alguns segundos para a página carregar completamente
# time.sleep(5)

# # Permita que o usuário interaja com a página
# input("Interaja com a página. Pressione Enter quando terminar.")

# # Fechar o navegador
# driver.quit()

# # Configuração do serviço do ChromeDriver
TIMER_SLEEP = 2
chrome_options = Options()
chrome_service = ChromeService(executable_path='chromedriver.exe')
chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument("--headless")
driver = webdriver.Chrome(service=chrome_service, options=chrome_options)

# 1° Acesse o link -
url = "https://busca.inpi.gov.br/pePI/jsp/marcas/Pesquisa_num_processo.jsp"
driver.get(url)
wait = WebDriverWait(driver, 10)
continuar_link = wait.until(EC.visibility_of_element_located((By.XPATH, "//a[contains(text(), 'Continuar...')]")))
continuar_link.click()
url = "https://busca.inpi.gov.br/pePI/jsp/marcas/Pesquisa_num_processo.jsp"
driver.get(url)
time.sleep(5)

# CODIGO QUE VAI SE REPETIR
def buscar_marca(num_pedido):
    print(f"Esperando {TIMER_SLEEP} segundos...")
    time.sleep(TIMER_SLEEP)
    num_processo = ""
    nome_marca = ""
    situacao = ""
    apresentacao = ""
    natureza = ""
    classificacao_nice = ""
    data_de_deposito = ""
    classificacao_de_viena = ""
    texto_do_despacho = ""

    url = "https://busca.inpi.gov.br/pePI/jsp/marcas/Pesquisa_num_processo.jsp"
    driver.get(url)

    try:
        # 2° Busque o input com name="NumPedido" e insira o código
        # num_pedido = "927361949"  # Substitua pelo número de pedido desejado
        input_num_pedido = driver.find_element(By.NAME, "NumPedido")
        input_num_pedido.send_keys(num_pedido)
        input_num_pedido.send_keys(Keys.RETURN)
    except Exception as e:
        print("823 - Erro ao procurar por processo - ", e)

    try:
        # Encontre o elemento <a> com o atributo href contendo "/pePI/servlet/MarcasServletController?Action=detail&amp"
        elemento_a = driver.find_element(By.XPATH,f"//a[contains(text(), '{num_pedido}')]")
        href = elemento_a.get_attribute("href")
        driver.get(href)
    except Exception as e:
        print("753 - Não encontrado na busca - ", e)

    try:
    # 4° Na página aberta, faça um crawler das informações da penúltima tabela
    # Aguarde até que a penúltima tabela seja visível
        tabelas = wait.until(EC.presence_of_all_elements_located((By.TAG_NAME, "table")))

        informacoes = {}
        # Localize os elementos <td> que contêm os rótulos
        rotulos = driver.find_elements(By.XPATH,"//td[contains(font, 'Nº do Processo:') or contains(font, 'Marca:') or contains(font, 'Situação:') or contains(font, 'Apresentação:') or contains(font, 'Natureza:') or contains(font, 'Apostila :')]")
        # Itere sobre os rótulos e capture os valores correspondentes
        for rotulo in rotulos:
            chave = rotulo.find_element(By.XPATH, "font").text.strip().replace(":", "")
            valor = rotulo.find_element(By.XPATH, "following-sibling::td/font").text.strip()
            informacoes[chave] = valor

        # Exiba as informações coletadas
        for chave, valor in informacoes.items():
            num_processo = informacoes['Nº do Processo']
            nome_marca = informacoes['Marca']
            situacao = informacoes['Situação']
            apresentacao = informacoes['Apresentação']
            natureza = informacoes['Natureza']

            # print(f"{chave}: {valor}")
    except Exception as e:
        print(f"423 - Erro ao coletar informações da tabela: {num_pedido} - ", e)

    # CAPTURAR CLASSIFICAÇÃO NICE
    try:
        class_nice = driver.find_element(By.XPATH, '//*[@id="principal"]/div[3]/div/div/table/tbody/tr/td[1]/font/a')
        classificacao_nice = class_nice.text
        # print("Classificacao NICE: ", class_nice.text)
    except Exception as e:
        print("831 - Erro ao capturar classificação NICE - ", e)

    #CAPTURAR A DATA DE DEPOSITO
    try:
       # Encontre a tabela com o elemento font que contém 'Data de Depósito'
        table_element = driver.find_element(By.XPATH, "//table[contains(.//font, 'Data de Depósito')]")

        # Em seguida, encontre o primeiro elemento th dentro do tbody
        th_element = table_element.find_element(By.XPATH, ".//tbody/tr/th[1]")

        # Capture o texto do primeiro th
        data_de_deposito = th_element.text
    except Exception as e:
        print("712 - Erro ao capturar data de deposito - ", e)

    # CAPTURAR CLASSIFICAÇÃO DE VIENA
    try:
        # Encontre a tabela usando o XPath especificado
        tabela = driver.find_element(By.XPATH, "//*[@id='principal']/div[4]/div/div/table")
        # Encontre todas as linhas da tabela dentro do corpo (elementos <tr>)
        linhas = tabela.find_elements(By.XPATH, ".//tbody/tr")
        # Itere sobre as linhas para obter os valores
        valores_totais = []
        for linha in linhas:
            # Encontre as colunas em cada linha (elementos <td>)
            colunas = linha.find_elements(By.XPATH, ".//td")
            # Obtenha os valores de cada coluna
            valores = [coluna.text for coluna in colunas]

            valores_totais.append(valores)
            # Faça algo com os valores, por exemplo, imprima-os
            # print("Classificacao de VIENA: ", valores)

        classificacao_de_viena = valores_totais
    except Exception as e:
        print("483 - Erro ao capturar classificação de VIENA - ", e)

    try:
        # Definir o padrão de regex
        padrao = r"A marca reproduz ou imita os seguintes registros de terceiros[^;]*"
        correspondencias = re.search(padrao, driver.page_source)
        if correspondencias:
            texto_do_despacho = correspondencias.group(0)
            # print("Texto completo: ", texto_do_despacho)

        else:
            print("Padrão não encontrado na página.")

    except Exception as e:
        print("9123 - Erro ao capturar detalhe do despacho:", e)

    try:
        #SALVANDO IMAGEM
        time.sleep(1)
        url = driver.current_url
        match = re.search(r'CodPedido=(\d+)', url)
        cod_pedido = ""
        if match:
            cod_pedido = match.group(1)
            # print("CodPedido:", cod_pedido)

        linkimg = f"https://busca.inpi.gov.br/pePI/servlet/LogoMarcasServletController?Action=image&codProcesso={cod_pedido}".split("/")[-1]

        elemento_img = driver.find_element(By.CSS_SELECTOR, f"img[src*='{linkimg}']")
        # print(elemento_img, linkimg)

        with open(f'imgs/imgs_revista/{num_pedido}.png', 'wb') as file:
            file.write(elemento_img.screenshot_as_png)
    except Exception as e:
        print(f"923 - Processo {cod_pedido} sem imagem ")

    return num_processo, nome_marca, situacao, apresentacao, natureza, classificacao_nice, data_de_deposito, classificacao_de_viena, texto_do_despacho

cabecalho = [
                'N° Processo',
                'Marca Indeferida',
                'Situação',
                'Apresentação',
                'Natureza',
                'Classe de Nice',
                'Classificação de Viena',
                'Data de Deposito',

                'N° Processo',
                'Marca Registrada',
                'Situação',
                'Apresentação',
                'Natureza',
                'Classe de Nice',
                'Classificação de Viena',
                'Data de Deposito',

                'Despacho',
                'Revista',
                'Visual',
                'Escrita',
                'Fonética'
            ]

def processo_ja_existe(processo_indeferido, processo_deferido, nome_arquivo):
    with open(nome_arquivo, 'r', newline='', encoding='utf-8') as arquivo_csv:
        leitor = csv.reader(arquivo_csv, delimiter='\t')

        # Pula o cabeçalho
        next(leitor, None)

        # Verifica se o número do processo já existe nas colunas 1 e 9
        for linha in leitor:
            if linha[0] == processo_indeferido or linha[8] == processo_deferido:
                return True
    return False

for revista in os.listdir('revistas'):
    print(f"Analisando revista {revista}")

    total_analisado = 0
    dados = []

    try:
        with open(f'revistas/{revista}', newline='', encoding='utf-8') as arquivo_csv:
            leitor_csv = csv.reader(arquivo_csv, delimiter="\t")
            next(leitor_csv)
            for linha in leitor_csv:
                processo_indeferido = linha[1]

                texto_complementar = linha[3]
                numeros_de_processo = re.findall(r'Processo (\d+)', texto_complementar)
                print("Processo Indeferido: ", processo_indeferido, "Processos deferidos: ", numeros_de_processo)

                # busco pelo processo indeferido
                num_processo_in, nome_marca_in, situacao_in, apresentacao_in, natureza_in, classificacao_nice_in, data_de_deposito_in, classificacao_de_viena_in, texto_do_despacho_in = buscar_marca(processo_indeferido)
                print("Marca indeferida: ", num_processo_in, nome_marca_in, situacao_in, apresentacao_in, natureza_in, classificacao_nice_in, data_de_deposito_in, classificacao_de_viena_in )
                # agora busco pelos processo deferido (marcasregistradas)

                for processo_deferido in numeros_de_processo:
                #crawler para buscar processos indeferidos
                    if not processo_ja_existe(processo_indeferido, processo_deferido,   f'{revista}_analisada.csv'):
                        num_processo, nome_marca, situacao, apresentacao, natureza, classificacao_nice, data_de_deposito, classificacao_de_viena, texto_do_despacho = buscar_marca(processo_deferido)
                        print("Marca deferida: ", num_processo, nome_marca, situacao, apresentacao, natureza, classificacao_nice, data_de_deposito, classificacao_de_viena)

                        dados.append(
                            [num_processo_in, nome_marca_in, situacao_in, apresentacao_in, natureza_in, classificacao_nice_in, classificacao_de_viena_in, data_de_deposito_in,
                            num_processo, nome_marca, situacao, apresentacao, natureza, classificacao_nice, classificacao_de_viena, data_de_deposito,
                            texto_do_despacho_in, revista, 0, 0, 0]
                        )

                        novo_registro = [num_processo_in, nome_marca_in, situacao_in, apresentacao_in, natureza_in, classificacao_nice_in, classificacao_de_viena_in, data_de_deposito_in,
                            num_processo, nome_marca, situacao, apresentacao, natureza, classificacao_nice, classificacao_de_viena, data_de_deposito,
                            texto_do_despacho_in, revista, 0, 0, 0]

                        nome_arquivo = f'{revista}_analisada.csv'
                        arquivo_existe = os.path.exists(nome_arquivo)

                        with open(nome_arquivo, 'a', newline='', encoding='utf-8') as arquivo_csv:
                            escritor = csv.writer(arquivo_csv, delimiter='\t')

                            if not arquivo_existe or arquivo_csv.tell() == 0:
                                escritor.writerow(cabecalho)

                            escritor.writerow(novo_registro)

                    else:
                        print(f"Processo {processo_indeferido} e processo {processo_deferido} já analisado!")

                    total_analisado += 1

                print("Total Analisado: ", total_analisado)

    except Exception as e:
        print(f"Erro ao analisar revista {revista}. Total Analisado: {total_analisado} -- ", e)

    print("Salvando informações...")

    # cabecalho = [
    #             'N° Processo',
    #             'Marca Indeferida',
    #             'Situação',
    #             'Apresentação',
    #             'Natureza',
    #             'Classe de Nice',
    #             'Classificação de Viena',
    #             'Data de Deposito',

    #             'N° Processo',
    #             'Marca Registrada',
    #             'Situação',
    #             'Apresentação',
    #             'Natureza',
    #             'Classe de Nice',
    #             'Classificação de Viena',
    #             'Data de Deposito',

    #             'Despacho',
    #             'Revista',
    #             'Visual',
    #             'Escrita',
    #             'Fonética'
    #         ]

    # nome_arquivo = f'{revista}_analisada.csv'
    # # Crie e escreva o arquivo CSV
    # with open(nome_arquivo, 'w', newline='', encoding='utf-8') as arquivo_csv:
    #     escritor = csv.writer(arquivo_csv, delimiter='\t')
    #     escritor.writerow(cabecalho)
    #     for linha in dados:
    #         escritor.writerow(linha)


"""
mnist_average_darkness
~~~~~~~~~~~~~~~~~~~~~~

A naive classifier for recognizing handwritten digits from the MNIST
data set.  The program classifies digits based on how dark they are
--- the idea is that digits like "1" tend to be less dark than digits
like "8", simply because the latter has a more complex shape.  When
shown an image the classifier returns whichever digit in the training
data had the closest average darkness.

The program works in two steps: first it trains the classifier, and
then it applies the classifier to the MNIST test data to see how many
digits are correctly classified.

Needless to say, this isn't a very good way of recognizing handwritten
digits!  Still, it's useful to show what sort of performance we get
from naive ideas."""

#### Libraries
# Standard library
from collections import defaultdict

# My libraries
import mnist_loader

def main():

    # On charge les datasets
    print("\n>> 1. DATASETS <<\n")
    training_data, validation_data, test_data = mnist_loader.load_data()

    # On compte les moyennes d'obscurité des images pour chaque nombre a prédire pour le dataset d'apprentissage
    print("\n>> 2. \"TRAINING\" <<\n")
    print("\"Training\"...")
    avgs = avg_darknesses(training_data)
    print("\"Trained\"...")

    # On compte le nombre de chiffres correctement prédits pour le dataset de test
    print("\n>> 3. \"TESTING\" <<\n")
    print("Testing...")
    num_correct = sum(int(guess_digit(image, avgs) == digit) for image, digit in zip(test_data[0], test_data[1]))
    print("Tested...")

    n_test_data = len(test_data[1])
    print("Baseline classifier using average darkness of image.")
    print("{0} of {1} ({2}%) values are correct.".format(num_correct, n_test_data, num_correct * 100.0 / n_test_data))
    print("Tested...")

def avg_darknesses(training_data):
    """ Return a defaultdict whose keys are the digits 0 through 9.
    For each digit we compute a value which is the average darkness of
    training images containing that digit.  The darkness for any
    particular image is just the sum of the darknesses for each pixel."""

    digit_counts = defaultdict(int)
    darknesses = defaultdict(float)

    # Pour chaque couple, image / nombre à prédire
    for image, digit in zip(training_data[0], training_data[1]):

        # On compte le nombre d'images pour chaque chiffre
        digit_counts[digit] += 1

        # On compte l'obscurité de l'image pour chaque chiffre
        darknesses[digit] += sum(image)

    avgs = defaultdict(float)

    # Pour chaque chiffre
    for digit, n in digit_counts.items():

        # On associe chaque chiffre à une moyenne d'obscurité d'image
        avgs[digit] = darknesses[digit] / n

    return avgs

def guess_digit(image, avgs):
    """Return the digit whose average darkness in the training data is
    closest to the darkness of ``image``.  Note that ``avgs`` is
    assumed to be a defaultdict whose keys are 0...9, and whose values
    are the corresponding average darknesses across the training data."""

    # On calcule l'obscurité de l'image
    darkness = sum(image)

    # On calcule les différences entre l'obscurité de l'image et la moyenne d'obscurité de chaque chiffre
    distances = {k: abs(v-darkness) for k, v in avgs.items()}

    # On retourne le chiffre correspondant à une obscurité la plus proche de celle calculée
    return min(distances, key=distances.get)

if __name__ == "__main__":
    main()


import torch
from torch.utils.data import DataLoader

class VariableLengthLoader:
    def __init__(self, loader, num_iters):
        self.loader = loader
        self.num_iters = num_iters
        self._cur_iters = 0

    def __iter__(self):
        while True:
            for elem in self.loader:
                yield elem
                self._cur_iters += 1
                if self._cur_iters == self.num_iters:
                    self._cur_iters = 0
                    return

    def __len__(self):
        return self.num_iters

def get_dataloaders(train_ds, val_ds, config):
    train_loader = None
    if train_ds is not None:
        train_loader = DataLoader(
            train_ds, config.train_batch_size, shuffle=True,
            num_workers=config.train_num_workers, collate_fn=id_collator
        )
        train_loader = VariableLengthLoader(train_loader,
                                            config.epoch_num_iters)

    val_loader = None
    if val_ds is not None:
        val_loader = DataLoader(
            val_ds, config.val_batch_size, shuffle=True,
            num_workers=config.val_num_workers, collate_fn=id_collator
        )
    return train_loader, val_loader

def id_collator(objects):
    '''
    stacks objects along batch axis
    '''
    x, y = list(zip(*objects))
    return torch.stack(x, dim=0), torch.stack(y, dim=0)


import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
x = np.array([5,7,8,7,2,17,2,9,4,11,12,9,6])
x=x.reshape(-1,1)
y = np.array([99,86,87,88,111,86,103,87,94,78,77,85,86])
y=y.reshape(-1,1)
model=LinearRegression()
model=model.fit(x,y)
x_test=np.array(np.linspace(0,20)).reshape(-1,1)
y_pred=model.predict(x_test)
plt.scatter(x,y)
plt.plot(x_test,y_pred)
plt.show()


from ase.calculators.espresso import Espresso
from ase.dft import DOS
import matplotlib.pyplot as plt
from ase.io import read

# Load structure
si = read('Opt.traj@-1')

# Set up calculator and calculation parameters
pseudopotentials = {'Si': 'Si.vbc.UPF'}
input_data = {
    'control': {
        'calculation': 'scf',
        'restart_mode': 'from_scratch',
        'prefix': 'si_scf',
        'outdir': './',
    },
    'system': {
        'ecutwfc': 50,
        'ecutrho': 200,
        'occupations': 'smearing',
        'smearing': 'gaussian',
        'degauss': 0.01,
        'input_dft': 'PBE',
    },
    'electrons': {
        'conv_thr': 1.0e-8,
    },
}

# Set up the Quantum ESPRESSO calculator
calc = Espresso(pseudopotentials=pseudopotentials,
                tstress=True, tprnfor=True, kpts=(6, 6, 6), input_data=input_data)

# Attach the calculator to the Silicon atoms
si.calc = calc

# Perform SCF calculation
si.get_potential_energy()

# Calculate DOS
dos = DOS(calc, width=0.2)
dos.get_dos()

# Get energies and DOS values
energies = dos.get_energies()
dos_values = dos.get_dos()

# Plot DOS
plt.figure(figsize=(8, 6))
plt.plot(energies, dos_values, color='red')
plt.xlabel('Energy (eV)')
plt.ylabel('Density of States')
plt.title('Electronic DOS of Silicon')

# Plot Fermi level line at 0 energy
fermi_level = calc.get_fermi_level()
# For testing
#print(fermi_level)
plt.axvline(x=0, color='k', linestyle='--', label=f'Fermi Level: {fermi_level:.2f} eV')

plt.legend()
plt.savefig('DOS.png')
plt.show()


#!/usr/bin/env python
# Execute crab command for multiple tasks.
from __future__ import print_function
import argparse
import os
import subprocess
import sys

parser = argparse.ArgumentParser(description='Execute crab command for multiple tasks.',
                  formatter_class = lambda prog: argparse.HelpFormatter(prog,width=90))
parser.add_argument('--workArea', required=True, type=str, help="Working area")
parser.add_argument('--cmd', required=True, type=str, help="CRAB command")
parser.add_argument('cmd_args', type=str, nargs='*', help="Arguments for the CRAB command (if any)")
args = parser.parse_args()

if args.cmd == 'submit':
    print("ERROR: Please, use crab_submit.py to run the submit command.")
    sys.exit(1)
cmd_args_str = ' '.join(args.cmd_args)

def sh_call(cmd):
    sep = '-' * (len(cmd) + 3)
    print('{}\n>> {}'.format(sep, cmd))
    result = subprocess.call([cmd], shell=True)
    if result != 0:
        print('ERROR: failed to run "{}"'.format(cmd))
        sys.exit(1)

for dir in os.listdir(args.workArea):
    task_dir = os.path.join(args.workArea, dir)
    if not os.path.isdir(task_dir): continue

    cmd_line = 'crab {} -d {} {}'.format(args.cmd, task_dir, cmd_args_str)
    sh_call(cmd_line)


from django.test import TestCase

from webscraper.admin import ChannelAdminForm, entry_title_with_link, entry_site, channel_feed_link
from webscraper.models import Channel, Entry
from .util import CHANNEL_DEFAULTS

class ChannelAdminFormTestCase(TestCase):

    def test_slug_field_disabled(self):
        form = ChannelAdminForm()
        slug_field = form.fields['slug']
        self.assertTrue(slug_field.disabled)

class AdminHelpersTestCase(TestCase):

    def test_entry_title_with_link(self):
        e = Entry(title='blah', url='http://host.com')
        rv = entry_title_with_link(e)
        self.assertIn(e.title, rv)
        self.assertIn(e.url, rv)

    def test_entry_site(self):
        e = Entry(url='http://somehost.com/doc.html')
        rv = entry_site(e)
        self.assertIn('somehost.com', rv)

    def test_channe_feed_link(self):
        c = Channel(**CHANNEL_DEFAULTS)
        c.save()    # slug generated on first save
        rv = channel_feed_link(c)
        self.assertIn(c.slug, rv)


#%%
import numpy as np
import librosa
import scipy as sp
import IPython.display as ipd
import matplotlib.pyplot as plt
import os

import json
import codecs

import argparse
import random
import time

from pythonosc import udp_client

################## insert here the name of the cover you want to segment ###############
song_name = "Florida"
######################################################################################

import DEF_features as DEF_features
#%%############## DEFINITION OF FEATURES FUNCTIONS ##########################
#ZCR
def compute_zcr(win, Fs):
    win_sign = np.sign(win)

    N = win.shape[0]

    sign_diff = np.abs(win_sign[:-1] - win_sign[1:])

    zcr = len(sign_diff[sign_diff != 0]) * Fs / N

    # equivalent to

    zcr2 = np.sum(sign_diff) * Fs / (2 * N)
    return zcr
#SPECTRAL CENTROID
def compute_speccentr(spec):
    k_axis = np.arange(1, spec.shape[0] + 1)
    centr = np.sum(np.transpose(k_axis) * np.abs(spec)) / (np.sum(np.abs(spec))+np.finfo(np.float32).eps)
    return centr

def compute_specdec(spec):
    mul_fact  = 1 / (np.sum(np.abs(spec[1:]))+np.finfo(np.float32).eps)
    num = np.abs(spec[1:]) - np.tile(np.abs(spec[0]), len(spec) - 1)
    den = np.arange(1, len(spec))
    spectral_decrease = mul_fact * np.sum(num / den)
    return spectral_decrease

#beat tracking
from librosa.beat import beat_track

def compute_beats(y, sr):
    return beat_track(y, sr=sr)

#not using this because imprecise
def compute_beats_samp(y, sr):
    return beat_track(y, sr=sr, units='samples')

#this function is not used
def first_beat(y, sample_beats):#   ASSUMING WE HAVE A 4/4
    size4= sample_beats.size-(sample_beats.size%4) # closest multiple of 4
    beats4 = np.reshape(sample_beats[:size4],(-1,4))
    energy4 = np.mean(np.power(y[beats4],2), axis=0)
    return np.argmax(energy4)

#I'm  using this because I can set beat_track to output samples directlyb but they are not precise
from librosa import frames_to_samples
def beats_to_sample(beats, y, sr):
    """
    Aligning supposed beats to the peak of energy in the y signal

    beats:  np.ndarray
        frames index where beats are supposed to be
    y: np.ndarray
        input signal
    sr: int
        samplerate

    Returns
    y_beat: np.ndarray
        array with 1 when there is a beat
    beats_indices: np.ndarray
        array with indices of the beats
    """

    y_beat = np.zeros(y.shape)
    margin=int(0.1 * sr)
    for beat in frames_to_samples(beats):
        bs_index = beat-margin +np.argmax(np.abs(y[beat-margin:beat+margin]))
        y_beat[bs_index]=1
    return y_beat, np.where(y_beat==1)[0]

def compute_rms(S):
  return np.sqrt(np.mean(np.abs(S)**2, axis=0, keepdims=True))

def compute_specroll(spec, perc, freq):
  spec= np.abs(spec)
  total_energy = np.cumsum(spec, 0)
  threshold = perc * total_energy[-1]
  ind = np.where(total_energy < threshold, np.nan, 1)
  return np.nanmin(ind*freq)

#%%

song_name_format=song_name + ".mp3"
DATA_DIR="data"
assert os.path.exists(DATA_DIR), "wrong data dir"

filename_in=os.path.join(DATA_DIR, song_name_format) # put whatever you like
audio, Fs = librosa.load(filename_in, sr=None)

ipd.Audio(audio, rate=Fs)

#%%
######################################SETTING PARAMETERS#
win_length = int(np.floor(0.01 * Fs))
hop_size = int(np.floor(0.075 * Fs))

window = sp.signal.get_window(window='hanning', Nx=win_length)

features_names = ['Zero Crossing Rate', 'Spectral Decrease', 'Spectral Centroid', 'Root mean square', 'Spectral Roll off']

#for beat tracking
t=np.arange(audio.size)/Fs

#%%
######################################COMPUTE FEATURES
win_number = int(np.floor((audio.shape[0] - win_length) / hop_size))

n_features = np.asarray(features_names).shape[0]

features = np.zeros((win_number, n_features))
for i in np.arange(win_number):
    frame = audio[i * hop_size : i * hop_size + win_length]
    frame_wind = frame * window

    spec = np.fft.fft(frame_wind)
    nyquist = int(np.floor(spec.shape[0] / 2))
    spec = spec[1:nyquist]
    freq = np.fft.fftfreq(n=win_length, d= 1/Fs)
    freq = freq[1:nyquist]

    features[i, 0] = compute_zcr(frame_wind, Fs)
    features[i, 1] = compute_specdec(spec)
    features[i, 2] = compute_speccentr(spec)
    features[i, 3] = compute_rms(spec)
    features[i, 4] = compute_specroll(spec, 0.85, freq)

#bpm, sample_beats = compute_beats_samp(audio, sr=Fs)

bpm, beats = compute_beats(audio, sr=Fs)
y_beats,sample_beats= beats_to_sample(beats, audio, sr=Fs)
print(bpm)

#%%
###################################### Plot the selected signal and features
plt.figure(1, figsize=(16, 8))

plt.subplot(n_features+1,1,1)
time_axis = np.arange(audio.shape[0]) / Fs
plt.plot(time_axis, audio)
plt.grid(True)
plt.title('audio')

plt.subplot(n_features+1,1,2)
feat_time_axis = np.arange(features.shape[0]) * hop_size / Fs
plt.title(features_names[0])
plt.plot(feat_time_axis, features[:, 0])
plt.grid(True)

plt.subplot(n_features+1,1,3)
feat_time_axis = np.arange(features.shape[0]) * hop_size / Fs
plt.title(features_names[1])
plt.plot(feat_time_axis, features[:, 1])
plt.grid(True)

plt.subplot(n_features+1,1,4)
feat_time_axis = np.arange(features.shape[0]) * hop_size / Fs
plt.title(features_names[2])
plt.plot(feat_time_axis, features[:, 2])
plt.grid(True);

plt.subplot(n_features+1,1,5)
feat_time_axis = np.arange(features.shape[0]) * hop_size / Fs
plt.title(features_names[3])
plt.plot(feat_time_axis, features[:, 3])
plt.grid(True);

plt.subplot(n_features+1,1,6)
feat_time_axis = np.arange(features.shape[0]) * hop_size / Fs
plt.title(features_names[4])
plt.plot(feat_time_axis, features[:, 4])
plt.grid(True);

#See if beats are correct
plt.figure(2, figsize=(16, 8))
plt.plot(t, audio, label="y")
plt.scatter(t[sample_beats], audio[sample_beats], label="beats", color="red")
plt.xlim([0, 5])
plt.xlabel("Time [s]")
plt.legend()

#%%
###################################### Smoothing process
#low pass
smooth_factor = 0.1
features_s=features
for i in range(win_number-1):
  for a in range(features.shape[1]):
    features_s[i+1][a]=smooth_factor*features_s[i+1][a]+(1-smooth_factor)*features_s[i][a]
features_s.shape

plt.figure(3, figsize=(16, 8))

plt.subplot(n_features+1,1,1)
time_axis = np.arange(audio.shape[0]) / Fs
plt.plot(time_axis, audio)
plt.grid(True)
plt.title('audio')

plt.subplot(n_features+1,1,2)
feat_time_axis = np.arange(features_s.shape[0]) * hop_size / Fs
plt.title(features_names[0])
plt.plot(feat_time_axis, features_s[:, 0])
plt.grid(True)

plt.subplot(n_features+1,1,3)
feat_time_axis = np.arange(features_s.shape[0]) * hop_size / Fs
plt.title(features_names[1])
plt.plot(feat_time_axis, features_s[:, 1])
plt.grid(True)

plt.subplot(n_features+1,1,4)
feat_time_axis = np.arange(features_s.shape[0]) * hop_size / Fs
plt.title(features_names[2])
plt.plot(feat_time_axis, features_s[:, 2])
plt.grid(True);

plt.subplot(n_features+1,1,5)
feat_time_axis = np.arange(features_s.shape[0]) * hop_size / Fs
plt.title(features_names[3])
plt.plot(feat_time_axis, features_s[:, 3])
plt.grid(True);

plt.subplot(n_features+1,1,6)
feat_time_axis = np.arange(features_s.shape[0]) * hop_size / Fs
plt.title(features_names[4])
plt.plot(feat_time_axis, features_s[:, 4])
plt.grid(True);

#%%
###################################### Mean, Max, min evaluation
features_mean = np.mean(features_s, 0)
features_Max= np.max(features_s, 0)
features_min = np.min(features_s, 0)
print(features_mean)
print(features_Max)
print(features_min)

print(features_s.shape)
# %%
for x in range(features_s.shape[1]):
    features_s[:, x] = (features_s[:, x]-features_min[x])/(features_Max[x]-features_min[x])

# %%
data0= features_s[:, 0].tolist()
data1= features_s[:, 1].tolist()
data2= features_s[:, 2].tolist()
data3= features_s[:, 3].tolist()
data4= features_s[:, 4].tolist()

sample_beats_s = sample_beats/Fs
data5 = sample_beats_s.tolist()

file_path = song_name+".json" ## your path variable
file_path=os.path.join(DATA_DIR, file_path)
json.dump({'zcr': data0, 'specdec': data1, 'speccentr': data2, 'rms': data3, 'specroll': data4, 'hopsize_s': hop_size/Fs, 'beats': data5}, codecs.open(file_path, 'w',
    encoding='utf-8'), separators=(',', ':'), sort_keys=False, indent=4)

# %%
#if __name__ == "__main__":
#
#    parser = argparse.ArgumentParser()
#    parser.add_argument("--ip", default="127.0.0.1",
#      help="The ip of the OSC server")
#    parser.add_argument("--port", type=int, default=5005,
#      help="The port the OSC server is listening on")
#    args = parser.parse_args()
#
#    client = udp_client.SimpleUDPClient(args.ip, args.port)
#    client.send_message("/Fs", Fs)
#    client.send_message("/t", t)
#
#    for x in range(features_s.shape[1]):
#        client.send_message("/features_s", features_s[:][x])
#    for x in range(sample_beats.shape):
#        client.send_message("/beats", sample_beats[x])

#plt.show()
# %%


import requests as req
import json

from config import *

def bytes_to_str(b: bytes):
   result = ''
   for byte in b:
      result = result + "%" + hex(byte)[2:]
   return result

def cond_input(text: str):
   while True:
      answer = input(text)
      if answer.lower()[0] == 'y': return True
      elif answer.lower()[0] == 'n': return False

def uri_format(data: dict):
   result = ''

   temp = ''
   for k, v in data.items():
      result = result + temp + k + "=" + str(v)
      temp = '&'

   return result

class YandexSpeechKitTest:
   def __init__(self, token: str, folder_id: str, api_ver: int = 1) -> None:
      self._ya_token = token
      self._folder_id = folder_id
      self._api_ver = api_ver
      self.receive_iam_token()

   def receive_iam_token(self):
      response = req.post("https://iam.api.cloud.yandex.net/iam/v1/tokens", data=json.dumps({ "yandexPassportOauthToken": self._ya_token }))

      if not (response.status_code >= 200 and response.status_code <= 299):
         print("failed to receive IAM token: request failed (" + str(response.status_code) + ")")
         print("content: ", response.content)
         exit(1)

      json_response = response.json()

      self._iam_token = json_response["iamToken"]
      self._iam_session = req.Session()
      self._iam_session.headers = {
         "Authorization": "Bearer " + self._iam_token,
         "x-folder-id": self._folder_id
      }

   def speech_synthesis(self, text: str, out: str, voice: str = "zahar", emotion: str = "neutral", speed: float = 1.0, format: str = None):
      if format is None:
         # determine format
         if out.endswith(".ogg"):
            format = "oggopus"
         elif out.endswith(".wav"):
            format = "lpcm"
         elif out.endswith(".mp3"):
            format = "mp3"
         else:
            format = "oggopus" # by default

      request_body = {
         "text": bytes_to_str(text.encode('utf-8')),
         "lang": "ru-RU",
         "voice": voice,
         "emotion": emotion,
         "speed": speed,
         "format": format,
         "folderId": self._folder_id
      }

      print('request body ', uri_format(request_body))

      response = self._iam_session.post("https://tts.api.cloud.yandex.net/speech/v1/tts:synthesize", data=uri_format(request_body))

      if not (response.status_code >= 200 and response.status_code <= 299):
         print("failed to speech_synthesis(): request failed (" + str(response.status_code) + ")")
         print("content: ", response.content)
         exit(1)

      with open(out, "wb+") as f:
         f.write(response.content)
         f.close()

def main():
   if cond_input("do you want to see token?[y/n] "):
      print("token: '" + CFG_YA_TOKEN + "'")

   app = YandexSpeechKitTest(CFG_YA_TOKEN, CFG_YA_FOLDER)

   text = input("Enter text: ")

   out_filename = "output.ogg"
   app.speech_synthesis(text, out_filename)

   print('writen to ' + out_filename)

   return 0

if __name__ == "__main__":
   exit(main())


from pytube import YouTube
from pathlib import Path
import os
print('Youtube Downloader.')
print()

def download():

    # This function converts videos into mp3.
    link = input("Type in the videos's url that you want to download:\n")
    youtube_video = YouTube(link, use_oauth=True, allow_oauth_cache=True)
    youtube_video = youtube_video.streams.get_audio_only()
    try:
        out_file = youtube_video.download(str(Path.home() / "Downloads"))
        base, ext = os.path.splitext(out_file)
        new_file = base + '.mp3'
        os.rename(out_file, new_file)
    except: return 'Sorry, something went wrong.'
    return 'Download successfully.'

print(download())


# -*- coding: utf-8 -*-
"""
Created on Wed Sep 22 08:32:41 2021

@author: abz1e14
"""

from pandas import read_excel
from matplotlib import pyplot
from statsmodels.tsa.seasonal import seasonal_decompose
#series = read_excel('CementProduction.xls', sheetname='Data', header=0,
#              index_col=0, parse_dates=True, squeeze=True)
#result = seasonal_decompose(series, model='additive')
series = read_excel('ClayBricks.xls', sheet_name='BRICKSQ', header=0,
              index_col=0, parse_dates=True, squeeze=True)
#series = read_excel('Electricity.xls',
#              sheetname='ELEC', header=0,
#              index_col=0, parse_dates=True, squeeze=True)
#
#series = read_excel('TreasuryBills.xls',
#              sheetname='USTREAS', header=0,
#              index_col=0, parse_dates=True, squeeze=True)

#series = read_excel('DowJones.xls', sheet_name='Data2', header=0, index_col=0, parse_dates=True, squeeze=True)

result = seasonal_decompose(series, model='additive')
result.plot()
pyplot.show()

# the following optional commands can be used to extract the values of the decomposition components
residual = result.resid
seasonal = result.seasonal
trend = result.trend

LearningResource/Forecasting/Chapter 1 Demonstrations Material/Chapter 1 Demonstrations Material/Demo 1.7/Decomposition.py


import cv2
import imutils
import numpy as np
from imutils.object_detection import non_max_suppression
from imutils import paths
import cvController

class BodyModule(object):

    def __init__(self):
        self.isBody = False
        self.numBodies = 0
        self.hog = cv2.HOGDescriptor()
        self.hog.setSVMDetector(cv2.HOGDescriptor_getDefaultPeopleDetector())
        self.toPrint = True
        self.currentFrame = None

    def run(self,image,condition):

        #ret,image = self.cap.read()
        #image = imutils.resize(image, width=min(400, image.shape[1]))

        orig = image.copy()

        # detect people in the image
        (rects, weights) = self.hog.detectMultiScale(image, winStride=(4, 4),
        padding=(8, 8), scale=1.01)

        # draw the original bounding boxes
        #if self.toPrint:
        #    for (x, y, w, h) in rects:
        #        cv2.rectangle(orig, (x, y), (x + w, y + h), (0, 0, 255), 2)

        # apply non-maxima suppression to the bounding boxes using a
        # fairly large overlap threshold to try to maintain overlapping
        # boxes that are still people
        rects = np.array([[x, y, x + w, y + h] for (x, y, w, h) in rects])
        pick = non_max_suppression(rects, probs=None, overlapThresh=0.65)

        # draw the final bounding boxes
        #if self.toPrint:
        #    for (xA, yA, xB, yB) in pick:
        #        cv2.rectangle(image, (xA, yA), (xB, yB), (0, 255, 0), 2)

        self.numBodies = len(pick)

        if(len(pick) > 0):
            self.isBody = True
        else:
            self.isBody = False

        # show the output images
        #cv2.imshow("Output", image)
        self.currentFrame = image

        return (self.testCondition(condition),pick)

    def testCondition(self,condition):
        variable = condition["var"]
        comparison = condition["comp"]
        equality = condition["eq"]

        result = False
        if(variable == "NumberOfBodies"):
            if comparison == "=":
                if(self.numBodies == int(equality)):
                    result = True
            elif comparison == ">":
                if(self.numBodies > int(equality)):
                    result = True
            elif comparison == ">=":
                if(self.numBodies >= int(equality)):
                    result = True
            elif comparison == "<":
                if(self.numBodies < int(equality)):
                    result = True
            elif comparison == "<=":
                if(self.numBodies <= int(equality)):
                    result = True
            elif comparison == "!=":
                if(self.numBodies != int(equality)):
                    result = True
        elif (variable == "ThereIsAFace"):
            eqFlag = False
            if(equality == "True"):
                eqFlag = True

            if comparison == "=":
                if(self.isBody == eqFlag):
                    result = True
            elif comparison == "!=":
                if(self.isBody != eqFlag):
                    result = True
        return result


import oci

config = oci.config.from_file("~/.oci/config","DEFAULT")
identity = oci.identity.IdentityClient(config)
user = identity.get_user(config["user"]).data
print(user)

management_client = oci.core.ComputeManagementClient(config)
update = oci.core.models.UpdateInstancePoolDetails(size=2)

management_client.update_instance_pool("ocid1.instancepool.oc1.eu-frankfurt-1.aaaaaaaazmnyykpa5aysclbwc6x5imghmtjirmh2xikjgmn2vn3ywvyf6dwq", update)


# -*- coding: utf-8 -*-
"""
Created on Wed Nov 24 20:43:17 2021

COIT29224 Evolutionary Computation
Assessment 1 - Neural Network Weight Optimisation using
                Particle Swarms

@author: Daniel Roi Agustin
"""

# ---- import dependencies
import random
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import MinMaxScaler

# import assessment related modules
from src import NeuralNetwork, PSO

import warnings
warnings.filterwarnings("ignore")

# for reproducibility
random.seed(100)

# ---- data preprocessing

# load dataset and split into training and testing sets
dataset = load_iris()
x_ = dataset.data
y_ = dataset.target

x_train, x_test, y_train, y_test = train_test_split( x_, y_, test_size = 0.3, random_state = 123, stratify = y_ )

# scale x_ values to optimise model training
scaler = MinMaxScaler()
x_train_scaled = scaler.fit_transform( x_train )

x_test_scaled = scaler.transform( x_test )

# ---- application

# initialise neural network
nn = NeuralNetwork( 4, 30, 3, activation_func="tanh" ) # include x and y here

# neural network self optimisation
print( "Starting Neural Network Optimisation" )
nn.self_optimise(x_train_scaled, y_train, 0.05, 500)
nn_y_pred = nn.predict( x_test_scaled )
nn_test_acc = ( nn_y_pred == y_test ).mean()

print( "Neural network done" )
print( "\n" )

# neural network is also trained INSIDE this PSO instance
print( "Starting Particle Swarm Optimisation" )
pso = PSO( 100, 30 ) # only swarms here
pso.optimise_swarm( nn, x_train_scaled, y_train, 500 ) # nn and iters here

# apply trained nn to test data
pso_y_pred = nn.predict( x_test_scaled )
pso_test_acc = ( pso_y_pred == y_test ).mean()

print( "PSO done" )
print("\n")

print("Test Parameters\n"
      f"Neural Network Accuracy = {nn_test_acc:.4f}\n"
      f"Particle Swarm Opsimisation Accuracy = {pso_test_acc:.4f}")


from __future__ import (absolute_import, division, print_function,
                        unicode_literals)

import time
from collections import deque
from datetime import datetime

import backtrader as bt
from backtrader.feed import DataBase
from backtrader.utils.py3 import with_metaclass

from .ccxtstore import CCXTStore

class MetaCCXTFeed(DataBase.__class__):
    def __init__(cls, name, bases, dct):
        '''Class has already been created ... register'''
        # Initialize the class
        super(MetaCCXTFeed, cls).__init__(name, bases, dct)

        # Register with the store
        CCXTStore.DataCls = cls

class CCXTFeed(with_metaclass(MetaCCXTFeed, DataBase)):
    """
    CryptoCurrency eXchange Trading Library Data Feed.
    Params:
      - ``historical`` (default: ``False``)
        If set to ``True`` the data feed will stop after doing the first
        download of data.
        The standard data feed parameters ``fromdate`` and ``todate`` will be
        used as reference.
      - ``backfill_start`` (default: ``True``)
        Perform backfilling at the start. The maximum possible historical data
        will be fetched in a single request.

    Changes From Ed's pacakge

        - Added option to send some additional fetch_ohlcv_params. Some exchanges (e.g Bitmex)
          support sending some additional fetch parameters.
        - Added drop_newest option to avoid loading incomplete candles where exchanges
          do not support sending ohlcv params to prevent returning partial data

    """

    params = (
        ('historical', False),  # only historical download
        ('backfill_start', False),  # do backfilling at the start
        ('fetch_ohlcv_params', {}),
        ('ohlcv_limit', 20),
        ('drop_newest', False),
        ('debug', False)
    )

    _store = CCXTStore

    # States for the Finite State Machine in _load
    _ST_LIVE, _ST_HISTORBACK, _ST_OVER = range(3)

    # def __init__(self, exchange, symbol, ohlcv_limit=None, config={}, retries=5):
    def __init__(self, **kwargs):
        # self.store = CCXTStore(exchange, config, retries)
        self.store = self._store(**kwargs)
        self._data = deque()  # data queue for price data
        self._last_id = ''  # last processed trade id for ohlcv
        self._last_ts = 0  # last processed timestamp for ohlcv

    def start(self, ):
        DataBase.start(self)

        if self.p.fromdate:
            self._state = self._ST_HISTORBACK
            self.put_notification(self.DELAYED)
            self._fetch_ohlcv(self.p.fromdate)

        else:
            self._state = self._ST_LIVE
            self.put_notification(self.LIVE)

    def _load(self):
        if self._state == self._ST_OVER:
            return False

        while True:
            if self._state == self._ST_LIVE:
                if self._timeframe == bt.TimeFrame.Ticks:
                    return self._load_ticks()
                else:
                    self._fetch_ohlcv()
                    ret = self._load_ohlcv()
                    if self.p.debug:
                        print('----     LOAD    ----')
                        print('{} Load OHLCV Returning: {}'.format(datetime.utcnow(), ret))
                    return ret

            elif self._state == self._ST_HISTORBACK:
                ret = self._load_ohlcv()
                if ret:
                    return ret
                else:
                    # End of historical data
                    if self.p.historical:  # only historical
                        self.put_notification(self.DISCONNECTED)
                        self._state = self._ST_OVER
                        return False  # end of historical
                    else:
                        self._state = self._ST_LIVE
                        self.put_notification(self.LIVE)
                        continue

    def _fetch_ohlcv(self, fromdate=None):
        """Fetch OHLCV data into self._data queue"""
        granularity = self.store.get_granularity(self._timeframe, self._compression)

        if fromdate:
            since = int((fromdate - datetime(1970, 1, 1)).total_seconds() * 1000)
        else:
            if self._last_ts > 0:
                since = self._last_ts
            else:
                since = None

        limit = self.p.ohlcv_limit

        while True:
            dlen = len(self._data)

            if self.p.debug:
                # TESTING
                since_dt = datetime.utcfromtimestamp(since // 1000) if since is not None else 'NA'
                print('---- NEW REQUEST ----')
                print('{} - Requesting: Since TS {} Since date {} granularity {}, limit {}, params'.format(
                    datetime.utcnow(), since, since_dt, granularity, limit, self.p.fetch_ohlcv_params))
                data = sorted(self.store.fetch_ohlcv(self.p.dataname, timeframe=granularity,
                                                     since=since, limit=limit, params=self.p.fetch_ohlcv_params))
                try:
                    for i, ohlcv in enumerate(data):
                        tstamp, open_, high, low, close, volume = ohlcv
                        print('{} - Data {}: {} - TS {} Time {}'.format(datetime.utcnow(), i,
                                                                        datetime.utcfromtimestamp(tstamp // 1000),
                                                                        tstamp, (time.time() * 1000)))
                        # ------------------------------------------------------------------
                except IndexError:
                    print('Index Error: Data = {}'.format(data))
                print('---- REQUEST END ----')
            else:

                data = sorted(self.store.fetch_ohlcv(self.p.dataname, timeframe=granularity,
                                                     since=since, limit=limit, params=self.p.fetch_ohlcv_params))

            # Check to see if dropping the latest candle will help with
            # exchanges which return partial data
            if self.p.drop_newest:
                del data[-1]

            for ohlcv in data:

                # for ohlcv in sorted(self.store.fetch_ohlcv(self.p.dataname, timeframe=granularity,
                #                                           since=since, limit=limit, params=self.p.fetch_ohlcv_params)):

                if None in ohlcv:
                    continue

                tstamp = ohlcv[0]

                # Prevent from loading incomplete data
                # if tstamp > (time.time() * 1000):
                #    continue

                if tstamp > self._last_ts:
                    if self.p.debug:
                        print('Adding: {}'.format(ohlcv))
                    self._data.append(ohlcv)
                    self._last_ts = tstamp

            if dlen == len(self._data):
                break

    def _load_ticks(self):
        if self._last_id is None:
            # first time get the latest trade only
            trades = [self.store.fetch_trades(self.p.dataname)[-1]]
        else:
            trades = self.store.fetch_trades(self.p.dataname)

        for trade in trades:
            trade_id = trade['id']

            if trade_id > self._last_id:
                trade_time = datetime.strptime(trade['datetime'], '%Y-%m-%dT%H:%M:%S.%fZ')
                self._data.append((trade_time, float(trade['price']), float(trade['amount'])))
                self._last_id = trade_id

        try:
            trade = self._data.popleft()
        except IndexError:
            return None  # no data in the queue

        trade_time, price, size = trade

        self.lines.datetime[0] = bt.date2num(trade_time)
        self.lines.open[0] = price
        self.lines.high[0] = price
        self.lines.low[0] = price
        self.lines.close[0] = price
        self.lines.volume[0] = size

        return True

    def _load_ohlcv(self):
        try:
            ohlcv = self._data.popleft()
        except IndexError:
            return None  # no data in the queue

        tstamp, open_, high, low, close, volume = ohlcv

        dtime = datetime.utcfromtimestamp(tstamp // 1000)

        self.lines.datetime[0] = bt.date2num(dtime)
        self.lines.open[0] = open_
        self.lines.high[0] = high
        self.lines.low[0] = low
        self.lines.close[0] = close
        self.lines.volume[0] = volume

        return True

    def haslivedata(self):
        return self._state == self._ST_LIVE and self._data

    def islive(self):
        return not self.p.historical