from django.http import request
from Core.tests import BaseTestCase
from Core.models import ExternalProject
from rest_framework import status
from django.test import tag
import PIL, tempfile

@tag('Users', 'external-projects-tests')
class ExternalProjectTestCase(BaseTestCase):

    def setUp(self):
        self.baseSetup()
        self.project = ExternalProject.objects.create(user=self.user,name="prova")

    @tag('get', 'auth')
    def test_external_projects_user_list_auth(self):
        response = self.client.get("/api/user/external-projects/")
        self.assertEquals(response.status_code, status.HTTP_200_OK)

    @tag('get', 'auth')
    def test_external_projects_users_list_auth(self):
        response = self.client.get(f"/api/users/{self.user.slug}/external-projects/")
        self.assertEquals(response.status_code, status.HTTP_200_OK)

    @tag('post','auth')
    def test_external_project_create_auth(self):
        response = self.client.post("/api/user/external-projects/", data={"name":"sdasd"})
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        response = self.client.post("/api/user/external-projects/",format="json",data=[{"name":"sdasd"},{"name":"sdasd"}])
        self.assertEqual(response.status_code, status.HTTP_201_CREATED)
        self.assertIsInstance(response.data, list)

    @tag('post','unauth')
    def test_external_project_create_unauth(self):
        self.client.logout()
        response = self.client.post("/api/user/external-projects/", data={"name":"sdasd"})
        self.assertEqual(response.status_code, status.HTTP_401_UNAUTHORIZED)

    @tag('delete','auth')
    def test_external_project_delete_auth(self):
        response = self.client.delete(f"/api/user/external-projects/{self.project.id}/")
        self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)

    @tag('put','auth')
    def test_external_project_update_auth(self):
        data = {"name":"test"}
        response = self.client.patch(f"/api/user/external-projects/{self.project.id}/",data=data)
        self.assertEqual(response.status_code, status.HTTP_200_OK)
        self.assertDictEqual(data, dict(response.json()))

    @tag('put','auth','file')
    def test_external_project_update_image_auth(self):
        image = PIL.Image.new('RGB', size=(1, 1))
        file = tempfile.NamedTemporaryFile(suffix=".jpg")
        image.save(file)
        with open(file.name, 'rb') as file_open:
            response = self.client.patch(f"/api/user/external-projects/{self.project.id}/",
                                         data={"image": file_open}, format='multipart')
        self.assertEqual(response.status_code, status.HTTP_200_OK)


import zmq
import sys
import random
import hashlib
import ast

partSize = 1024 * 1024 * 1

def uploadFile2(filename, socket):
    with open(filename, "rb") as f:
        finished = False
        part = 0
        while not finished:
            print("Uploading part {}".format(part))
            f.seek(part*partSize)
            bt = f.read(partSize)
            socket.send_multipart([filename, bt])
            response = socket.recv()
            print("Received reply  [%s ]" % (response))
            part = part + 1
            if len(bt) < partSize:
                finished = True

def computeHashFile(filename):
    BUF_SIZE = 65536  # lets read stuff in 64kb chunks!
    sha1 = hashlib.sha1()

    with open(filename, 'rb') as f:
        while True:
            data = f.read(BUF_SIZE)
            if not data:
                break
            sha1.update(data)
    return sha1.hexdigest()

def computeHash(bytes):
    sha1 = hashlib.sha1()
    sha1.update(bytes)
    return sha1.hexdigest()

def uploadFile(context, filename, servers, username, proxy):
    sockets = []
    for ad in servers:
        s = context.socket(zmq.REQ)
        s.connect("tcp://"+ ad.decode('ascii'))
        sockets.append(s)

    with open(filename, "rb") as f:
        completeSha1= bytes(computeHashFile(filename), 'ascii')
        finished = False
        part = 0
        index = open("1.txt","w+")
        index.write(filename.decode('ascii') + "\n")
        # index.write(completeSha1.decode('ascii') + "\n")
        auxDict = {}

        while not finished:
            print("Uploading part {}".format(part))
            f.seek(part*partSize)
            bt = f.read(partSize)
            sha1bt = bytes(computeHash(bt), 'ascii')
            #auxDict llave = sha1 de la parte; valor = servidor de la parte
            auxDict[sha1bt] = servers[part % len(sockets)]
            index.write(sha1bt.decode('ascii') + "\n")
            s = sockets[part % len(sockets)]
            s.send_multipart([b"upload", filename, bt, sha1bt, completeSha1])
            response = s.recv()
            # print(response)
            print("Received reply for part {} ".format(part))
            part = part + 1
            if len(bt) < partSize:
                finished = True

        index.close()
        ShaIndex = bytes(computeHashFile("1.txt"), 'ascii')
        print(ShaIndex)
        #Envio del archivo index a uno de los servidores
        serverIndex = random.randint(0, len(sockets)-1)
        with open("1.txt", "rb") as f:
            s = sockets[serverIndex]
            s.send_multipart([b"upload", b".txt", f.read(), ShaIndex, completeSha1])

        #Envio de auxDict, el sha1 de index y quien lo tiene,
        proxy.send_multipart([b"finished", bytes(str(auxDict),'ascii'), ShaIndex, servers[serverIndex], bytes(username, 'ascii'), filename])

def downloadFile(context, shaIndex, dictIndex,locatIndex, servers):
    sockets = {}

    for ad in servers:
        s = context.socket(zmq.REQ)
        s.connect("tcp://"+ ad.decode('ascii'))
        sockets[ad] = s

    sockets[locatIndex].send_multipart([b"download", shaIndex])
    msg = sockets[locatIndex].recv_multipart()
    fileIndex = msg[0].decode('ascii')
    fileIndex = fileIndex.split("\n")
    # print(fileIndex)
    # print(dictIndex)
    for i, line in enumerate(fileIndex):
        if i < 1:
            finFile = open(line, "ab")
            name = line
            continue
        if i < len(fileIndex)-1:
            key = line.encode('ascii')
            locatPart = dictIndex[key]
            print(locatPart)
            sock = sockets[locatPart]
            print(sock)
            sock.send_multipart([b"download", key])
            partFile = sock.recv_multipart()
            finFile.write(partFile[0])
    finFile.close()
    return name

def main():
    if len(sys.argv) != 4:
        print("Must be called with a filename")
        print("Sample call: python ftclient <username> <operation> <filename>")
        exit()

    username = sys.argv[1]
    operation = sys.argv[2]
    filename = sys.argv[3].encode('ascii')

    context = zmq.Context()
    proxy = context.socket(zmq.REQ)
    proxy.connect("tcp://localhost:6666")

    print("Operation: {}".format(operation))
    if operation == "upload":
        proxy.send_multipart([b"availableServers"])
        servers = proxy.recv_multipart()
        print("There are {} available servers".format(len(servers)))
        uploadFile(context, filename, servers, username, proxy)
        print("File {} was uploaded.".format(filename.decode('ascii')))
    elif operation == "download":
        proxy.send_multipart([b"serverIndex", filename])
        infDownload = proxy.recv_multipart()
        shaIndex = infDownload[0]
        dictIndex = eval(infDownload[1])
        locatIndex = infDownload[2]
        servers = eval(infDownload[3])
        # print(shaIndex)
        # print("###########")
        # print(dictIndex)
        # print("###########")
        # print(locatIndex)
        # print("###########")
        # print(servers)
        nameFile = downloadFile(context, shaIndex, dictIndex,locatIndex, servers)
        print("The {} has been downloaded". format(nameFile))

    elif operation == "tolist":
        proxy.send_multipart([b"tolist", bytes(username, 'ascii')])
        msg = proxy.recv_multipart()
        listFiles = eval(msg[0].decode('ascii'))
        for i in listFiles:
            print(i)

    elif operation == "share":
        print("Not implemented yet")

    else:
        print("Operation not found!!!")

if __name__ == '__main__':
    main()


import numpy as np
from scipy.cluster.hierarchy import linkage, fcluster
from scipy.spatial.distance import pdist, squareform
import scipy.linalg as Linalg
from scipy.stats import multivariate_normal as mvn

class Cluster(object):
    def __init__(self, X):
        """Return a new object to cluster data based on selected clutering
        algorithm.
        Example usage: clusterObj = Cluster(X)
        X:     numpy array, shape (n_samples, n_features)"""
        self._X = X
        self._nsample = X.shape[0]
        self._nfeature = X.shape[1]

    def assign_label(self, Centers):
        """ Assign labels to the data points
            Input:
                self
                Centers:  numpy array, shape (n_clusters, n_features) the centers of each cluster
            Output:
                clusters: the index of data points in each class
                labels: the label of each class

        """
        numClusters = Centers.shape[0]
        clusters = {}
        labels = []

        for sample_idx in range(self._nsample):
            x = self._X[sample_idx, :]
            dist = []

            label_x = min( [(i, np.linalg.norm( x-Centers[i,:] ) ) for i in range(numClusters)], key = lambda t:t[1])[0]
            try:
                clusters[label_x].append(sample_idx)
            except KeyError:
                clusters[label_x] = [sample_idx]

            labels.append(label_x)

        return clusters, labels

    def kMeans(self, nClusters, maxIter=300):
        """
        K-means clustering algorithm.

        Function usage: kMeans(nClusters, maxIter, nInit)

        Inputs:
        nClusters : int
                    The number of clusters to form as well as the number of
                    centroids to generate.
        maxIter : int, optional, default 300
                    Maximum number of iterations of the k-means algorithm to run.

        Returns:
        centroid :  float ndarray with shape (k, n_features)
                    Centroids found at the last iteration of k-means.
        label : integer ndarray with shape (n_samples,)
                    label[i] is the code or index of the centroid the i-th
                    observation is closest to.
        clusters : identity of the data point in the cluster
        """

        # Initialize K-means algorithm by randomly sampling k points
        idx = np.random.choice(range(self._nsample), size= nClusters, replace=False)
        centroid = self._X[idx, :]

        # fix centrod, get the label of each data point
        old_clusters, old_labels  = self.assign_label( Centers = centroid )

        # set flag of convergence
        flag_converge = False
        nite = 0 # iteration counter

        while not flag_converge:
            nite = nite + 1
            if nite > maxIter:
                raise RuntimeError('Exceeds maximum number of iterations')
            # obtain new estimate of clusters
            for i in range(nClusters):
                class_index = old_clusters[i]
                centroid[i,:] = np.mean(self._X[class_index], axis = 0)
                new_clusters, new_labels = self.assign_label( Centers = centroid)
                if old_labels == new_labels:
                    flag_converge = True
                old_labels = new_labels
                old_clusters = new_clusters

        clusters = new_clusters
        labels = new_labels
        return centroid, labels, clusters

    def H_clustering(self, nClusters):
        """
        Performe hierarchical clustering
        """

        # construct hierarchical clustering matrix
        Z = linkage(self._X, metric='euclidean', method = 'ward')
        # obtain labels
        labels = fcluster(Z, nClusters, criterion='maxclust')
        clusters = {}
        centroid = np.zeros( (nClusters, self._nfeature) )
        for i in range(nClusters):
            class_index  = np.where( labels == i)[0]
            clusters[i]= class_index
            centroid[i,:] = np.mean(self._X[class_index, :], axis = 0)
        return centroid, labels, clusters

    # Gaussian mixture clustering using EM algorithm
    def Gaussian_mixture(self, nClusters, max_iter = 300):
        # Initialize EM algorithm by randomly sampling k points as centers
        idx = np.random.choice(range(self._nsample), size= nClusters, replace=False)
        centroid = self._X[idx, :] # initial mean vectors

        # initialize the covariance matrices for each gaussians
        Sigma= [np.eye(self._nfeature)] * nClusters

        # initialize the probabilities/weights for each gaussians
        w = [1./nClusters] * nClusters

        # responsibility matrix is initialized to all zeros
        # we have responsibility for each of n points for eack of k gaussians
        R = np.zeros((self._nsample, nClusters))

        ### log_likelihoods
        log_likelihoods = []

        P = lambda mu, s: np.linalg.det(s) ** -.5 ** (2 * np.pi) ** (-self._X.shape[1]/2.) \
                * np.exp(-.5 * np.einsum('ij, ij -> i',\
                        self._X  - mu, np.dot(np.linalg.inv(s) , (self._X - mu).T).T ) )

        # Iterate till max_iters iterations
        while len(log_likelihoods) < max_iter:

            # E - Step

            ## Vectorized implementation of e-step equation to calculate the
            ## membership for each of k -gaussians
            for k in range(nClusters):
                R[:, k] = w[k] * P(centroid[k], Sigma[k])

            ### Likelihood computation
            log_likelihood = np.sum(np.log(np.sum(R, axis = 1)))

            log_likelihoods.append(log_likelihood)

            ## Normalize so that the responsibility matrix is row stochastic
            R = (R.T / np.sum(R, axis = 1)).T

            ## The number of datapoints belonging to each gaussian
            N_ks = np.sum(R, axis = 0)

            # M Step
            ## calculate the new mean and covariance for each gaussian by
            ## utilizing the new responsibilities
            for k in range(nClusters):

                ## means
                centroid[k] = 1. / N_ks[k] * np.sum(R[:, k] * self._X.T, axis = 1).T
                x_mu = np.matrix(self._X  - centroid[k])

                ## covariances
                Sigma[k] = np.array(1 / N_ks[k] * np.dot(np.multiply(x_mu.T,  R[:, k]), x_mu))

                ## and finally the probabilities
                w[k] = 1. / self._nsample * N_ks[k]
            # check for onvergence
            if len(log_likelihoods) < 2 : continue
            if np.abs(log_likelihood - log_likelihoods[-2]) < 1e-6: break

        clusters, labels  = self.assign_label( Centers = centroid )

        return clusters, labels, centroid, Sigma, w

    def Spectral(self, nClusters = 5, cluster_metric = 'euclidean', sigma = 0.05 ):
        """ Spectral Clustering
            cluster_metric is the metric used to compute the affinity matrix

            sigma is the standard deviation used in the Gaussian kernel

        """

        num_cls = nClusters
        # compute the affinity matrix
        aff_mat = squareform( pdist (self._X, metric = cluster_metric)/sigma )
        np.fill_diagonal(  aff_mat, 1)
        aff_mat = 1 / aff_mat
        np.fill_diagonal(  aff_mat, 0)

        # construct D^{-1/2} by taking the square root of the sum of column of A
        #print(np.sqrt( np.sum(aff_mat, axis = 0)))
        D_mat = np.diag( 1 / np.sqrt( np.sum(aff_mat, axis = 0)) )

        # graph Laplacian, an n by n matrix
        L = np.dot( np.dot(D_mat, aff_mat), D_mat )

        # Now that we have the graph Laplacian, spectral clustering does eigen decomposition on L and obtain the first k eigenvectors
        _ , X_embed =  Linalg.eigh(L, eigvals = (self._nsample - nClusters, self._nsample-1))

        # X_embed can be viewd as the embedding of data

        # normalize the rows of X_embed to unit norm
        row_norm = np.linalg.norm( X_embed, axis = 1).reshape( self._nsample, 1)
        Y = np.divide(X_embed, row_norm)  # n by k matrix, feed to K means
        model1 = Cluster(Y)
        _, labels, clusters = model1.kMeans(nClusters = num_cls)
        return labels, clusters, X_embed


from keras.applications import vgg16
from keras.models import Model, Sequential
from keras.layers import Conv2D, Conv2DTranspose, Input, Cropping2D, add, Dropout, Reshape, Activation
from keras.optimizers import Adam
import numpy as np
import matplotlib.pyplot as plt
from pylab import *
import os
import sys
from functools import partial
from keras.models import Model
from keras.regularizers import l2
from keras.layers import *
from keras.engine import Layer
from keras.applications.vgg16 import *
from keras.models import *
from keras.applications.imagenet_utils import _obtain_input_shape
import keras.backend as K
import tensorflow as tf
from keras import optimizers

nClasses = 2
sgd = optimizers.SGD(lr=1E-2, decay=5**(-4), momentum=0.9, nesterov=True)

# compute dsc
def dice_coef(y_true, y_pred, smooth=1.):
    y_true_f = K.flatten(y_true)
    y_pred_f = K.flatten(y_pred)
    intersection = K.sum(y_true_f * y_pred_f)
    return (2. * intersection + smooth) / (K.sum(y_true_f) + K.sum(y_pred_f) + smooth)

# proposed loss function
def dice_coef_loss(y_true, y_pred):
    distance = 0
    for label_index in range(1,nClasses):
        dice_coef_class = dice_coef(y_true[:,:,:,label_index], y_pred[:, :,:,label_index])
        distance = 1 - dice_coef_class + distance
    return distance

# dsc per class
def label_wise_dice_coefficient(y_true, y_pred, label_index):
    return dice_coef(y_true[:,:,:,label_index], y_pred[:, :,:,label_index])

# get label dsc
def get_label_dice_coefficient_function(label_index):
    f = partial(label_wise_dice_coefficient, label_index=label_index)
    f.__setattr__('__name__', 'label_{0}_dice_coef'.format(label_index))
    return f

def focal_loss(gamma=2., alpha=.25):
	def focal_loss_fixed(y_true, y_pred):
		y_pred = K.clip(y_pred, 0.0001, 1-0.0001)
		pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))
		pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))
		return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))
	return focal_loss_fixed

def FCN8(nClasses, input_height=192, input_width=192):
    ## input_height and width must be devisible by 32 because maxpooling with filter size = (2,2) is operated 5 times,
    ## which makes the input_height and width 2^5 = 32 times smaller
    img_input = Input(shape=(input_height, input_width,
                             3))  ## Assume 224,224,3
    IMAGE_ORDERING = "channels_last"
    ## Block 1
    x = Conv2D(64, (3, 3),
               activation='relu',
               padding='same',
               name='block1_conv1',
               data_format=IMAGE_ORDERING)(img_input)
    x = Conv2D(64, (3, 3),
               activation='relu',
               padding='same',
               name='block1_conv2',
               data_format=IMAGE_ORDERING)(x)
    x = MaxPooling2D((2, 2),
                     strides=(2, 2),
                     name='block1_pool',
                     data_format=IMAGE_ORDERING)(x)
    f1 = x

    # Block 2
    x = Conv2D(128, (3, 3),
               activation='relu',
               padding='same',
               name='block2_conv1',
               data_format=IMAGE_ORDERING)(x)
    x = Conv2D(128, (3, 3),
               activation='relu',
               padding='same',
               name='block2_conv2',
               data_format=IMAGE_ORDERING)(x)
    x = MaxPooling2D((2, 2),
                     strides=(2, 2),
                     name='block2_pool',
                     data_format=IMAGE_ORDERING)(x)
    f2 = x

    # Block 3
    x = Conv2D(256, (3, 3),
               activation='relu',
               padding='same',
               name='block3_conv1',
               data_format=IMAGE_ORDERING)(x)
    x = Conv2D(256, (3, 3),
               activation='relu',
               padding='same',
               name='block3_conv2',
               data_format=IMAGE_ORDERING)(x)
    x = Conv2D(256, (3, 3),
               activation='relu',
               padding='same',
               name='block3_conv3',
               data_format=IMAGE_ORDERING)(x)
    x = MaxPooling2D((2, 2),
                     strides=(2, 2),
                     name='block3_pool',
                     data_format=IMAGE_ORDERING)(x)
    pool3 = x

    # Block 4
    x = Conv2D(512, (3, 3),
               activation='relu',
               padding='same',
               name='block4_conv1',
               data_format=IMAGE_ORDERING)(x)
    x = Conv2D(512, (3, 3),
               activation='relu',
               padding='same',
               name='block4_conv2',
               data_format=IMAGE_ORDERING)(x)
    x = Conv2D(512, (3, 3),
               activation='relu',
               padding='same',
               name='block4_conv3',
               data_format=IMAGE_ORDERING)(x)
    pool4 = MaxPooling2D((2, 2),
                         strides=(2, 2),
                         name='block4_pool',
                         data_format=IMAGE_ORDERING)(x)  ## (None, 14, 14, 512)

    # Block 5
    x = Conv2D(512, (3, 3),
               activation='relu',
               padding='same',
               name='block5_conv1',
               data_format=IMAGE_ORDERING)(pool4)
    x = Conv2D(512, (3, 3),
               activation='relu',
               padding='same',
               name='block5_conv2',
               data_format=IMAGE_ORDERING)(x)
    x = Conv2D(512, (3, 3),
               activation='relu',
               padding='same',
               name='block5_conv3',
               data_format=IMAGE_ORDERING)(x)
    pool5 = MaxPooling2D((2, 2),
                         strides=(2, 2),
                         name='block5_pool',
                         data_format=IMAGE_ORDERING)(x)  ## (None, 7, 7, 512)

    n = 4096
    o = (Conv2D(n, (7, 7),
                activation='relu',
                padding='same',
                name="conv6",
                data_format=IMAGE_ORDERING))(pool5)
    conv7 = (Conv2D(n, (1, 1),
                    activation='relu',
                    padding='same',
                    name="conv7",
                    data_format=IMAGE_ORDERING))(o)

    ## 4 times upsamping for pool4 layer
    conv7_4 = Conv2DTranspose(nClasses,
                              kernel_size=(4, 4),
                              strides=(4, 4),
                              use_bias=False,
                              data_format=IMAGE_ORDERING)(conv7)
    ## (None, 224, 224, 10)
    ## 2 times upsampling for pool411
    pool411 = (Conv2D(nClasses, (1, 1),
                      activation='relu',
                      padding='same',
                      name="pool4_11",
                      data_format=IMAGE_ORDERING))(pool4)
    pool411_2 = (Conv2DTranspose(nClasses,
                                 kernel_size=(2, 2),
                                 strides=(2, 2),
                                 use_bias=False,
                                 data_format=IMAGE_ORDERING))(pool411)

    pool311 = (Conv2D(nClasses, (1, 1),
                      activation='relu',
                      padding='same',
                      name="pool3_11",
                      data_format=IMAGE_ORDERING))(pool3)

    o = Add(name="add")([pool411_2, pool311, conv7_4])
    o = Conv2DTranspose(nClasses,
                        kernel_size=(8, 8),
                        strides=(8, 8),
                        use_bias=False,
                        data_format=IMAGE_ORDERING)(o)

    o = (Activation('softmax'))(o)

    fine_model = Model(img_input, o)

    metrics = dice_coef
    include_label_wise_dice_coefficients = True;
    if not isinstance(metrics, list):
        metrics = [metrics]

    if include_label_wise_dice_coefficients and nClasses > 1:
        label_wise_dice_metrics = [get_label_dice_coefficient_function(index) for index in range(nClasses)]
        if metrics:
            metrics = metrics + label_wise_dice_metrics
        else:
            metrics = label_wise_dice_metrics

    fine_model.compile(optimizer=sgd,

                  loss='categorical_crossentropy',
                  metrics=['accuracy'])

    #fine_model.load_weights('/data/ydeng1/pancreatitis/fcn/fine_weights.h5')
#    fine_model.compile(optimizer=sgd,

#                  loss=[focal_loss(alpha=.25, gamma=2)],
#                  metrics=['accuracy'])

    return fine_model

fine_model = FCN8(nClasses=2, input_height=192, input_width=192)
fine_model.summary()


import multiprocessing as mp
from functools import partial

import country_converter as coco
import pandas as pd
from tqdm import tqdm

cc = coco.CountryConverter()

ktoe_to_twh = 0.011630

# name in JRC-IDEES Energy Balances
sector_sheets = {
    "Integrated steelworks": "cisb",
    "Electric arc": "cise",
    "Alumina production": "cnfa",
    "Aluminium - primary production": "cnfp",
    "Aluminium - secondary production": "cnfs",
    "Other non-ferrous metals": "cnfo",
    "Basic chemicals": "cbch",
    "Other chemicals": "coch",
    "Pharmaceutical products etc.": "cpha",
    "Basic chemicals feedstock": "cpch",
    "Cement": "ccem",
    "Ceramics & other NMM": "ccer",
    "Glass production": "cgla",
    "Pulp production": "cpul",
    "Paper production": "cpap",
    "Printing and media reproduction": "cprp",
    "Food, beverages and tobacco": "cfbt",
    "Transport Equipment": "ctre",
    "Machinery Equipment": "cmae",
    "Textiles and leather": "ctel",
    "Wood and wood products": "cwwp",
    "Mining and quarrying": "cmiq",
    "Construction": "ccon",
    "Non-specified": "cnsi",
}

fuels = {
    "All Products": "all",
    "Solid Fuels": "solid",
    "Total petroleum products (without biofuels)": "liquid",
    "Gases": "gas",
    "Nuclear heat": "heat",
    "Derived heat": "heat",
    "Biomass and Renewable wastes": "biomass",
    "Wastes (non-renewable)": "waste",
    "Electricity": "electricity",
}

eu28 = cc.EU28as("ISO2").ISO2.tolist()

jrc_names = {"GR": "EL", "GB": "UK"}

def industrial_energy_demand_per_country(country, year, jrc_dir):
    jrc_country = jrc_names.get(country, country)
    fn = f"{jrc_dir}/JRC-IDEES-2015_EnergyBalance_{jrc_country}.xlsx"

    sheets = list(sector_sheets.values())
    df_dict = pd.read_excel(fn, sheet_name=sheets, index_col=0)

    def get_subsector_data(sheet):
        df = df_dict[sheet][year].groupby(fuels).sum()

        df["ammonia"] = 0.0

        df["other"] = df["all"] - df.loc[df.index != "all"].sum()

        return df

    df = pd.concat(
        {sub: get_subsector_data(sheet) for sub, sheet in sector_sheets.items()}, axis=1
    )

    sel = ["Mining and quarrying", "Construction", "Non-specified"]
    df["Other Industrial Sectors"] = df[sel].sum(axis=1)
    df["Basic chemicals"] += df["Basic chemicals feedstock"]

    df.drop(columns=sel + ["Basic chemicals feedstock"], index="all", inplace=True)

    df *= ktoe_to_twh

    return df

def add_ammonia_energy_demand(demand):
    # MtNH3/a
    fn = snakemake.input.ammonia_production
    ammonia = pd.read_csv(fn, index_col=0)[str(year)] / 1e3

    def get_ammonia_by_fuel(x):
        fuels = {
            "gas": params["MWh_CH4_per_tNH3_SMR"],
            "electricity": params["MWh_elec_per_tNH3_SMR"],
        }

        return pd.Series({k: x * v for k, v in fuels.items()})

    ammonia_by_fuel = ammonia.apply(get_ammonia_by_fuel).T
    ammonia_by_fuel = ammonia_by_fuel.unstack().reindex(
        index=demand.index, fill_value=0.0
    )

    ammonia = pd.DataFrame({"ammonia": ammonia * params["MWh_NH3_per_tNH3"]}).T

    demand["Ammonia"] = ammonia.unstack().reindex(index=demand.index, fill_value=0.0)

    demand["Basic chemicals (without ammonia)"] = (
        demand["Basic chemicals"] - ammonia_by_fuel
    )

    demand["Basic chemicals (without ammonia)"].clip(lower=0, inplace=True)

    demand.drop(columns="Basic chemicals", inplace=True)

    return demand

def add_non_eu28_industrial_energy_demand(countries, demand):
    non_eu28 = countries.difference(eu28)
    if non_eu28.empty:
        return demand
    # output in MtMaterial/a
    fn = snakemake.input.industrial_production_per_country
    production = pd.read_csv(fn, index_col=0) / 1e3

    # recombine HVC, Chlorine and Methanol to Basic chemicals (without ammonia)
    chemicals = ["HVC", "Chlorine", "Methanol"]
    production["Basic chemicals (without ammonia)"] = production[chemicals].sum(axis=1)
    production.drop(columns=chemicals, inplace=True)

    eu28_production = production.loc[countries.intersection(eu28)].sum()
    eu28_energy = demand.groupby(level=1).sum()
    eu28_averages = eu28_energy / eu28_production

    demand_non_eu28 = pd.concat(
        {k: v * eu28_averages for k, v in production.loc[non_eu28].iterrows()}
    )

    return pd.concat([demand, demand_non_eu28])

def industrial_energy_demand(countries, year):
    nprocesses = snakemake.threads
    disable_progress = snakemake.config["run"].get("disable_progressbar", False)
    func = partial(
        industrial_energy_demand_per_country, year=year, jrc_dir=snakemake.input.jrc
    )
    tqdm_kwargs = dict(
        ascii=False,
        unit=" country",
        total=len(countries),
        desc="Build industrial energy demand",
        disable=disable_progress,
    )
    with mp.Pool(processes=nprocesses) as pool:
        demand_l = list(tqdm(pool.imap(func, countries), **tqdm_kwargs))

    return pd.concat(demand_l, keys=countries)

if __name__ == "__main__":
    if "snakemake" not in globals():
        from _helpers import mock_snakemake

        snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today")

    params = snakemake.params.industry
    year = params.get("reference_year", 2015)
    countries = pd.Index(snakemake.params.countries)

    demand = industrial_energy_demand(countries.intersection(eu28), year)

    demand = add_ammonia_energy_demand(demand)

    demand = add_non_eu28_industrial_energy_demand(countries, demand)

    # for format compatibility
    demand = demand.stack(dropna=False).unstack(level=[0, 2])

    # style and annotation
    demand.index.name = "TWh/a"
    demand.sort_index(axis=1, inplace=True)

    fn = snakemake.output.industrial_energy_demand_per_country_today
    demand.to_csv(fn)


from ajax_select.fields import AutoCompleteSelectField
from ajax_select.fields import AutoCompleteSelectMultipleField
from ajax_select import make_ajax_field
from django import forms
from django.forms import Form, ModelForm
from django.contrib import messages
from newsroom.models import Article, Author
from payment.models import Commission, Invoice, Fund, COMMISSION_DESCRIPTION_CHOICES
from payment.models import PayeRequisition

BIRTH_YEAR_CHOICES = range(1920,2016)
COMMISSION_YEAR_CHOICES = range(2012,2050)

class BaseInvoiceForm(ModelForm):
    dob = forms.DateField(widget=
                          forms.SelectDateWidget(years=BIRTH_YEAR_CHOICES, attrs={"type": "date"}),
                          label="Date of birth",
                          required=False)
    address = forms.CharField(widget=forms.Textarea(attrs={'rows': '4'}),
                              required=False,
                              help_text="Required by SARS")

class InvoiceStaffForm(BaseInvoiceForm):
    fund = forms.ModelChoiceField(queryset=Fund.objects.filter(ledger=False).
                                filter(deprecated=False),
                                required=False)
    merge = forms.ChoiceField(required=False)

    class Meta:
        model = Invoice
        fields = ['identification', 'dob', 'invoicing_company', 'address',
                  'bank_name', 'bank_account_number',
                  'bank_account_type', 'bank_branch_name', 'bank_branch_code',
                  'swift_code', 'iban', 'tax_no', 'tax_percent', 'vat',
                  'level', 'query', 'additional_emails',
                  'requisition', 'requisition_number', 'payment_method',
                  'description', 'fund', 'vouchers_attached',
                  'prepared_by', 'approved_by', 'authorised_by', 'merge',]

class InvoiceForm(BaseInvoiceForm):
    identification = forms.CharField(max_length=20, required=True,
                                     help_text=
                                     "SA ID, passport or some form "
                                     "of official identification")
    dob = forms.DateField(widget=
                          forms.SelectDateWidget(years=BIRTH_YEAR_CHOICES, attrs={'type': 'date'}),
                          label="Date of birth",
                          required=True)
    address = forms.CharField(widget=forms.Textarea(attrs={'rows': '4'}),
                              required=True,
                              help_text="Required by SARS")

    bank_name = forms.CharField(max_length=20, required=True)
    bank_account_number = forms.CharField(max_length=20, required=True)

    class Meta:
        model = Invoice
        fields = ['identification', 'dob', 'invoicing_company', 'address',
                   'bank_name', 'bank_account_number',
                   'bank_account_type', 'bank_branch_name', 'bank_branch_code',
                   'swift_code', 'iban', 'tax_no', 'tax_percent', 'vat',
                   'query',
        ]

    def clean(self):
        cleaned_data = super(InvoiceForm, self).clean()
        query = cleaned_data.get("query")
        if "query_button" in self.data and query == "":
            self.add_error("query",
                           "Please explain why you're querying "
                           "the invoice")

class CommissionForm(ModelForm):
    author = AutoCompleteSelectField("authors", required=False,
                                     help_text=None, label="Payee")
    article = AutoCompleteSelectField('articles', required=False,
                                      help_text=None)
    fund = forms.ModelChoiceField(
        queryset=Fund.objects.filter(ledger=True).filter(deprecated=False),
        label='Pastel', required=False)

    def clean_author(self):
        data = self.cleaned_data['author']
        if data is None:
            raise forms.ValidationError("Please enter a user")
        return data

    class Meta:
        model = Commission
        fields = ['author', 'article', 'fund', 'description', 'notes',
                  'commission_due', 'taxable', 'vatable', 'vat_amount' ]

class CommissionFormset(ModelForm):
    fund = forms.ModelChoiceField(
        # queryset=Fund.objects.filter(ledger=True).filter(deprecated=False),
        queryset=Fund.objects.filter(deprecated=False),
        label='Ledger', required=False)

    def clean_fund(self):
        fund = self.cleaned_data['fund']
        if 'commission_due' in self.cleaned_data:
            due = self.cleaned_data['commission_due']
        else:
            due = None
        if due and due != 0.0 and fund is None:
            raise forms.ValidationError("Please select a fund")
        return fund

class CommissionAnalysisForm(ModelForm):
    descriptions = forms.MultipleChoiceField(choices=COMMISSION_DESCRIPTION_CHOICES,
                                            help_text="You can select multiple choices "
                                            "using the ctrl button and left click.",
                                            required=False)
    funds = forms.MultipleChoiceField(choices=Fund.get_funds,
                                      help_text="You can select multiple choices "
                                            "using the ctrl button and left click.",
                                      required=False)
    date_from = forms.DateField(widget=forms.SelectDateWidget
                                (empty_label=("Year", "Month", "Day"),
                                 years=COMMISSION_YEAR_CHOICES,
                                 attrs={'class': 'date_field', 'type': 'date'}),
                                required=False)
    date_to = forms.DateField(widget=forms.SelectDateWidget
                                (empty_label=("Year", "Month", "Day"),
                                 years=COMMISSION_YEAR_CHOICES,
                                 attrs={'class': 'date_field', 'type': 'date'}),
                                required=False)
    authors = AutoCompleteSelectMultipleField('authors', required=False,
                                             help_text="Enter text to search. "
                                              "You can select multiple payees",
                                              label="Payees")

    class Meta:
        model = Commission
        fields = ['descriptions', 'funds', 'authors', 'date_from', 'date_to', ]

class PayeRequisitionForm(ModelForm):

    def generate_invoices(self):
        PayeRequisition.generate_invoices(self.cleaned_data['payee'],
                                          self.cleaned_data['date_from'],
                                          self.cleaned_data['date_to'])

    class Meta:
        model = PayeRequisition
        fields = ['payee', 'date_from', 'date_to', ]


import time
import json
import nxppy
import picamera
from subprocess import Popen
from threading import Thread
from flask import Flask, request, send_from_directory
from os import path

app = Flask(__name__)
camera = picamera.PiCamera()

records = [
    {"id": 0, "url": "/assets/img_0.jpg"}
]

data = {
    "0": "Zach's Card"
}

currentProc = None

@app.route('/')
def start():
    return send_from_directory("client", "index.html")

@app.route('/js/<file_name>')
def send_js (file_name):
    return send_from_directory("client/js", file_name)

@app.route('/css/<file_name>')
def send_css (file_name):
    return send_from_directory("client/css", file_name)

@app.route('/assets/<file_name>')
def send_asset (file_name):
    return send_from_directory("assets", file_name)

@app.route('/data')
def send_data():
    return json.dumps(records)

@app.route('/add')
def add_figure():
    global records, data, camera
    mifare = nxppy.Mifare()
    print("Starting")
    while True:
        try:
            val = mifare.select()
            value = ""
            for i in range(64):
                try:
                    value += mifare.read_block(i)
                except Exception:
                    break
            time.sleep(1)
            name = path.join("assets", "img_{}.jpg".format(len(records)))
            camera.capture(name)
            data[str(len(records))] = value
            records.append({"id": len(records), "url": name})
            return ""
        except Exception as e:
            print (e)
            pass
        time.sleep(1)
    time.sleep(4)

@app.route('/emu/<id_name>')
def start_emulation(id_name):
    global currentProc
    if currentProc is not None:
        currentProc.kill()
        currentProc = None
    try:
            currentProc = Popen(["explorenfc-cardemulation", "-t", data[id_name]])
    except Exception:
        pass
    return ""

@app.route('/cancel')
def stop_emulation():
    global currentProc
    if currentProc is not None:
        currentProc.kill()
    return ""

if __name__ == "__main__":
    app.run("0.0.0.0", 1337)


import os
from cassandra.cluster import Cluster
from cassandra.auth import PlainTextAuthProvider
from cassandra.query import SimpleStatement, BatchStatement

dir_path = os.path.dirname(os.path.realpath(__file__))
BUNDLE_PATH = os.path.join(dir_path, 'secure-connect-gesture-prediction.zip')

cloud_config = {
    'secure_connect_bundle': BUNDLE_PATH
}
auth_provider = PlainTextAuthProvider('UzJkZZyFwGSAxJCcKGLXtoEu',
                                      'ZntHG+Xa01.deSCmpuEGt1fRY_G+ptJgZZR9SluCpAk9Zz2Cqtkg2v7Q4FwpXi3-1qD7I9UMQzsNZsONamSju6f9fRGy4D8xWRJZJjvvGDYYaXiv1wmZxkqUk6W4bNhM')
cluster = Cluster(cloud=cloud_config, auth_provider=auth_provider)
session = cluster.connect()

def delete_data():
    query = f"""
            TRUNCATE gesture_prediction.gesture_prediction
            """
    session.execute(query)

def add_data(raw_df):
    delete_data()
    df = raw_df.copy()
    col = """id, time, channel1, channel2, channel3,
          channel4, channel5, channel6, channel7, channel8,
          class"""
    # for _, row in df.iterrows():
    #     query = f"""
    #     INSERT INTO gesture_prediction.gesture_prediction ({col})
    #     VALUES (uuid(), {int(row['time'])},{row['channel1']},{row['channel2']},
    #             {row['channel3']},{row['channel4']},{row['channel5']},
    #             {row['channel6']},{row['channel7']},{row['channel8']},{int(row['class'])})
    #     """
    #     session.execute(query)

    batch = BatchStatement()

    for _, row in df.iterrows():
        INSERT_STMT = f"""
            INSERT INTO gesture_prediction.gesture_prediction ({col})
            VALUES (uuid(), {int(row['time'])},{row['channel1']},{row['channel2']},
                    {row['channel3']},{row['channel4']},{row['channel5']},
                    {row['channel6']},{row['channel7']},{row['channel8']},{int(row['class'])})
            """
        batch.add(SimpleStatement(INSERT_STMT))

    session.execute(batch)

src/databaseconnection - not using/connect_database.py


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
import cv2
import pickle
import random

dataset = pd.read_csv('E:/celeba-dataset/list_attr_celeba.csv')
X = dataset.iloc[:, :-1].values
y = dataset.iloc[:,[0,21]].values
print(y[0][1])

DATADIR="E:/celeba-dataset/img_align_celeba/img_align_celeba"
categories=["Female","Male"]

training_data=[]
img_size=50

def create_training_data():
    i=0
    m=0
    f=0
    for img in os.listdir(DATADIR):
        try:
            if(y[i][1]==1 or (y[i][1]==-1 and f<20946)):
                img_array=cv2.imread(os.path.join(DATADIR,img),cv2.IMREAD_GRAYSCALE)
                new_array=cv2.resize(img_array,(img_size,img_size))
                if y[i][1]==1 :
                    gender=1
                    m=m+1
                else:
                    gender=0
                    f=f+1
                # i=i+1
                training_data.append([new_array,gender])
        except Exception as e:
            pass
        finally:
            if(i%100==0):
                print(i)
            i=i+1
            if i==50000:
                print("breaking")
                break
    print(m)
    print(f)

create_training_data()
print(training_data[0])

random.shuffle(training_data)

X=[]
y=[]

for a,b in training_data:
    X.append(a)
    y.append(b)

X=np.array(X).reshape(-1,img_size,img_size,1)
# X=X/255.0
f=open("X_pickle1.pkl","wb")
pickle.dump(X,f)
f.close()

w=open("y_pickle1.pkl","wb")
pickle.dump(y,w)
w.close()


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('bp_cupid', '0024_praxen_aktualisiere_zeitraeume'),
    ]

    operations = [
        migrations.AlterModelOptions(
            name='platz',
            options={'ordering': ['zeitraum'], 'verbose_name': 'Platz', 'verbose_name_plural': 'Plätze'},
        ),
    ]


import telebot #pip install pyTelegramBotAPI
import configure #Token in another python file

client = telebot.TeleBot(configure.config['token'])

@client.message_handler(content_types =  ['text'])
def get_text(message):
    if message.text.lower() == 'hello':
        client.send_message(message.chat.id,'hello unknown user')
    elif message.text.lower() == 'how are you':
        client.send_message(message.chat.id,'I am okay,and you 😁')

client.polling(none_stop=True,interval=0)


""" Main Datamaker Entrypoint"""
from __future__ import print_function
import argparse
from pathlib import Path

import datamaker.db as db

from multiprocessing import Pool

def _get_hist(pair_id, path, range):
    session = db.Session()
    pair = session.query(db.CurrencyPair).get(pair_id)
    return pair.download_historical_data(path, range)

def get_historical(path=Path(".")):
    """ Get historical"""
    parser = argparse.ArgumentParser(
        description="Datamaker CLI Tool",
        usage='''dm-hist [args]'''
    )
    parser.add_argument(
        "range", type=float, help="Range (in years) of data to get")
    args = parser.parse_args()

    session = db.Session()
    pairs = session.query(db.CurrencyPair).all()
    pool = Pool()
    for pair in pairs:
        pool.apply_async(_get_hist, args=(pair.id, path, args.range))
    pool.close()
    pool.join()

    for pair in pairs:
        print(len(pair.get_historical_database(path).ohlcv))
        pair.deduplicate_historical(path)
        print(len(pair.get_historical_database(path).ohlcv))

if __name__ == "__main__":
    get_historical()


#!/usr/bin/env python

import argparse
import csv
import json
import os
from pathlib import Path
import sys

import pysam

class Variant:
    def __init__(self, contig, position, reference, alt):
        self.contig = contig
        self.position = position
        self.reference = reference
        self.alt = alt
        self.name = None

    def key(self):
        return ",".join([str(x) for x in [self.contig, self.position, self.reference, self.alt]])

def load_vcf(filename):
    variants = list()
    f = pysam.VariantFile(filename,'r')
    for record in f:
        if len(record.alts) > 1:
            sys.stderr.write("Multi-allelic VCF not supported\n")
            sys.exit(1)

        # Skip any line in the .vcf that starts with '#'
        if not record.chrom.startswith('#'):
            v = Variant(record.chrom, record.pos, record.ref, record.alts[0])
            if "Name" in record.info:
                v.name = record.info["Name"]
            variants.append(v)

    return variants

def parse_sample_ids(sample_ids_path):
    """
    input: "path/to/sample_ids.tsv"
    output: ["sample-01", "sample-02", ...]
    """
    with open(sample_ids_path, 'r') as f:
        all_sample_ids = [line.strip() for line in f]

    return all_sample_ids

def parse_ncov_watch_ouput_by_sample_id(ncov_watch_output_path, all_sample_ids):
    """
    input: "/path/to/ncov_watch_output.tsv", ["sample-01", "sample-02", ...]
    output: {
              "sample-01": {
                             "MN908947.3,11287,GTCTGGTTTT,G": "ORF1ab:del3675-3677",
                             "MN908947.3,23063,A,T": "S:N501Y"
                           },
              "sample-02": {
                             ...
                           },
              ...
            }
    """
    ncov_watch_output_by_sample_id = {}

    for sample_id in all_sample_ids:
        ncov_watch_output_by_sample_id[sample_id] = {}

    try:
        with open(ncov_watch_output_path, 'r') as f:
            reader = csv.DictReader(f, delimiter='\t')
            for record in reader:
                sample_id = record['sample'].split('.')[0]
                if sample_id not in ncov_watch_output_by_sample_id.keys():
                    ncov_watch_output_by_sample_id[sample_id] = {}
                concatenated_key = ','.join([record['contig'], record['position'], record['reference'], record['alt']])
                ncov_watch_output_by_sample_id[sample_id][concatenated_key] = record['mutation']
    except:
        pass

    return ncov_watch_output_by_sample_id

def main(args):
    all_sample_ids = parse_sample_ids(args.sample_ids)

    ncov_watch_output_by_sample_id = parse_ncov_watch_ouput_by_sample_id(args.ncov_watch_output, all_sample_ids)

    watch_variants = load_vcf(args.watchlist)

    # Only count watchlist mutations with unique names
    unique_mutation_names_in_watchlist = list({var.name for var in watch_variants})
    num_unique_mutation_names_in_watchlist = len(unique_mutation_names_in_watchlist)

    print("\t".join(["sample_id", "watchlist_id", "num_observed_mutations", "num_mutations_in_watchlist", "proportion_watchlist_mutations_observed"]))
    for sample_id, observed_mutations in ncov_watch_output_by_sample_id.items():
        # Only count observed mutations with unique names
        unique_observed_mutation_names = list({mut for mut in observed_mutations.values()})
        num_unique_observed_mutation_names = len(unique_observed_mutation_names)
        proportion_watchlist_mutations_observed = num_unique_observed_mutation_names / num_unique_mutation_names_in_watchlist
        sample_summary_record = '\t'.join([
            sample_id, args.watchlist_id, str(num_unique_observed_mutation_names), str(num_unique_mutation_names_in_watchlist), "{:.3f}".format(proportion_watchlist_mutations_observed)
        ])
        print(sample_summary_record)

if __name__ == "__main__":
    description = 'Summarize mutations detected per sample and watchlist'
    parser = argparse.ArgumentParser(description=description)
    parser.add_argument('-s', '--sample-ids', help='file containing a single list of all sample IDs')
    parser.add_argument('-w', '--watchlist', help='file containing the mutations to screen for (.vcf format)')
    parser.add_argument('-i', '--watchlist-id', help='identifier for the watchlist')
    parser.add_argument('ncov_watch_output', help='output from ncov-watch.py')
    args = parser.parse_args()
    main(args)


import matplotlib.pyplot as plt
import math

dt = 0.001
Cm = 1
VNa = 50
VK = -77
VL = -54.387
gNa = 120
gK = 36
gL = 0.3
v = -65
m = 0.053
h = 0.6
n = 0.318

alfa = 2
beta = 1
Esyn = 0
T = 0
r = 0
gsyn = 1
Isyn = 0

vres = []
mres = []
hres = []
nres = []
time = []
rres = []
Isynres = []

f = open("h-h-model.txt", "w")
f.write(f'{str(v)},{str(m)},{str(h)},{str(n)},0,{str(r)}')

for i in range(int(100/dt)):
    milisegundo = int(i*dt)
    if milisegundo == 30 or milisegundo == 40 or milisegundo == 50:
        T = 1
    else:
        T = 0

    vres.append(v)
    mres.append(m)
    hres.append(h)
    nres.append(n)
    time.append(i*dt)
    rres.append(r)
    Isynres.append(Isyn)

    aM = (0.1 * (-v-40)) / (math.exp((-v-40)/10) - 1)
    bM = 4 * math.exp((-v-65)/18)
    aH = 0.07 * math.exp((-v-65)/20)
    bH = 1 / (math.exp((-v-35)/10) + 1)
    aN = (0.01 * (-v-55)) / (math.exp((-v-55)/10) - 1)
    bN = 0.125 * math.exp((-v-65)/80)

    Isyn = gsyn * r * (v - Esyn)
    vNext = v + dt*(-gL*(v - VL) - gNa*(m**3)*h*(v - VNa) - gK*(n**4)*(v - VK) - Isyn)
    mNext = m + dt*(aM*(1-m) - bM*m)
    hNext = h + dt*(aH*(1-h) - bH*h)
    nNext = n + dt*(aN*(1-n) - bN*n)
    r = r + dt*((alfa*T) * (1-r) - (beta*r))

    v = vNext
    m = mNext
    h = hNext
    n = nNext

    f.write(f'{str(v)},{str(m)},{str(h)},{str(n)},{str(i*dt)},{str(r)}\n')
f.close()

fig, axes = plt.subplots(2,1)
axes[0].plot(time, vres, label='Potencial de membrana')
axes[0].plot(time, Isynres, label = 'Corriente sináptica')
axes[0].set_title('Potencial de membrana / t', fontsize=24, loc='left')
axes[0].legend(loc="upper right", fontsize=18)
axes[0].set_ylabel('Potencial (mV)', fontsize=18)
axes[0].set_xlabel('Tiempo (ms)', fontsize=18)

axes[1].plot(time, rres)
axes[1].set_title('r / t', fontsize=24, loc='left')
axes[1].set_ylabel('r', fontsize=24)
axes[1].set_xlabel('Tiempo (ms)', fontsize=18)
axes[1].set_ylim(ymin=0, ymax=1)

plt.show()


import json
import re
import requests

class InvalidInput(Exception):
    pass

class Product:
    def __init__(self, product_url: str):
        self.product_url = product_url

    def url_validation(self):
        try:
            if type(self.product_url) is str and type(int(self.product_url.split("/")[-2])):

                if "myntra.com" in re.findall('://www.([\w\-\.]+)', self.product_url):
                    pass

                else:
                    raise InvalidInput("Provided input is not Myntra's product link")

        except Exception as e:
            raise InvalidInput(e)

    def fetch_data(self):
        from Headers import fetched_header
        self.url_validation()

        product_code = self.product_url.split("/")[-2]

        url = f"https://www.myntra.com/gateway/v2/product/{product_code}"
        fetched_header = fetched_header()

        response = requests.request("GET", url, cookies=fetched_header['cookies'],
                                    headers=fetched_header['headers'], data=fetched_header['payload'])

        raw_data = json.loads(response.text)

        product_name = raw_data['style']['brand']['name']
        image = raw_data['style']['media']['albums'][0]['images'][0]['src']
        to_replace = [["($height)", "732"], ["($qualityPercentage)", "90"], ["($width)", "600"]]

        # Resizing  product image to save network bandwidth
        for _ in range(3):
            image = image.replace(to_replace[_][0], to_replace[_][1])

        # print(raw_data)

        discounted_price = "None"
        mrp = raw_data['style']['mrp']

        for i in raw_data['style']['sizes']:
            if "sizeSellerData" in i:
                for j in i['sizeSellerData']:
                    if "discountedPrice" in j:
                        discounted_price = j['discountedPrice']
                        break

        available_count = 0

        for total in raw_data['style']['sizes']:
            if "sizeSellerData" in total:
                for each in total['sizeSellerData']:
                    if "availableCount" in each:
                        available_count += each['availableCount']

        # If encountered a product with no rating, handling with exception
        try:
            average_rating = raw_data['style']['ratings']['averageRating']
        except TypeError:
            average_rating = "None"

        return json.dumps(
            {'product_name': product_name, "image": image, "discounted_Price": discounted_price, "MRP": mrp,
             "available_Count": available_count, "average_Rating": format(average_rating, '.2f'),
             'product_code': product_code, 'product_url': self.product_url}, indent=4)

def product(url: str):
    return Product(url).fetch_data()


import os
import argparse
import logging
import pickle
from tqdm import tqdm
from sklearn.cluster import KMeans
import torch

from symbolnet_utils import dataloaders, networks, memory, folder_tool, emnist_tools

def register_memory(folder, data_path, classes, **kwargs):
    """
    Creates the file 'memory.pkl' inside 'folder' and saves the feature representations of images from the classes
    specified by 'classes' in it. This memory file contains a pickled dictionary (let's call it memory_dict) whose keys
    are class names. For instance, if the class 'A' is specified in 'classes', then memory_dict['A'] contains the output
    features obtained when mapping 'num_mem' randomly selected images labeled 'A' through the network's feature_network
    (except when using K-means, i.e. when 'num_see' > 'num_mem', see below). These feature representations can then be
    compared, using the network's evaluator, to the features of images to be classified (see e.g. classification_test).

    Note that it is possible to memorize classes that were not used during training!

    REQUIRED PARAMETERS
        - folder: A folder containing the network state dictionary for which to register memory, as well as the
                  architecture file for that state dictionary. (Both the required files 'network_state_dict.pth'
                  and 'network_info.pkl' are created when running 'train.py'.)
        - data_path: Path where the EMNIST dataset is located. The data_root path must lead to a folder containing
                     a folder called EMNIST which itself contains the dataset. If the dataset is not there, then
                     it gets downloaded automatically.
        - classes: A string specifying which classes to use for training. Basic options are 'digits', 'uppercases'
                   and 'lowercases'. See symbolnet_utils.emnist_tools.EmnistClassLabelsSetup for a detailed explanation
                   of how to specify other sets of classes.
                   TYPE: string

    OPTIONAL PARAMETERS
        - device: Can be either torch.device('cpu') or torch.device('cuda').
                  TYPE: torch.device
                  DEFAULT: torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')
        - emnist_split: Which split of the EMNIST dataset to use. Can be either 'byclass', 'bymerge', 'balanced'
                        or 'digits' (the 'letters' split is not supported).
                        TYPE: string
                        DEFAULT: 'byclass'
        - train: True means we use EMNIST training data, False means we use EMNIST test data.
                 TYPE: bool
                 DEFAULT: True
        - num_mem: Number of sets features to memorize for each class. See K-MEANS explanation below
                   TYPE: int
                   DEFAULT: 10
        - num_see: Number of images to look at for each at for each class. See K-MEANS explanation below
                   TYPE: int
                   DEFAULT: 10

    K-MEANS OPTION
        When 'num_mem' == 'num_see' (the default option), each set of features being memorized is obtained by mapping
        one randomly selected image from the dataset through the newtwork's feature_network module. When 'num_mem' <
        'num_see', 'num_see' images are mapped through the feature network for each class, and a K-means algorithm with
        'num_mem' centroids is performed on the resulting features. The 'num_mem' centroid features are then saved.
    """
    # -------------------------- SETTING UP --------------------------
    # Folder
    required_files = ['network_state_dict.pth', 'network_info.pkl']
    resume = folder_tool.check_folder(folder, required_files)
    if not resume:
        raise ValueError('...')

    # Device
    if 'device' in kwargs.keys():
        device = kwargs['device']
    else:
        device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')

    # Dataloader - batch_size=None by default, so we load one element at a time.
    train = kwargs['train']        if ('train'        in kwargs) else True
    split = kwargs['emnist_split'] if ('emnist_split' in kwargs) else 'byclass'
    loader = dataloaders.EmnistSinglesLoader(data_path = data_path,
                                             split = split,
                                             classes_string = classes,
                                             inds_save_folder = folder,
                                             train=train,
                                             device = device)
    loader_it = iter(loader)
    classes = loader.classes
    class_labels = loader.class_labels
    train_or_test = 'train' if train else 'test'

    # Network
    with open(os.path.join(folder, 'network_info.pkl'), 'rb') as net_dict_file:
        net_dict = pickle.load(net_dict_file)
    arch      = net_dict['architecture']
    comp_mode = net_dict['compare_mode']
    blocks    = net_dict['blocks']
    arch   = getattr(networks, arch)
    blocks = [int(b) for b in blocks.split(',')]
    net = arch(blocks=blocks, compare_mode=comp_mode).to(device)
    net.load_state_dict(torch.load(os.path.join(folder, 'network_state_dict.pth'), map_location=device))

    if 'memorized_classes' in net_dict.keys():
        mem_classes = net_dict['memorized_classes'] + classes
    else:
        mem_classes = classes
    mem_classes = emnist_tools.sort_classes(mem_classes)
    net_dict['memorized_classes'] = mem_classes

    with open(os.path.join(folder, 'network_info.pkl'), 'wb') as net_dict_file:
            pickle.dump(net_dict, net_dict_file, protocol=pickle.HIGHEST_PROTOCOL)

    # Numbers of images
    num_mem = kwargs['num_mem'] if ('num_mem' in kwargs) else 10
    num_see = kwargs['num_see'] if ('num_see' in kwargs) else 10
    compute_kmeans = (num_see > num_mem)

    # Logging
    log_file = os.path.join(folder, 'log.txt')

    logging.basicConfig(level = 0)
    logger = logging.getLogger(__name__) # We use a single logger with 2 handlers.
    logger.propagate = False
    logger.setLevel(0)

    console_handler = logging.StreamHandler() # Writes to console
    console_handler.setFormatter(logging.Formatter('%(message)s'))
    console_handler.setLevel(1)
    logger.addHandler(console_handler)

    file_handler = logging.FileHandler(log_file) # Writes to log.txt
    file_handler.setFormatter(logging.Formatter('%(message)s'))
    file_handler.setLevel(2)
    logger.addHandler(file_handler)

    # Initial message
    message  = f"Registering memory of the following classes:\n{classes}\n"
    if compute_kmeans:
        message += f"For each class, looking at {num_see} images and memorizing {num_mem} elements through a K-means algorithm.\n"
        message += f"Images are taken from the '{split} {train_or_test}' EMNIST split."
    else:
        message += f"For each class, memorizing {num_mem} images taken at random from the '{split} {train_or_test}' EMNIST split."
    message +=  "\n...\n"
    logger.log(2, message)

    # -------------------------- FINDING IMAGES TO MEMORIZE --------------------------
    logger.log(1, "Finding images to memorize.\n")
    # Get examples from each class
    examples = {class_ : [] for class_ in classes}
    while any(len(mem_list) < num_see for mem_list in examples.values()):
        try:
            im, label = next(loader_it)
            class_ = classes[class_labels.index(label)]
            if len(examples[class_]) < num_see:
                examples[class_].append(im)
        except StopIteration:
            break
    for class_ in classes:
        examples[class_] = torch.stack(examples[class_], dim=0).to(device)

    # -------------------------- MEMORIZING --------------------------
    logger.log(1, "Memorizing elements from each class.\n")
    # Map examples to features, perform K-means if required, save in memory file.
    with torch.no_grad():
        for class_ in tqdm(classes):
            batch = examples[class_]
            features = net.feature_network(batch)
            if compute_kmeans:
                for feat_key in features.keys():
                    np_feats = features[feat_key].cpu().numpy()
                    shape = list(np_feats.shape)
                    np_feats = np_feats.reshape((shape[0], -1))
                    feat_kmeans = KMeans(n_clusters=num_mem).fit(np_feats).cluster_centers_
                    shape[0] = num_mem
                    feat_kmeans = feat_kmeans.reshape(shape)
                    feat_kmeans = torch.from_numpy(feat_kmeans)
                    features[feat_key] = feat_kmeans
            memory.register_features(folder, class_, features)

    # -------------------------- EPILOGUE --------------------------
    message  = "DONE"
    message += "\n\n\n\n----------------------\n\n\n"
    logger.log(2, message)

if __name__ == '__main__':
    parser = argparse.ArgumentParser('...')
    # General
    parser.add_argument('--folder',      type=str,  required=True, help="Folder to create or from which to resume")
    # Dataset
    parser.add_argument('--data_path',    type=str, required=True,     help="Where the data is.")
    parser.add_argument('--classes',      type=str, required=True,     help="String specifying which classes to use.")
    parser.add_argument('--emnist_split', type=str, default='byclass', help="Which split of the EMNIST dataset to use.")
    # Memorize
    parser.add_argument('--num_mem', type=int, default=10, help='Number of elements to store in memories for each class.')
    parser.add_argument('--num_see', type=int, default=10, help='Number of images from each class to see while memorizing.')

    args = parser.parse_args()
    register_memory(**args.__dict__)


import setuptools

with open("README.md", "r") as fh:
    long_description = fh.read()

setuptools.setup(
    name="show_month_day",
    version="0.0.1",
    author="Tom",
    author_email="s099064146@gmail.com",
    description="Input string of starting year, month, date and ending year, month, date.",
    long_description=long_description,
    long_description_content_type="text/markdown",
    url="https://github.com/s099064146/Practice",
    packages=setuptools.find_packages(),
    classifiers=(
        "Programming Language :: Python :: 3",
        "License :: OSI Approved :: MIT License",
        "Operating System :: OS Independent",
    ),
)


from rest_framework.serializers import (ModelSerializer, SlugRelatedField,
                                        SerializerMethodField)
from .models import PhotoAlbum, Image

class ImageSerializer(ModelSerializer):

    class Meta:
        model = Image
        exclude = ('id',)

class PhotoAlbumSerializer(ModelSerializer):
    image_data= SerializerMethodField()
    thumbnail_data = SerializerMethodField()
    thumbnail = SlugRelatedField(slug_field='uuid',
                                 queryset=Image.objects.all())

    class Meta:
        model = PhotoAlbum
        exclude = ('id',)

    def get_image_data(self, obj):
        image_urls = [{'src':image.image.url, 'width':image.image.width,
                       'height':image.image.height}
                      for image in obj.images.select_related()]
        return image_urls

    def get_thumbnail_data(self, obj):
        return {'uuid':obj.thumbnail.uuid,
                'src':obj.thumbnail.image.url,
                'width':obj.thumbnail.width,
                'height':obj.thumbnail.image.height
               }


from typing import List
class Solution:
    def searchRange(self, nums: List[int], target: int) -> List[int]:
        left = 0
        right = len(nums)
        res = []
        while left < right:
            mid = left + (right - left) // 2
            if nums[mid] == target:
                right = mid
            elif nums[mid] < target:
                left = mid + 1
            elif nums[mid] > target:
                right = mid
        if left == len(nums) or nums[left] != target:
            res += [-1, -1]
            return res
        else:
            res.append(left)

        left = 0
        right = len(nums)
        while left < right:
            mid = left + (right - left) // 2
            if nums[mid] == target:
                left = mid + 1
            elif nums[mid] < target:
                left = mid + 1
            elif nums[mid] > target:
                right = mid
        res.append(right - 1)
        return res

if __name__ == '__main__':
    solution = Solution()
    res = solution.searchRange([5,7,7,8,8,10],8)
    print(res)


import logging
import global_vars
import time
import threading
import socket
import pickle
import server
import inspect
import ctypes
# membership_list = []  # membership_list = [(ip,port,time,type,listening),()]

class Spout():
    def __init__(self):
        self.ip = socket.gethostbyname(socket.gethostname())
        self.file_length = 0
        self.stop = global_vars.fault_stop
        self.is_stop = False
        self.line_total = []
        self.lines = 0
        self.sk = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)

    def start(self):
        self.stop = False
        if self.ip == global_vars.MASTER:
            if not self.is_ready():
                logging.info("There is no topology, please assign it.")

            else:
                self.read_stream()
                self.spout_work()

    def spout_work(self):
        global id
        spout_thread = threading.Thread(target=self.spout_thread)
        spout_thread.start()
        id = spout_thread.ident
        logging.info(spout_thread)

    def terminate(self):
        global id
        if self.ip == global_vars.MASTER:
            logging.info("start to stop")
            if not global_vars.result:
                self.stop_thread(id)
            self.sk.close()
            self.stop = True
            global_vars.stream_list.queue.clear()
            sk = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
            logging.info("********terminate**********")
            message = {
                'type': 'stream_data',
                'send_ip': socket.gethostbyname(socket.gethostname()),
                'data': 'TERMINATE'
            }
            for member in global_vars.membership_list:
                ip_port = (member[0], 7000)
                sk.sendto(pickle.dumps(message), ip_port)
            sk.close()

    def is_ready(self):
        flag = True
        if len(global_vars.membership_list) < 2:
            flag = False
        return flag

    def read_stream(self):
        file_name = ""
        if global_vars.membership_list[0][5] == 0:
            logging.info("Please select application.")
        elif global_vars.membership_list[0][5] == "1":
            file_name = global_vars.file_name_one
        elif global_vars.membership_list[0][5] == "2":
            file_name = global_vars.file_name_two
        elif global_vars.membership_list[0][5] == "3":
            file_name = global_vars.file_name_three
        while file_name:
            f = open(file_name, "r")
            lines = f.readlines()
            if not lines:
                break
            for line in lines:
                global_vars.stream_list.put(line)
            self.file_length = len(lines)
            f.close()
            break

    def spout_thread(self):
        time.sleep(0.5)
        while not self.stop and not self.is_stop:
            target_ip =[]
            for member in global_vars.membership_list:
                if member[4] == global_vars.BOLT_FILTER:
                    target_ip.append(member[0])
            ip_length = len(target_ip)
            logging.info("spout out doing now")
            logging.info(target_ip)
            # only read one line
            num = self.file_length // ip_length
            logging.info(num)
            for ips in target_ip:
                ip_port = (ips, 7000)
                count = 0
                if ips != target_ip[-1]:
                    logging.info("SpoutWorker to the filter bolt: " + ips + " start")
                    while count < num:
                        logging.info('+++++++++++++++')
                        send_message = global_vars.stream_list.get()
                        message = {
                            'type': 'stream_data',
                            'send_ip': socket.gethostbyname(socket.gethostname()),
                            'target_ip':ips,
                            'data': send_message
                        }
                        logging.info(message)
                        self.sk.sendto(pickle.dumps(message), ip_port)
                        time.sleep(0.005)
                        count += 1
                    message = {
                        'type': 'stream_data',
                        'send_ip': socket.gethostbyname(socket.gethostname()),
                        'target_ip': ips,
                        'data': global_vars.Stream_Stop
                    }
                    self.sk.sendto(pickle.dumps(message), ip_port)
                    logging.info(message)

                else:
                    logging.info("SpoutWorker to the filter bolt: " + ips + " start")
                    remain = self.file_length - num * (ip_length - 1)
                    while count < remain:
                        logging.info('+++++++++++++++')
                        send_message = global_vars.stream_list.get()
                        message = {
                            'type': 'stream_data',
                            'send_ip': socket.gethostbyname(socket.gethostname()),
                            'target_ip': ips,
                            'data': send_message
                        }
                        logging.info(message)
                        self.sk.sendto(pickle.dumps(message), ip_port)
                        time.sleep(0.006)
                        count += 1
                    message = {
                        'type': 'stream_data',
                        'send_ip': socket.gethostbyname(socket.gethostname()),
                        'target_ip': ips,
                        'data': global_vars.Stream_Stop
                    }
                    self.sk.sendto(pickle.dumps(message), ip_port)
                    logging.info(message)

            break
        global_vars.stream_list.queue.clear()
        logging.info("Stream concurrent list is empty, all the message have been sent out.")
        self.sk.close()

    def listen_thread(self):
        sk2 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
        sk2.bind(('0.0.0.0', 5050))
        target_ip = []
        for member in global_vars.membership_list:
            if member[4] == global_vars.BOLT_FILTER:
                target_ip.append(member[0])
        listen_ip = target_ip[-1]
        while True:
            msg, address = sk2.recvfrom(32768)
            receive_msg = pickle.loads(msg)
            if receive_msg['type'] == 'ack':
                send_i = receive_msg['data']
                if send_i in self.line_total:
                    self.line_total.remove(send_i)
            break
        ip_port = (listen_ip, 7000)
        if len(self.line_total) != 0:
            for k in self.line_total:
                message = {
                    'type': 'stream_data',
                    'send_ip': socket.gethostbyname(socket.gethostname()),
                    'target_ip': listen_ip,
                    'data': self.lines[k],
                    'lines': k
                }

                sk2.sendto(pickle.dumps(message), ip_port)
                logging.info(message)
        message = {
            'type': 'stream_data',
            'send_ip': socket.gethostbyname(socket.gethostname()),
            'target_ip': listen_ip,
            'data': global_vars.Stream_Stop,
            'lines': 0
        }
        sk2.sendto(pickle.dumps(message), ip_port)
        logging.info(message)

    def _async_raise(self,tid, exctype):
        """raises the exception, performs cleanup if needed"""
        tid = ctypes.c_long(tid)
        if not inspect.isclass(exctype):
            exctype = type(exctype)
        res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))
        if res == 0:
            raise ValueError("invalid thread id")
        elif res != 1:
            # """if it returns a number greater than one, you're in trouble,
            # and you should call it again with exc=NULL to revert the effect"""
            ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)
            raise SystemError("PyThreadState_SetAsyncExc failed")

    def stop_thread(self,id):
        self._async_raise(id, SystemExit)


from flask import request
from flask_restx import marshal_with, fields
from dependency_injector.wiring import Provide, inject
from wtforms import Form as BaseForm, fields as forms_fields, validators

from app.core.presentation.v1.helpers import AbstractResource, ResultEnum, ResultField, marshal_error_fields, jwt_required
from app.core.di.container import Container
from app.core.domain.search.entity import SearchFilter
from app.core.useCase.search import SearchUseCase

class Form(BaseForm):
    title = forms_fields.StringField('Title', [validators.Length(min=2)])
    year = forms_fields.IntegerField('Year')

class Search(AbstractResource):
    data = marshal_error_fields.copy()
    data.update({
        'items': fields.List(
            fields.Nested(
                {
                    'id': fields.Integer,
                    'titles': fields.List(
                        fields.Nested({
                            'language': fields.String(attribute='language.name'),
                            'value': fields.String
                        })
                    ),
                    'year': fields.Integer,
                    'rating': fields.Float
                },
                allow_null=True,
                skip_none=True
            )
        )
    })
    resource_fields = {
        'status': ResultField,
        'data': fields.Nested(data, skip_none=True)
    }

    @jwt_required
    @inject
    @marshal_with(fields=resource_fields, skip_none=True)
    def post(self, searchUseCase: SearchUseCase = Provide[Container.searchUseCase]):
        form = Form(data=request.get_json())
        if not form.validate():
            return self._send_form_error(form.errors, status=400)

        items = searchUseCase.get(
            SearchFilter(
                title=form.title.data,
                year=form.year.data
            )
        )

        return {
            'status': ResultEnum.success,
            'data': {'items': items}
        }


from socket import *
from datetime import *

#This object controls the TCP Client methods.
#There are slight variations between sending with UDP/TCP, so I split them into classes.

class TCPClient():

    def receiveMessage(self, client):
        modifiedMessage, serverAddress = client.recvfrom(2048)
        print(modifiedMessage.decode())

    def send(self, client, message):
        print(message)
        if message == "1":
            client.send('IP'.encode())
            self.receiveMessage(client)

        elif message == "2":
            client.send('PORT'.encode())
            self.receiveMessage(client)

        elif message == "3":
            current = 'TIMEDELAY' + str(datetime.utcnow())
            client.send(str(current).encode())

            modifiedMessage, serverAddress = client.recvfrom(2048)
            splitMessage = modifiedMessage.decode().split(' ')

            print('Delay SENDING: ' + splitMessage[0])  #Calculating time delay
            convertedServerTime = datetime.strptime(splitMessage[1]+' '+splitMessage[2],"%Y-%m-%d %H:%M:%S.%f")
            totalDelay = str(datetime.utcnow() - convertedServerTime)
            print('Delay RECEIVING: ' + totalDelay)

        elif message == '4':
            client.close()
            print("Exiting the client.")
            exit()

        else:
            client.send(message.encode())
            self.receiveMessage(client)
        # else:
        #     print('Invalid selection.')


from typing import NamedTuple, List, Iterable

from ..core import datetime_aware, Res, LazyLogger
from ..core.compat import removeprefix

logger = LazyLogger(__name__)

class Watched(NamedTuple):
    url: str
    title: str
    when: datetime_aware

    @property
    def eid(self) -> str:
        return f'{self.url}-{self.when.isoformat()}'

# todo define error policy?
# although it has one from google takeout module.. so not sure

def watched() -> Iterable[Res[Watched]]:
    try:
        from ..google.takeout.parser import events
        from google_takeout_parser.models import Activity
    except ModuleNotFoundError as ex:
        logger.exception(ex)
        from ..core.warnings import high
        high("Please set up my.google.takeout.parser module for better youtube support. Falling back to legacy implementation.")
        yield from _watched_legacy()
        return

    YOUTUBE_VIDEO_LINK = '://www.youtube.com/watch?v='

    # TODO would be nice to filter, e.g. it's kinda pointless to process Location events
    for e in events():
        if isinstance(e, Exception):
            yield e

        if not isinstance(e, Activity):
            continue

        url = e.titleUrl
        header = e.header
        title = e.title

        if url is None:
            continue

        if header in {'Image Search', 'Search', 'Chrome'}:
            # sometimes results in youtube links.. but definitely not watch history
            continue

        if header not in {'YouTube', 'youtube.com'}:
            # TODO hmm -- wonder if these would end up in dupes in takeout? would be nice to check
            # perhaps this would be easier once we have universal ids
            if YOUTUBE_VIDEO_LINK in url:
                # TODO maybe log in this case or something?
                pass
            continue

        if header == 'youtube.com' and title.startswith('Visited '):
            continue

        if title.startswith('Searched for') and url.startswith('https://www.youtube.com/results'):
            # search activity, don't need it here
            continue

        if title.startswith('Subscribed to') and url.startswith('https://www.youtube.com/channel/'):
            # todo might be interesting to process somewhere?
            continue

        # all titles contain it, so pointless to include 'Watched '
        # also compatible with legacy titles
        title = removeprefix(title, 'Watched ')

        if YOUTUBE_VIDEO_LINK not in url:
            if e.details == ['From Google Ads']:
                # weird, sometimes results in odd
                continue
            if title == 'Used YouTube' and e.products == ['Android']:
                continue

            yield RuntimeError(f'Unexpected url: {e}')
            continue

        yield Watched(
            url=url,
            title=title,
            when=e.time,
        )

from ..core import stat, Stats
def stats() -> Stats:
    return stat(watched)

### deprecated stuff (keep in my.media.youtube)

get_watched = watched

def _watched_legacy() -> Iterable[Watched]:
    from ..google.takeout.html import read_html
    from ..google.takeout.paths import get_last_takeout

    # todo looks like this one doesn't have retention? so enough to use the last
    path = 'Takeout/My Activity/YouTube/MyActivity.html'
    last = get_last_takeout(path=path)
    if last is None:
        return []

    watches: List[Watched] = []
    for dt, url, title in read_html(last, path):
        watches.append(Watched(url=url, title=title, when=dt))

    # todo hmm they already come sorted.. wonder if should just rely on it..
    return list(sorted(watches, key=lambda e: e.when))


from flask import request
from bson.objectid import ObjectId
from .controller import Controller
from app.models.user import User
from app.messages import HTTP_400_BAD_REQUEST

class UsersController(Controller):

    @classmethod
    def index(self, _id):
        try:
            if(_id != None):
                users = User.find_one(_id)
            else:
                users = User.find()
            return self.buildResponse(200, users)
        except Exception as e:
            print(e)
            return self.handleErrors()

    @classmethod
    def create(self):
        try:
            jsonData = request.json
            if ("name" in jsonData and "email" in jsonData and "password" in jsonData):
                (message, statusCode) = User.create(jsonData)
                return self.buildResponse(statusCode, {"message": message})
            else:
                return self.buildResponse(400, { 'message': HTTP_400_BAD_REQUEST })
        except Exception as e:
            print(e)
            return self.handleErrors()

    @classmethod
    def update(self, _id):
        try:
            if(not len(_id) == 24):
                return self.buildResponse(400, { 'message': 'ID inváĺido!' })
            jsonData = request.json
            (message, statusCode) = User.update(_id, jsonData)
            return self.buildResponse(statusCode, {"message": message})
        except Exception as e:
            print(e)
            return self.handleErrors()

    @classmethod
    def delete(self, _id):
        try:
            if(not len(_id) == 24):
                return self.buildResponse(400, { 'message': 'ID inváĺido!' })
            (message, statusCode) = User.delete(_id)
            return self.buildResponse(statusCode, {"message": message})
        except Exception as e:
            print(e)
            return self.handleErrors()


# Starter code for CS 165B HW4

"""
Implement the testing procedure here.

Inputs:
    Given the folder named "hw4_test" that is put in the same directory of your "predictio.py" file, like:
    - Main folder
        - "prediction.py"
        - folder named "hw4_test" (the exactly same as the uncompressed hw4_test folder in Piazza)
    Your "prediction.py" need to give the following required output.

Outputs:
    A file named "prediction.txt":
        * The prediction file must have 10000 lines because the testing dataset has 10000 testing images.
        * Each line is an integer prediction label (0 - 9) for the corresponding testing image.
        * The prediction results must follow the same order of the names of testing images (0.png – 9999.png).
    Notes:
        1. The teaching staff will run your "prediction.py" to obtain your "prediction.txt" after the competition ends.
        2. The output "prediction.txt" must be the same as the final version you submitted to the CodaLab,
        elsewise you will be given 0 score for your hw4.

**!!!!!!!!!!Important Notes!!!!!!!!!!**
    To open the folder "hw4_test" or load other related files,
    please use open('./necessary.file') instaed of open('some/randomly/local/directory/necessary.file').

    For instance, in the student Jupyter's local computer, he stores the source code like:
    - /Jupyter/Desktop/cs165B/hw4/prediction.py
    - /Jupyter/Desktop/cs165B/hw4/hw4_test
    If he use os.chdir('/Jupyter/Desktop/cs165B/hw4/hw4_test'), this will cause an IO error
    when the teaching staff run his code under other system environments.
    Instead, he should use os.chdir('./hw4_test').

    If you use your local directory, your code will report an IO error when the teaching staff run your code,
    which will cause 0 socre for your hw4.
"""
import os
import skimage
from skimage import data
from glob import glob
import numpy as np
from skimage.color import rgb2gray
from skimage import transform
import tensorflow as tf

def load_test_data(label_directory):
    images = []
    files = [os.path.join(label_directory, f)
             for f in os.listdir(label_directory)
             if f.endswith(".png")]
    files.sort()
    for f in files:
        images.append(skimage.data.imread(f))

    return images

def load_training_data(data_directory):
    directories = [d for d in os.listdir(data_directory)
                   if os.path.isdir(os.path.join(data_directory, d))]

    labels = []
    images = []
    directories.sort()
    for d in directories:
        label_directory = os.path.join(data_directory, d)
        files = [os.path.join(label_directory, f)
                 for f in os.listdir(label_directory)
                 if f.endswith(".png")]
        files.sort()
        for f in files:
            images.append(skimage.data.imread(f))
            labels.append(int(d))

    return images, labels

images, labels = load_training_data("./hw4_train")

images = np.array(images)
#images = rgb2gray(images)

x = tf.placeholder(dtype = tf.float32, shape = [None, 28, 28])
y = tf.placeholder(dtype = tf.int32, shape = [None])

images_flat = tf.contrib.layers.flatten(x)

logits = tf.contrib.layers.fully_connected(images_flat, 62, tf.nn.relu)

# Define a loss function
loss = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(labels = y,
                                                                    logits = logits))
# Define an optimizer
train_op = tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss)

# Convert logits to label indexes
correct_pred = tf.argmax(logits, 1)

# Define an accuracy metric
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

tf.set_random_seed(1234)

sess = tf.Session()

sess.run(tf.global_variables_initializer())

for i in range(201):
    _, accuracy_val = sess.run([train_op, accuracy], feed_dict={x:images, y:labels})
    #if i%10 == 0:
     #   print(accuracy_val)

test_images = load_test_data("./hw4_test")

test_images = np.array(test_images)
predicted = sess.run([correct_pred], feed_dict={x:test_images})[0]

print(predicted)


#Takes a Summary_counts file from TargetScan and either a miRNA name or a seed sequence.  Returns list of that are targeted by that miRNA or seed sequence.  Optionally, takes a list of genes of interest and will make special note of those genes that are targeted.

#Usage: python miRNAtargetgenes.py --help

import sys
import os
import argparse

def geneswithsites(targetscanfile, genelist, queryseedsite, querymiRNA):
    genes_with_sites = []
    interesting_genes_with_sites = []
    interesting_genes_in_targetscan = 0
    genes_with_seedsites = []
    interesting_genes_with_seedsites = []
    genes_with_miRNAtargets = []
    interesting_genes_with_miRNAtargets = []
    targetscanfh = open(targetscanfile, 'r')
    targetscanlines = []
    genes_with_queryseedsite = []
    for line in targetscanfh:
        line = line.strip().split('\t')
        if line[0] != 'Transcript ID': #skip header line
            targetscanlines.append(line)
    targetscanfh.close()

    if genelist:
        genelistfh = open(genelist, 'r')
        genes_of_interest = []
        for line in genelistfh:
            line = line.strip()
            genes_of_interest.append(line)
        genelistfh.close()

        for gene in genes_of_interest:
            for line in targetscanlines:
                if gene == line[1]:
                    interesting_genes_in_targetscan +=1
                    break
        sys.stderr.write('Found {0} of {1} supplied genes in TargetScan file.\n'.format(interesting_genes_in_targetscan, len(genes_of_interest)))

    for line in targetscanlines:
        transcript = line[0]
        gene = line[1]
        seed = line[2]
        targetsites = int(line[4]) + int(line[5]) + int(line[6]) + int(line[7]) + int(line[8]) + int(line[9]) + int(line[10]) + int(line[11])
        miRNA = line[12]
        if queryseedsite:
            if queryseedsite == seed and targetsites > 0:
                genes_with_seedsites.append(gene)
                if genelist:
                    if gene in genes_of_interest:
                        interesting_genes_with_seedsites.append(gene)
        if querymiRNA:
            if querymiRNA == miRNA and targetsites > 0:
                genes_with_miRNAtargets.append(gene)
                if genelist:
                    if gene in genes_of_interest:
                        interesting_genes_with_miRNAtargets.append(gene)

    if queryseedsite:
        if genelist:
            #Remove duplicates. They could be hits on multiple transcripts of the same gene.
            interesting_genes_with_seedsites = list(set(interesting_genes_with_seedsites))
            genes_with_seedsites = list(set(genes_with_seedsites))
            return genes_with_seedsites, interesting_genes_with_seedsites
        else:
            genes_with_seedsites = list(set(genes_with_seedsites))
            return genes_with_seedsites, None # just easier to always return two things
    if querymiRNA:
        if genelist:
            interesting_genes_with_miRNAtargets = list(set(interesting_genes_with_miRNAtargets))
            genes_with_miRNAtargets = list(set(genes_with_miRNAtargets))
            return genes_with_miRNAtargets, interesting_genes_with_miRNAtargets
        else:
            genes_with_miRNAtargets = list(set(genes_with_miRNAtargets))
            return genes_with_miRNAtargets, None

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--targetscan', type = str, help = 'TargetScan summary counts file')
    parser.add_argument('--genelist', type = str, help = 'Optional. List of genes of particular interest. E.g. Localized genes')
    parser.add_argument('--output', type = str, help = 'Output file.')
    group = parser.add_mutually_exclusive_group()
    group.add_argument('--queryseedsite', type = str, help = 'Seed site to search for. Corresponds to 3rd field of TargetScan file.')
    group.add_argument('--querymiRNA', type = str, help = 'miRNA to search for.  Corresponds to 13th field of TargetScan file.')
    args = parser.parse_args()

    outfh = open(args.output, 'w')

    genes_with_hits, interesting_genes_with_hits = geneswithsites(args.targetscan, args.genelist, args.queryseedsite, args.querymiRNA)
    if args.genelist:
        outfh.write('***All genes with hits***\n\n')
        for gene in genes_with_hits:
            outfh.write(gene + '\n')
        outfh.write('\n***Interesting genes with hits***\n\n')
        for gene in interesting_genes_with_hits:
            outfh.write(gene + '\n')
    elif args.genelist == None:
        for gene in genes_with_hits:
            outfh.write(gene + '\n')
    outfh.close()