from pprint import pprint
# from django.shortcuts import render
from django.contrib.auth import login, logout, get_user
from django.utils.decorators import method_decorator
from django.views.decorators.csrf import ensure_csrf_cookie, csrf_protect
from django_filters.rest_framework import DjangoFilterBackend
from rest_framework import viewsets, permissions, filters, status
from django.views.generic import TemplateView
# from .models import Car, Account
from rest_framework.authentication import BasicAuthentication
from rest_framework.permissions import IsAuthenticated, AllowAny
from rest_framework.response import Response
from rest_framework.views import APIView

from .serializers import *

# Auth views
ensure_csrf = method_decorator(ensure_csrf_cookie)
csrf_protect_method = method_decorator(csrf_protect)

class SetCSRFCookie(APIView):
    permission_classes = []
    authentication_classes = []

    @ensure_csrf
    def get(self, request):
        return Response("CSRF Cookie set.")

class LoginView(APIView):
    permission_classes = [AllowAny]
    authentication_classes = []

    def get(self, request):
        return Response(UserSerializer(get_user(request), context={'request': request}).data)

    @csrf_protect_method
    def post(self, request):
        serializer = LoginSerializer(data=request.data)
        serializer.is_valid(raise_exception=True)
        user = serializer.validated_data['user']
        login(request, user)
        return Response(UserSerializer(get_user(request), context={'request': request}).data)

class LogoutView(APIView):
    def get(self, request, format=None):
        # using Django logout
        logout(request)
        return Response(status=status.HTTP_200_OK)

# Libraries
class LibraryView(APIView):
    permission_classes = [AllowAny]
    # authentication_classes = []

    def get(self, request):
        result = {'techniqueModel': dict([(elem.id, elem.name) for elem in TechniqueModel.objects.all()])}
        result.update({'engineModel': dict([(elem.id, elem.name) for elem in EngineModel.objects.all()])})
        result.update({'transmissionModel': dict([(elem.id, elem.name) for elem in TransmissionModel.objects.all()])})
        result.update({'driveAxleModel': dict([(elem.id, elem.name) for elem in DriveAxleModel.objects.all()])})
        result.update({'steerableAxleModel': dict([(elem.id, elem.name) for elem in SteerableAxleModel.objects.all()])})
        result.update({'maintenanceType': dict([(elem.id, elem.name) for elem in MaintenanceType.objects.all()])})
        result.update({'failure': dict([(elem.id, elem.name) for elem in Failure.objects.all()])})
        result.update({'recovery': dict([(elem.id, elem.name) for elem in Recovery.objects.all()])})
        result.update({'car': dict([(field.name, field.verbose_name) for field in Car._meta.fields])})
        result.update({'maintenance': dict([(field.name, field.verbose_name) for field in Maintenance._meta.fields
                                            if field.name != 'id'])})
        result.update({'reclamation': dict([(field.name, field.verbose_name) for field in Reclamation._meta.fields
                                            if field.name != 'id'])})
        return Response(result)

# Auth model views
class UserViewset(viewsets.ReadOnlyModelViewSet):
    # authentication_classes = [BasicAuthentication]
    permission_classes = [IsAuthenticated]
    lookup_field = 'username'
    queryset = User.objects.all()
    serializer_class = UserSerializer
    filter_backends = [DjangoFilterBackend]
    filterset_fields = ['groups']

class GroupViewset(viewsets.ReadOnlyModelViewSet):
    permission_classes = [IsAuthenticated]
    lookup_field = 'name'
    queryset = Group.objects.all()
    serializer_class = GroupSerializer

# Main app view
class AppView(TemplateView):
    template_name = 'default.html'

    # def get_context_data(self, **kwargs):
    #     context = super(AppView, self).get_context_data(**kwargs)
    #     context['custom_context'] = 'some custom context'
    #     return context

# REST permissions
class AdminPermission(permissions.BasePermission):
    def has_permission(self, request, view):
        if request.user.is_staff:
            return True
        return request.user.is_authenticated and request.method in permissions.SAFE_METHODS

class ManagerPermission(permissions.BasePermission):
    def has_permission(self, request, view):
        return request.user.groups.filter(name='Manager').exists()

class ClientPermission(permissions.BasePermission):
    def has_permission(self, request, view):
        return request.user.groups.filter(name='Client').exists()

class ServicePermission(permissions.BasePermission):
    def has_permission(self, request, view):
        if view.basename == 'maintenance':
            model = Maintenance
        elif view.basename == 'reclamation':
            model = Reclamation
        else:
            return request.user.groups.filter(name='Service').exists()
        obj = model.objects.filter(**view.kwargs)
        is_contractor = obj[0].serviceCompany == request.user
        pprint(is_contractor)
        return is_contractor or request.user.groups.filter(name='Service').exists() and \
               request.method not in ['PATCH', 'PUT', 'DELETE']

class ReadonlyPermission(permissions.BasePermission):
    def has_permission(self, request, view):
        return request.method in permissions.SAFE_METHODS

# REST viewsets
# class AccountViewset(viewsets.ModelViewSet):
#     permission_classes = [AdminPermission]
#     lookup_field = 'user'
#     queryset = Account.objects.all()
#     serializer_class = AccountSerializer

class CarViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ManagerPermission | ReadonlyPermission]
    lookup_field = 'carNumber'
    queryset = Car.objects.all()
    filter_backends = [filters.OrderingFilter, filters.SearchFilter, DjangoFilterBackend]
    ordering_fields = ['shippingDate']
    ordering = ['shippingDate']
    filterset_fields = ['carNumber', 'client', 'serviceCompany']
    search_fields = ['carNumber', 'client__username', 'serviceCompany__username']

    # Client filtering
    def get_queryset(self):
        queryset = self.queryset
        user = self.request.user
        if user.groups.filter(name='Client').exists():
            queryset = queryset.filter(client=user)
        return queryset

    # def get_queryset(self):
    #     queryset = self.queryset
    #     params = self.request.query_params
    #     field_names = [field.name for field in queryset.model._meta.fields]
    #     for param in params:
    #         if param in field_names:
    #             queryset = queryset.filter(**{param: params[param]})
    #             # pprint(f'{param}: {params[param]} is a field')
    #         # pprint(queryset)
    #     return queryset

    def get_serializer_class(self):
        if self.request.user.is_authenticated:
            # print('-- auth')
            return CarSerializer
        # print('-- No auth')
        return CarBasicSerializer

class MaintenanceViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ClientPermission | ServicePermission | ManagerPermission]
    lookup_field = 'id'
    queryset = Maintenance.objects.all()
    serializer_class = MaintenanceSerializer
    filter_backends = [filters.OrderingFilter, filters.SearchFilter, DjangoFilterBackend]
    ordering_fields = ['date']
    ordering = ['date']
    filterset_fields = ['type', 'car', 'serviceCompany']
    search_fields = ['type__name', 'car__carNumber', 'serviceCompany__username']

    # Client filtering
    def get_queryset(self):
        queryset = self.queryset
        user = self.request.user
        if user.groups.filter(name='Client').exists():
            queryset = queryset.filter(car__client=user)
        return queryset

    # def get_queryset(self):
    #     queryset = self.queryset
    #     params = self.request.query_params
    #     field_names = [field.name for field in queryset.model._meta.fields]
    #     for param in params:
    #         if param in field_names:
    #             queryset = queryset.filter(**{param: params[param]})
    #             # pprint(f'{param}: {params[param]} is a field')
    #         # pprint(queryset)
    #     return queryset

class ReclamationViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ServicePermission | ManagerPermission]
    lookup_field = 'pk'
    queryset = Reclamation.objects.all()
    serializer_class = ReclamationSerializer
    filter_backends = [filters.OrderingFilter, filters.SearchFilter, DjangoFilterBackend]
    ordering_fields = ['failureDate']
    ordering = ['failureDate']
    filterset_fields = ['failure', 'recovery', 'car', 'serviceCompany']
    search_fields = ['failure__name', 'recovery__name', 'car__carNumber', 'serviceCompany__username']

    # Client filtering
    def get_queryset(self):
        queryset = self.queryset
        user = self.request.user
        if user.groups.filter(name='Client').exists():
            queryset = queryset.filter(car__client=user)
        return queryset
    # def get_queryset(self):
    #     queryset = self.queryset
    #     params = self.request.query_params
    #     field_names = [field.name for field in queryset.model._meta.fields]
    #     for param in params:
    #         if param in field_names:
    #             queryset = queryset.filter(**{param: params[param]})
    #             # pprint(f'{param}: {params[param]} is a field')
    #         # pprint(queryset)
    #     return queryset

class TechniqueModelViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ManagerPermission]
    lookup_field = 'pk'
    queryset = TechniqueModel.objects.all()
    serializer_class = TechniqueModelSerializer

class EngineModelViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ManagerPermission]
    lookup_field = 'pk'
    queryset = EngineModel.objects.all()
    serializer_class = EngineModelSerializer

class TransmissionModelViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ManagerPermission]
    lookup_field = 'pk'
    queryset = TransmissionModel.objects.all()
    serializer_class = TransmissionModelSerializer

class DriveAxleModelViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ManagerPermission]
    lookup_field = 'pk'
    queryset = DriveAxleModel.objects.all()
    serializer_class = DriveAxleModelSerializer

class SteerableAxleModelViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ManagerPermission]
    lookup_field = 'pk'
    queryset = SteerableAxleModel.objects.all()
    serializer_class = SteerableAxleModelSerializer

class MaintenanceTypeViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ManagerPermission]
    lookup_field = 'pk'
    queryset = MaintenanceType.objects.all()
    serializer_class = MaintenanceTypeSerializer

class FailureViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ManagerPermission]
    lookup_field = 'pk'
    queryset = Failure.objects.all()
    serializer_class = FailureSerializer

class RecoveryViewset(viewsets.ModelViewSet):
    permission_classes = [AdminPermission | ManagerPermission]
    lookup_field = 'pk'
    queryset = Recovery.objects.all()
    serializer_class = RecoverySerializer


from django.http import HttpResponse
from django.shortcuts import render, redirect, get_object_or_404
from .models import *
from .forms import *

def add_item(request):
    if request.method == 'POST':
        form = ItemForm(request.POST)
        form.save()
        return redirect('calc:items')
    else:
        form = ItemForm()
    context = {
        'form': form,
    }
    return HttpResponse(render(request, 'calc/add_item.html', context=context))


#!/usr/bin/env python3

import sys
import argparse

def main():
    args = parse_args()
    draw(args.fields)

def draw(fields):
    drawaux(fields, '┌', '┬', '┐', lambda s: '─' * len(s))
    drawaux(fields, '│', '│', '│', lambda s: s)
    drawaux(fields, '└', '┴', '┘', lambda s: '─' * len(s))

def drawaux(fields, start, middle, end, value):
    for i, s in enumerate(fields):
        if i == 0:
            print(start, end='')
        else:
            print(middle, end='')
        print(value(s), end='')
    else:
        print(end)

def parse_args():
    ap = argparse.ArgumentParser()
    ap.add_argument("-w", "--width",
                    type=int)
    ap.add_argument("-c", "--count",
                    type=int)
    ap.add_argument("fields",
                    nargs='+')

    args = ap.parse_args()
    if args.count is None:
        args.count = len(args.fields)

    if args.width is not None:
        for i in range(len(args.fields)):
            args.fields[i] = '%*s' % (args.width, args.fields[i])

    while args.count > len(args.fields):
        if args.width is None:
            args.fields.append('')
        else:
            args.fields.append(' ' * args.width)

    return args

if __name__ == '__main__':
    main()


# Import render from django shortcuts
from django.shortcuts import render

#Import check_email function, cv, model from Email_Spaming_Model.py
from Email_Spaming_Model import check_email,cv,model

# Making prediction function.
def predict_email(msg):
    # Here msg is going through check_email function.
    email_pre = check_email(msg)

    # Then passes through CountVectorizer.
    email_arr = cv.transform(email_pre)

    if (email_arr.shape == (0,7274)):
        email_post = ['ham']
    else:
        # Then Predict the email (spam or ham)
        email_post = model.predict(email_arr)

    # return the output.
    return email_post[0]

# Making email spaming function.
def emailhit(request):
    if request.method == 'POST':

        # receive input by post function.
        check_msg = request.POST['T1']

        #checking output is spam or not.
        if predict_email(check_msg) == 'spam':
            data = 'spam'

#           return spam to html file.
            return render(request, 'home.html', {'data': data})
        else:
            data = 'ham'
#           return ham to html file.
            return render(request, 'home.html', {'data': data})
    else:
#       return error if page is not load.
        return render(request, 'home.html', {'data': "Page Loading Fail"})


from django.conf import settings
from rest_framework import serializers
from rest_framework.exceptions import ValidationError
from students.models import Course

class CourseSerializer(serializers.ModelSerializer):
    class Meta:
        model = Course
        fields = ("id", "name", "students")

    def validate(self, attr):
        http_method = self.context["request"].method
        if http_method == 'POST':
            student_count = len(attr.get('students'))
            if student_count >= settings.MAX_STUDENTS_PER_COURSE:
                raise ValidationError(f'Вы превысили максимальное число студентов на курсе:'
                                      f' {settings.MAX_STUDENTS_PER_COURSE}')

        return attr


import numpy as np
from scipy.special import gamma

def between(x, y, xmin, xmax):
    """
    Set to zero the values of x that are outside the interval [xmin, xmax]

    Args:
        x (array-like): The coordinates of the array to be cut
        y (array-like): The array to be cut
        xmin (scalar): The minimum value of x
        xmax (scalar): The maximum value of x

    Returns:
        y (array-like): values outside of the domain [xmin, xmax] are set to 0
    """

    return np.heaviside(x-xmin,1)*np.heaviside(-x-xmax,1)*y

class DSD(object):
    def __init__(self, Dmin=0.0, Dmax=np.inf):

        if (Dmin > Dmax):
            raise AttributeError('The minimum size of any distribution must be \
                                  smaller then its maximum size, passed Dmin is\
                                  larger than Dmax')
        if (Dmin < 0):
            raise AttributeError('Dmin < 0 implies that negative diameters are \
                                  possible whereas DSDs domains are strictly \
                                  semidefinite positive')
        self.Dmin = Dmin
        self.Dmax = Dmax

    def __call__(self, D):
        if np.shape(D) == ():
            return 0.0
        else:
             return np.zeros_like(D)

    def generator(self, N):
        """
        Generate N samples of the distribution (probably easiest method is
        the cumulative normalized)
        """
        pass

    def normalizedCumulative(self):
        """
        Numerical method to implement cumulative distribution
        """
        pass

    def moment(self, x, N=1.e5):
        """
        Calculate numerically the moment of order x from N samples
        """

class InverseExponential(DSD):
    """Inverse exponential drop size distribution (DSD).

    Callable class to provide an inverse exponential DSD:
    N(D) = N0 * exp(-Lambda*D)

    Attributes:
        N0: the intercept parameter
        Lambda: the inverse scale parameter
        D_max: the maximum diameter to consider (defaults to 11/Lambda,
            i.e. approx. 3*D0, if None) # TODO: set to 99% of DSD volume

    Args (call):
        D: the particle diameter.

    Returns (call):
        The PSD value for the given diameter.
        Returns 0 for all diameters larger than D_max.
    """

    def __init__(self, N0=1.0, Lambda=1.0, D_max=None):
        self.N0 = float(N0)
        self.Lambda = float(Lambda)
        self.D_max = 11.0/Lambda if D_max is None else D_max

    def __call__(self, D):
        psd = self.N0 * np.exp(-self.Lambda*D)
        if np.shape(D) == ():
            if D > self.D_max:
                return 0.0
        else:
            psd[D > self.D_max] = 0.0
        return psd

class GammaPSD(InverseExponential):
    """Unnormalized Gamma drop size distribution (DSD).

    Callable class to provide an gamma DSD with the given parameters:
    N(D) = N0 * D**mu * exp(-Lambda*D)

    Attributes:
        N0: the intercept parameter.
        Lambda: the inverse scale parameter
        mu: the shape parameter
        D_max: the maximum diameter to consider (defaults to 11/Lambda,
            i.e. approx. 3*D0, if None)

    Args (call):
        D: the particle diameter.

    Returns (call):
        The PSD value for the given diameter.
        Returns 0 for all diameters larger than D_max.
    """

    def __init__(self, N0=1.0, Lambda=1.0, mu=0.0, D_max=None):
        super(UnnormalizedGammaPSD, self).__init__(N0=N0, Lambda=Lambda,
            D_max=D_max)
        self.mu = mu

    def __call__(self, D):
        # For large mu, this is better numerically than multiplying by D**mu
        psd = self.N0 * np.exp(self.mu*np.log(D)-self.Lambda*D)
        if np.shape(D) == ():
            if (D > self.D_max) or (D==0):
                return 0.0
        else:
            psd[(D > self.D_max) | (D == 0)] = 0.0
        return psd

class Lognormal(DSD):
    """Lognormal drop size distribution (DSD)

    Callable object to provide a lognormal drop size distribution with the given
    parameters

    The DSD form is:
    N(D) = Nt/(sqrt(2*pi)*g(D-theta)) * exp(-(ln(D-theta)-mu)**2 / (2*sigma**2))

    Attributes:
        Nt:
        g:
        theta:
        mu:
        sigma:

    """

    def __init__(self, Nt=1., g=1., theta=1., mu=0., sigma=1.):
        """
        """
        self.Nt = Nt
        self.g = g
        self.theta = theta
        self.mu = mu
        self.sigma = sigma

    def __call__(self, D):
        coeff = Nt/(np.sqrt(2*np.pi) * g * (D - theta))
        expon = np.exp(-(np.log(D - theta) -mu)**2 / (2. * sigma**2))
        psd = coeff * expon
        psd[D > Dmax] = 0.0
        psd[D < Dmin] = 0.0
        return psd

class NormalizedGamma(DSD):
    """Normalized gamma particle size distribution (DSD).

    Callable class to provide a normalized gamma DSD with the given
    parameters.

    TODO: the value of 3.67 comes from observation that RR is proportional to
    3.67th moment of the DSD, not sure if that is still ok

    The PSD form is:
    N(D) = Nw * f(mu) * (D/D0)**mu * exp(-(3.67+mu)*D/D0)
    f(mu) = 6/(3.67**4) * (3.67+mu)**(mu+4)/Gamma(mu+4)

    Attributes:
        D0: the median volume diameter.
        Nw: the intercept parameter.
        mu: the shape parameter.
        D_max: the maximum diameter to consider (defaults to 3*D0 when
            if None)

    Args (call):
        D: the particle diameter.

    Returns (call):
        The DSD value for the given diameter.
        Returns 0 for all diameters larger than D_max.
    """

    def __init__(self, D0=1.0, Nw=1.0, mu=0.0, D_max=None):
        self.D0 = float(D0)
        self.mu = float(mu)
        self.D_max = 3.0*D0 if D_max is None else D_max
        self.Nw = float(Nw)
        self.nf = Nw * 6.0/3.67**4 * (3.67+mu)**(mu+4)/gamma(mu+4)

    def __call__(self, D):
        d = (D/self.D0)
        psd = self.nf * np.exp(self.mu*np.log(d)-(3.67+self.mu)*d)
        if np.shape(D) == ():
            if (D > self.D_max) or (D==0.0):
                return 0.0
        else:
            psd[(D > self.D_max) | (D==0.0)] = 0.0
        return psd

class Binned(DSD):
    """Binned drop size distribution (DSD).

    Binned DSD given the bin edges and DSD values per bin.

    Args (constructor):
        The first argument to the constructor should specify n+1 bin edges,
        and the second should specify n bin_psd values.

    Args (call):
        D: the particle diameter.

    Returns (call):
        The PSD value for the given diameter.
        Returns 0 for all diameters outside the bins.
    """

    def __init__(self, bin_edges, bin_psd):
        if len(bin_edges) != len(bin_psd)+1:
            raise ValueError("There must be n+1 bin edges for n bins.")

        self.bin_edges = bin_edges
        self.bin_psd = bin_psd

    def psd_for_D(self, D):
        if not (self.bin_edges[0] < D <= self.bin_edges[-1]):
            return 0.0

        # binary search for the right bin
        start = 0
        end = len(self.bin_edges)
        while end-start > 1:
            half = (start+end)//2
            if self.bin_edges[start] < D <= self.bin_edges[half]:
                end = half
            else:
                start = half

        return self.bin_psd[start]

    def __call__(self, D):
        if np.shape(D) == (): # D is a scalar
            return self.psd_for_D(D)
        else:
            return np.array([self.psd_for_D(d) for d in D])


import torch
from torch.optim import AdamW
from torch.optim.lr_scheduler import LambdaLR
from torch.utils.data import DataLoader
from tqdm import tqdm
from transformers import PreTrainedModel

from .utils import get_prediction

def train_epoch(
    model: PreTrainedModel, train_loader: DataLoader, optim: AdamW, scheduler: LambdaLR
) -> torch.float64:
    """
    Train model epochs
    """
    # Set model to train mode
    model.train()

    total_loss: torch.float64 = 0

    pbar = tqdm(train_loader)

    for batch in pbar:
        optim.zero_grad()

        outputs = get_prediction(model, batch)
        loss = outputs[0]

        # backwards pass
        loss.backward()

        # update weights
        optim.step()

        # update the learning rate
        scheduler.step()

        total_loss += loss.item()

        pbar.set_postfix({"batch loss": loss.item()})

    total_loss /= len(train_loader)

    return total_loss


import re
import time
import os
from copy import deepcopy

import numpy as np
import json as js

class Logger(object):
  def __init__(self, filename=None, overwrite=True):
    self.filename = filename
    if filename is None:
      self.log_to_file = False
    else:
      self.log_to_file = True
      if overwrite:
        self.file_obj = open(filename, "w", buffering=1)
      else:
        self.file_obj = open(filename, "r+", buffering=1)
      self.file_obj.seek(0)

  def js_dumpstring(self, obj):
    """Dump json string with special NumpyEncoder"""
    return js.dumps(obj, sort_keys=True, indent=2, cls=NumpyEncoder)

  def log_trainable_variables(self, name_list):
    """
    Use logging to write names of trainable variables in model
    Inputs:
      name_list: list containing variable names
    """
    js_str = self.js_dumpstring(name_list)
    self.log_info("<train_vars>"+js_str+"</train_vars>")

  def log_params(self, params):
    """
    Use logging to write model params
    Inputs:
      params: [dict] containing parameters values
    """
    out_params = deepcopy(params)
    if "rand_state" in out_params.keys():
      del out_params["rand_state"]
    js_str = self.js_dumpstring(out_params)
    self.log_info("<params>"+js_str+"</params>")

  def log_schedule(self, sched):
    """Use logging to write current schedule"""
    js_str = self.js_dumpstring(sched)
    self.log_info("<schedule>"+js_str+"</schedule>")

  def log_info(self, string):
    """Log input string"""
    now = time.localtime(time.time())
    time_str = time.strftime("%m/%d/%y %H:%M:%S", now)
    out_str = "\n" + time_str + " -- " + str(string)
    if self.log_to_file:
      self.file_obj.write(out_str)
    else:
      print(out_str)

  def load_file(self, filename=None):
    """
    Load log file into memory
    Outputs:
      log_text: [str] containing log file text
    """
    if filename is None:
      self.file_obj.seek(0)
    else:
      self.file_obj = open(filename, "r", buffering=1)
    text = self.file_obj.read()
    return text

  def read_js(self, tokens, text):
    """
    Read js string encased by tokens and convert to python object
    Outpus:
      output: converted python object
    Inputs:
      tokens: [list] of length 2 with [0] entry indicating start token and [1]
        entry indicating end token
      text: [str] containing text to parse, can be obtained by calling load_file()
    TODO: Verify that js_matches is the same type for both conditionals at the end
      I believe js_matches should be a list at all times. That way when e.g. read_params
      is called the output is a list no matter how many params specifications there are
      in the logfile.
    """
    assert type(tokens) == list, ("Input variable tokens must be a list")
    assert len(tokens) == 2, ("Input variable tokens must be a list of length 2")
    matches = re.findall(re.escape(tokens[0])+"([\s\S]*?)"+re.escape(tokens[1]), text)
    if len(matches) > 1:
      js_matches = [js.loads(match) for match in matches]
    else:
      js_matches = [js.loads(matches[0])]
    return js_matches

  def read_trainable_variables(self, text):
    """
    Read params from text file and return as a dictionary
    Outpus:
      params: converted python object
    Inputs:
      text: [str] containing text to parse, can be obtained by calling load_file()
    """
    tokens = ["<train_vars>", "</train_vars>"]
    trainable_var_names = self.read_js(tokens, text)[-1]
    return trainable_var_names

  def read_params(self, text):
    """
    Read params from text file and return as a params object or list of params objects
    Outpus:
      params: converted python object
    Inputs:
      text: [str] containing text to parse, can be obtained by calling load_file()
    """
    tokens = ["<params>", "</params>"]
    params = self.read_js(tokens, text)
    param_list = []
    for param_dict in params:
      param_obj = type("param_obj", (), {})()
      for key, val in param_dict.items():
        setattr(param_obj, key, val)
      param_list.append(param_obj)
    return param_list

  def read_schedule(self, text):
    """
    Read schedule from text file and return as a list of dictionaries
    Outpus:
      schedule: converted python object
    Inputs:
      text: [str] containing text to parse, can be obtained by calling load_file()
    """
    tokens = ["<schedule>", "</schedule>"]
    return self.read_js(tokens, text)[-1]

  def read_stats(self, text):
    """
    Generate dictionary of lists that contain stats from log text
    Outpus:
      stats: [dict] containing run statistics
    Inputs:
      text: [str] containing text to parse, can be obtained by calling load_file()
    """
    tokens = ["<stats>", "</stats>"]
    js_matches = self.read_js(tokens, text)
    stats = {}
    for js_match in js_matches:
      if type(js_match) is str:
        js_match = {js_match:js_match}
      for key in js_match.keys():
        if key in stats:
          stats[key].append(js_match[key])
        else:
          stats[key] = [js_match[key]]
    return stats

  def __del__(self):
    if self.log_to_file and hasattr(self, "file_obj"):
      self.file_obj.close()

class NumpyEncoder(js.JSONEncoder):
  def default(self, obj):
    if(callable(obj)):
      return None
    elif(isinstance(obj, np.integer)):
      return int(obj)
    elif(isinstance(obj, np.floating)):
      return float(obj)
    elif(isinstance(obj, np.ndarray)):
      return obj.tolist()
    else:
      return super(NumpyEncoder, self).default(obj)


# coding: utf-8
from setuptools import setup, find_packages
import os

version = '1.0b2.dev0'
description = "A package to convert every image stored and uploaded to a \
Plone site to PNG format."
long_description = open("README.txt").read() + "\n" + \
                   open(os.path.join("docs", "INSTALL.txt")).read() + "\n" + \
                   open(os.path.join("docs", "CREDITS.txt")).read() + "\n" + \
                   open(os.path.join("docs", "HISTORY.txt")).read()

setup(name='openmultimedia.imagewatchdog',
      version=version,
      description=description,
      long_description=long_description,
      classifiers=[
        "Development Status :: 4 - Beta",
        "Environment :: Web Environment",
        "Framework :: Plone",
        "Framework :: Plone :: 4.1",
        "Framework :: Plone :: 4.2",
        "Framework :: Plone :: 4.3",
        "Intended Audience :: System Administrators",
        "License :: OSI Approved :: GNU General Public License v2 (GPLv2)",
        "Operating System :: OS Independent",
        "Programming Language :: Python",
        "Programming Language :: Python :: 2.6",
        "Programming Language :: Python :: 2.7",
        "Topic :: Software Development :: Libraries :: Python Modules",
        ],
      keywords='plone images jpeg gif png',
      author='Juan Pablo Giménez',
      author_email='jpg@rosario.com',
      url='https://github.com/OpenMultimedia/openmultimedia.imagewatchdog',
      license='GPLv2',
      packages=find_packages('src'),
      package_dir={'': 'src'},
      namespace_packages=['openmultimedia', ],
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          'setuptools',
          'Pillow',
          'Products.CMFPlone',
      ],
      extras_require={'test': ['plone.app.testing']},
      entry_points="""
      # -*- Entry points: -*-
      [z3c.autoinclude.plugin]
      target = plone
      """,
      )


import sql.sql_backend as sql
import distance.top_users as tu
import re

###################################################################################################
#                                                                                #
#                                      UTILIZACIÓN DEL SCRIPT                                     #
#                                                                                #
###################################################################################################
#                                                                                #
# 1. Ejecutar el script con python3 reduced_db.py                                                 #
#    + El script actualizará user.sql y mention.sql (los archvios importantes para las consultas) #
#    + El script irá mostrando el progreso de la ejecución                                        #
#    + Hay que esperar a que termine la ejecución                                                 #
# 2. Editar el archivo user.sql:                                                                  #
#    + Añadir la siguiente linea antes de la inserción de los usuarios: (línea 41)                #
#      INSERT INTO `user` (`id`, `user_id`, `created_at`, `retrieved_at`) VALUES                  #
#    + En la última inserción, quitar ',' y añadir ';'                                            #
# 3. Editar el archivo mention.sql:                                                               #
#    + Abrir el archivo, buscar y eliminar todas las líneas líneas que concuerden con:            #
#      INSERT INTO `mention` (`id`, `tweet_id`, `user_id`, `retrieved_at`) VALUE;                 #
#    + En la última inserción, quitar ',' y añadir ';'                                            #
#                                                                                #
###################################################################################################

def main():
    prueba_db = sql.get_engine('phpmyadmin')

    # Obtenemos los top users
    print("Obteniondo top users...")
    top_users = tu.top_users(prueba_db)
    not_delete_users = []

    print("Leyendo usuarios...")

    with open('/home/rubgarfue/Descargas/original_db/user.sql') as f:
        # Obtenemos todas las lineas del fichero
        lines = f.readlines()
        delete_lines = []
        for i in range(len(lines)):
            print("Linea " + str(i+1) + "/" + str(len(lines)))
            # Si la linea es la creación de un nuevo usuario
            if re.match(r"^\([0-9]+, ", lines[i]):
                split = lines[i].split(',')
                user = split[1][2:-1]
                # Si el usuario a crear no está en top lo eliminamos
                if user not in top_users:
                    delete_lines.append(i)
                else:
                    not_delete_users.append(split[0][1:])
            elif re.match(r"^INSERT INTO `user`", lines[i]):
                delete_lines.append(i)

        print("\nBorrando lineas...")
        delete_len = len(delete_lines)

        # Borramos las lineas indicadas
        write_lines = lines[:delete_lines[0]]
        for i in range(delete_len-1):
            print(str(i+1) + "/" + str(len(delete_lines)) + " borrado")
            write_lines += lines[delete_lines[i]+1:delete_lines[i+1]]
        write_lines += lines[delete_lines[-1]+1:]

    print("\n" + str(delete_len) + " lineas borradas\n")
    print("Escribiendo usuarios...")

    # Escribimos el nuevo fichero
    with open('/home/rubgarfue/Descargas/reduced_db/user.sql', 'w') as f:
        f.write(''.join(write_lines))

    ###############################################################################################

    print("Leyendo mentions...")

    with open('/home/rubgarfue/Descargas/original_db/mention.sql') as f:
        # Obtenemos todas las lineas del fichero
        lines = f.readlines()
        delete_lines = []
        for i in range(len(lines)):
            print("Linea " + str(i+1) + "/" + str(len(lines)))
            # Si la linea es la creación de una nueva mención
            if re.match(r"^\([0-9]+, ", lines[i]):
                user = lines[i].split(',')[2][1:]
                # Si la mencion a crear menciona a un usaurio no creado la eliminamos
                if user not in not_delete_users:
                    delete_lines.append(i)

        print("\nBorrando lineas...")
        delete_len = len(delete_lines)

        # Borramos las lineas indicadas
        write_lines = lines[:delete_lines[0]]
        for i in range(delete_len-1):
            print(str(i+1) + "/" + str(len(delete_lines)) + " borrado")
            write_lines += lines[delete_lines[i]+1:delete_lines[i+1]]
        write_lines += lines[delete_lines[-1]+1:]

        print("\nLeyendo líneas de inserción...")
        # Leemos las lineas del texto a escribir
        insert_change = []
        for i in range(len(write_lines)):
            print("Linea " + str(i+1) + "/" + str(len(write_lines)))
            # Si la linea es la inserción de los usuarios la añadimos al cambio
            if re.match(r"^INSERT INTO `mention`", write_lines[i]):
                insert_change.append(i)

        print("\nCambiando líneas de inserción...")
        # Cambiamos las líneas de inserción
        insert_change_len = len(insert_change)
        for i in range(1, insert_change_len-1):
            print(str(i) + "/" + str(len(insert_change)))
            write_lines[insert_change[i]-1] = write_lines[insert_change[i]-1][:-2] + ';\n'

    print("\n" + str(len(delete_lines)) + " lineas borradas\n")
    print("Escribiendo mentions...")

    # Escribimos el nuevo fichero
    with open('/home/rubgarfue/Descargas/reduced_db/mention.sql', 'w') as f:
        f.write(''.join(write_lines))

if __name__ == '__main__':
    main()


#!/usr/bin/python3

from copy import copy
from collections import Counter
import sys
from time import time

name='average'
params=('a','b')

# For each symbol: (isFunction, numParams, isCommutative)
grammar={
	'b':(0,0,0),
	'a':(0,0,0),
	'sum':(1,2,1),
	'half':(1,1,0),
}

# The only limit is the number of primitive functions used controlled by length param

# F ->N1'('P')'|N2'('P','P')'
# N1 ->'half'
# N2 ->'sum'
# P ->'a'|'b'|F

tables=[ [1,2], [1,2] ]
row=[ 'X' for i in range(len(tables))]

def pScalar (tables,row,tableIndex=None,commutative=False):

	global cache

	indexCache = str(tables)
	if indexCache in cache:
		return cache[indexCache]

	if tableIndex==None:
		tableIndex=len(row)

	if tableIndex==1:
		results=[]
		for row[tableIndex-1] in tables[tableIndex-1]:
			results.append(copy(row))
		#print("Opciones:",results)
		return results
	else:
		results=[]
		#print("Empieza el bucle")
		for row[tableIndex-1] in tables[tableIndex-1]:
			if not commutative:
				results.extend(pScalar(tables,row,tableIndex-1))
			else:
				pack=pScalar(tables,row,tableIndex-1)
				#print("Pack:",pack)
				for i in pack:
					pset=Counter(i)
					#print("Set: ",results)
					#print("Lalal",pset)
					exists=False
					for j in results:
						if pset==Counter(j):
							exists=True
							#print("Existe!",pset,j)
					if not exists:
						results.append(i)
		#print("Termina el bucle con ",results)
		cache[str(tables)]=results
		return results

"""
def pScalarGen (tables,row,tableIndex=None):

	if tableIndex==None:
		tableIndex=len(row)

	if tableIndex==1:
		results=[]
		for row[tableIndex-1] in tables[tableIndex-1]:
			results.append(copy(row))
		yield results
	else:
		results=[]
		for row[tableIndex-1] in tables[tableIndex-1]:
			results+=list(pScalarGen(tables,row,tableIndex-1))[0]
		yield results

print('Generator')
time0=time()
for i in pScalarGen(tables,row):
	pass
	print('PScalar',i)
print('Elapsed generator',time()-time0)
"""
"""
time1=time()
print('Recursive')
for i in pScalar(tables,row,commutative=True):
	print('PScalar',i)
print('Elapsed recursive',time()-time1)
sys.exit()
"""

cache={}

# Primitives, params of the parent, and the number of funcions to use inside
def getNodeKey (grammar,maxLevel,minLevel=0):

	global cache

	# Retrieve options that fits in the actual level
	options=[ i for i in grammar if grammar[i][0]<= maxLevel and maxLevel>=minLevel]

	#print( "Options en ",maxLevel,": ",options)
	for i in options:
		isFunction,params,isCommutative=grammar[i]
		#print("pilla "+str(i))
		if isFunction:
			menu=list(getNodeKey(grammar, maxLevel-1,minLevel))
			if menu:
				#indexCache = str(menu)
				#if indexCache in cache:
				if not menu:
					scalarResults=cache[indexCache]
					#print("Hit!",)
				else:
					#print("-menu",menu)
					parts=[ menu for i in range(params) ]
					#print("parts:",parts)
					body=[ 'X' for i in range(len(parts))]
					#print("From parts:",parts)
					scalarResults=pScalar(parts,body,commutative=isCommutative)
					#cache[indexCache]=scalarResults
				for j in scalarResults:
					#print("   Chosen: ",j)
					yield i + '(' + ','.join(j) + ')'
		else:               	# its a parameter
			yield i
	return

time0=time()
cnt=0
maxLevel=int(sys.argv[1])
if len(sys.argv)==3:
	minLevel=int(sys.argv[2])
else:
	minLevel=0
for i in getNodeKey(grammar,maxLevel,minLevel):
	print ("Result: ",i)
	cnt=cnt+1
print("Total:", cnt," en ",time()-time0)

#for i in cache:
#	print("Cache ",i,cache[i])


#!/usr/bin/env python3
import pickle
import re
import csv
import random
import string
import mojimoji
import MeCab
from gensim.models.doc2vec import Doc2Vec
from gensim.models.doc2vec import TaggedDocument

# mecabのneologd辞書のファイルパス
MECAB_NEOLOGD = '/usr/local/lib/mecab/dic/mecab-ipadic-neologd'

def preprocessing_text(text):
    # 半角・全角の統一
    text = mojimoji.han_to_zen(text)
    # 改行、半角スペース、全角スペースを削除
    text = re.sub('\r', '', text)
    text = re.sub('\n', '', text)
    text = re.sub('　', '', text)
    text = re.sub(' ', '', text)
    # 数字文字の一律「0」化
    text = re.sub(r'[0-9 ０-９]+', '0', text)  # 数字

    # カンマ、ピリオド以外の記号をスペースに置換
    for p in string.punctuation:
        if (p == ".") or (p == ","):
            continue
        else:
            text = text.replace(p, " ")

    return text

# 分かち書き
def tokenizer_mecab(text):
    m_t = MeCab.Tagger('-Owakati')
    try:
        f = open(MECAB_NEOLOGD, 'r')
        f.close()
    except IsADirectoryError:
         m_t = MeCab.Tagger('-Owakati -d '+MECAB_NEOLOGD)
    except FileNotFoundError:
        pass

    text = m_t.parse(text)  # これでスペースで単語が区切られる
    ret = text.strip().split()  # スペース部分で区切ったリストに変換
    return ret

# 前処理と分かち書きをまとめた関数を定義
def tokenizer_with_preprocessing(text):
    text = preprocessing_text(text)  # 前処理の正規化
    ret = tokenizer_mecab(text)  # Mecabの単語分割
    return ret

# 基本設計
class BaseCh():
    position = 0
    position_dic = {
        '村人': 0, '占い師': 1, '霊媒師': 2, '狩人': 3,
        '人狼': 4, '狂人': 5, '共有者': 6, '妖狐': 7
    }
    status = 0
    corpus = {
        'Zzz…。':[0], 'ひかり':[0], '腹筋！':[0], 'やっほー':[0],
        'こんばんはー。':[0], 'は？':[0], 'やあ':[0], 'どきどき':[0],
        'ﾑｼｬﾑｼｬﾑｼｬﾑｼｬ((´)艸(｀ ))｡ﾟ｡':[0], 'ずさー。':[0], 'おはようございます。':[0],
        'よろしくお願いします。':[0], 'こんばんわー':[0], '呼んだ？':[0], 'ズサー':[0],
        'よろしく':[0], 'ずさー':[0], '人狼なんているわけないじゃん。みんな大げさだなあ':[0],
        'ふぁーあ……ねむいな……寝てていい？':[0]
    }
    predict_positions = {}

    def __init__(self):
        pass

    # 役職から役職の番号へ
    def ch2position(self, ch):
        return self.position_dic[ch]

    # ch2positionの逆変換
    def position2ch(self, position):
        keys = [ k for k,v in self.position_dic.items() if v == position ]
        return keys[0] if keys else None

    # 状態からコーパス検索
    def status2seq(self):
        seq = [ s for s,v in self.corpus.items() if v[0] == self.status ]
        if seq:
            index = random.randint(0,len(seq)-1)
            return seq[index]
        return None

    # 出力
    def output(self, s):
        print(s)

class Position0(BaseCh):
    def say(self):
        seq = self.status2seq()
        self.output(seq)

if __name__ == '__main__':
    '''
    documents = [[],[]]
    with open('corpus.csv', 'r') as f:
        for row in csv.reader(f, delimiter='\t'):
            documents[0].append(tokenizer_with_preprocessing(row[0]))
            documents[1].append(row[1])

    vec = [ {} for i in range(len(documents)) ]
    for i in set(documents[0]):
        for j in range(len(documents)):
            vec[j][i] = tokenized[j].count(i)

    print(vec)

    ue = st1 = st2 = 0
    for i in tokens:
        ue += vec[0][i] * vec[1][i]
        st1 += vec[0][i] ** 2
        st2 += vec[1][i] ** 2

    print(ue / (math.sqrt(st1) * math.sqrt(st2)))
    '''

    position0 = Position0()
    while True:
        try:
            s = input()
            if s == 'quit':
                break
        except EOFError:
            break
    position0.say()


import scipy
from keras.preprocessing import image
from keras.applications import inception_v3
from keras import backend as K
import numpy as np

def resize_img(img, size):
    img = np.copy(img)
    factors = (1,
            float(size[0]) / img.shape[1],
            float(size[1]) / img.shape[2],
            1)
    return scipy.ndimage.zoom(img, factors, order=1)

def deprocess_image(x):
    # 通用函数，将一个张量转换为有效图像
    if K.image_data_format() == 'channels_first':
        x = x.reshape((3, x.shape[2], x.shape[3]))
        x = x.transpose((1, 2, 0))
    else:
        x = x.reshape((x.shape[1], x.shape[2], 3))

    x /= 2.
    x += 0.3
    x *= 255.
    x = np.clip(x, 0, 255).astype('uint8')
    return x

def save_img(img, fname):
    pil_img = deprocess_image(np.copy(img))
    scipy.misc.imsave(fname, pil_img)

# 用于打开图像、改变图像大小以及将图像格式转化为 Inception V3 模型能够处理的张量
def preprocess_image(image_path):
    img = image.load_img(image_path)
    img = image.img_to_array(img)
    img = np.expand_dims(img, axis=0)
    img = inception_v3.preprocess_input(img)
    return img


import h5py
import matplotlib.pyplot as plt
import numpy as np
import argparse
from scipy.optimize import curve_fit

fig_dict={0:(0,0),1:(0,1),2:(0,2),3:(0,3),4:(0,4),5:(0,5),6:(0,6),7:(0,7),
        8:(1,0),9:(1,1),10:(1,2),11:(1,3),12:(1,4),13:(1,5),14:(1,6),15:(1,7),
        16:(2,0),17:(2,1),18:(2,2),19:(2,3),20:(2,4),21:(2,5),15:(2,6),16:(2,7),
        24:(3,0),25:(3,1),26:(3,2),27:(3,3),28:(3,4),29:(3,5),30:(3,6),31:(3,7),
        32:(4,0),33:(4,1),34:(4,2),35:(4,3),36:(4,4),37:(4,5),38:(4,6),39:(4,7),
        40:(5,0),41:(5,1),42:(5,2),43:(5,3),44:(5,4),45:(5,5),46:(5,6),47:(5,7),
        48:(6,0),49:(6,1),50:(6,2),51:(6,3),52:(6,4),53:(6,5),54:(6,6),55:(6,7),
        56:(7,0),57:(7,1),58:(7,2),59:(7,3),60:(7,4),61:(7,5),62:(7,6),63:(7,7)}
ch_dict={7:(0,0),11:(0,1),15:(0,2),23:(0,3),
         27:(1,0),30:(1,1),39:(1,2),43:(1,3),
         45:(2,0),55:(2,1),59:(2,2),60:(2,3)}

def gauss(x, A, mu, sigma): return A*np.exp(-(x-mu)**2/(2*sigma**2))

def basic_parsing(f):
    parity_mask=f['packets'][:]['valid_parity']==1
    data_mask=f['packets'][:]['packet_type']==0
    mask = np.logical_and(parity_mask,data_mask)
    return f['packets'][mask]

def plot_adc_dist(p, nrows=3, ncols=4):
    unique_channels = np.unique(p['channel_id'])
    fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(24,12))
    bins=np.linspace(0,255,256)
    for uc in unique_channels:
        mask = p['channel_id']==uc
        adc=p[mask]['dataword']
        ax[ch_dict[uc][0]][ch_dict[uc][1]].hist(adc,bins=bins)
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_xlabel('ADC')
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_ylabel('Packet Count')
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_yscale('log')
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_title('Channel '+str(uc))
    plt.tight_layout()
    plt.show()

def plot_timestamp_vs_adc(p, nrows=3, ncols=4):
    unique_channels = np.unique(p['channel_id'])
    fig, ax = plt.subplots(nrows=nrows, ncols=ncols, figsize=(24,12))
    for uc in unique_channels:
        mask = p['channel_id']==uc
        adc=p[mask]['dataword']
        timestamp = p[mask]['timestamp']
        ax[ch_dict[uc][0]][ch_dict[uc][1]].scatter(timestamp,adc)
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_xlabel('Timestamp')
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_ylabel('ADC')
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_title('Channel '+str(uc))
    plt.tight_layout()
    plt.show()

def find_extrema(l, out):
    temp = max(l)
    if abs(min(l))>temp: temp=abs(min(l))
    out.append(temp)
    return

def find_static_linearity(p, nrows=3, ncols=4):
    fig, ax = plt.subplots(nrows=nrows, ncols=ncols, sharex=True, sharey=True, figsize=(30,22))
    fig1, ax1 = plt.subplots(nrows=1, ncols=2, figsize=(12,6))
    fig2, ax2 = plt.subplots(nrows=1, ncols=2, figsize=(10,4))
    fig3, ax3 = plt.subplots(nrows=1, ncols=2, figsize=(10,4))
    dnl_extrema, inl_extrema = [[] for i in range(2)]
    unique_channels = np.unique(p['channel_id'])
    for uc in unique_channels:
        mask = p['channel_id']==uc
        adc=p[mask]['dataword']
        vals, bin_edges = np.histogram(adc, bins=256, range=(0,256))

        meanDict, runningSum = [{} for k in range(2)]
        dnl_val, dnl_adc, inl_val = [{} for k in range(3)]
        for k in range(1,255): # exclude first and last ADC bins
            meanDict[k]=0; runningSum[k]=0
            dnl_val[k]=[]; dnl_adc[k]=[]; inl_val[k]=[]

        for k in range(1,255): # exclude first and last ADC bins
            for l in range(1,255):
                if l>=k: meanDict[l] += vals[k]

        for key in meanDict.keys(): meanDict[key] = meanDict[key]/key

        for k in range(1,255):
            for l in range(1,255):
                if l>=k:
                    dnl = (vals[k]-meanDict[l])/meanDict[l]
                    dnl_val[l].append( dnl )
                    runningSum[l]+=dnl
                    inl_val[l].append( runningSum[l] )
                    dnl_adc[l].append(k)

        find_extrema(dnl_val[254], dnl_extrema)
        find_extrema(inl_val[254], inl_extrema)

        ax[ch_dict[uc][0]][ch_dict[uc][1]].plot(dnl_adc[254],dnl_val[254])
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_xlabel('ADC')
        if ch_dict[uc][1]==0:
            ax[ch_dict[uc][0]][ch_dict[uc][1]].set_ylabel('DNL [LSB]', \
                                                          color='tab:blue', \
                                                          fontweight='bold')
        ax[ch_dict[uc][0]][ch_dict[uc][1]].legend(title='Channel '+str(uc), loc='upper center')
        ax[ch_dict[uc][0]][ch_dict[uc][1]].grid(True)
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_xlim(0,255)
        ax[ch_dict[uc][0]][ch_dict[uc][1]].set_ylim(-0.5,0.5)
        axtwin = ax[ch_dict[uc][0]][ch_dict[uc][1]].twinx()
        axtwin.plot(dnl_adc[254],inl_val[254], color='k')
        if ch_dict[uc][1]==3:
            axtwin.set_ylabel('INL [LSB]', fontweight='bold')
        axtwin.set_ylim(-4,4)
        if ch_dict[uc][1]!=3:
            axtwin.set_yticklabels([])
        if uc==7:
            ax2[0].plot(dnl_adc[254], dnl_val[254])
            ax3[0].plot(dnl_adc[254], inl_val[254])

    fig.subplots_adjust(wspace=0, hspace=0)
#    fig.tight_layout()
    bins=np.linspace(0.2,0.4,21)
    hist, bin_edges = np.histogram(dnl_extrema, bins=20, \
                                   range=(0.2,0.4))
    params, cov = curve_fit(gauss, bin_edges[:-1], hist, \
                            p0=[max(hist), np.mean(dnl_extrema),
                                np.std(dnl_extrema)])
    ax1[0].hist(dnl_extrema, bins=bins)
    ax1[0].plot(bin_edges[:-1], gauss(bin_edges[:-1], *params), \
                '-', label=r'Fit'+'\n'+'$\mu$={:.2f}$\pm${:.2f}'.format(params[1], np.sqrt(cov[1][1]))+'\n'+'$\sigma$={:.2f}$\pm${:.2f}'.format(params[2], np.sqrt(cov[2][2])))

    ax1[0].set_xlabel('DNL Extrema')
    ax1[0].set_ylabel('Channel Count / 0.01 LSB')
    ax1[0].grid(True)
    ax1[0].set_xlim(0.2,0.4)
    ax1[0].legend()

    ax2[1].hist(dnl_extrema, bins=bins)
    ax2[1].plot(bin_edges[:-1], gauss(bin_edges[:-1], *params), \
                '-', label=r'Fit'+'\n'+'$\mu$={:.2f}$\pm${:.2f}'.format(params[1], np.sqrt(cov[1][1]))+'\n'+'$\sigma$={:.2f}$\pm${:.2f}'.format(params[2], np.sqrt(cov[2][2])))

    bins=np.linspace(2,4,21)
    hist, bin_edges = np.histogram(inl_extrema, bins=20, \
                                   range=(2,4))
    params, cov = curve_fit(gauss, bin_edges[:-1], hist, \
                            p0=[max(hist), np.mean(inl_extrema),
                                np.std(inl_extrema)])
    ax1[1].hist(inl_extrema, bins=bins)
    ax1[1].plot(bin_edges[:-1], gauss(bin_edges[:-1], *params), \
                '-', label=r'Fit'+'\n'+'$\mu$={:.2f}$\pm${:.2f}'.format(params[1], np.sqrt(cov[1][1]))+'\n'+'$\sigma$={:.2f}$\pm${:.2f}'.format(params[2], np.sqrt(cov[2][2])))
    ax1[1].set_xlabel('INL Extrema')
    ax1[1].set_ylabel('Channel Count / 0.1 LSB')
    ax1[1].grid(True)
    ax1[1].set_xlim(2,4)
    ax1[1].legend()

    ax3[1].hist(inl_extrema, bins=bins)
    ax3[1].plot(bin_edges[:-1], gauss(bin_edges[:-1], *params), \
                '-', label=r'Fit'+'\n'+'$\mu$={:.2f}$\pm${:.2f}'.format(params[1], np.sqrt(cov[1][1]))+'\n'+'$\sigma$={:.2f}$\pm${:.2f}'.format(params[2], np.sqrt(cov[2][2])))
#    fig1.tight_layout()

    for i in range(2):
        ax2[i].grid(True)
        ax3[i].grid(True)
    ax2[0].set_xlabel('ADC')
    ax2[0].set_ylabel('DNL [LSB]')
    ax3[0].set_xlabel('ADC')
    ax3[0].set_ylabel('INL [LSB]')
    ax2[0].set_xlim(0,255)
    ax2[0].set_ylim(-0.5,0.5)
    ax3[0].set_xlim(0,255)
    ax3[0].set_ylim(-3,3)
    ax2[1].set_xlabel('DNL Extrema')
    ax3[1].set_xlabel('INL Extrema')
    ax2[1].set_ylabel('Channel Count')
    ax2[1].set_xlim(0.2,0.4)
    ax3[1].set_ylabel('Channel Count')
    ax3[1].set_xlim(2,4)
    ax2[1].legend()
    ax3[1].legend()

    plt.show()

def main(**kwargs):
    f = h5py.File('/home/russell/LArPix/v2_ASIC_paper/data/v2a/adc-linearity/f.h5','r');
    p = basic_parsing(f)
#    plot_adc_dist(p)
#    plot_timestamp_vs_adc(p)
    find_static_linearity(p)
    return

if __name__=='__main__':
    parser = argparse.ArgumentParser()
    args = parser.parse_args()
    main(**vars(args))


import matplotlib.pyplot as plt
import pandas as pd
from sklearn.metrics import roc_curve, auc
import glob
from scipy import interp
import numpy as np
import os
import random
from matplotlib.ticker import FuncFormatter

def fsave(list_results, file, addn=False):
    if addn == True:
        list_results = [i+'\n' for i in list_results]
    with open(file, 'w') as f:
        f.writelines(list_results)
        f.close()

def fread(file, removen=True, remove1strow=False):
    with open(file, 'r') as f:
        contents = f.readlines()
        f.close()
    if removen == True:
        contents = [i.replace('\n','') for i in contents]
    if remove1strow == True:
        contents.pop(0)
    return contents

def calculate_single(id, method, results):
    with open('BindingDB_human_DATA.csv','r') as f:
        contents = f.readlines()
        f.close()
    targets = []

    for line in contents:
        if id == line.split(',')[0]:
            if float(line.split(',')[2]) < 10000:
                if not ':' in line.split(',')[3][:-1]:
                    targets.append(line.split(',')[3][:-1])
                else:
                    tmp_list = line.split(',')[3][:-1].split(':')
                    for tmp_i in tmp_list:
                        targets.append(tmp_i)
    unique_target_list = list(set(targets))
    unique_target_list = ['P27487', 'P23219', 'P16083', 'O75762', 'P35354', 'P05067', 'Q16678', 'P09884', 'P09917', 'P06276', 'P22303', 'P27338', 'O60341', 'P04798', 'Q13153', 'P21397', 'P11926', 'Q04206', 'P11511', 'P00519', 'P11274', 'P18031', 'P07711', 'P01106', 'P08183', 'P28907', 'P07339', 'P06239', 'Q07817', 'P09960', 'P14679', 'P08684', 'P03372', 'Q9BYT3', 'P17252', 'Q96EB6', 'Q8IXJ6', 'Q9NTG7', 'P00915', 'P00918', 'P07451', 'P22748', 'P35218', 'Q9Y2D0', 'P23280', 'P43166', 'Q16790', 'O43570', 'Q8N1Q1', 'Q9ULX7', 'Q16236', 'P02766']
    if '' in unique_target_list:
        unique_target_list.remove('')
    unique_target_length = len(unique_target_list)
    if method == 'SEA':
        with open('uID-Gene_name.tsv','r') as f:
            uID_Gene_list = f.readlines()
            f.close()
        for line in results:
            if 'HUMAN' not in line:
                results.remove(line)
        count = 0
        x_list = [0]
        y_list = [0]
        appeared_list = []
        Gene_uID_SEA_results = []
        for index in range(len(results)):
            whole_gene_id = results[index].split(',')[1]
            gene_id = whole_gene_id.split('_')[0]
            # print(gene_id)
            for gene_uID in uID_Gene_list:
                if gene_id in gene_uID:
                    Gene_uID_SEA_results.append(gene_uID)
        # print(Gene_uID_SEA_results)
        for index in range(len(Gene_uID_SEA_results)):
            for uID in unique_target_list:
                if uID in Gene_uID_SEA_results[index]:
                    if uID not in appeared_list:
                        appeared_list.append(uID)
                        count += 1
                        y_list.append(count/len(unique_target_list))
                        x_list.append(index/len(Gene_uID_SEA_results))
                    elif uID in appeared_list:
                        pass
        x_list.append(1)
        y_list.append(y_list[-1])
    elif method == 'Swiss':
        count = 0
        x_list = [0]
        y_list = [0]
        appeared_list = []
        for index in range(len(results)):
            for uID in unique_target_list:
                if uID in results[index]:
                    if uID not in appeared_list:
                        appeared_list.append(uID)
                        count += 1
                        y_list.append(count/len(unique_target_list))
                        x_list.append(index/len(results))
                    elif uID in appeared_list:
                        pass
        x_list.append(1)
        y_list.append(y_list[-1])
    elif method == 'MAI_30000_docking':
        if id == '6866':
            pass
        else:
            count = 0
            x_list = [0]
            y_list = [0]
            appeared_list = []
            for index in range(len(results)):
                for uID in unique_target_list:
                    if uID in results[index]:
                        if uID not in appeared_list:
                            appeared_list.append(uID)
                            count += 1
                            y_list.append(count/len(unique_target_list))
                            x_list.append(index/len(results))
                        elif uID in appeared_list:
                            pass
            x_list.append(1)
            y_list.append(y_list[-1])
    elif method == 'MAI_30000_params':
        if id =='6866':
            pass
        else:
            file = f'30000_resveratrol_hybrid_results_sorted'
            with open(file, 'r') as f:
                results = f.readlines()
                f.close()
            count=0
            x_list = [0]
            y_list = [0]
            appeared_list = []
            for index in range(len(results)):
                for uID in unique_target_list:
                    if uID in results[index]:
                        if uID not in appeared_list:
                            appeared_list.append(uID)
                            count += 1
                            y_list.append(count/len(unique_target_list))
                            x_list.append(index/len(results))
                        elif uID in appeared_list:
                            pass
            x_list.append(1)
            y_list.append(y_list[-1])
    return [x_list, y_list]

Swiss_results = fread('SwissTargetPrediction.csv', remove1strow=True)
SEA_results = fread('SEA-resutls.csv', remove1strow=True)
MAI_TF_results = fread('MAI_TargetFisher_results.csv', remove1strow=True)
[swiss_x, swiss_y] = calculate_single('23926', 'Swiss', Swiss_results)
[SEA_x, SEA_y] = calculate_single('23926', 'SEA', SEA_results)
[MAI_x, MAI_y] = calculate_single('23926', 'MAI_30000_docking', MAI_TF_results)
[MAI_h_x, MAI_h_y] = calculate_single('23926', 'MAI_30000_params', None)

from matplotlib.ticker import FuncFormatter
plt.figure(figsize = (6.4,4.8), dpi = 300)
plt.xlim([0,1])
plt.ylim([0,1])
plt.rc('font', family='Helvetica')
def to_percent(temp, position):
  return '%1.0f'%(100*temp) + '%'
plt.gca().yaxis.set_major_formatter(FuncFormatter(to_percent))
plt.gca().xaxis.set_major_formatter(FuncFormatter(to_percent))
plt.plot(MAI_h_x, MAI_h_y, label='MAI-TargetFisher_weighted', linewidth=3,  color='#E47C7C')
plt.plot(MAI_x, MAI_y, label='MAI-TargetFisher_unweighted', linewidth=3, color=(0.22, 0.50, 0.57))
plt.plot(SEA_x, SEA_y, label='SEA', color=(0.96, 0.73, 0.47))
plt.plot(swiss_x, swiss_y, label='SwissTarget', linewidth=3, color=(0.56, 0.77, 0.58))
plt.title(f'Average recall rate of test dataset', fontsize='19')
plt.xlabel('Result', fontsize='16')
plt.ylabel('Identified Tatgets', fontsize='16')
plt.xticks(size='12')
plt.yticks(size='12')
font_legend={'family' : 'Helvetica','style':'normal', 'weight':'bold'}
plt.legend(loc='best', markerfirst=False, shadow=True, fancybox=True, prop=font_legend)
plt.savefig(f'ave_all_4_methods_nolegand.eps')


import cv2
import sys
if __name__ == '__main__':
    faceCascade = cv2.CascadeClassifier("Resources/haarcascade_frontalface_default.xml")
    cap = cv2.VideoCapture(0)
    fourcc = cv2.VideoWriter_fourcc(*'XVID')
    while cap.isOpened():
        ret, frame = cap.read()
        gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
        faces = faceCascade.detectMultiScale(
            gray,
            scaleFactor=1.3,
            minNeighbors=3,
            minSize=(30, 30),
            flags=cv2.ORB_HARRIS_SCORE
        )
        for (x, y, w, h) in faces:
            cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
        if ret == True:
            cv2.imshow('frame', frame)
            if cv2.waitKey(1) & 0xff == ord('q'):
                break


from __future__ import absolute_import, division, print_function

__metaclass__ = type

DOCUMENTATION = '''
---
module: cs_pod
short_description: Manages pods on Apache CloudStack based clouds.
description:
    - Create, update, delete pods.
author: René Moser (@resmo)
version_added: 0.1.0
options:
  name:
    description:
      - Name of the pod.
    type: str
    required: true
  id:
    description:
      - uuid of the existing pod.
    type: str
  start_ip:
    description:
      - Starting IP address for the Pod.
      - Required on I(state=present)
    type: str
  end_ip:
    description:
      - Ending IP address for the Pod.
    type: str
  netmask:
    description:
      - Netmask for the Pod.
      - Required on I(state=present)
    type: str
  gateway:
    description:
      - Gateway for the Pod.
      - Required on I(state=present)
    type: str
  zone:
    description:
      - Name of the zone in which the pod belongs to.
    type: str
    required: true
  state:
    description:
      - State of the pod.
    type: str
    default: present
    choices: [ present, enabled, disabled, absent ]
extends_documentation_fragment:
- ngine_io.cloudstack.cloudstack
'''

EXAMPLES = '''
- name: Ensure a pod is present
  ngine_io.cloudstack.cs_pod:
    name: pod1
    zone: ch-zrh-ix-01
    start_ip: 10.100.10.101
    gateway: 10.100.10.1
    netmask: 255.255.255.0

- name: Ensure a pod is disabled
  ngine_io.cloudstack.cs_pod:
    name: pod1
    zone: ch-zrh-ix-01
    state: disabled

- name: Ensure a pod is enabled
  ngine_io.cloudstack.cs_pod:
    name: pod1
    zone: ch-zrh-ix-01
    state: enabled

- name: Ensure a pod is absent
  ngine_io.cloudstack.cs_pod:
    name: pod1
    zone: ch-zrh-ix-01
    state: absent
'''

RETURN = '''
---
id:
  description: UUID of the pod.
  returned: success
  type: str
  sample: 04589590-ac63-4ffc-93f5-b698b8ac38b6
name:
  description: Name of the pod.
  returned: success
  type: str
  sample: pod01
start_ip:
  description: Starting IP of the pod.
  returned: success
  type: str
  sample: 10.100.1.101
end_ip:
  description: Ending IP of the pod.
  returned: success
  type: str
  sample: 10.100.1.254
netmask:
  description: Netmask of the pod.
  returned: success
  type: str
  sample: 255.255.255.0
gateway:
  description: Gateway of the pod.
  returned: success
  type: str
  sample: 10.100.1.1
allocation_state:
  description: State of the pod.
  returned: success
  type: str
  sample: Enabled
zone:
  description: Name of zone the pod is in.
  returned: success
  type: str
  sample: ch-gva-2
'''

from ansible.module_utils.basic import AnsibleModule

from ..module_utils.cloudstack import (AnsibleCloudStack, cs_argument_spec,
                                       cs_required_together)

class AnsibleCloudStackPod(AnsibleCloudStack):

    def __init__(self, module):
        super(AnsibleCloudStackPod, self).__init__(module)
        self.returns = {
            'endip': 'end_ip',
            'startip': 'start_ip',
            'gateway': 'gateway',
            'netmask': 'netmask',
            'allocationstate': 'allocation_state',
        }
        self.pod = None

    def _get_common_pod_args(self):
        args = {
            'name': self.module.params.get('name'),
            'zoneid': self.get_zone(key='id'),
            'startip': self.module.params.get('start_ip'),
            'endip': self.module.params.get('end_ip'),
            'netmask': self.module.params.get('netmask'),
            'gateway': self.module.params.get('gateway')
        }
        state = self.module.params.get('state')
        if state in ['enabled', 'disabled']:
            args['allocationstate'] = state.capitalize()
        return args

    def get_pod(self):
        if not self.pod:
            args = {
                'zoneid': self.get_zone(key='id')
            }

            uuid = self.module.params.get('id')
            if uuid:
                args['id'] = uuid
            else:
                args['name'] = self.module.params.get('name')

            pods = self.query_api('listPods', **args)
            if pods:
                for pod in pods['pod']:
                    if not args['name']:
                        self.pod = self._transform_ip_list(pod)
                        break
                    elif args['name'] == pod['name']:
                        self.pod = self._transform_ip_list(pod)
                        break
        return self.pod

    def present_pod(self):
        pod = self.get_pod()
        if pod:
            pod = self._update_pod()
        else:
            pod = self._create_pod()
        return pod

    def _create_pod(self):
        required_params = [
            'start_ip',
            'netmask',
            'gateway',
        ]
        self.module.fail_on_missing_params(required_params=required_params)

        pod = None
        self.result['changed'] = True
        args = self._get_common_pod_args()
        if not self.module.check_mode:
            res = self.query_api('createPod', **args)
            pod = res['pod']
        return pod

    def _update_pod(self):
        pod = self.get_pod()
        args = self._get_common_pod_args()
        args['id'] = pod['id']

        if self.has_changed(args, pod):
            self.result['changed'] = True

            if not self.module.check_mode:
                res = self.query_api('updatePod', **args)
                pod = res['pod']
        return pod

    def absent_pod(self):
        pod = self.get_pod()
        if pod:
            self.result['changed'] = True

            args = {
                'id': pod['id']
            }
            if not self.module.check_mode:
                self.query_api('deletePod', **args)
        return pod

    def _transform_ip_list(self, resource):
        """ Workaround for 4.11 return API break """
        keys = ['endip', 'startip']
        if resource:
            for key in keys:
                if key in resource and isinstance(resource[key], list):
                    resource[key] = resource[key][0]
        return resource

    def get_result(self, resource):
        resource = self._transform_ip_list(resource)
        super(AnsibleCloudStackPod, self).get_result(resource)
        return self.result

def main():
    argument_spec = cs_argument_spec()
    argument_spec.update(dict(
        id=dict(),
        name=dict(required=True),
        gateway=dict(),
        netmask=dict(),
        start_ip=dict(),
        end_ip=dict(),
        zone=dict(required=True),
        state=dict(choices=['present', 'enabled', 'disabled', 'absent'], default='present'),
    ))

    module = AnsibleModule(
        argument_spec=argument_spec,
        required_together=cs_required_together(),
        supports_check_mode=True
    )

    acs_pod = AnsibleCloudStackPod(module)
    state = module.params.get('state')
    if state in ['absent']:
        pod = acs_pod.absent_pod()
    else:
        pod = acs_pod.present_pod()

    result = acs_pod.get_result(pod)

    module.exit_json(**result)

if __name__ == '__main__':
    main()


#!/usr/bin/env python
import random, os, re
from .backend import BackendBase
from telegram.ext import Updater, CommandHandler, MessageHandler, Filters
from nlptools.audio.speech import Speech_Deepspeech
from nlptools.audio.utils import *

class TelegramBackend(BackendBase):
    def __init__(self, session_config, token, deepspeech=None, tts=None, **args):
        super().__init__(session_config=session_config, **args)
        self.updater = Updater(token=token, use_context=True)
        self.dispatcher = self.updater.dispatcher
        self.dispatcher.add_handler(CommandHandler("start", TelegramBackend.start))
        self.dispatcher.add_handler(CommandHandler("reset", self.reset))
        self.dispatcher.add_handler(CommandHandler("tts", self.get_tts))
        self.dispatcher.add_handler(MessageHandler(Filters.text, self.query))
        self.dispatcher.add_handler(MessageHandler(Filters.voice, self.voice))
        if deepspeech:
            self.ds_model = Speech_Deepspeech(**deepspeech)
        else:
            self.ds_model = None
        if tts:
            from nlptools.audio.tts import MozillaTTS
            self.tts_model = MozillaTTS(**tts)
        else:
            self.tts_model = None

    @staticmethod
    def start(update, context):
        chat_id = update.message.chat_id
        context.bot.send_message(chat_id=chat_id, text="Elsa is here ~")

    def reset(self, update, context):
        chat_id = update.message.chat_id
        self.init_session()
        context.bot.send_message(chat_id=chat_id, text="Chatbot Reset")

    def get_tts(self, update, context):
        chat_id = update.message.chat_id
        text = update.message.text.strip()
        text = re.split("\s", text, maxsplit=1)
        if len(text) < 2:
            return
        text = text[1]

        random_id = random.randint(0,100000)

        wav_base = "/tmp/{}_{}_%i.wav".format(chat_id, random_id)
        wav_file = "/tmp/{}_{}.wav".format(chat_id, random_id)
        ogg_file = "/tmp/{}_{}.ogg".format(chat_id, random_id)

        wav_files = []

        import spacy
        nlp = spacy.load("en")

        if self.tts_model:
            doc = nlp(text)
            for i, sent in enumerate(doc.sents):
                wavefile = wav_base%i
                wav_files.append(wavefile)
                text = sent.text.strip()
                self.tts_model(text, wavefile)

            concat_wavs(wav_files, wav_file, 0.6)
            wav2ogg(wav_file, ogg_file)
            with open(ogg_file, "rb") as oggf:
                context.bot.send_voice(chat_id=chat_id, voice=oggf)
            os.remove(wav_file)
            os.remove(ogg_file)
            for f in wav_files:
                os.remove(f)
        else:
            context.bot.send_message(chat_id=chat_id, text=text)

    def voice(self, update, context):
        chat_id = update.message.chat_id
        if not self.ds_model:
            context.bot.send_voice(chat_id=chat_id, voice=update.message.voice)
            return

        #convert voice to text
        random_id = random.randint(0,100000)
        ogg_file = "/tmp/{}_{}.ogg".format(chat_id, random_id)
        wav_file = "/tmp/{}_{}.wav".format(chat_id, random_id)
        response_wave_file = "/tmp/response_{}_{}.wav".format(chat_id, random_id)
        response_ogg_file = "/tmp/response_{}_{}.ogg".format(chat_id, random_id)

        update.message.voice.get_file().download(ogg_file)
        try:
            ogg2wav(ogg_file, wav_file)
        except Exception as err:
            context.bot.send_message(chat_id=chat_id, text=err)
            os.remove(ogg_file)
            return

        text = self.ds_model(wav_file)
        os.remove(ogg_file)
        os.remove(wav_file)

        #query
        chat_id, response, score = self.session(text, session_id=chat_id)

        #tts
        if self.tts_model:
            self.tts_model(response, response_wave_file)
            wav2ogg(response_wave_file, response_ogg_file)
            with open(response_ogg_file, "rb") as oggf:
                context.bot.send_voice(chat_id=chat_id, voice=oggf)
            os.remove(response_wave_file)
            os.remove(response_ogg_file)
        else:
            context.bot.send_message(chat_id=chat_id, text=response)

    def query(self, update, context):
        if update.message is None:
            return
        chat_id = update.message.chat_id
        text = update.message.text.strip()
        chat_id, response, score = self.session(text, session_id=chat_id)
        context.bot.send_message(chat_id=chat_id, text=response)

    def run(self):
        import logging
        logging.basicConfig(level=logging.WARNING, format='%(levelname)-8s %(message)s')
        self.updater.start_polling()


import copy
import random
import warnings
import sys
sys.path.append("..")
import time
import os
import pickle
import torch
from torch import nn
import numpy as np
import pandas as pd
from torch.utils.data import DataLoader
from sklearn import metrics

import utils
import models
from data_utils import CustomDataset

# Set cuda
DEVICE = torch.device("cpu")
if torch.cuda.is_available():
    cuda_num = random.randint(0, (torch.cuda.device_count()-1))
    cuda_name = "cuda:" + str(cuda_num)
    DEVICE = torch.device(cuda_name)

# -----------------------------------
def make_dir(path, dir_name):
    try:
        out_path = path + "/" + dir_name
        if not os.path.exists(out_path):
            os.makedirs(out_path)
            return True
        else:
            print("Directory %s already exists" % dir_name)
            return False
    except OSError:
        print('Error: Creating directory. ' + out_path)
        sys.exit(0)

# extract the corresponding testing data based on the class
def testing_data_extraction(testing_data, label):
    (x_test, y_test_bin) = testing_data
    un_label = np.unique(label)
    # convert to df
    df = pd.DataFrame(x_test)
    df["label"] = y_test_bin
    # extract testing based on label
    new_df = df[df['label'].isin(un_label)]
    # convert back to nd array
    new_testing_X = new_df.iloc[:, :-1].to_numpy()
    new_testing_y = new_df.iloc[:, -1].to_numpy()
    # Verify
    unique, counts = np.unique(new_testing_y, return_counts=True)
    print("New client testing data distribution:", dict(zip(unique, counts)))
    return new_testing_X, new_testing_y

# -----------------------------------
# Get model performance
def get_performance(y_pred, y_test, classification_method):
    accuracy = metrics.accuracy_score(y_test, y_pred)
    if classification_method == "Binary":
        '''Binary classification score'''
        # precision = metrics.precision_score(y_test, y_pred)
        # recall = metrics.recall_score(y_test, y_pred)
        # f1 = metrics.f1_score(y_test, y_pred)
    elif classification_method == "Multi":
        '''Muti class classification'''
        precision_all_classes = metrics.precision_score(y_test, y_pred, average=None)
        recall_all_classes = metrics.recall_score(y_test, y_pred, average=None)
        f1_all_classes = metrics.f1_score(y_test, y_pred, average=None)
        print('Precision for all classes =', precision_all_classes)
        print('Recall for all classes =', recall_all_classes)
        print('F1 for all classes =', f1_all_classes)
        precision = metrics.precision_score(y_test, y_pred, average='macro')
        recall = metrics.recall_score(y_test, y_pred, average='macro')
        f1 = metrics.f1_score(y_test, y_pred, average='macro')
    else:
        print("Classification method does not found.")
        sys.exit()
    print("Classification Metrics: ")
    # print(metrics.confusion_matrix(y_test, y_pred))
    # print(metrics.classification_report(y_test, y_pred))
    print('Accuracy =', accuracy)
    print('Precision =', precision)
    print('Recall =', recall)
    print('F1 =', f1)
    return accuracy, f1, precision, recall

# ------------------------------------------------
def test(model, loss_fn, test_loader, curr_path, classification_method="Multi", device="cpu"):
    results_path = curr_path + "/results/cent-binary.csv"
    model_path = curr_path + "/models/cent-binary" + ".pth"

    # model.load_state_dict(torch.load(model_path))
    model.to(device)
    # test the model
    model.eval()
    y_pred_list = []
    loss = 0.0

    all_true_values = []
    all_pred_values = []
    print("Test the global model")
    with torch.no_grad():
        for data, target in test_loader:
            data, target = data.to(device), target.to(device)
            output = model(data)
            if classification_method == "Binary":
                '''Attach Binary classification label'''
                # y_pred_tag = torch.round(output)
                # y_pred_tensor = y_pred_tag.flatten()
                # y_pred_np_int = y_pred_tensor.type(torch.float32)
                # pred_values = y_pred_np_int.tolist()
                # all_pred_values.extend(pred_values)
                # loss += loss_fn(output, target.reshape(-1, 1)).item()      # Binary classification

            '''Attach true label'''
            act_values = target.tolist()
            all_true_values.extend(act_values)

            # write_predictions(results_path, act_values, pred_values)
            if classification_method == "Multi":
                '''Attach Muti class classification label'''
                if device == "cpu":
                    _, predictions = torch.max(output, 1)
                else:
                    _, predictions = torch.max(output.cpu(), 1)
                all_pred_values.extend(predictions)
                loss = loss_fn(output, target).item()                   # Muti class classification
    accuracy, f1, precision, recall  = get_performance(all_pred_values, all_true_values, classification_method)
    losses = loss/len(test_loader)
    return losses, accuracy, f1, precision, recall

if __name__ == "__main__":
    data_dir = '../2017_data/'
    # hyper-parameters
    epochs = 50
    learning_rate = 0.01
    batch_size = 64
    MLP_first_hidden = 64
    MLP_second_hidden = 128
    num_classes = 8
    num_clients = 20
    training_data_name = str(num_clients) + '_training.pkl'
    classification = "Multi"
    neural_network = "MLP_Mult"
    # create df for results recording
    res_list = []
    # -------------------Create folder for model----------------------
    curr_path = os.getcwd()
    make_dir(curr_path, "../results")
    # -------------- load datasets ----------------------
    print("Loading data...")
    partitioned_data, testing_data, _ = utils.load_data(data_dir, training_data=training_data_name)
    for i in range(num_clients):
        print("\nClient ", str(i+1), "training...")
        (X_train, y_train) = partitioned_data[i]
        # --------------Print Shape-----------------
        # Verify
        unique, counts = np.unique(y_train, return_counts=True)
        print("Client ", str(i+1), "data distribution:", dict(zip(unique, counts)))
        # -------------- Preprocess datasets -----------------
        train_data = CustomDataset(X_train, y_train, neural_network)
        train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)
        x_test_new, y_test_new = testing_data_extraction(testing_data, y_train)
        test_data = CustomDataset(x_test_new, y_test_new, neural_network)
        test_loader = DataLoader(test_data, batch_size=batch_size, shuffle=False)
        # -------------- Define model ----------------------
        if classification == "Binary":
            model = models.MLP(input_shape=X_train.shape[1]).to(DEVICE).train()
            optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
            loss_fn = nn.BCELoss()  # Binary classification
        elif classification == "Multi":
            model = models.MLP_Mult(input_shape=X_train.shape[1], first_hidden=MLP_first_hidden, second_hidden=MLP_second_hidden, num_classes=num_classes).to(DEVICE).train()
            optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)
            loss_fn = nn.CrossEntropyLoss()  # Muti class classification
        else:
            print("Classification method does not found.")
            sys.exit()
        # -------------------Training model----------------------
        train_time = time.time()
        model_weights, _ = utils.train(model, optimizer, loss_fn, train_loader, epochs, neural_network, i, device=DEVICE)
        training_time = (time.time() - train_time) / 60
        print("---Training time: %s minutes. ---" % training_time)
        # -------------------Testing model----------------------
        test_time = time.time()
        model.load_state_dict(copy.deepcopy(model_weights))
        loss, accuracy, f1, precision, recall = test(model, loss_fn, test_loader, curr_path, device=DEVICE)
        res_list.append([accuracy, precision, recall, f1, loss, training_time])
        print("---Testing time: %s minutes. ---" % ((time.time() - test_time) / 60))
    res_df = pd.DataFrame(res_list, columns=['Accuracy', 'Precision', 'Recall', 'F1', 'Loss', 'Training_time(min)'])
    res_df.to_csv("results/2017_clients_data_res.csv")


from math import fabs, sin, cos, radians

import cv2
import numpy as np

from mmdet.datasets.pipelines import Pad, Normalize
from mmdet.datasets.builder import PIPELINES
from davarocr.davar_common.datasets.pipelines import DavarResize, RandomRotate, ResizeNormalize, ColorJitter

@PIPELINES.register_module()
class ConsistResize(DavarResize):
    """ Same with DavarResize, the only difference is ConsistResize support results(list) contain multiple
    instance and  resize to same size for all the instances in results(list)

    """

    def __init__(self,
                 img_scale=None,
                 multiscale_mode='range',
                 ratio_range=None,
                 keep_ratio=True,
                 bbox_clip_border=True,
                 backend='cv2',
                 override=False):
        """
        Args:
            img_scale (tuple or list[tuple]): Images scales for resizing.
            multiscale_mode (str): Either "range" or "value".
            ratio_range (tuple[float]): (min_ratio, max_ratio)
            keep_ratio (bool): Whether to keep the aspect ratio when resizing the
                image.
            bbox_clip_border (bool, optional): Whether clip the objects outside
                the border of the image. Defaults to True.
            backend (str): Image resize backend, choices are 'cv2' and 'pillow'.
                These two backends generates slightly different results. Defaults
                to 'cv2'.
            override (bool, optional): Whether to override `scale` and
                `scale_factor` so as to call resize twice. Default False. If True,
                after the first resizing, the existed `scale` and `scale_factor`
                will be ignored so the second resizing can be allowed.
                This option is a work-around for multiple times of resize in DETR.
                Defaults to False.
        """

        super().__init__(img_scale=img_scale, multiscale_mode=multiscale_mode, ratio_range=ratio_range,
                         keep_ratio=keep_ratio, bbox_clip_border=bbox_clip_border, backend=backend, override=override)

    def __call__(self, results):
        """ Main process of ConsistResize

        Args:
            results(dict | list(dict)): input data flow.

        Returns:
            dict | list(dict): updated data flow.
        """

        # Deal with results(dict) contains single instance
        if isinstance(results, dict):
            results = super().__call__(results)
            return results

        # Deal with results(list(dict)) contains multiple instances
        scale = None
        scale_idx = None
        results_ = []
        for instance in results:

            # Use same scale to resize all instances in results(list(dict))
            if scale:
                instance['scale'] = scale
                instance['scale_idx'] = scale_idx
            instance = super().__call__(instance)
            scale = instance['scale']
            scale_idx = instance['scale_idx']
            results_.append(instance)

        return results_

@PIPELINES.register_module()
class ConsistRandomRotate(RandomRotate):
    """same with RandomRotate, the only difference is ConsistRandomRotate support results(list) contain multiple
    instance and  resize to same size for all the instances in results(list)
    """

    def __init__(self, angles=None):
        """
        Args:
            angles(list | tuple):   the rotated degree, contains a single value or multiple values
                                    - If `angles` contrains single value, this value represents `rotating fixed angle`;
                                    - If `angles` contains two values(tuple or lis), angles represents `rotating
                                        random angle in the interval ranged by these two values`
                                    -  If angles contains more than two values, angles represents `randomly choosing
                                        a value in this list as rotation angle`
        """
        super().__init__(angles)

    def _rotate_flow(self, results):
        """Randomly rotating optical flow

        Args:
            results(dict): data flow in pipeline

        Returns:
            dict: updated data flow in pipeline
        """

        angle = self.angle
        result_flows = []
        if 'flow' in results['img_info']:
            flows = results['img_info']['flow']
            for k in range(flows.shape[0]):
                h_flow, w_flow, _ = flows[k].shape

                # Compute height and width of new image after rotation
                height_new = int(w_flow * fabs(sin(radians(angle))) + h_flow * fabs(cos(radians(angle))))
                width_new = int(h_flow * fabs(sin(radians(angle))) + w_flow * fabs(cos(radians(angle))))

                # Compute rotation matrix
                mat_rotation = cv2.getRotationMatrix2D((w_flow / 2, h_flow / 2), angle, 1)
                mat_rotation[0, 2] += (width_new - w_flow) / 2
                mat_rotation[1, 2] += (height_new - h_flow) / 2

                # Rotating flow
                flow_rotation = cv2.warpAffine(flows[k], mat_rotation, (width_new, height_new),
                                               borderValue=(0, 0, 0))
                result_flows.append(flow_rotation)

            results['img_info']['flow'] = np.stack(result_flows)

    def __call__(self, results):
        """ Main process of ConsistRandomRotate

        Args:
            results(dict): input data flow

        Returns:
            dict or list(dict): updated data flow, `img` , boxes, flow will be transformed.
        """
        # Angles must be a list or tuple
        assert isinstance(self.angles, (list, tuple))
        if len(self.angles) == 1:
            angle = self.angles[0]
        elif len(self.angles) == 2:
            angle_max = max(self.angles)
            angle_min = min(self.angles)
            angle = np.random.randint(angle_min, angle_max)
        else:
            angle = np.random.choice(self.angles)
        self.angle = angle

        # Deal with results(dict) contains single instance
        if isinstance(results, dict):
            results = super().__call__(results)
            return results

        # Deal with results(list(dict)) contains multiple instances
        results_ = []
        for instance in results:

            # Rotate img and gt_bboxes
            self._rotate_img(instance)

            # Rotate the optical flow
            self._rotate_flow(instance)
            results_.append(instance)
        return results_

@PIPELINES.register_module()
class ConsistColorJitter(ColorJitter):
    """Same with ColorJitter, the only difference is ConsistColorJitter support results(list) contain multiple
    instance and  do same operation for all the instances in results(list)"""

    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def __call__(self, results):
        """Main process of color jitter, refer to mmocr

        Args:
            results(dict or list): input data flow

        Returns:
            results(dict or list): updated data flow, key of `img` will be update

        """
        # Deal with results(dict) contains single instance
        if isinstance(results, dict):
            results = super().__call__(results)
            return results

        # Deal with results(list(dict)) contains multiple instances
        results_ = []
        for instance in results:
            instance = super().__call__(instance)
            results_.append(instance)
        return results_

@PIPELINES.register_module()
class ConsistResizeNormalize(ResizeNormalize):
    """ Same with ResizeNormalize, the only difference is ConsistResizeNormalize support results(list) contain multiple
    instance and  do same operation for all the instances in results(list) """

    def __init__(self, size,
                 interpolation=0,
                 mean=(127.5, 127.5, 127.5),
                 std=(127.5, 127.5, 127.5),):
        """
        Args:
            size (tuple): image resize size
            interpolation (int): interpolation type, including [0, 1, 2, 3]
            mean (tuple): image normalization mean
            std (tuple): image normalization std
        """
        super().__init__(size=size, interpolation=interpolation, mean=mean, std=std)

    def __call__(self, results):
        """
        Args:
            results (dict or list(dict)): images

        Returns:
            dict or list(dict): dict with images after resize and normalization
        """
        # Deal with results(dict) contains single instance
        if isinstance(results, dict):
            results = super().__call__(results)
            return results

        # Deal with results(list(dict)) contains multiple instances
        results_ = []
        for instance in results:
            instance = super().__call__(instance)
            results_.append(instance)
        return results

@PIPELINES.register_module()
class ConsistPad(Pad):
    """ Same with Pad, the only difference is ConsistPad support results(list) contain multiple instance and do same
    operation for all the instances in results(list)
    """

    def __init__(self, size=None, size_divisor=None, pad_val=0):
        super().__init__(size=size, size_divisor=size_divisor, pad_val=pad_val)

    def __call__(self, results):
        """Call function to pad images, masks, semantic segmentation maps.

        Args:
            results (dict | list(dict)): Results from loading pipeline.

        Returns:
            dict | list(dict): Updated results.

        """
        # Deal with results(dict) contains single instance
        if isinstance(results, dict):
            results = super().__call__(results)
            return results

        # Deal with results(list(dict)) contains multiple instances
        results_ = []
        for instance in results:
            instance = super().__call__(instance)
            results_.append(instance)

        return results_

@PIPELINES.register_module()
class ConsistNormalize(Normalize):
    """ Same with Normalize. the only difference is ConsistNormalize support results(list) contain multiple instance
    and do same operation for all the instances in results(list)
    """

    def __init__(self, mean, std, to_rgb=True):
        super().__init__(mean=mean, std=std, to_rgb=to_rgb)

    def __call__(self, results):
        """Call function to normalize images.

        Args:
            results (dict | list(dict)): Results from loading pipeline.

        Returns:
            dict | list(dict): Normalized results, 'img_norm_cfg' key is added into result dict.

        """
        # Deal with results(dict) contains single instance
        if isinstance(results, dict):
            results = super().__call__(results)
            return results

        # Deal with results(list(dict)) contains multiple instances
        results_ = []
        for instance in results:
            instance = super().__call__(instance)
            results_.append(instance)
        return results_


import sys
import cv2 as cv
import numpy as np
from numpy import *
import matplotlib.pyplot as plt
from PIL import Image

#Arxika tha paro se metablites ta orismata pou moy dinei
arxiki_eikona = sys.argv[1]
teliki_eikona = sys.argv[2]

#twra tha diabaso tin eikona kai tha thn balw se enan pinaka
image=Image.open(arxiki_eikona)
img=np.array(image)

print("Kleikare mou ta 4 shmeia ksekinontas apo to pano aristera meta sto pano deksia synexizoume sto kato deksia kai telos sto kato aristera")

position=[] #pinakas o opoios krataei ta 4 shmeia
def four_clicks(event,x,y,flags,param):
    if event == cv.EVENT_LBUTTONDOWN: #an pathsei aristero klik pairno tis sintetgmenes tou simeiou
    	position.append([x,y])

cv.namedWindow("granma",cv.WINDOW_NORMAL)
cv.resizeWindow("granma",800,600) #emfanizo tim eikona se diastaseis pou mporo na tis epekserasto
cv.setMouseCallback("granma",four_clicks)

while(1):
    cv.imshow("granma",img)
    if cv.waitKey(1) and len(position)==4: #otan paro ta 4 shmeia stamatao kai kleino to parathiro
       break
cv.destroyAllWindows()

#--------SEIMIOSI--------
#Kano klik stin eikona: Arxika pano aristera meta pano deksia
# amesos meta kato deksia kai telos kato aristera
#--------#TELOS SEIMIOSIS--------

A=np.array([ #dimiourgo ton pinaka a opws stis diafaneies
[position[0][0],position[0][1],1,0,0,0,-position[0][0]*0,-position[0][1]*0],
[0,0,0,position[0][0],position[0][1],1,-position[0][0]*0,-position[0][1]*0],
[position[1][0],position[1][1],1,0,0,0,-position[1][0]*1000,-position[1][1]*1000],
[0,0,0,position[1][0],position[1][1],1,-position[1][0]*0,-position[1][1]*0],
[position[2][0],position[2][1],1,0,0,0,-position[2][0]*1000,-position[2][1]*1000],
[0,0,0,position[2][0],position[2][1],1,-position[2][0]*1000,-position[2][1]*1000],
[position[3][0],position[3][1],1,0,0,0,-position[3][0]*0,-position[3][1]*0],
[0,0,0,position[3][0],position[3][1],1,-position[3][0]*1000,-position[3][1]*1000],
],np.int32)

'''
x1',y1'=0,0
x2',y2'=1000,0
x3',y3'=1000,1000
x4',y4'=0,1000
'''
b=np.array([0,0,1000,0,1000,1000,0,1000]) #dimiourgo ton pinaka b opws stis diafaneies
A=np.linalg.inv(A) #ypologizo ton antistrofo tou A
x=np.matmul(A,b) #polaplasiazo ton A antistrofo me to b oste na bro to dianisma x

probolikos=np.array([ #dimiourgo ton pinaka 3*3 opws stis diafaneies
[x[0],x[1],x[2]],
[x[3],x[4],x[5]],
[x[6],x[7],1],
],np.float32)

warp=cv.warpPerspective(img,probolikos,(1000,1000)) #efarmozo ton metasximatismo me tin sinartisi warpPerspective
cv.namedWindow("WARP",cv.WINDOW_NORMAL)
cv.resizeWindow("WARP",800,600)
while(1):
	cv.imshow("WARP",warp)
	if cv.waitKey(1) & 0xFF==ord('x'): #emfanizo tin teliki eikona kai OTAN PATISEI x KLEINEI TO PARATHIRO kai termatizei to programma
		break
cv.destroyAllWindows()

#apothikeuo tin teliki eikona
teliko=Image.fromarray(warp.astype(np.uint8))
teliko.save(teliki_eikona)


# coding=utf-8
import logging

__author__ = 'Tarzan'
_logger = logging.getLogger(__name__)

from marshmallow import (
    Schema,
    SchemaOpts,
    validates,
    validates_schema,
    pre_dump,
    post_dump,
    pre_load,
    post_load,
    pprint,
    ValidationError,
    missing,
)

from . import fields
from . import validators


# !/usr/bin/env python

import csv
import sys
from enum import Enum

class Candidate(Enum):
    BOTH_CANDIDATE = 'Both Candidate'
    DONALD_TRUMP = 'Donald Trump'
    JOE_BIDEN = 'Joe Biden'

class Source(Enum):
    TWITTER_WEB_APP = 'Twitter Web App'
    TWITTER_FOR_IPHONE = 'Twitter for iPhone'
    TWITTER_FOR_ANDROID = 'Twitter for Android'

def read_csv(file):
    reader = csv.reader(file)
    next(reader)  # titles
    for row in reader:
        yield row

def get_tweet_candidate(tweet):
    tweet = tweet.lower()

    if ('trump' in tweet) and ('biden' in tweet):
        candidate = Candidate.BOTH_CANDIDATE
    elif 'trump' in tweet:
        candidate = Candidate.DONALD_TRUMP
    elif 'biden' in tweet:
        candidate = Candidate.JOE_BIDEN
    else:
        return None

    return candidate.value

def get_source_list(source):
    return [source == Source.TWITTER_WEB_APP.value, source == Source.TWITTER_FOR_IPHONE.value,
            source == Source.TWITTER_FOR_ANDROID.value]

def get_output(candidate, likes, retweet_count, source):
    sources = get_source_list(source)
    candidate = candidate.replace(' ', ';-;')

    return candidate, int(float(likes)), int(float(retweet_count)), sources[0], sources[1], sources[2]

def main():
    rows = read_csv(sys.stdin)

    for row in rows:
        candidate = get_tweet_candidate(row[2])
        if candidate is None:
            continue

        print('%s %d %d %d %d %d' % get_output(candidate, row[3], row[4], row[5]))

if __name__ == "__main__":
    main()


import requests
import argparse
import os

parser = argparse.ArgumentParser()
parser.add_argument("prompt", help="enter your prompt to send to OpenAI", type=str)
parser.add_argument("file_name", help="enter a file name for the results of the prompt", type=str)

args = parser.parse_args()

api_endpoint = "https://api.openai.com/v1/completions"
api_key = os.getenv("OPENAI_API_KEY")

content_headers ={
    "Content-Type": "application/json",
    "Authorization": "Bearer "+ api_key
}
request_data ={
    'model': 'text-davinci-003',
    'prompt': f"write python script to {args.prompt}",
    "max_tokens": 500,
    "temperature": 0.5
}
response =  requests.post(api_endpoint,headers=content_headers,json=request_data)

print(response.status_code)
if response.status_code == 200:
    response_text = response.json()["choices"][0]["text"]
    with open(args.file_name, "w") as file:
        file.write(response_text)
else:
    print(f"request fail with status code: {str(response.status_code)}")