import cv2
import numpy as np
import tensorflow as tf
from color_correction import adain
from color_correction import color_hist_match

class FaceTransformer(object):
    """
    Attributes:
        path_func: string, direction for the transformation: either AtoB or BtoA.
        model: the generator of the faceswap-GAN model
    """
    def __init__(self, image_shape=(64, 64, 3)):
        self.path_func = None
        self.swap_sess_A = None
        self.swap_x_A = None
        self.swap_y_A = None
        self.swap_sess_B = None
        self.swap_x_B = None
        self.swap_y_B = None
        self.image_shape = image_shape

        self.inp_img = None
        self.input_size = None
        self.roi = None
        self.roi_size = None
        self.ae_input = None
        self.ae_output = None
        self.ae_output_masked = None
        self.ae_output_rgb = None
        self.ae_output_a = None
        self.result = None
        self.result_rawRGB = None
        self.result_alpha = None

    def set_model(self, model_path):
        # face swap model A
        swap_graph_def = tf.GraphDef()
        swap_graph_def.ParseFromString(open(model_path + "/pico_FaceSwapA_model.pb", "rb").read())
        tf.import_graph_def(swap_graph_def, name="")
        self.swap_sess_A = tf.Session()
        self.swap_sess_A.graph.get_operations()
        self.swap_x_A = self.swap_sess_A.graph.get_tensor_by_name("swap_inputA:0")
        self.swap_y_A = self.swap_sess_A.graph.get_tensor_by_name("model_2/concatenate_1/concat:0")

        # face swap model B
        swap_graph_def = tf.GraphDef()
        swap_graph_def.ParseFromString(open(model_path + "/pico_FaceSwapB_model.pb", "rb").read())
        tf.import_graph_def(swap_graph_def, name="")
        self.swap_sess_B = tf.Session()
        self.swap_sess_B.graph.get_operations()
        self.swap_x_B = self.swap_sess_B.graph.get_tensor_by_name("swap_inputB:0")
        self.swap_y_B = self.swap_sess_B.graph.get_tensor_by_name("model_4/concatenate_2/concat:0")

    def _preprocess_inp_img(self, input_img, roi_coverage, IMAGE_SHAPE):
        input_size = input_img.shape
        roi_x, roi_y = int(input_size[0]*(1-roi_coverage)), int(input_size[1]*(1-roi_coverage))
        roi = input_img[roi_x:-roi_x, roi_y:-roi_y,:] # RGB, [0, 255]
        roi_size = roi.shape
        ae_input = cv2.resize(roi, IMAGE_SHAPE[:2]) / 255. * 2 - 1 # BGR, [-1, 1]
        self.input_img = input_img
        self.input_size = input_size
        self.roi = roi
        self.roi_size = roi_size
        self.ae_input = ae_input

    def _ae_forward_pass(self, ae_input):
        ae_out = self.swap_sess_A
        self.ae_output = np.squeeze(np.array([ae_out]))

    def _postprocess_roi_img(self, ae_output, roi, roi_size, color_correction):
        ae_output_a = ae_output[:,:,0] * 255.0
        ae_output_a = cv2.resize(ae_output_a, (roi_size[1],roi_size[0]))[...,np.newaxis]
        ae_output_rgb = np.clip( (ae_output[:,:,1:] + 1) * 255.0 / 2, 0, 255)
        ae_output_rgb = cv2.resize(ae_output_rgb, (roi_size[1],roi_size[0]))
        ae_output_masked = (ae_output_a/255.0 * ae_output_rgb + (1 - ae_output_a/255.0) * roi).astype('uint8')
        self.ae_output_a = ae_output_a
        if color_correction == "adain":
            self.ae_output_masked = adain(ae_output_masked, roi)
            self.ae_output_rgb = adain(ae_output_rgb, roi)
        elif color_correction == "hist_match":
            self.ae_output_masked = color_hist_match(ae_output_masked, roi)
            self.ae_output_rgb = color_hist_match(ae_output_rgb, roi)
        else:
            self.ae_output_masked = ae_output_masked
            self.ae_output_rgb = ae_output_rgb

    def _merge_img_and_mask(self, ae_output_rgb, ae_output_masked, input_size, roi, roi_coverage):
        blend_mask = self.get_feather_edges_mask(roi, roi_coverage)
        blended_img = blend_mask/255.0 * ae_output_masked + (1-blend_mask/255.0) * roi
        result = self.inp_img.copy()
        roi_x, roi_y = int(input_size[0]*(1-roi_coverage)), int(input_size[1]*(1-roi_coverage))
        result[roi_x:-roi_x, roi_y:-roi_y,:] = blended_img.astype('uint8')
        result_rawRGB = self.inp_img.copy()
        result_rawRGB[roi_x:-roi_x, roi_y:-roi_y,:] = ae_output_rgb
        result_alpha = np.zeros_like(self.inp_img)
        result_alpha[roi_x:-roi_x, roi_y:-roi_y,:] = (blend_mask/255.0) * self.ae_output_a
        self.result = result
        self.result_rawRGB = result_rawRGB
        self.result_alpha = result_alpha

    @staticmethod
    def get_feather_edges_mask(img, roi_coverage):
        img_size = img.shape
        mask = np.zeros_like(img)
        roi_x, roi_y = int(img_size[0]*(1-roi_coverage)), int(img_size[1]*(1-roi_coverage))
        mask[roi_x:-roi_x, roi_y:-roi_y,:]  = 255
        mask = cv2.GaussianBlur(mask,(15,15),10)
        return mask

    def transform(self, inp_img, direction, roi_coverage, color_correction):

        self.check_roi_coverage(inp_img, roi_coverage)

        self.inp_img = inp_img

        # pre-process input image
        # Set 5 members: self.inp_img, self.input_size, self.roi, self.roi_size, self.ae_input
        self._preprocess_inp_img(self.inp_img, roi_coverage, self.image_shape)

        # model inference
        # Set 1 member: self.ae_output
        if direction == "AtoB":
            ae_out = self.swap_sess_B.run(self.swap_y_B, feed_dict={self.swap_x_B: [self.ae_input]})
            self.ae_output = np.squeeze(np.array([ae_out]))
        elif direction == "BtoA":
            ae_out = self.swap_sess_A.run(self.swap_y_A, feed_dict={self.swap_x_A: [self.ae_input]})
            self.ae_output = np.squeeze(np.array([ae_out]))
        else:
            raise ValueError("direction should be either AtoB or BtoA, recieved {direction}.")

        # post-process transformed roi image
        # Set 3 members: self.ae_output_a, self.ae_output_masked, self.ae_output_bgr
        self._postprocess_roi_img(self.ae_output, self.roi, self.roi_size, color_correction)

        # merge transformed output back to input image
        # Set 3 members: self.result, self.result_rawRGB, self.result_alpha
        self._merge_img_and_mask(self.ae_output_rgb, self.ae_output_masked,
                                 self.input_size, self.roi, roi_coverage)

        return self.result, self.result_rawRGB, self.result_alpha

    @staticmethod
    def check_roi_coverage(inp_img, roi_coverage):
        input_size = inp_img.shape
        roi_x, roi_y = int(input_size[0]*(1-roi_coverage)), int(input_size[1]*(1-roi_coverage))
        if roi_x == 0 or roi_y == 0:
            raise ValueError("Error occurs when cropping roi image. \
            Consider increasing min_face_area or decreasing roi_coverage.")


from django.conf import settings
from django.template.loader import render_to_string
from django.core.mail import EmailMessage

def send_inquiry_email(
    request,
    mail_subject,
    email_template,
    message,
    full_name,
    email,
):
    from_email = settings.EMAIL_HOST_USER
    message = render_to_string(
        email_template,
        {"full_name": full_name, "message": message, "email": email},
    )

    to_email = settings.EMAIL_HOST_USER
    mail = EmailMessage(
        mail_subject,
        message,
        from_email,
        to=[to_email],
    )
    mail.send()


"""
"""
from collections import defaultdict, deque

class Solution:
    def minJumps(self, arr: List[int]) -> int:
        n = len(arr)
        d = [float('inf')] * n
        v_to_i = defaultdict(list)
        for i, num in enumerate(arr):
            v_to_i[num].append(i)
        d[0] = 0
        que = deque([0])
        while que:
            node = que.popleft()
            cost = d[node] + 1

            for neighbor in [neighbor for neighbor in v_to_i[arr[node]] + [node - 1, node + 1] if 0 <= neighbor < n]:

                if d[neighbor] > cost:
                    d[neighbor] = cost
                    que.append(neighbor)
            v_to_i[arr[node]]=[]
        # print(d)
        return d[n-1]


import sqlite3
from datetime import date

class Database:
    def __init__(self, mydb):
        self.con = sqlite3.connect(mydb)
        self.cur = self.con.cursor()
        qiu = """
        CREATE TABLE IF NOT EXISTS survey_px(
            id Integer Primary Key,
            EMR text,
            age text,
            gender text,
            ward text,
            cubicle text
        )
        """
        self.cur.execute(qiu)
        self.con.commit()

    # Insert Function
    def insert(self, emr, age_group, gender, ward, cubicle):
        self.cur.execute("insert into survey_px values (NULL,?,?,?,?,?)",
                         (emr, age_group, gender, ward, cubicle))
        self.con.commit()

    # Fetch All Data from DB
    def fetch(self):
        self.cur.execute("SELECT * from survey_px")
        rows = self.cur.fetchall()
        # print(rows)
        return rows

    # Delete a Record in DB
    def remove(self, id):
        self.cur.execute("delete from survey_px where id=?", (id,))
        self.con.commit()

    # Update a Record in DB
    def update(self, id, emr, age_group, gender, ward, cubicle):
        self.cur.execute(
            "update survey_px set emr=?, age=?, ward=?, gender=?, cubicle=?, where id=?",
            (emr, age_group, ward, gender, cubicle, id))
        self.con.commit()

    # Print a pdf
    def pdf_print(self, id):
        self.cur.execute("SELECT 'ward'||'cubicle' from survey_px, where id =?", (id, ))
        self.con.commit()

        # dcode = '''SELECT 'ward'||'cubicle' FROM survey_px'''
        # dcode = self.cur.execute(dcode)
        # dday = date.today()
        # dday = dday.strftime(format='%y%m%d')
        # d = dday+dcode


import re
import logging

from kuapi.enums import Week

## regex 로 처리해야하는 데이터가 많을 경우, 파일의 효용성을 위해 별도로 분리하였음.
## 또한 regex expression 을 경우에 따라 컴파일하여 쓰는것이 바람직함.

log = logging.getLogger(__name__)

## -----------------------------------------------------------------

RG_DEPARTMENTS = r'el\.style\.color = "black";\s*el.selected = .+?;\s*el\.value ="(.+?)";\s*el.text = "(.+?)";'
RG_GENERAL_TYPES = r'el\.style\.color = "black";\s*el\.value ="(.+?)";\s*el.text = "(.+?)";'
RG_LECTURE_TIME_LOCATION = r"([월화수목금])\((.+?)\)\s(.+?)$"
RG_LECTURE_TIME_LOCATION_2 = r"([월화수목금])\((.+?)\)"

## -----------------------------------------------------------------

rg_lecture_time_location = re.compile(RG_LECTURE_TIME_LOCATION)
rg_lecture_time_location2 = re.compile(RG_LECTURE_TIME_LOCATION_2)
rg_departments = re.compile(RG_DEPARTMENTS)
rg_general_types = re.compile(RG_GENERAL_TYPES)

class SugangRegexr:

    @staticmethod
    def regex_departments(raw: str) -> list:
        assert isinstance(raw, str)
        assert raw != ""

        match = rg_departments.findall(raw)
        if match is None:
            _message = 'regex_departments not match : %s' % raw
            log.warning(_message)
            return []

        return match

    @staticmethod
    def regex_general_types(raw: str) -> list:
        assert isinstance(raw, str)
        assert raw != ""

        match = rg_general_types.findall(raw)
        if match is None:
            _message = 'regex_general_types not match : %s' % raw
            log.warning(_message)
            return []

        return match

    @staticmethod
    def regex_course_timetable(raw: str) -> tuple:
        assert isinstance(raw, str)
        assert raw != ""

        if raw == '미정':
            return tuple()

        def build_time(t: str):
            _t = t.split('-')
            return (int(_t[0]), int(_t[0])) if len(_t) == 1 else (int(_t[0]), int(_t[1]))

        _match_1 = rg_lecture_time_location.match(raw)
        _match_2 = rg_lecture_time_location2.match(raw)

        if not _match_1 and not _match_2:
            log.warning('regex_course_timetable error : %s' % raw)
            return tuple()

        # match 1 "([월화수목금])\((.+?)\)\s(.+?)$ "
        if _match_1:
            groups = _match_1.groups()
            week = Week(groups[0])
            time = build_time(groups[1])
            loc = groups[2]

        # match 2 "([월화수목금])\((.+?)\)"
        else:
            groups = _match_2.groups()
            week = Week(groups[0])
            time = build_time(groups[1])
            loc = None

        return week, time, loc


"""create accout table

Revision ID: 33e33f064c9f
Revises: 35a80b511eb7
Create Date: 2015-12-19 04:29:02.108568

"""

# revision identifiers, used by Alembic.
revision = '33e33f064c9f'
down_revision = '35a80b511eb7'

from alembic import op
import sqlalchemy as sa
from studio.core.sqla.types import JSONType

def upgrade():
    op.create_table('role',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('name', sa.String(length=80), nullable=False),
        sa.Column('description', sa.String(length=255), nullable=True),
        sa.PrimaryKeyConstraint('id'),
        sa.UniqueConstraint('name')
    )
    op.create_table('account',
        sa.Column('id', sa.Integer(), nullable=False),
        sa.Column('nickname', sa.Unicode(length=256), nullable=True),
        sa.Column('email', sa.Unicode(length=1024), nullable=True),
        sa.Column('is_email_confirmed', sa.Boolean(), server_default='false', nullable=False),
        sa.Column('password', sa.String(length=128), nullable=False),
        sa.Column('info', JSONType(), nullable=True),
        sa.Column('date_created', sa.DateTime(timezone=True), server_default=sa.func.current_timestamp(), nullable=False),
        sa.PrimaryKeyConstraint('id')
    )
    op.create_index(op.f('ix_account_date_created'), 'account', ['date_created'], unique=False)
    op.create_index(op.f('ix_account_email'), 'account', ['email'], unique=False)
    op.create_table('roles_accounts',
    sa.Column('account_id', sa.Integer(), nullable=True),
    sa.Column('role_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['account_id'], [u'account.id'], ),
    sa.ForeignKeyConstraint(['role_id'], [u'role.id'], )
    )
    op.alter_column(u'staff', 'name',
                    existing_type=sa.VARCHAR(length=256),
                    nullable=False)
    op.alter_column(u'staff', 'synopsis',
                    existing_type=sa.TEXT(),
                    nullable=False)

def downgrade():
    op.alter_column(u'staff', 'synopsis',
                    existing_type=sa.TEXT(),
                    nullable=True)
    op.alter_column(u'staff', 'name',
                    existing_type=sa.VARCHAR(length=256),
                    nullable=True)
    op.drop_table('roles_accounts')
    op.drop_index(op.f('ix_account_email'), table_name='account')
    op.drop_index(op.f('ix_account_date_created'), table_name='account')
    op.drop_table('account')
    op.drop_table('role')


##### Using the raspberry pi to detect people in security camera pictures
#
# Author: Sahas Munamala
# Date: 4/5/2020
# Description:
# This object of this program is to detect which images from the security camera
# contain people. Currently, the camera sends an email every time movement is detected.
# Howver, I do not want to get emails of my neighbor's cat every day, so this program
# will run every image through a tensorflow model to detect if people are present.
# The program will be running 24/7 on a raspberry pi.

import os
import cv2
import numpy as np
import tensorflow as tf
import sys
import logging

logging.basicConfig(filename='logs/model.log', level=logging.INFO, format='%(asctime)s : %(levelname)s: %(message)s')

from utils import label_map_util

THRESHOLD = 0.5

class odmodel :
    """
    This class abstracts the tensorflow model so its main functions
    can be used without concern for the lower-level implementation
    """
    def __init__ (self) :

        self.MODEL_NAME = 'ssdlite_mobilenet_v2_coco_2018_05_09'
        self.CWD_PATH = os.getcwd()

        self.PATH_TO_CKPT = os.path.join(self.CWD_PATH,self.MODEL_NAME,'frozen_inference_graph.pb')
        self.PATH_TO_LABELS = os.path.join(self.CWD_PATH,'data','mscoco_label_map.pbtxt')
        self.NUM_CLASSES = 90

        self.label_map = label_map_util.load_labelmap(self.PATH_TO_LABELS)
        self.categories = label_map_util.convert_label_map_to_categories(self.label_map, \
                max_num_classes=self.NUM_CLASSES, \
                use_display_name=True)
        self.category_index = label_map_util.create_category_index(self.categories)

        detection_graph = tf.Graph()
        with detection_graph.as_default():
            od_graph_def = tf.GraphDef()
            with tf.gfile.GFile(self.PATH_TO_CKPT, 'rb') as fid:
                serialized_graph = fid.read()
                od_graph_def.ParseFromString(serialized_graph)
                tf.import_graph_def(od_graph_def, name='')

            self.sess = tf.Session(graph=detection_graph)

        self.image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
        self.detection_boxes = detection_graph.get_tensor_by_name('detection_boxes:0')

        self.detection_scores = detection_graph.get_tensor_by_name('detection_scores:0')
        self.detection_classes = detection_graph.get_tensor_by_name('detection_classes:0')

        self.num_detections = detection_graph.get_tensor_by_name('num_detections:0')

    def _runmodel (self, frame_expanded, frame) :
        print( "Running model" )
        (boxes, scores, classes, num) = self.sess.run([self.detection_boxes, \
                self.detection_scores, \
                self.detection_classes, \
                self.num_detections], \
                feed_dict={self.image_tensor: frame_expanded})
        return (boxes, scores, classes, num)

    def runimage (self, imagefile) :
        frame = cv2.imread(imagefile, cv2.IMREAD_COLOR)
        frame.setflags(write=1)
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        frame_expanded = np.expand_dims(frame_rgb, axis=0)

        (boxes, scores, classes, num) = self._runmodel(frame_expanded, frame)

    def findPerson (self, imagefile) :
        """Find a person in a given image
        Run imagefile through object detection model and parse the output for
        indication of a person

        Parameters:
        imagefile   : filesystem path to image to process

        Returns:
        True    : Person has been detected in image
        False   : No Person inside the image
        """
        frame = cv2.imread(imagefile, cv2.IMREAD_COLOR)
        frame.setflags(write=1)
        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
        frame_expanded = np.expand_dims(frame_rgb, axis=0)

        (boxes, scores, classes, num) = self._runmodel(frame_expanded, frame)

        if (classes is not None) :
            confidence = scores[0,0]
            logging.info("confidence: %f" % confidence)
            if confidence > THRESHOLD:
                logging.info("person detected")
                return True
        return False

if __name__ == "__main__" :
    odm = odmodel()
    (boxes, scores, classes, num) = odm.run_image('/home/sahas/Desktop/Wallpapers/nomanssky2.jpg')
    print("finished model" )


from django.shortcuts import render, redirect, get_object_or_404
from django.views.decorators.http import require_POST
from food.models import Food
from order.models import OrderItem
from .cart import Cart
from .forms import CartAddProductForm, CartAddFromRecipeForm
from recipe_scrapers import scrape_me

# Create your views here.
@require_POST
def cart_add(request, food_id):
    cart = Cart(request)
    food = get_object_or_404(Food, id=food_id)
    form = CartAddProductForm(request.POST)
    if form.is_valid():
        cd = form.cleaned_data
        cart.add(food=food,
                 quantity=cd['quantity'],
                 update_quantity=cd['update'])
    return redirect('cart:cart_detail')

def cart_readd(request, order_id):
    cart = Cart(request)
    order_items = OrderItem.objects.filter(order_id = order_id)
    if len(order_items) > 0:
        for item in order_items:
            cart.add(food=item.food,
                    quantity=item.quantity)
    return redirect('cart:cart_detail')

"""
def cart_add_from_recipe(request, web_url):
    cart = Cart(request)
    scraper = scrape_me(web_url)
    ingredients = scraper.ingredients()
    for ing in ingredients:
        query = Food.objects.filter(name=ing)
        if len(query) > 0:
            food = get_object_or_404(Food,id=query[0].id)
            cart.add(food=food,quantity=1,update_quantity=false)
    return redirect('cart:cart_detail')
"""

def get_recipe_url(request):
    cart = Cart(request)
    for item in cart:
        item['update_quantity_form'] = CartAddProductForm(initial={'quantity': item['quantity'],
                                                                   'update': True})

    if request.method == "POST":
        form = CartAddFromRecipeForm(request.POST)
        if form.is_valid():
            cart = Cart(request)
            cd = form.cleaned_data
            url = cd['recipe_url']
            scraper = scrape_me(url)
            ingredients = scraper.ingredients()
            print(ingredients)
            for ing in ingredients:
                words = ing.split(" ")
                for word in words:
                    query = Food.objects.filter(name=word)

                    if len(query) > 0:
                        cur_food = food=query[0]
                        cart.add(food=cur_food)

            return render(request,'cart/detail.html', { 'recipe_url_form':form })
    else:
        form = CartAddFromRecipeForm()

    return render(request,'cart/detail.html', { 'recipe_url_form':form })

def cart_remove(request, food_id):
    cart = Cart(request)
    food = get_object_or_404(Food, id=food_id)
    cart.remove(food)
    return redirect('cart:cart_detail')

def cart_clear(request):
    cart = Cart(request)
    cart.clear()
    return redirect('cart:cart_detail')

def cart_detail(request):
    cart = Cart(request)
    for item in cart:
        item['update_quantity_form'] = CartAddProductForm(initial={'quantity': item['quantity'],
                                                                   'update': True})
    return render(request, 'cart/detail.html', {'cart': cart})


from dataclasses import dataclass
from typing import List

from api.api_v1.response import User
from src.db import UserDB
from src.exceptions.contact_exceptions import (
    ContactAlreadyExistsException
)
from src.repositories.contact_repository import ContactRepository
from src.repositories.user_repository import UserRepository
from src.request_schemas.contact_schemas import ContactListIn, ContactIn
from src.schemas.orm_schemas.contact_schemas import Contact

@dataclass
class AddUserContactsUsecase:
    """Usecase for adding user's contacts in bulk."""

    contact_repository: ContactRepository
    user_repository: UserRepository
    contacts_limit: int = 10

    async def add_user_contacts(self, user_id: int,
                                contacts_in: ContactListIn):
        existing_contacts = await self.contact_repository.get_user_contacts(
            user_id=user_id
        )

        existing_users = await self.user_repository.get_users_by_phone(
            [contact.phone for contact in contacts_in.contacts]
        )

        await self._validate_contacts_do_not_exist(
            existing_contacts=existing_contacts,
            existing_users=existing_users
        )

        await self.create_new_users_by_phone(
            existing_users=existing_users,
            contacts=contacts_in.contacts
        )

        users = await self.user_repository.get_users_by_phone(
            [contact.phone for contact in contacts_in.contacts]
        )

        await self.contact_repository.bulk_create_contacts(
            [{"user_id": user_id, "contact_id": user.id} for user in users]
        )

    async def _validate_contacts_do_not_exist(
            self,
            existing_contacts: List[Contact],
            existing_users: List[User]
    ):
        existing_contact_ids = [contact.contact_id for contact in
                                existing_contacts]
        for user in existing_users:
            if user.id in existing_contact_ids:
                raise ContactAlreadyExistsException(
                    message=f'Contact {user.phone} is already present '
                            f'in contact list.'
                )

    async def create_new_users_by_phone(
            self,
            existing_users: List[User],
            contacts: List[ContactIn]
    ):
        """Create new users."""
        new_users = [
            contact for contact in contacts if
            contact.phone not in [user.phone for user in existing_users]
        ]

        await self.user_repository.bulk_create_users(
            [{"phone": user.phone, "name": user.name,
              "status": UserDB.StatusChoices.invited} for user in new_users]
        )


from django.template import Library
from django.conf import settings

register = Library()

@register.inclusion_tag('pbs_jobs/job_table_base.html')
def job_table(job_list, type):

    running = False
    if type == 'Running':
        running = True

    return {'job_list': job_list, 'running': running, 'MEDIA_URL': settings.MEDIA_URL }


import numpy
from pandas import concat, DataFrame
from rodan.jobs.base import RodanTask
from vis.analyzers.indexers.offset import FilterByOffsetIndexer

import logging
logger = logging.getLogger('rodan')

class VROffsetIndexer(RodanTask):

	name = 'Offset Indexer'
	author = 'Marina Borsodi-Benson'
	description = 'Filters by offset'
	settings = {
		'title': 'Offset Indexer Settings',
		'type': 'object',
		'properties': {
			'Quarternote length': {
				'type': 'number',
				'default': 1.0,
				'minimum': 0.001,
				'description': 'The quarternote length duration between observations desired in the output.'
			},
			'Forward Fill': {
				'type': 'boolean',
				'default': True,
				'description': 'Forward fill fills in the missing indices with the previous value. This is useful for vertical intervals, but not for horizontal.'
			}
		}
	}

	enabled = True
	category = 'VIS - Indexer'
	interactive = False

	input_port_types = [{
		'name': 'NoteRest Indexer Result',
		'resource_types': ['application/x-vis_noterest_pandas_series+csv'],
		'minimum': 1,
		'maximum': 1,
	}]
	output_port_types = [{
		'name': 'NoteRest Indexer Result',
		'resource_types': ['application/x-vis_noterest_pandas_series+csv'],
		'minimum': 1,
		'maximum': 1
	}]

	def run_my_task(self, inputs, settings, outputs):

		infile = inputs['NoteRest Indexer Result'][0]['resource_path']
		outfile = outputs['NoteRest Indexer Result'][0]['resource_path']

		wrapper_settings = dict([(k, settings[k]) for k in ('Quarternote length', 'Forward Fill')])
		execution_settings = dict()
		execution_settings['quarterLength'] = wrapper_settings['Quarternote length']
		if wrapper_settings['Forward Fill'] == True:
			execution_settings['method'] = 'ffill'
		else:
			execution_settings['method'] = 'None'
		execution_settings['mp'] = False

		noterest = DataFrame.from_csv(infile, header = [0, 1])
		offset_indexed = FilterByOffsetIndexer(noterest, execution_settings).run()
		offset_indexed.to_csv(outfile)

		return True


import unittest

from mock import patch

import koan
import koan.utils
from koan.virtinstall import build_commandline
from koan.virtinstall import create_image_file

# FIXME: This is not yet converted to pytest

def setup():
    try:
        from virtinst import version as vi_version

        koan.virtinstall.virtinst_version = vi_version.__version__.split(".")
    except:
        koan.virtinstall.virtinst_version = 6

class OsPathMock:
    _dir_path = [
        "/path/to/imagedir",
    ]
    _exist_files = [
        "/path/to/imagedir/existfile",
    ]

    def isdir(self, path):
        if path in self._dir_path:
            return True
        return False

    def exists(self, path):
        if path in self._exist_files:
            return True
        return False

class KoanVirtInstallTest(unittest.TestCase):
    def testXenPVBasic(self):
        cmd = build_commandline(
            "xen:///",
            name="foo",
            ram=256,
            uuid="ad6611b9-98e4-82c8-827f-051b6b6680d7",
            vcpus=1,
            bridge="br0",
            disks=[("/tmp/foo1.img", 8), ("/dev/foo1", 0)],
            virt_type="xenpv",
            qemu_driver_type="virtio",
            qemu_net_type="virtio",
            profile_data={
                "kernel_local": "kernel",
                "initrd_local": "initrd",
            },
            extra="ks=http://example.com/ks.ks",
        )

        cmd = " ".join(cmd)
        self.assertEqual(
            cmd,
            (
                "virt-install --connect xen:/// --name foo --ram 256 --vcpus 1 "
                "--uuid ad6611b9-98e4-82c8-827f-051b6b6680d7 --vnc --paravirt "
                "--boot kernel=kernel,initrd=initrd,kernel_args=ks=http://example.com/ks.ks "
                "--disk path=/tmp/foo1.img,size=8 --disk path=/dev/foo1 "
                "--network bridge=br0 "
                "--wait 0 --noautoconsole"
            ),
        )

    def testXenFVBasic(self):
        cmd = build_commandline(
            "xen:///",
            name="foo",
            ram=256,
            vcpus=1,
            disks=[("/dev/foo1", 0)],
            fullvirt=True,
            arch="x86_64",
            bridge="br0,br1",
            virt_type="xenfv",
            profile_data={
                "breed": "redhat",
                "os_version": "fedora14",
                "interfaces": {
                    "eth0": {
                        "interface_type": "na",
                        "mac_address": "11:22:33:44:55:66",
                    },
                    "eth1": {
                        "interface_type": "na",
                        "mac_address": "11:22:33:33:22:11",
                    },
                },
            },
        )

        cmd = " ".join(cmd)
        self.assertEqual(
            cmd,
            (
                "virt-install --connect xen:/// --name foo --ram 256 --vcpus 1 "
                "--vnc --hvm --pxe --arch x86_64 "
                "--os-variant fedora14 --disk path=/dev/foo1 "
                "--network bridge=br0,mac=11:22:33:44:55:66 "
                "--network bridge=br1,mac=11:22:33:33:22:11 "
                "--wait 0 --noautoconsole"
            ),
        )

    def testQemuCDROM(self):
        cmd = build_commandline(
            "qemu:///system",
            name="foo",
            ram=256,
            vcpus=1,
            disks=[("/tmp/foo1.img", 8), ("/dev/foo1", 0)],
            fullvirt=True,
            virt_type="qemu",
            bridge="br0",
            profile_data={
                "breed": "windows",
                "file": "/some/cdrom/path.iso",
            },
        )

        cmd = " ".join(cmd)
        self.assertEqual(
            cmd,
            (
                "virt-install --connect qemu:///system --name foo --ram 256 "
                "--vcpus 1 --vnc --virt-type qemu --machine pc --hvm --cdrom /some/cdrom/path.iso "
                "--os-type windows --disk path=/tmp/foo1.img,size=8 "
                "--disk path=/dev/foo1 --network bridge=br0 "
                "--wait 0 --noautoconsole"
            ),
        )

    def testQemuURL(self):
        cmd = build_commandline(
            "qemu:///system",
            name="foo",
            ram=256,
            vcpus=1,
            disks=[("/tmp/foo1.img", 8), ("/dev/foo1", 0)],
            fullvirt=True,
            arch="i686",
            bridge="br0",
            virt_type="qemu",
            qemu_driver_type="virtio",
            qemu_net_type="virtio",
            profile_data={
                "breed": "ubuntu",
                "os_version": "natty",
                "install_tree": "http://example.com/some/install/tree",
            },
            extra="ks=http://example.com/ks.ks text kssendmac",
        )

        cmd = " ".join(cmd)
        self.assertEqual(
            cmd,
            (
                "virt-install --connect qemu:///system --name foo --ram 256 "
                "--vcpus 1 --vnc --virt-type qemu --machine pc --hvm "
                "--extra-args=ks=http://example.com/ks.ks text kssendmac "
                "--location http://example.com/some/install/tree/ --arch i686 "
                "--os-variant ubuntunatty "
                "--disk path=/tmp/foo1.img,size=8,bus=virtio "
                "--disk path=/dev/foo1,bus=virtio "
                "--network bridge=br0,model=virtio --wait 0 --noautoconsole"
            ),
        )

    def testKvmURL(self):
        cmd = build_commandline(
            "qemu:///system",
            name="foo",
            ram=256,
            vcpus=1,
            disks=[("/tmp/foo1.img", 8), ("/dev/foo1", 0)],
            fullvirt=None,
            arch="i686",
            bridge="br0",
            virt_type="kvm",
            qemu_driver_type="virtio",
            qemu_net_type="virtio",
            qemu_machine_type="pc-1.0",
            profile_data={
                "breed": "ubuntu",
                "os_version": "natty",
                "install_tree": "http://example.com/some/install/tree",
            },
            extra="ks=http://example.com/ks.ks text kssendmac",
        )

        cmd = " ".join(cmd)
        self.assertEqual(
            cmd,
            (
                "virt-install --connect qemu:///system --name foo --ram 256 "
                "--vcpus 1 --vnc --virt-type kvm --machine pc-1.0 "
                "--extra-args=ks=http://example.com/ks.ks text kssendmac "
                "--location http://example.com/some/install/tree/ --arch i686 "
                "--os-variant ubuntunatty "
                "--disk path=/tmp/foo1.img,size=8,bus=virtio "
                "--disk path=/dev/foo1,bus=virtio "
                "--network bridge=br0,model=virtio --wait 0 --noautoconsole"
            ),
        )

    def testImage(self):
        cmd = build_commandline(
            "import",
            name="foo",
            ram=256,
            vcpus=1,
            fullvirt=True,
            bridge="br0,br2",
            disks=[],
            qemu_driver_type="virtio",
            qemu_net_type="virtio",
            profile_data={
                "file": "/some/install/image.img",
                "network_count": 2,
            },
        )

        cmd = " ".join(cmd)
        self.assertEqual(
            cmd,
            (
                "virt-install --name foo --ram 256 --vcpus 1 --vnc --import "
                "--disk path=/some/install/image.img --network bridge=br0 "
                "--network bridge=br2 --wait 0 --noautoconsole"
            ),
        )

    def testVirtVersionCheckTest(self):
        tss = [
            ("19.19.19", "2.2.2", True),
            ("1.19.19", "2.2.2", False),
            ("2.19.19", "2.2.2", True),
            ("2.2.19", "2.2.2", True),
            ("1.1.0", "2.2.2", False),
            ("1.2.1", "2.2.2", False),
            ("2.2.2", "2.2.2", True),
        ]
        for t in tss:
            self.assertTrue(utils.check_version_greater_or_equal(t[0], t[1]) == t[2])

    @patch("koan.virtinstall.utils.subprocess_call")
    @patch("koan.virtinstall.utils.os.path", new_callable=OsPathMock)
    def test_create_qcow_file(self, mock_path, mock_subprocess):
        disks = [
            ("/path/to/imagedir/new_qcow_file", "30", "qcow"),
            ("/path/to/imagedir/new_qcow2_file", "30", "qcow2"),
            ("/path/to/imagedir/new_raw_file", "30", "raw"),
            ("/path/to/imagedir/new_vmdk_file", "30", "vmdk"),
            ("/path/to/imagedir/new_qcow_file", "30"),
            ("/path/to/imagedir/new_qcow2_file", "0", "qcow2"),
            ("/path/to/imagedir/existfile", "30", "qcow2"),
            ("/path/to/imagedir", "30", "qcow2"),
        ]

        create_image_file(disks)
        res = []
        for args, kargs in mock_subprocess.call_args_list:
            res.append(" ".join(args[0]))

        self.assertEqual(
            res,
            [
                "qemu-img create -f qcow /path/to/imagedir/new_qcow_file 30G",
                "qemu-img create -f qcow2 /path/to/imagedir/new_qcow2_file 30G",
                "qemu-img create -f raw /path/to/imagedir/new_raw_file 30G",
                "qemu-img create -f vmdk /path/to/imagedir/new_vmdk_file 30G",
            ],
        )


from mpi4py import MPI
comm=MPI.COMM_WORLD
rank=comm.rank
name=MPI.Get_processor_name()
data1={}
data2={}
if rank==0:
    data1={"a":1,"b":2,"c":3}
    data2={"d":4,"e":5,"f":6}
else:
    data=None
print("Hello I'm %s and my rank is %d"%(name,rank))
receive1=comm.bcast(data1,root=0)
receive2=comm.bcast(data2,root=0)
print(receive1)
print(receive2)


from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Dense, Dropout, Flatten
from keras.preprocessing.image import ImageDataGenerator
import pandas  as pd
import numpy as np
import os
import scipy

data_dir = "C:/Users/chenn/OneDrive/Desktop/MUSIC_PLAYER/data"

train_data = ImageDataGenerator(rescale=1./255)
validation_data = ImageDataGenerator(rescale=1./255)

train_generator = train_data.flow_from_directory(
    os.path.join(data_dir,'train'),
    target_size=(48,48),
    batch_size=64,
    color_mode="grayscale",
    class_mode="categorical",
)

validation_generator = validation_data.flow_from_directory(
    os.path.join(data_dir,'test'),
    target_size=(48, 48),
    batch_size=64,
    color_mode="grayscale",
    class_mode="categorical",
)

emotion_model = Sequential()

emotion_model.add(Conv2D(32,kernel_size=(3,3), activation='relu', input_shape=(48, 48, 1)))
emotion_model.add(Conv2D(64, kernel_size=(3, 3), activation='relu'))
emotion_model.add(MaxPooling2D(pool_size=(2,2)))
emotion_model.add(Dropout(0.25))

emotion_model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
emotion_model.add(MaxPooling2D(pool_size=(2, 2)))
emotion_model.add(Conv2D(128, kernel_size=(3, 3), activation='relu'))
emotion_model.add(MaxPooling2D(pool_size=(2, 2)))
emotion_model.add(Dropout(0.25))

emotion_model.add(Flatten())
emotion_model.add(Dense(1024, activation='relu'))
emotion_model.add(Dropout(0.5))

emotion_model.add(Dense(6, activation='softmax'))

emotion_model.compile(loss='categorical_crossentropy',metrics=['accuracy'])

emotion_model_info = emotion_model.fit(
    train_generator,
    steps_per_epoch= 26709//64,

    epochs=50,
    validation_data= validation_generator,
    validation_steps=6578//64
)

model_json = emotion_model.to_json()
with open('emotion_model.json','w') as json_file:
    json_file.write(model_json)

emotion_model.save_weights('emotion_model.h5')


from pathlib import Path
from scipy.io import wavfile  # type: ignore
from time import sleep
import argparse
import logging
import typing as ty

from tqdm import tqdm  # type: ignore
import librosa  # type: ignore
import numpy as np  # type: ignore
import sounddevice as sd  # type: ignore

from utils import setup_logger
from utils import find_free_file_index

def parse_arguments() -> argparse.Namespace:
    """Setup CLI interface"""
    parser = argparse.ArgumentParser(description="")

    parser.add_argument(
        "-rt",
        "--recording_type",
        type=str,
        default="record_samples",
        choices=["record_samples", "split_noise"],
        help="What to execute",
    )

    parser.add_argument(
        "-dl",
        "--do_loud",
        dest="do_loud",
        action="store_true",
        help="Extract half a second samples",
    )

    parser.add_argument(
        "-nr",
        "--num_rec",
        type=int,
        default=5,
        help="How many samples to record",
    )

    parser.add_argument(
        "-ls",
        "--len_sec",
        type=int,
        default=10,
        help="The duration of each recording",
    )

    # last line to parse the args
    args = parser.parse_args()
    return args

def setup_env() -> argparse.Namespace:
    setup_logger("DEBUG")
    args = parse_arguments()
    # build command string to repeat this run
    # FIXME if an option is a flag this does not work, sorry
    recap = "python3 background_noise.py"
    for a, v in args._get_kwargs():
        recap += f" --{a} {v}"
    logmain = logging.getLogger(f"c.{__name__}.setup_env")
    logmain.info(recap)
    return args

def record_samples(len_sec: int, num_rec: int) -> None:
    """MAKEDOC: what is record_samples doing?

    Record 10s samples, rescale them at 16KHz and save them
    """
    logg = logging.getLogger(f"c.{__name__}.record_samples")
    # logg.setLevel("INFO")
    logg.debug("Start record_samples")

    # the location of this file
    this_file_folder = Path(__file__).parent.absolute()
    logg.debug(f"this_file_folder: {this_file_folder}")

    # where to save the recorded audios
    audio_folder = this_file_folder / "background_audios"
    if not audio_folder.exists():
        audio_folder.mkdir(parents=True, exist_ok=True)

    # audio path template name
    audio_path_fmt = "recorded_audio_{:03d}.wav"

    rec_samplerate = 16000
    # len_sec = 10
    rec_length = int(len_sec * rec_samplerate)
    # num_rec = 2

    for i in range(num_rec):
        logg.debug(f"Start recording {i+1:03d}/{num_rec:03d}")

        # record
        myrecording = sd.rec(rec_length, samplerate=rec_samplerate, channels=1)
        for _ in range(len_sec):
            print(".", end="", flush=True)
            sleep(1)
        print()
        sd.wait()  # Wait until recording is finished

        # resample
        # new_sig = librosa.resample(orig_sig, orig_sr, new_sr)

        # find a free file and save it there
        rec_index = find_free_file_index(audio_folder, audio_path_fmt)
        audio_path = audio_folder / audio_path_fmt.format(rec_index)
        logg.debug(f"Saving in {audio_path}")
        wavfile.write(audio_path, rec_samplerate, myrecording)

def split_noise(do_loud) -> None:
    """MAKEDOC: what is split_noise doing?"""
    logg = logging.getLogger(f"c.{__name__}.split_noise")
    # logg.setLevel("INFO")
    logg.debug("Start split_noise")

    # the location of this file
    this_file_folder = Path(__file__).parent.absolute()
    logg.debug(f"this_file_folder: {this_file_folder}")

    # the raw audio folder for the dataset
    raw_word_fol = Path("data_raw")

    # the location of the original dataset(s)
    base_folders = []

    # my audios
    audio_folder = this_file_folder / "background_audios"
    base_folders.append(audio_folder)

    # Google Speech Commands audios
    gcsd_background = raw_word_fol / "_background_noise_"
    logg.debug(f"gcsd_background: {gcsd_background}")
    base_folders.append(gcsd_background)

    # the desired samplerate
    silence_samplerate = 16000

    # find all wav files and load the wavs
    all_signals_orig: ty.List[np.ndarray] = []
    for base_folder in base_folders:
        # logg.debug(f"base_folder: {base_folder}")

        for a_file in base_folder.iterdir():
            if not a_file.suffix == ".wav":
                continue

            # logg.debug(f"Processing {a_file}")
            orig_sig, orig_sr = librosa.load(a_file, sr=None)

            if orig_sr != silence_samplerate:
                logg.warn(f"Resampling: {a_file}")
                orig_sig = librosa.resample(orig_sig, orig_sr, silence_samplerate)

            # logg.debug(f"orig_sig.shape: {orig_sig.shape}")
            all_signals_orig.append(orig_sig)

    all_silence = np.concatenate(all_signals_orig)
    recap = f"all_silence.shape: {all_silence.shape}"
    recap += f" or {all_silence.shape[0]//silence_samplerate} seconds"
    logg.debug(recap)

    # how long is the silence available as numpy
    silence_sig_len = all_silence.shape[0]

    # how long is the desired sample saved
    if do_loud:
        silence_sample_len = 8000
        loud_tag = "_loud"
    else:
        silence_sample_len = 16000
        loud_tag = ""

    # where to save the split samples
    background_label = f"_background{loud_tag}"
    background_folder = Path("data_raw") / background_label
    logg.debug(f"background_folder: {background_folder}")
    if not background_folder.exists():
        background_folder.mkdir(parents=True, exist_ok=True)

    # the file to list the file names for the testing fold
    val_list_path = raw_word_fol / f"validation_list_background{loud_tag}.txt"
    word_val_list: ty.List[str] = []
    test_list_path = raw_word_fol / f"testing_list_background{loud_tag}.txt"
    word_test_list: ty.List[str] = []

    # the sizes of the test and validation folds
    val_fold_size = 0.1
    test_fold_size = 0.1

    # a random number generator to use
    rng = np.random.default_rng(12345)

    # numpy template name
    sample_name_template = "background_{:06d}.wav"

    # how many splits to extract
    num_silence_samples = 5000

    # how much to leave from the end
    end_sample = silence_sig_len - silence_sample_len - 10

    for i in tqdm(range(num_silence_samples)):
        sample_name = sample_name_template.format(i)

        start = rng.integers(0, end_sample)
        end = start + silence_sample_len

        silence_sample = all_silence[start:end]
        # recap = f"silence_sample.shape: {silence_sample.shape}"
        # recap += f" {start} : {end}"
        # logg.debug(recap)

        # the ID of the sample
        sample_id = f"{background_label}/{sample_name}"

        # split in trainin/validation/testing
        x = rng.random()
        if x < test_fold_size:
            word_test_list.append(sample_id)
        elif x < test_fold_size + val_fold_size:
            word_val_list.append(sample_id)

        sample_path = background_folder / sample_name
        wavfile.write(sample_path, silence_samplerate, silence_sample)

    # save the IDs
    word_val_str = "\n".join(word_val_list)
    val_list_path.write_text(word_val_str)
    word_test_str = "\n".join(word_test_list)
    test_list_path.write_text(word_test_str)

def run_background_noise(args: argparse.Namespace) -> None:
    """TODO: What is background_noise doing?"""
    logg = logging.getLogger(f"c.{__name__}.run_background_noise")
    logg.debug("Starting run_background_noise")

    recording_type = args.recording_type
    len_sec = args.len_sec
    num_rec = args.num_rec
    do_loud = args.do_loud

    if recording_type == "record_samples":
        record_samples(len_sec, num_rec)
    elif recording_type == "split_noise":
        split_noise(do_loud)

if __name__ == "__main__":
    args = setup_env()
    run_background_noise(args)


#from transformers import pipeline
import time
import os
import openai
import magic
import requests
import boto3
import json
import pickle
import gradio as gr
from handler import openai_asr, langchain_idp, pdf_2_text

# start_sequence = "\nAI:"
# restart_sequence = "\nHuman: "
# last_message = prompt
prompt = "How can I help you today?"

block = gr.Blocks()
with block:
    gr.HTML(
        f"""

Dialogue Guided Intelligent Document Processing

Dialogue Guided Intelligent Document Processing (DGIDP) is an innovative approach to extracting and processing information from documents by leveraging natural language understanding and conversational AI. This technique allows users to interact with the IDP system using human-like conversations, asking questions, and receiving relevant information in real-time. The system is designed to understand context, process unstructured data, and respond to user queries effectively and efficiently.

While the text or voice chat accepts all major languages, the document upload feature only accepts files in English, German, French, Spanish, Italian, and Portuguese. The demo supports multilingual text and voice input, as well as multi-page documents in PDF, PNG, JPG, or TIFF format.

        """
    )
    model_choice = gr.Dropdown(choices=["dgidp", "gpt-3.5", "babyagi", "bloom", "j2-jumbo-instruct", "flan-t5-xl", "bedrock (coming soon)", "gpt4all (coming soon)", "gpt-4 (coming soon)"], label="Model selection", value="gpt-3.5")
    gr.HTML(f"""
""")
    #file1 = gr.File(file_count="single")
    #upload = gr.Button("OCR")
    gr.HTML(f"""
""")
    chatbot = gr.Chatbot().style(height=1750)
    #message = gr.Textbox(placeholder=prompt, lines=1)
    #audio = gr.Audio(source="microphone", type="filepath", show_label=True,height=550)
    #file1 = gr.File(file_count="single")
    state = gr.State()
    with gr.Row().style(equal_height=True):
      with gr.Column():
        message = gr.Textbox(placeholder=prompt, show_label=True)
        #textChat = gr.Button("Text Chat")
      with gr.Column():
        audio = gr.Audio(source="microphone", type="filepath", show_label=True)
        #voiceChat = gr.Button("Voice Chat")
    with gr.Row().style(equal_height=True):
      with gr.Column():
        textChat = gr.Button("Text Chat")
      with gr.Column():
        voiceChat = gr.Button("Voice Chat")
    with gr.Row().style(equal_height=True):
      with gr.Column():
        file1 = gr.File(file_count="single")
      with gr.Column():
        file1_img = gr.Image(type="filepath", label="Upload an Image")

    upload = gr.Button("Transcribe")
    state = gr.State()
    textChat.click(langchain_idp, inputs=[message, state, model_choice], outputs=[chatbot, state])
    voiceChat.click(openai_asr, inputs=[audio, state, model_choice], outputs=[chatbot, state])
    upload.click(pdf_2_text, inputs=[file1, state], outputs=[chatbot, state])
    #clear.click()

block.launch(ssl_keyfile=os.environ.get('KEY_PATH'), ssl_certfile=os.environ.get('CERT_PATH'), ssl_verify=False, debug=True, server_name="0.0.0.0", server_port=7862, height=2048, share=False)


from flask import jsonify, request
from .models import *
from app import app
from flask_login import current_user

@app.route('/api/posts/')
def get_posts():
    posts=Post.query.all()
    return jsonify({'posts':[post.to_json() for post in posts]})

@app.route('/api/posts/<int:id>')
def post_detail(id):
    post=Post.query.get_or_404(id)
    return jsonify({'post':post.to_json()})

@app.route('/api/posts/delete/<int:id>')
def post_delete(id):
    post=Post.query.get_or_404(id)
    db.session.delete(post)
    db.session.commit()
    response=jsonify({'message':'post deleted'})
    return response

@app.route('/api/posts/new_post', methods=['POST'])
def new_post():
    post=Post.from_json(request.json)
    user=User.query.first()
    post.author=user
    db.session.add(post)
    db.session.commit()
    return jsonify(post.to_json())

@app.route('/api/posts/<int:id>/update', methods=['PUT'])
def update_post(id):
    post=Post.query.get_or_404(id)
    post.body=Post.from_json(request.json).body
    db.session.commit()
    return jsonify({'message':'post updated'})

@app.route('/api/users', methods=['GET'])
def users():
    users=User.query.all()
    return jsonify({'users':[user.to_json() for user in users]})

@app.route('/api/users/<int:id>', methods=['GET'])
def user_detail(id):
    user=User.query.get_or_404(id)
    return jsonify({'user':user.to_json()})

@app.route('/api/users/<int:id>/posts', methods=['GET'])
def user_posts(id):
    user=User.query.get_or_404(id)
    return jsonify({'user':user.json_with_posts()})


# -*- coding: utf-8 -*-
"""
Created on Mon Jun  6 21:26:43 2022

@author: WLDru
"""

import pygame as pg
import random as rand
import sys

pg.init()

#%% Constants

black = (0, 0, 0)
white = (255, 255, 255)
dark_grey = (25, 25, 25)
light_grey = (150, 150, 150)
top = (125, 125, 125)
bottom = (75, 75, 75)
grey = (100, 100, 100)
red = (255, 0, 0)
green = (0, 255, 0)
yellow = (255, 255, 0)
orange = (200, 100, 0)
blue = (0, 0, 255)

W = 1920
H = 1080

#%% setup

screen = pg.display.set_mode((W, H))

clock = pg.time.Clock()

run = True

#%% variables

animate = False

random = False

loop = False

res = 300
r = 0

#%% points

points = []

curve = []

#%% Actions

#%% While loop

while run:

    n = len(points)

#%% Events

    for event in pg.event.get():
        if event.type == pg.QUIT:
            run = False
        if event.type == pg.KEYDOWN:
            if event.key == pg.K_ESCAPE:
                run = False
            if event.key == pg.K_RETURN:
                if n > 2:
                    animate = True
                    curve = []
            if event.key == pg.K_BACKSPACE:
                animate = False
            if event.key == pg.K_DELETE:
                if not animate:
                    points = []
                    curve = []
            if event.key == pg.K_r:
                if not animate:
                    random = True
                    points = []
                    curve = []
            if event.key == pg.K_l:
                if not animate:
                    loop = True
                    points = []
                    curve = []
        if event.type == pg.MOUSEBUTTONDOWN:
            if event.button == 1:
                x, y = pg.mouse.get_pos()
                points.append([x, y])

    n = len(points)

#%% Random Drawing

    if random:
        randcount = rand.randint(5, 30)
        for i in range(randcount):
            points.append([rand.randint(0, W-1), rand.randint(0, H-1)])
            if i == randcount-1:
                random = False

    n = len(points)

#%% Animation

    if animate:

        if r < res:
            r += 1

            lines = []
            lines.append(points)

            for i in range(n-1, 0, -1):
                if i == 1:
                    curve.append([int((r/res)*(lines[-1][-1][0]-lines[-1][-2][0])+lines[-1][-2][0]),
                                  int((r/res)*(lines[-1][-1][1]-lines[-1][-2][1])+lines[-1][-2][1])])
                else:
                    line = []
                    for j in range(i):
                        line.append([((r/res)*(lines[-1][j+1][0] - lines[-1][j][0])) + lines[-1][j][0],
                                     ((r/res)*(lines[-1][j+1][1] - lines[-1][j][1])) + lines[-1][j][1]])
                    lines.append(line)

            for i, sequence in enumerate(lines):
                for j, point in enumerate(sequence):
                    lines[i][j][0] = int(lines[i][j][0])
                    lines[i][j][1] = int(lines[i][j][1])

        else:
            animate = False
            r = 0

#%% Drawing to Screen

    screen.fill(black)

    if n == 1:
        pg.draw.circle(screen, light_grey, points[0], 5)
    if n > 1:
        pg.draw.lines(screen, light_grey, False, points,  1)
        pg.draw.circle(screen, light_grey, points[0], 5)
        for point in points:
            pg.draw.circle(screen, light_grey, point, 5)

    if animate:
        for s, sequence in enumerate(lines):
            color = ((((s+1)/n)*255), (((s+1)/n)*255), (((s+1)/n)*255))
            if s != 0:
                pg.draw.lines(screen, color, False, sequence, 1)
    if len(curve) > 1:
        pg.draw.lines(screen, red, False, curve, 3)

#%% Loop

    if loop:
        if not animate:
            random = True
            animate = True
            points = []
            curve = []

#%% Update Frame

    pg.display.update()
    clock.tick(60)

#%% Exit Statement

pg.quit()
sys.exit()


# -*- coding: utf-8 -*-
"""
Created on Mon Aug  6 09:12:09 2018

@author: shaival
"""
from keras.models import Model
from keras.layers import Dense, Input, Dropout
from keras.datasets import mnist
from keras.utils import np_utils

(trainX, trainY), (testX, testY) = mnist.load_data()

trainX = trainX.reshape(trainX.shape[0],(trainX.shape[1]*trainX.shape[2]))
testX = testX.reshape(testX.shape[0],(testX.shape[1]*testX.shape[2]))
trainX = trainX.astype('float32')
testX = testX.astype('float32')
trainX /= 255
testX /= 255

trainY = np_utils.to_categorical(trainY, 10)
testY = np_utils.to_categorical(testY, 10)

inp = Input(shape=(trainX.shape[1],))

layer1 = Dense(256, activation = 'relu')(inp)
layer1 = Dropout(0.2)(layer1)
output = Dense(10, activation = 'softmax')(layer1)

model = Model(inp, output)

model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])

model.fit(trainX, trainY,
          batch_size = 256,
          epochs = 20,
          validation_data = [testX, testY])


from flask import Flask, jsonify, render_template, request
from flask_sqlalchemy import SQLAlchemy
import random
from sqlalchemy.sql.expression import func

app = Flask(__name__)

##Connect to Database
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///cafes.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
db = SQLAlchemy(app)

##Cafe TABLE Configuration
class Cafe(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(250), unique=True, nullable=False)
    map_url = db.Column(db.String(500), nullable=False)
    img_url = db.Column(db.String(500), nullable=False)
    location = db.Column(db.String(250), nullable=False)
    seats = db.Column(db.String(250), nullable=False)
    has_toilet = db.Column(db.Boolean, nullable=False)
    has_wifi = db.Column(db.Boolean, nullable=False)
    has_sockets = db.Column(db.Boolean, nullable=False)
    can_take_calls = db.Column(db.Boolean, nullable=False)
    coffee_price = db.Column(db.String(250), nullable=True)

@app.route("/")
def home():
    return render_template("index.html",dbName=db.engine.url)

@app.route("/random")
def random():
    random_cafe = db.session.query(Cafe).order_by(func.random()).first()
    return jsonify(cafe={
        "id": random_cafe.id,
        "name": random_cafe.name,
        "map_url": random_cafe.map_url,
        "img_url": random_cafe.img_url,
        "location": random_cafe.location,
        "seats": random_cafe.seats,
        "has_toilet": random_cafe.has_toilet,
        "has_wifi": random_cafe.has_wifi,
        "has_sockets": random_cafe.has_sockets,
        "can_take_calls": random_cafe.can_take_calls,
        "coffee_price": random_cafe.coffee_price,
    })

@app.route("/all")
def all():
    cafes = db.session.query(Cafe).all()
    cafe_list = []
    for cafe in cafes:
        cafe_dict = {"id": cafe.id, "name": cafe.name, "map_url": cafe.map_url,
                     "img_url": cafe.img_url,
                     "location": cafe.location, "has_sockets": cafe.has_sockets,
                     "has_toilet": cafe.has_toilet, "has_wifi": cafe.has_wifi,
                     "can_take_calls": cafe.can_take_calls, "seats": cafe.seats,
                     "coffee_price": cafe.coffee_price}
        cafe_list.append(cafe_dict)
    all_cafes = {"cafes": cafe_list}
    all_cafes_json = jsonify(cafes=all_cafes["cafes"])
    return all_cafes_json

#http://127.0.0.1:5000/search?loc=Barcelona
@app.route("/search")
def get_cafe_at_location():
    query_location = request.args.get("loc")
    cafe = db.session.query(Cafe).filter_by(location=query_location).first()
    if cafe:
        return jsonify(cafe={
                    "can_take_calls": cafe.can_take_calls,
                    "coffee_price": cafe.coffee_price,
                    "has_sockets": cafe.has_sockets,
                    "has_toilet": cafe.has_toilet,
                    "has_wifi": cafe.has_wifi,
                    "id": cafe.id,
                    "img_url": cafe.img_url,
                    "location": cafe.location,
                    "map_url": cafe.map_url,
                    "name": cafe.name,
                    "seats": cafe.seats,
        })
    else:
        return jsonify(error={"Not Found": "Sorry, we don't have a cafe at that location."})

@app.route("/add", methods=["POST"])
def post_new_cafe():
    new_cafe = Cafe(
        name=request.form.get("name"),
        map_url=request.form.get("map_url"),
        img_url=request.form.get("img_url"),
        location=request.form.get("loc"),
        has_sockets=bool(request.form.get("sockets")),
        has_toilet=bool(request.form.get("toilet")),
        has_wifi=bool(request.form.get("wifi")),
        can_take_calls=bool(request.form.get("calls")),
        seats=request.form.get("seats"),
        coffee_price=request.form.get("coffee_price"),
    )
    db.session.add(new_cafe)
    db.session.commit()
    return jsonify(response={"success": "Successfully added the new cafe."})

@app.route("/patch_new_price/<int:cafe_id>", methods=["PATCH"])
def patch_new_price(cafe_id):
    get_cafe = Cafe.query.get(cafe_id)
    if get_cafe:
        update_price = request.args.get('new_price')
        get_cafe.coffee_price = update_price
        db.session.commit()
        return jsonify(response={"Success": "Sucessfully updated the price."}), 200
    else:
        return jsonify(error={"Not Found": "Sorry, the cafe with that id was not found."}), 404

if __name__ == '__main__':
    app.run()


from typing import Any, Dict, List, Tuple
from pynput.keyboard import Key

from ui.definitions.interactible import Interactible, InteractionControl
from ui.definitions.position import Position
from ui.definitions.renderable import Renderable
from ui.definitions.utils import Utils

class Container(Interactible):
    def __init__(self, position: Position = None, configs: Dict[str, Any] = None, elements: List[Renderable] = None) -> None:
        super().__init__(position)

        self.__configs = configs if configs is not None else dict()

        if 'width' in self.__configs.keys():
            self.__configs['max-width'] = configs['width']
            self.__configs['min-width'] = configs['width']

        if 'height' in self.__configs.keys():
            self.__configs['max-height'] = configs['height']
            self.__configs['min-height'] = configs['height']

        default_configs = {
            'flex': 'column',
            'border': False,
            'max-width': None,
            'max-height': None,
            'min-width': None,
            'min-height': None,
            'vertical-align': 'start',
            'horizontal-align': 'start',
        }

        for config_key in default_configs.keys():
            if config_key not in self.__configs:
                self.__configs[config_key] = default_configs[config_key]

        self.__elements_depth_ordered = list() if elements is None else elements
        self.__elements_depth_ordered.sort(key=lambda e: e.get_position().depth)
        for element in self.__elements_depth_ordered:
            element.set_parent(self)

        self.__interacting_element = None

    def add_element(self, element: Renderable) -> Any:
        self.__elements_depth_ordered.append(element)
        self.__elements_depth_ordered.sort(key=lambda e: e.get_position().depth)
        element.set_parent(self)
        return self

    def __mod__(self, other):
        self.add_element(other)
        return self

    def render(self) -> List[str]:
        current_render = []

        for element in self.__elements_depth_ordered:
            current_render = self.__add_element_to_render(element, current_render)

        current_render = self.__pad_lines(current_render)
        current_render = self.__add_borders(current_render)
        return current_render

    def __pad_lines(self, current_render: List[str]) -> List[str]:
        # Pad all lines to get the same size
        expected_size = self.__get_render_sizes(current_render)
        width = expected_size[0]
        height = expected_size[1]

        min_width = self.__configs['min-width']
        if min_width is not None:
            width = max(width, min_width)

        max_width = self.__configs['max-width']
        if max_width is not None:
            if self.__configs['border']:
                max_width -= 2
            width = min(width, max_width)

        min_height = self.__configs['min-height']
        if min_height is not None:
            height = max(height, min_height)

        max_height = self.__configs['max-height']
        if max_height is not None:
            if self.__configs['border']:
                max_height -= 2
            height = min(height, max_height)

        vertical_align = self.__configs['vertical-align']
        horizontal_align = self.__configs['horizontal-align']

        vertical_base = max(0 if vertical_align != 'center' else (height - len(current_render)) // 2, 0)

        padded_render = list()
        for line_idx in range(0, height):
            if line_idx < vertical_base or line_idx - vertical_base >= len(current_render):
                padded_render.append(' ' * width)
                continue

            if line_idx >= len(padded_render):
                padded_render.append('')

            current_render_line_raw = current_render[line_idx - vertical_base]
            current_render_line = Utils.remove_escape_seqs(current_render_line_raw)
            horizontal_base = max(0 if horizontal_align != 'center' else (width - len(current_render_line)) // 2, 0)

            padded_render_line = ' ' * horizontal_base + current_render_line_raw
            padded_render_line += ' ' * (width - (horizontal_base + len(current_render_line)))
            padded_render[line_idx] = padded_render_line

        current_render = padded_render
        return current_render

    def __get_render_sizes(self, current_render: List[str]) -> Tuple[int, int]:
        if len(current_render) == 0:
            return (0, 0)

        max_row_len = max(map(lambda line: len(Utils.remove_escape_seqs(line)), current_render))
        return (max_row_len, len(current_render))

    def __add_borders(self, current_render: List[str]) -> List[str]:
        if self.__configs['border']:
            current_size = self.__get_render_sizes(current_render)
            up_down_borders = '―' * current_size[0]
            current_render.insert(0, up_down_borders)
            current_render.append(up_down_borders)

            for line_idx in range(0, len(current_render)):
                render_line = current_render[line_idx]

                if line_idx == 0:
                    render_line = '┌' + render_line + '┐'
                elif line_idx == len(current_render) - 1:
                    render_line = '└' + render_line + '┘'
                else:
                    render_line = '│' + render_line + '│'

                current_render[line_idx] = render_line

        return current_render

    def __add_element_to_render(self, element: Renderable, current_render: List[str]) -> List[str]:
        if self.__configs['flex'] == 'column':
            element.set_position(Position(len(current_render), element.get_position().col))
        if self.__configs['flex'] == 'row':
            max_col = max(map(lambda line: len(Utils.remove_escape_seqs(line)), current_render)) if len(current_render) > 0 else 0
            element.set_position(Position(element.get_position().row, max_col))

        element_rendered = element.render()
        element_position = element.get_position()

        # Bottom pad the current render with empty lines until
        # there are enough lines to put the element lines
        current_render += [''] * ((len(element_rendered) + element_position.row) - len(current_render))

        for line_idx in range(0, len(element_rendered)):
            element_rendered_line_raw = element_rendered[line_idx]
            element_rendered_line = Utils.remove_escape_seqs(element_rendered_line_raw)
            current_render_line = current_render[line_idx + element_position.row]

            # Right pad the current render with spaces until there is enough
            # space to put the element rendered line
            current_render_line += ' ' * ((len(element_rendered_line) + element_position.col) - len(current_render_line))

            current_render_line = Utils.replace_keeping_escape_seqs(current_render_line, element_rendered_line_raw, element_position.col)
            current_render[line_idx + element_position.row] = current_render_line

        return current_render

    def __get_next_element(self, direction: str):
        interactibles = list(filter(lambda e: Interactible.is_interactible(e), self.__elements_depth_ordered))

        if len(interactibles) == 0:
            return None

        if len(interactibles) == 1 and self.__interacting_element is not None:
            return None

        if self.__interacting_element is None:
            return interactibles[0]

        filtered: List[Interactible] = []
        current_row = self.__interacting_element.get_position().row
        current_col = self.__interacting_element.get_position().col

        if direction in ['up', 'down']:
            if direction == 'up':
                filtered = list(filter(lambda e: e.get_position().row < current_row, interactibles))
            else:
                filtered = list(filter(lambda e: e.get_position().row > current_row, interactibles))

            if len(filtered) == 0:
                return None

            filtered.sort(key=lambda e: abs(current_row - e.get_position().row))
            return filtered[0]
        elif direction in ['left', 'right']:
            if direction == 'left':
                filtered = list(filter(lambda e: e.get_position().col < current_col, interactibles))
            else:
                filtered = list(filter(lambda e: e.get_position().col > current_col, interactibles))

            if len(filtered) == 0:
                return None

            filtered.sort(key=lambda e: abs(current_col - e.get_position().col))
            return filtered[0]

        return None

    def handle_key(self, key: Key, control: InteractionControl):
        next_element: Interactible = None

        if key == Key.up:
            next_element = self.__get_next_element('up')
        elif key == Key.down:
            next_element = self.__get_next_element('down')
        elif key == Key.left:
            next_element = self.__get_next_element('left')
        elif key == Key.right:
            next_element = self.__get_next_element('right')
        else:
            return

        if next_element is None:
            if self._parent is not None:
                self.__interacting_element = None
                control.pass_control(self._parent)
            elif self.__interacting_element is not None:
                self.__interacting_element.set_parent(self)
                control.pass_control(self.__interacting_element)

            return

        control.key_to_handle = None
        self.__interacting_element = next_element
        self.__interacting_element.set_parent(self)
        control.pass_control(next_element)

    def take_control(self, control: InteractionControl):
        interactibles = list(filter(lambda e: Interactible.is_interactible(e), self.__elements_depth_ordered))

        if len(interactibles) == 0:
            return

        if self.__interacting_element is None:
            self.__interacting_element = interactibles[0]
            self.__interacting_element.set_parent(self)
            control.key_to_handle = None
            control.pass_control(self.__interacting_element)
            return

        if control.key_to_handle is not None:
            key_to_handle = control.key_to_handle
            self.handle_key(key_to_handle, control)


from datetime import datetime
from typing import Set, Optional, Dict

import requests
from django.core.cache import cache
from django.db.models.query import QuerySet
from django.utils import timezone
from requests import HTTPError, Request

from backend.settings import CURRENCY_ARCHIVE_URL, MAIN_CURRENCY, CURRENCY_URL
from currency_api.exceptions import DateNotFoundException
from currency_api.models import ExchangeRate

class CurrencyService:

    def __init__(self, currencies: Set[str], date: Optional[str]):
        self.currencies = currencies
        if date is None:
            self.date = str(timezone.now().date())
        else:
            self.date = date

        self.result: Dict = {
            'date': self.date,
            "currencies": []
        }

    def get_rates(self) -> Dict:
        """Public method to get currency rates."""
        is_archive: bool = self._is_date_archive()
        if is_archive:
            return self._get_data_from_archive()
        is_future: bool = self._is_date_future()
        if is_future:
            return self.result

        return self._get_data_from_db()

    def _get_data_from_db(self) -> Dict:
        """Private method to fetch rates from db."""
        cached_rates = self._get_data_from_cache()
        if cached_rates:
            return cached_rates

        rates: QuerySet = ExchangeRate.objects.filter(
            currency_from__name=MAIN_CURRENCY, currency_to__name__in=self.currencies, date=self.date
        ).values('currency_to__name', 'rate')

        if not rates:
            self._get_data_from_source()
        for rate_item in rates:
            self.result['currencies'].append(
                {'name': rate_item['currency_to__name'], 'rate': rate_item['rate']}
            )
        self._put_data_to_cache(self.result)
        return self.result

    def _get_data_from_archive(self) -> Dict:
        """Private method call cbrf endpoint to get archived rates data."""
        year, month, day = self.date.split('-')
        url = CURRENCY_ARCHIVE_URL.format(YEAR=year, MONTH=month, DAY=day)
        request = requests.get(url)
        try:
            request.raise_for_status()
            raw_data = request.json()
        except HTTPError as error:
            if request.status_code == 404:
                raise DateNotFoundException()
            raise error
        return self._prepare_raw_data(raw_data)

    def _is_date_future(self) -> bool:
        """Private method check if date is future date."""
        try:
            input_date: datetime.date = datetime.strptime(self.date, '%Y-%m-%d').date()
            current_date: datetime.date = datetime.now().date()

            if input_date > current_date:
                return True
            return False
        except ValueError:
            return False

    def _is_date_archive(self) -> bool:
        """Private method check if input_date less than current_date."""
        try:
            input_date: datetime.date = datetime.strptime(self.date, '%Y-%m-%d').date()
            current_date: datetime.date = datetime.now().date()

            if input_date < current_date:
                return True
            return False
        except ValueError:
            return False

    def _prepare_raw_data(self, raw_data: Dict) -> Dict:
        """Private method prepare raw result from cbrf archive."""
        for currency in self.currencies:
            if currency in raw_data['Valute']:
                self.result['currencies'].append(
                    {'name': currency, 'rate': raw_data['Valute'][currency]['Value']}
                )
        return self.result

    def _get_data_from_source(self) -> Dict:
        """Private method fetch data from cbrf and execute delayed task to save data to db."""
        url: str = CURRENCY_URL
        request: Request = requests.get(url)

        request.raise_for_status()
        raw_data: Dict = request.json()

        return self._prepare_raw_data(raw_data)

    def _get_data_from_cache(self) -> Optional[Dict]:
        """Private method fetch data from Redis cache."""
        cached_data = cache.get(self.date)
        if cached_data:
            return cached_data
        return None

    def _put_data_to_cache(self, data: Dict) -> None:
        """Private method put data to Redis cache."""
        cache.set(self.date, data)


import zlib
import asyncio
import os

class FileWatcher:
    """
    Watch for file change and call the callback when something happens

    :param paths: A path or a list of file to watch
    :param delay: Delay between file check (seconds)
    :param strategy: File change strategy (mtime: modification time, hash: hash compute)
    """

    def __init__(self, paths, callback, delay=1, strategy='mtime'):
        self._paths = []
        if not isinstance(paths, list):
            paths = [paths]
        for path in paths:
            if not isinstance(path, str):
                path = str(path)
            self._paths.append(path)

        self._callback = callback
        self._delay = delay
        self._closed = False
        self._strategy = strategy

        if self._strategy == 'mtime':
            # Store modification time
            self._mtime = {}
            for path in self._paths:
                try:
                    self._mtime[path] = os.stat(path).st_mtime_ns
                except OSError:
                    self._mtime[path] = None
        else:
            # Store hash
            self._hashed = {}
            for path in self._paths:
                try:
                    # Alder32 is a fast but insecure hash algorithm
                    self._hashed[path] = zlib.adler32(open(path, 'rb').read())
                except OSError:
                    self._hashed[path] = None
        asyncio.get_event_loop().call_later(self._delay, self._check_config_file_change)

    def __del__(self):
        self._closed = True

    def close(self):
        self._closed = True

    def _check_config_file_change(self):
        if self._closed:
            return
        changed = False

        for path in self._paths:
            if self._strategy == 'mtime':
                try:
                    mtime = os.stat(path).st_mtime_ns
                    if mtime != self._mtime[path]:
                        changed = True
                        self._mtime[path] = mtime
                except OSError:
                    self._mtime[path] = None
            else:
                try:
                    hashc = zlib.adler32(open(path, 'rb').read())
                    if hashc != self._hashed[path]:
                        changed = True
                        self._hashed[path] = hashc
                except OSError:
                    self._hashed[path] = None
            if changed:
                self._callback(path)
        asyncio.get_event_loop().call_later(self._delay, self._check_config_file_change)

    @property
    def callback(self):
        return self._callback

    @callback.setter
    def callback(self, val):
        self._callback = val


import pandas as pd
import lightgbm as lgb
import matplotlib.pyplot as plt
import shap
import numpy as np

from sklearn import metrics
from sklearn.model_selection import StratifiedKFold, KFold

from utils.general_functions import timeit

def evaluation_metric(true, pred):
    """
    We use absolute percentage error to evaluate model performance.
    """
    return abs(true - pred)*100/true

def evaluation_auc(true, pred):
    """Calculate ROC AUC for binary target

    Args:
        true ([int]): [binary target to predict ]
        pred ([float]): [predicted probability of the model]

    Returns:
        [float]: [AUC]
    """
    fpr, tpr, thresholds = metrics.roc_curve(true, pred, pos_label=1)
    auc = metrics.auc(fpr, tpr)
    return auc

class TrainModel(object):
    """
    Wrapper around LightGBM model responsible for
        * fitting the LightGBM model on in put data.
        * evaluating model performance on a tests dataset.
        * Training on the overall dataset if no test set is given

    Arguments:
        object {[type]} --
    """

    def __init__(self, configs, data):
        self.data = data
        self.target_name = configs["TARGET"].upper()
        self.feature_name = [x.upper() for x in configs["FEATURES"]]
        self.params = configs
        self.weight = None

        if "WEIGHT" in configs.keys():
            self.weight = configs["WEIGHT"]
        if "weight" in configs.keys():
            self.weight = configs["weight"]

    def evaluate_model(self, model, test_data, final=False):
        """
        Plot feature importances and log error metrics.
        """

        if final:
            # evaluate results with plot importance
            test_data["PREDICTION_" + self.target_name].hist(alpha = 0.7)
            test_data[self.target_name].hist(alpha = 0.7)
            plt.show()

            valid_x = test_data[self.feature_name]
            shap_values = shap.TreeExplainer(model).shap_values(valid_x)
            shap.summary_plot(shap_values, valid_x)
            plt.show()

        # evaluate results based on evaluation metrics
        if self.params["parameters"]["objective"] == "regression":
            print("_" * 60)
            for error in ["PREDICTION_" + self.target_name]:
                if error in test_data.columns:
                    test_data["ABS_BIAS_" + error] = abs(test_data[self.target_name] - test_data[error])
                    test_data["BIAS_" + error] = test_data[self.target_name] - test_data[error]
                    test_data["ERROR_" + error] = evaluation_metric(test_data[self.target_name],
                                                                    test_data[error])

                    print("MAPE {2} : {0:.2f} +/- {1:.2f} % || ABS_BIAS {3:.3f} || BIAS {4:.3f}".format(
                                np.mean(test_data["ERROR_" + error].loc[test_data[self.target_name]>0]),
                                np.std(test_data["ERROR_" + error].loc[test_data[self.target_name]>0]),
                                error,
                                test_data["ABS_BIAS_" + error].mean(),
                                test_data["BIAS_" + error].mean()))
            print("_" * 60)
        else:
            print("_" * 60)
            for error in ["PREDICTION_" + self.target_name]:
                if error in test_data.columns:
                    print("AUC {1} : {0:.4f} %".format(
                          evaluation_auc(test_data[self.target_name], test_data[error]),
                          error
                      ))
            print("_" * 60)

    def train_on_set(self,
                    train_data,
                    test_data=None,
                    init_score=None):
            """
            Creates LightGBM model instance and trains on a train dataset. If a tests set is provided,
            we validate on this set and use early stopping to avoid over-fitting.
            """

            if isinstance(test_data, pd.DataFrame):
                if "early_stopping_round" not in self.params["parameters"].keys():
                    self.params["parameters"]["early_stopping_round"] = 40

                if self.weight:
                    train_weight = train_data[self.weight]
                    test_weight  = test_data[self.weight]
                else:
                    train_weight = None
                    test_weight = None

                if init_score:
                    train_init_bias = train_data[init_score]
                    test_init_bias = test_data[init_score]
                else:
                    train_init_bias = None
                    test_init_bias = None

                # model training and prediction of val
                # have an idea of the error rate and use early stopping round
                train_data = lgb.Dataset(
                    train_data[self.feature_name],
                    label=train_data[self.target_name],
                    weight=train_weight,
                    init_score=train_init_bias,
                    categorical_feature=self.params["categorical_features"]
                )

                val_data = lgb.Dataset(
                    test_data[self.feature_name],
                    label=test_data[self.target_name],
                    weight=test_weight,
                    init_score=test_init_bias,
                    categorical_feature=self.params["categorical_features"]
                )

                model = lgb.train(self.params["parameters"],
                                   train_set=train_data,
                                    valid_sets=[train_data, val_data],
                                    valid_names=["data_train", "data_valid"],
                                    verbose_eval=1000)

            else:
                if "early_stopping_round" in self.params["parameters"]:
                    self.params["parameters"].pop("early_stopping_round", None)

                if self.weight:
                    sample_weight = train_data[self.weight]
                else:
                    sample_weight = None

                if init_score:
                    init_bias = train_data[init_score]
                else:
                    init_bias = None

                if self.params["parameters"]["objective"] == "binary":
                    model = lgb.LGBMClassifier(**self.params["parameters"])
                else:
                    model = lgb.LGBMRegressor(**self.params["parameters"])

                model.fit(train_data[self.feature_name],
                          train_data[self.target_name],
                          sample_weight= sample_weight,
                          init_score= init_bias,
                          verbose=-1,
                          categorical_feature = self.params["categorical_features"])

            return model

    @timeit
    def test_on_set(self, model, test_data, init_score=None):
        """
        Takes model and tests dataset as input, computes predictions for the tests dataset, and evaluates metric
        on the predictions. Returns tests dataset with added columns for pediction and metrics.
        """
        test_data["PREDICTION_" + self.target_name] = model.predict(test_data[self.feature_name],
                                                            categorical_feature=self.params["categorical_features"])
        if init_score:
            test_data["PREDICTION_" + self.target_name] = test_data["PREDICTION_" + self.target_name] + test_data[init_score]

        test_data["ERROR_MODEL"] = evaluation_metric(test_data[self.target_name],
                                                    test_data["PREDICTION_" + self.target_name])

        self.evaluate_model(model, test_data)

        return test_data

    def modelling_cross_validation(self, data=None, init_score=None):
        """
        Fits model using k-fold cross-validation on a train set.
        """

        if not isinstance(data, pd.DataFrame):
            data = self.data.reset_index(drop=True)
        else:
            data = data.reset_index(drop=True)

        # if self.params["parameters"]["objective"] == "binary":
        #     kf = StratifiedKFold(
        #             n_splits=self.params["n_splits"],
        #             random_state=self.params["seed"],
        #             shuffle=True
        #         )
        # else:
        kf = KFold(
                n_splits=self.params["n_splits"],
                random_state=self.params["seed"],
                shuffle=True
            )

        total_test = pd.DataFrame()
        for train_index, val_index in kf.split(data.index, data.index):

            train_data = data.loc[train_index]
            test_data = data.loc[val_index]

            model = self.train_on_set(train_data, test_data, init_score)
            x_val = self.test_on_set(model, test_data, init_score)

            # concatenate all test_errors
            total_test = pd.concat([total_test, x_val], axis=0).reset_index(drop=True)

        self.evaluate_model(model, total_test, final=False)
        print("TRAIN full model")
        model = self.train_on_set(data, init_score=init_score)
        self.total_test= total_test

        return total_test, model


from operator import mod
import random
import pandas as pd
import plotly.figure_factory as ff
import statistics as st
import plotly.graph_objects as go

df = pd.read_csv("data.csv")
readscore = df["reading score"].to_list()

mean = st.mean(readscore)
median = st.median(readscore)
mode = st.mode(readscore)
stdev = st.stdev(readscore)

first_start, first_end = mean - stdev, mean + stdev
second_start, second_end = mean - (2*stdev), mean + (2*stdev)
third_start, third_end = mean - (3*stdev), mean + (3*stdev)

readscore_within_first_stdev = [result for result in readscore if result > first_start and result < first_end]
readscore_within_second_stdev = [result for result in readscore if result > second_start and result < second_end]
readscore_within_third_stdev = [result for result in readscore if result > third_start and result < third_end]

print("{}% of data lies within first stdev ".format(len(readscore_within_first_stdev)*100/len(readscore)))
print("{}% of data lies within second stdev ".format(len(readscore_within_second_stdev)*100/len(readscore)))
print("{}% of data lies within third stdev ".format(len(readscore_within_third_stdev)*100/len(readscore)))


import os.path
from datetime import datetime

import pytz
from PIL import Image

import models
import skills
from bosses import bosses
from card import Card

def get_stats(player_with_gains):
    stats = dict()
    for skill in skills.skills_tuple:
        player_xps = []
        for pwg in player_with_gains:
            xp = pwg[1].data[skills.key_for(skill)]['experience']['gained']
            player_xps.append((pwg[0].name, xp))
        player_xps.sort(key=lambda x: x[1], reverse=True)
        tied = False
        tied_rank = 1
        stats[skill] = dict()
        for i, boss_kc in enumerate(player_xps):
            if i + 1 >= len(player_xps):
                stats[skill][boss_kc[0]] = tied_rank if tied else i
                break
            next_player_xp = player_xps[i + 1]
            if boss_kc[1] != next_player_xp[1]:
                tied_rank = i + 1
                tied = False
            else:
                tied = True
            stats[skill][boss_kc[0]] = tied_rank if tied else i
    for boss in bosses:
        boss_kcs = []
        for pwg in player_with_gains:
            xp = pwg[1].data[boss.first_key][boss.second_key]['gained']
            boss_kcs.append((pwg[0].name, xp))
        boss_kcs.sort(key=lambda x: x[1], reverse=True)
        tied = False
        tied_rank = 1
        stats[boss.first_key] = dict()
        for i, boss_kc in enumerate(boss_kcs):
            if i + 1 >= len(boss_kcs):
                stats[boss.first_key][boss_kc[0]] = tied_rank if tied else i
                break
            next_player_xp = boss_kcs[i + 1]
            if boss_kc[1] != next_player_xp[1]:
                tied_rank = i + 1
                tied = False
            else:
                tied = True
            stats[boss.first_key][boss_kc[0]] = tied_rank if tied else i
    return stats

def get_xp_percentages(player_with_gains):
    xp_percentages = dict()
    for skill in skills.skills_tuple:
        total_xp = 0
        xp_percentages[skill] = dict()
        for pwg in player_with_gains:
            xp = pwg[1].data[skills.key_for(skill)]['experience']['gained']
            total_xp += xp
        for pwg in player_with_gains:
            xp = pwg[1].data[skills.key_for(skill)]['experience']['gained']
            xp_percentages[skill][pwg[0].name] = 100 * xp / total_xp
    for boss in bosses:
        total_kc = 0
        xp_percentages[boss.first_key] = dict()
        for pwg in player_with_gains:
            kc = pwg[1].data[boss.first_key][boss.second_key]['gained']
            total_kc += kc
        for pwg in player_with_gains:
            kc = pwg[1].data[boss.first_key][boss.second_key]['gained']
            xp_percentages[boss.first_key][pwg[0].name] = 100 * kc / total_kc if total_kc else 0
    return xp_percentages

def format_time(timezone, year, month, day, hour):
    dt = timezone.localize(datetime(year, month, day, hour))
    dt = pytz.utc.normalize(dt)
    return dt.strftime('%Y-%m-%dT%H:%M:%S.%f')[:-3] + 'Z'

def main():
    timezone = pytz.timezone('US/Pacific')
    start = format_time(timezone, 2022, month=8, day=4, hour=10)
    end = format_time(timezone, 2022, month=8, day=22, hour=10)

    guthix = Image.open(os.path.join("images", "Guthix_symbol.png"))
    saradomin = Image.open(os.path.join("images", "Saradomin_symbol.png"))

    id = 14542
    comp = models.Competition(id)
    comp.load_file()
    if not comp.players:
        comp = models.Competition(id)
        comp.update()
        comp.save_file()
    player_with_gains = []
    player_with_gains_guthix = []
    player_with_gains_sara = []
    for player in comp.players.values():
        player_gains = models.PlayerGains(player.name, start, end)
        player_gains.load_file()
        if not player_gains.data:
            player_gains.update()
            player_gains.save_file()
        player_with_gains.append((player, player_gains))
        if player.team == "Guthix":
            player_with_gains_guthix.append((player, player_gains))
        else:
            player_with_gains_sara.append((player, player_gains))

    stats_all = get_stats(player_with_gains)
    stats_guthix = get_stats(player_with_gains_guthix)
    stats_sara = get_stats(player_with_gains_sara)

    xp_percentages_all = get_xp_percentages(player_with_gains)
    xp_percentages_guthix = get_xp_percentages(player_with_gains_guthix)
    xp_percentages_sara = get_xp_percentages(player_with_gains_sara)

    for pwg in player_with_gains:
        player = pwg[0]
        player_gains = pwg[1]

        if player.team == "Guthix":
            icon = guthix
            icon_pos = (1200, 415)
            icon_large = icon.resize((icon.width * 2, icon.height * 2))
            stats_team = stats_guthix
            xp_percentages_team = xp_percentages_guthix
        else:
            icon = saradomin
            icon_pos = (1156, 420)
            icon_large = icon.resize((int(icon.width * 1.8), int(icon.height * 1.8)))
            stats_team = stats_sara
            xp_percentages_team = xp_percentages_sara

        card = Card(player.name, player_gains, icon_large, icon_pos, stats_all, stats_team, xp_percentages_all, xp_percentages_team)
        card.output()

if __name__ == "__main__":
    main()


import warnings
warnings.filterwarnings("ignore")

import os
import time
import pandas as pd
import numpy as np

from utils import readChunk, toCSV

file = "all_month.csv"
usecols = ['USERID', 'CONTENT_TYPE', 'VIDEO_CATEGORY_TITLE', 'SESSION_STARTDT_MONTH', 'SESSION_STARTDT_DAY', 'STARTHOUR']
df = readChunk(file, usecols = usecols)

df = df.loc[df.SESSION_STARTDT_MONTH != '11']
convert = {'12':'0'}
df.replace({'SESSION_STARTDT_MONTH':convert}, inplace = True)
toint = ['SESSION_STARTDT_MONTH', 'SESSION_STARTDT_DAY', 'STARTHOUR']
for i in toint:
	df[i] = df[i].astype(int)
df['ORDER'] = (df.SESSION_STARTDT_MONTH*100)+(df.SESSION_STARTDT_DAY*10)+df.STARTHOUR

df.sort_values('ORDER', inplace = True)

print(len(df))
df.drop_duplicates(subset = ['USERID'], keep = 'first', inplace = True)
print(len(df))

labels = pd.read_csv('clustering_6.csv')
labels.columns = labels.columns.str.upper()

df = df.merge(labels, how = 'left', on = 'USERID')

for i in df.LABEL:
	print(df.label)
	temp = df.loc[df.label == i]
	new_df = pd.DataFrame(index = list(range(1, 11)), columns = ['COUNT'])
	for j in df.CONTENT_TYPE:
		temp2 = temp.loc[temp.CONTENT_TYPE == j]
		new_df.loc[int(j)]['COUNT'] = len(temp2)

	print(new_df)
	new_df.index.name = 'CONTENT_TYPE'
	toCSV(new_df, 'results/firstwatched/'+str(i)+'.csv')