import PySimpleGUI as sg
font = ("bitstream charter", 14, "")
import numpy as np
import matplotlib.pyplot as plt
hist_kwargs = {"edgecolor":"black"}

from Functions import *

def Open_File_Calibration(data_file):
    with open(data_file) as f:
        tot_ev = len(list(f))

    # loop over events and perform analysis
    Ev_list = []
    selected_ev = 0
    with open(data_file) as f:
        for line in f:
            # read event
            ev, evNum, hits = Read_Data(line)
            # select event
            sel, chambers, n_layers = Select_Events_Calibration(ev, hits)

            if sel: selected_ev += 1

            # save most important information, in order to plot or reperform the analysis
            # without reading the whole file again
            Ev_list.append(
                {
                    "Number"    : evNum,
                    "Dataframe" : ev,
                    "Hits"      : hits,
                    "Accepted"  : sel,
                    "Chambers"  : chambers,
                    "Layers"    : n_layers
                }
            )
            # GUI progress bar
            sg.OneLineProgressMeter('File Loader', len(Ev_list), tot_ev, 'Reading CALIBRATION file', orientation="h", size=(30,30))

    print("{:35} = {:d}"    .format("Total number of events in the Run", tot_ev))
    print("{:35} = {:d}"    .format("Number of accepted events"        , selected_ev))
    print("{:35} = {:.4f} %".format("Fraction of accepted events"      , selected_ev/tot_ev*100))
    # create a pop up for the GUI with summary information
    sg.Popup("{:} = {:d}"    .format("Total number of events in the Run", tot_ev),
             "{:} = {:d}"    .format("Number of accepted events"        , selected_ev),
             "{:} = {:.4f} %".format("Fraction of accepted events"      , selected_ev/tot_ev*100),
            title="Summary", font=font)
    return Ev_list

def Calibration(Ev_list):
    tot_ev = len(Ev_list)
    selected_ev = 0
    # number of hits per event
    hits_number = []
    # residuals from good events fit
    Ev_residuals = []
    # x position of the best combinations of points per chamber
    X_position = {"up" : [], "down" : []}
    # local fit differences between chambers
    lf_diff = {"slope" : [], "intercept" : []}

    print("Performing fit and analysis")
    for idx, event in enumerate(Ev_list):
        hits_number.append(event["Hits"])
        if event["Accepted"]:
            selected_ev += 1
            ev = event["Dataframe"]
            chambers = event["Chambers"]
            n_layers = event["Layers"]

            #Local linear fit
            lf_results = Local_Fit(ev, chambers, n_layers, exclusion_layer=True)
            #Global linear fit
            gf_results = Global_Fit_Calibration(ev, chambers, lf_results)
            Ev_residuals.append(gf_results["residuals"])

            # x coordinates for the points in the optimal combination after local fit per chamber
            X_position["up"]   += [x[1] for x in lf_results[0]["optimal_comb"]]
            X_position["down"] += [x[1] for x in lf_results[1]["optimal_comb"]]
            # local fit differences
            lf_diff["slope"]    .append(lf_results[0]["slope"]    -lf_results[1]["slope"])
            lf_diff["intercept"].append(lf_results[0]["intercept"]-lf_results[1]["intercept"])
        # GUI progress bar
        sg.OneLineProgressMeter('File Analyzer', idx+1, tot_ev, 'Analyzing CALIBRATION file', orientation="h", size=(30,30))

    fig, ax = plt.subplots(figsize=(8,5))
    ax.hist(hits_number, bins=14, range=(0,70), **hist_kwargs)
    ax.set_title("Hit distribution", fontsize=14)
    ax.set_xlabel("number of hits")
    fig.show()

    fig, (ax1, ax2) = plt.subplots(figsize=(15,5), ncols=2)
    fig.suptitle("X position in local chambers coordinates", fontsize=16)
    ax1.set_title("Up", fontsize=14)
    ax1.hist(X_position["up"]    , bins=30, **hist_kwargs)
    ax1.set_xlabel("x [mm]")
    #
    ax2.set_title("Down", fontsize=14)
    ax2.hist(X_position["down"], bins=30, **hist_kwargs)
    ax2.set_xlabel("x [mm]")
    fig.show()

    fig, (ax1, ax2) = plt.subplots(figsize=(15,5), ncols=2)
    fig.suptitle("Differences between local fits results", fontsize=16)
    ax1.set_title("Slope")
    Gaussian_Fit_Hist(ax1, lf_diff["slope"],     nbins=30, hist_range=(-0.5,0.5), **hist_kwargs)
    ax1.set_xlabel("Slope")
    #
    ax2.set_title("Intercept")
    Gaussian_Fit_Hist(ax2, lf_diff["intercept"], nbins=30, hist_range=(-100,100), **hist_kwargs)
    ax2.set_xlabel("[mm]")
    fig.show()

    Hit_6_layers = 0
    Hit_7_layers = 0
    Hit_8_layers = 0
    Tot_res      = []

    for res in Ev_residuals:
        if len(res) == 0:
            Hit_6_layers += 1
        elif len(res) == 1:
            Hit_7_layers += 1
        elif len(res) == 2:
            Hit_8_layers += 1
        else: continue
        Tot_res += res.tolist()

    fig, (ax1, ax2) = plt.subplots(figsize=(15,5), ncols=2)
    ax1.set_title("Residuals of the points in the excluded layers", fontsize=14)
    Gaussian_Fit_Hist(ax1, Tot_res, nbins= 30, hist_range=(-20,20), **hist_kwargs)
    ax1.set_xlabel("[mm]")
    #
    labels = ['Hit in 8 layers', 'Hit in 7 layers', 'Hit in 6 layers']
    sizes  = [Hit_8_layers/len(Ev_residuals)*100,
              Hit_7_layers/len(Ev_residuals)*100,
              Hit_6_layers/len(Ev_residuals)*100]
    patches, _, _ = ax2.pie(sizes, labels=labels, explode = [.05,.05,.05], autopct='%1.1f%%', shadow=True,
                               wedgeprops={"edgecolor":'black', "linewidth":1.25})
    ax2.legend(patches, labels, loc='upper right', fontsize=14)
    ax2.axis('equal')
    ax2.set_title("Percentage of excluded layers in the fit", fontsize=14)
    fig.show()

    return Ev_residuals, X_position, lf_diff

def Open_File_Physics(data_file):
    with open(data_file) as f:
        tot_ev = len(list(f))

    # loop over events and perform analysis
    Ev_list = []
    selected_ev = 0
    with open(data_file) as f:
        for line in f:
            # read event
            ev, evNum, hits = Read_Data(line)
            # filter by hit position
            ev_left, ev_right, hits_left, hits_right = Points_Filter(ev)
            # select event
            sel_left,  chambers_left,  n_layers_left  = Select_Events_Calibration(ev_left,  hits_left)
            sel_right, chambers_right, n_layers_right = Select_Events_Calibration(ev_right, hits_right)

            if sel_left and sel_right: selected_ev += 1

            # save most important information, in order to plot or reperform the analysis
            # without reading the whole file again
            Ev_list.append(
                {
                    "Number"    : evNum,
                    "Dataframe" : ev,
                    "Hits"      : hits,
                    "Accepted"  : sel_left and sel_right,
                    "Chambers"  : chambers_left+chambers_right,
                    "Layers"    : [n_layers_left, n_layers_right]
                }
            )
            sg.OneLineProgressMeter('File Loader', len(Ev_list), tot_ev, 'key', 'Loading ', orientation="h", size=(30,30))

    print("{:35} = {:d}"     .format("Total number of events in the Run", tot_ev))
    print("{:35} = {:d}"     .format("Number of accepted events"        , selected_ev))
    print("{:35} = {:.4f} %" .format("Fraction of accepted events"      , selected_ev/tot_ev*100))
    # create a pop up for the GUI with summary information
    sg.Popup("{:} = {:d}"    .format("Total number of events in the Run", tot_ev),
             "{:} = {:d}"    .format("Number of accepted events"        , selected_ev),
             "{:} = {:.4f} %".format("Fraction of accepted events"      , selected_ev/tot_ev*100),
            title="Summary", font=font)
    return Ev_list

def Physics(Ev_list):
    tot_ev = len(Ev_list)
    # number of hits selected events
    Ev_hits = {"left":[], "right":[]}
    # angular coefficient of fit
    Ev_slope = {"left":[], "right":[]}
    # residuals from good events fit
    Ev_residuals = {"left":[], "right":[]}

    print("Performing fit and analysis")
    for idx,event in enumerate(Ev_list):
        if event["Accepted"]:
            ev = event["Dataframe"]
            layers = event["Layers"]

            # filter by hit position
            ev_left, ev_right, hits_left, hits_right = Points_Filter(ev)
            # save hit numbers
            Ev_hits["left"] .append(hits_left )
            Ev_hits["right"].append(hits_right)

            # save slope values
            gf_results_left  = Global_Fit_Physics(ev_left , layers[0])
            gf_results_right = Global_Fit_Physics(ev_right, layers[1])
            Ev_slope["left"] .append(gf_results_left ["slope"])
            Ev_slope["right"].append(gf_results_right["slope"])

            # save residuals
            Ev_residuals["left"]  += gf_results_left ["residuals"]
            Ev_residuals["right"] += gf_results_right["residuals"]
        # GUI progress bar
        sg.OneLineProgressMeter('File Analyzer', idx+1, tot_ev, 'Analyzing PHYSICS file', orientation="h", size=(30,30))

    fig, (ax1, ax2) = plt.subplots(figsize=(15,5), ncols=2)
    fig.suptitle("Hit distribution", fontsize=16)
    ax1.hist(Ev_hits["left"] , **hist_kwargs)
    ax1.set_title("Left Side", fontsize=14)
    ax1.set_xlabel("Number of hits")
    ax2.hist(Ev_hits["right"], **hist_kwargs)
    ax2.set_title("Right Side", fontsize=14)
    ax2.set_xlabel("Number of hits")
    fig.show()

    fig, (ax1, ax2) = plt.subplots(figsize=(15,5), ncols=2)
    fig.suptitle("Slope distribution", fontsize=16)
    Gaussian_Fit_Hist(ax1, Ev_slope["left"] , nbins=30, **hist_kwargs)
    ax1.set_title("Left Side", fontsize=14)
    ax1.set_xlabel("Slope")
    Gaussian_Fit_Hist(ax2, Ev_slope["right"], nbins=30, **hist_kwargs)
    ax2.set_title("Right Side", fontsize=14)
    ax2.set_xlabel("Slope")
    fig.show()

    fig, (ax1, ax2) = plt.subplots(figsize=(15,5), ncols=2)
    fig.suptitle("Residuals", fontsize=16)
    Gaussian_Fit_Hist(ax1, Ev_residuals["left"] , nbins=30, hist_range=(-25,25), **hist_kwargs)
    ax1.set_title("Left Side", fontsize=14)
    ax1.set_xlabel("Residuals [mm]")
    Gaussian_Fit_Hist(ax2, Ev_residuals["right"], nbins=30, hist_range=(-25,25), **hist_kwargs)
    ax2.set_title("Right Side", fontsize=14)
    ax2.set_xlabel("Residuals [mm]")
    fig.show()

    return Ev_hits, Ev_slope, Ev_residuals

def Make_Plot_GUI(ev, calibration=False):
    '''Plots of the background and the hits

    Parameters
    ----------
    ev : dict
        dictionary with information about the event; it contains: Number, Dataframe, Hits, Accepted, Chambers, Layers
    calibration : bool
        wether the event is taken during calibration or physics run
    '''

    plt.figure(figsize = (20,10))
    plots = Plot_Events_GUI(ev["Dataframe"], ev["Number"])
    if ev["Accepted"]: Plot_Fit(ev, plots, calibration=calibration)
    return plt.gcf()

def Plot_Events_GUI(dataframe, evNumber):
    '''Plot the positions of the hits

    Parameters
    ----------
    dataframe : pandas.dataframe
        Pandas Dataframe with all the information about the Event;
        each row is a single hit recorded in the event
    event_number : int
        number of the Event in the run

    Returns
    -------
    event_display : list(pyplot.axes)
        images of the hits of the events
    '''

    # get the EvNumber as argument, because, if the dataframe is empty,
    # I can't get it from data
    plots = Plot_Background_GUI()
    plots[0].set_title("Event: {:d}".format(evNumber), {'size':'18'})
    if dataframe.empty == False:
        xL = dataframe["XL_global"]
        xR = dataframe["XR_global"]
        z  = dataframe["Z_global"]
        for image in plots:
            image.plot(xL, z, "bo", markersize=3)
            image.plot(xR, z, "ro", markersize=3)

    return plots

def Plot_Background_GUI():
    '''Makes the plot for the background of the event display

    Returns
    -------
    axes : list(pyplot.axes)
        background images of the detector and the chambers
    '''

    # create Pandas DataFrame for the chambers positions
    chamber_position = pd.DataFrame({
    'chamber' : [i for i in range(4)],
    'x_vertices' : [(global_x_shifts[i], global_x_shifts[i] - 720, global_x_shifts[i] - 720, global_x_shifts[i])
                    for i in range(4)],
    'y_vertices' : [(global_z_shifts[i], global_z_shifts[i], global_z_shifts[i] + 52, global_z_shifts[i] + 52)
                    for i in range(4)],
    })
    x_lim = [[-1000, 1000], # global detector
                [    0, 1000], # chamber 0
                [    0, 1000], # chamber 1
                [-1000,    0], # chamber 2
                [-1000,    0]] # chamber 3
    y_lim = [[-100, 1000],  # global detector
                [800 ,  900],  # chamber 0
                [ -25,   75],  # chamber 1
                [ 800,  900],  # chamber 2
                [ -25,   75]]  # chamber 3
    title = ["DETECTOR", "Chamber 0", "Chamber 1", "Chamber 2", "Chamber 3"]
    # create pyplot 'Axes' objects
    # different dimensions and positions with respect to the notebook one
    gridsize = (7, 10)
    ax_global = plt.subplot2grid(gridsize, (0, 0), colspan=4, rowspan=7)
    ax_0 = plt.subplot2grid(gridsize, (0, 8), colspan=2, rowspan=3) # top-right
    ax_1 = plt.subplot2grid(gridsize, (4, 8), colspan=2, rowspan=3) # bottom-right
    ax_2 = plt.subplot2grid(gridsize, (0, 5), colspan=2, rowspan=3) # top-left
    ax_3 = plt.subplot2grid(gridsize, (4, 5), colspan=2, rowspan=3) # bottom-left

    axes = [ax_global, ax_0, ax_1, ax_2, ax_3]

    if exists("./wire_pos_glob.txt"):
        wires = np.loadtxt("./wire_pos_glob.txt")
    else: wires = None

    for index, ax in enumerate(axes):
        ax.set_xlim(x_lim[index])
        ax.set_ylim(y_lim[index])
        ax.set_xlabel("x [mm]")
        ax.set_ylabel("z [mm]")
        if index == 0: ax.set_title(title[index])
        else: ax.set_title(title[index], pad=-20)
        # plot the 4 chambers in each 'Axes'
        for j in range(4):
            chamber = chamber_position[chamber_position["chamber"] == j]
            ax.fill(chamber["x_vertices"].values[0], chamber["y_vertices"].values[0], color='gray', fill=False)
        if wires is not None:
            ax.plot(wires[:,0], wires[:,1], marker=".", markersize=.5, linestyle="", color="gray")
    return axes


# Definition for singly-linked list.
# class ListNode(object):
#     def __init__(self, x):
#         self.val = x
#         self.next = None

class Solution(object):
    def getIntersectionNode(self, headA, headB):
        """
        :type head1, head1: ListNode
        :rtype: ListNode
        """
        dummyA = headA
        dummyB = headB
        sizeA = 0
        sizeB = 0

        while(dummyA):
            sizeA += 1
            dummyA = dummyA.next

        while(dummyB):
            sizeB += 1
            dummyB = dummyB.next

        dummyA = headA
        dummyB = headB

        if(sizeA > sizeB):
            while(sizeA != sizeB):
                dummyA = dummyA.next
                sizeA -= 1

        if(sizeB > sizeA):
            while(sizeA != sizeB):
                dummyB = dummyB.next
                sizeB -= 1

        while(dummyA and dummyB):
            if(dummyA == dummyB):
                return dummyA
            else:
                dummyA = dummyA.next
                dummyB = dummyB.next

        return None


# 2.) Create a test case that will open amazon BestSellers page: https://www.amazon.com/gp/bestsellers/?ref_=nav_cs_bestsellers
# And verify there are 5 links:

from behave import given, when, then
from selenium.webdriver.common.by import By

BEST_LINKS = (By.ID, "zg_tabs")

@given("Open Amazon BestSeller Page")
def best_seller(context):
    context.driver.get("https://www.amazon.com/gp/bestsellers/?ref_=nav_cs_bestsellers")

@then("Verify if {expected_count} links are present")
def verify_links(context, expected_count):
    links = len(context.driver.find_elements(*BEST_LINKS))
    assert int(expected_count) != links, f'Amount of links incorrect. Expected {expected_count}, but got {links}'
    very_fied = context.driver.find_element(*BEST_LINKS)
    print(very_fied.text)


import sys

def AddLogs(msg, logsPath):
    #logsPath = "C:/Users/sjg/Desktop/SipH Sample Count/source/Logs.txt"
    Path = logsPath[:logsPath.rindex('/')+1]
    logsPath = Path + 'Logs.txt'
    with open(logsPath, "a") as myfile:
        myfile.write(msg+'\n')
    return 0

def DictSampleType(key):
    SampleType = {}
    #EDTA
    SampleType['EDTA'] = ['EDTA']
    SampleType['EDTA (CHK)'] = ['EDTA']
    SampleType['EDTA1'] = ['EDTA']
    SampleType['EDTA2'] = ['EDTA']

    #Clotted Blood
    SampleType['Clotted Blood'] = ['Clotted Blood']
    SampleType['Clotted Blood (CHK)'] = ['Clotted Blood']
    SampleType['Clotted 1hr'] = ['Clotted Blood']
    SampleType['Clotted 2 hr'] = ['Clotted Blood']
    SampleType['Clotted 3hr'] = ['Clotted Blood']
    #NaF
    SampleType['NaF '] = ['NaF']
    SampleType['NaF (CHK)'] = ['NaF']
    #Citrate
    SampleType['Citrate '] = ['Citrate']
    #Heparin
    SampleType['Heparin'] = ['Heparin']
    #Urine
    SampleType['Urine '] = ['Urine']
    SampleType['Urine 24hr'] = ['Urine']
    SampleType['Urine (CHK)'] = ['Urine']
    SampleType['Urine Amph (CHK)'] = ['Urine']
    #Stool
    SampleType['Stool'] = ['Stool']
    SampleType['Stool (CHK)'] = ['Stool']
    #Body fluid
    SampleType['Body fluid'] = ['Body fluid']
    #CSF
    SampleType['CSF'] = ['CSF']
    #Semen
    SampleType['Semen'] = ['Semen']
    #Hematocrit 2 tube
    SampleType['Hematocrit 2 tube'] = ['Hematocrit 2 tube']
    #Sputum
    SampleType['Sputum'] = ['Sputum']
    #Other
    SampleType['Other(OUT)'] = ['Other']
    SampleType['ติดต่อห้อง LAB'] = ['Other']
    SampleType['Other'] = ['Other']
    SampleType['ระบุSpmในClin Inform'] = ['Other']
    SampleType['Nasopharyngeal Swab'] = ['Other']
    SampleType['Rectal Swab'] = ['Other']
    SampleType['Vaginal Swab'] = ['Other']
    SampleType['ก้อนนิ่ว'] = ['Other']
    SampleType['เม็ดยา'] = ['Other']
    SampleType['Throat Swab'] = ['Other']
    SampleType['Capillary(OUT)'] = ['Other']
    SampleType['Vagina'] = ['Other']
    SampleType['Skin scapping'] = ['Other']
    SampleType['Discharge'] = ['Other']
    SampleType['Capillary'] = ['Other']
    try:
        returnValue = SampleType[key]
    except:
        returnValue = ['Not Found']

    return returnValue

def DictSource(key):
    dictSource = {}

    try:
        returnValue = dictSource[key]
    except:
        returnValue = 'Not Found'

    return returnValue


import io
import sys

if len(sys.argv) > 1:
    filename = sys.argv[1]
    inp = ''.join(open(filename, "r").readlines())
    sys.stdin = io.StringIO(inp)

def main(array):
    aantalKeer = array[0]
    array.pop(0)
    lengte = array[0][0]
    breedte = array[0][1]
    array.pop(0)
    opl = losOp(array, lengte, breedte)
    for _ in range(lengte):
        array.pop(0)
    print(opl)

def losOp(array, lengte, breedte):
    # Zet input om in 2 arrays die resp de groepen en de waarden bevatten.
    groepen = []
    waarden = []
    for rij in range(lengte):
        groepenrij =[]
        waardenrij = []
        for kolom in range(breedte):
            groepenrij.append(array[rij][kolom])
            waardenrij.append(array[rij][kolom + 1])
        groepen.append(groepenrij)
        waarden.append(waardenrij)

str = input()
arr = []
while str:
    arr.append(str)
    try:
        str = input()
    except:
        str = False
main(arr)


#! /usr/bin/python
from collections import deque
from utils import *

INPUT = read_input_as_list('18')

CACHE = {}

def print_map(m, robots):
  minx = min(x[0] for x in m.keys())
  maxx = max(x[0] for x in m.keys())
  miny = min(x[1] for x in m.keys())
  maxy = max(x[1] for x in m.keys())
  for y in range(miny, maxy + 1):
    line = ''
    for x in range(minx, maxx + 1):
      if (x, y) in robots:
        line += '@'
      else:
        line += m[(x, y)]
    print(line)
  print()

def get_maps(inp):
  m = dict()
  robots = set()
  for y in range(len(INPUT)):
    for x in range(len(INPUT[y])):
      if INPUT[y][x] == '@':
        robots.add((x, y))
      m[(x, y)] = INPUT[y][x]
  return m, tuple(robots)

def get_next_moves(pos):
  return [(pos[0] + move[0], pos[1] + move[1]) for move in ((0, 1), (0, -1), (1, 0), (-1, 0))]

def search_keys(m, robot, collected):
  d = deque([robot])
  steps = {robot: 0}
  keys = {}
  while d:
    pos = d.popleft()
    for next_pos in get_next_moves(pos):
      if next_pos in steps:
        continue
      steps[next_pos] = steps[pos] + 1
      if m[next_pos] == '#':
        continue
      if m[next_pos].isupper() and m[next_pos].lower() not in collected:
        continue
      if m[next_pos].islower() and m[next_pos] not in collected:
        keys[m[next_pos]] = (next_pos, steps[next_pos])
      d.append(next_pos)
  return keys

def calculate(m, robots, collected):
  sc = ''.join(sorted(collected))
  if (robots, sc) in CACHE:
    return CACHE[(robots, sc)]

  keys = {}
  for i, robot in enumerate(robots):
    for k, (kpos, kdist) in search_keys(m, robot, collected).items():
      keys[k] = (i, kpos, kdist)

  if len(CACHE) % 20 == 0:
    print(sc)

  if len(keys) == 0:
    c = 0
  else:
    vals = set()
    for k, (ki, kpos, kdist) in keys.items():
      new_robots = tuple(kpos if i == ki else r for i, r in enumerate(robots))
      vals.add(calculate(m, new_robots, collected | {k}) + kdist)
    c = min(vals)

  CACHE[(robots, sc)] = c
  return c

@timer
def part1():
  m, robots = get_maps(INPUT)
  return calculate(m, robots, set())

@timer
def part2():
  m, robots = get_maps(INPUT)
  m[robots[0]] = '#'
  for y in range(-1, 2):
    for x in range(-1, 2):
      rx = robots[0][0] + x
      ry = robots[0][1] + y
      if x == 0 or y == 0:
        m[(rx, ry)] = '#'
      else:
        m[(rx, ry)] = '@'
        robots += ((rx, ry), )
  robots = robots[1:]
  return calculate(m, robots, set())

#print(part1())
print(part2())


# -*- coding: utf-8 -*-
# @Time    : 2021/6/10
# @Author  : kaka

import argparse
import logging
import os
from pathlib import Path
from transformers import BertConfig, BertModel
# from datasets import load_dataset

import torch
import torch.nn.functional as F
from torch.utils.data import DataLoader
from tqdm import tqdm
from transformers import BertTokenizer
from SimCSE import SimCSE
from CSECollator import CSECollator
from dataReader import DataReader
import time
os.environ['CUDA_VISIBLE_DEVICES'] = "1"

def parse_args():
    parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument("--train_file", type=str,default='./data/weikong/train_weikong_dataset.xlsx', help="train text file")
    parser.add_argument("--pretrained", type=str, default="./pretrain_models/chinese-bert-wwm-ext", help="huggingface pretrained model")
    parser.add_argument("--model_out", type=str, default="./output", help="model output path")
    parser.add_argument("--num_proc", type=int, default=5, help="dataset process thread num")
    parser.add_argument("--max_length", type=int, default=64, help="sentence max length")
    parser.add_argument("--batch_size", type=int, default=64, help="batch size")
    parser.add_argument("--epochs", type=int, default=1, help="epochs")
    parser.add_argument("--lr", type=float, default=1e-5, help="learning rate")
    parser.add_argument("--tao", type=float, default=0.05, help="temperature")
    parser.add_argument("--device", type=str, default="cuda", help="device")
    parser.add_argument("--display_interval", type=int, default=100, help="display interval")
    parser.add_argument("--save_interval", type=int, default=860, help="save interval")
    parser.add_argument("--pool_type", type=str, default="cls", help="pool_type")
    parser.add_argument("--dropout_rate", type=float, default=0.3, help="dropout_rate")
    args = parser.parse_args()
    return args

    # parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    # parser.add_argument("--train_file", type=str, help="train text file")
    # args = parser.parse_args()
    # return args

def load_data(args, tokenizer):
    # data_files = {"train": args.train_file}
    # ds = load_dataset("text", data_files=data_files)
    #
    # ds_tokenized = ds.map(lambda example: tokenizer(example["text"]), num_proc=args.num_proc)
    # collator = CSECollator(tokenizer, max_len=args.max_length)
    # dl = DataLoader(ds_tokenized["train"],
    #                 batch_size=args.batch_size,
    #                 collate_fn=collator.collate)

    data_files = args.train_file
    dataset = DataReader(tokenizer,data_files,100)
    collator = CSECollator(tokenizer, max_len=args.max_length)
    dl = DataLoader(dataset=dataset,collate_fn=collator.collate,batch_size=args.batch_size,shuffle=False)
    return dl

def compute_infoceLoss(y_pred, tao=0.05, device="cuda"):
    """

    :param y_pred: 模型输出，维度[B,H]
    :param tao: 温度系数
    :param device:
    :return:
    """
    idxs = torch.arange(0, y_pred.shape[0], device=device)

    y_true = idxs + 1 - idxs % 2 * 2

    t1 = time.time()
    #[B,1,H]
    a = y_pred.unsqueeze(1)
    # [1,B,H]
    b = y_pred.unsqueeze(0)
    #[B,B]
    similarities = F.cosine_similarity( a, b, dim=2)
    t2 = time.time()
    print('time is %.4f' % (t2 - t1)) #cpu情况下 B=64 time is 0.2021

    t1 = time.time()
    #自己实现的cos——similarity计算，貌似比torch.cosine_similarity()要快
    #[B,H]
    a_new = y_pred
    #[H,B]
    b_new = y_pred.T
    #[B,B]
    d = torch.matmul(a_new,b_new)
    # [B,B]
    length = torch.mm(torch.norm(a_new,dim=1).unsqueeze(1),torch.norm(b_new,dim=0).unsqueeze(0))
    cos = d/length
    t2 = time.time()
    print('time is %.4f'%(t2-t1)) #cpu情况下 B=64 time is 0.0348

    #单位对角矩阵——对角线上为1e12很大的值
    c = torch.eye(y_pred.shape[0], device=device) * 1e12

    # 单位对角矩阵——对角线上为1-1e12很小的值
    similarities = similarities - c

    similarities = similarities / tao

    loss = F.cross_entropy(similarities, y_true)

    return torch.mean(loss)

def train(args):
    # args.device  = torch.device("cuda" if torch.cuda.is_available()  else "cpu")
    args.device = torch.device("cpu")
    tokenizer = BertTokenizer.from_pretrained(args.pretrained, mirror="tuna")
    dl = load_data(args, tokenizer)

    conf = BertConfig.from_pretrained(args.pretrained)
    model = SimCSE(conf,args.pretrained, args.pool_type, args.dropout_rate).to(args.device)

    optimizer = torch.optim.AdamW(model.parameters(), lr=args.lr)

    model_out = Path(args.model_out)
    if not model_out.exists():
        os.mkdir(model_out)

    model.train()
    batch_idx = 0
    for epoch_idx in range(args.epochs):
        for data in tqdm(dl,ncols=50):
            batch_idx += 1

            pred = model(input_ids=data["input_ids"].to(args.device),
                         attention_mask=data["attention_mask"].to(args.device),
                         token_type_ids=data["token_type_ids"].to(args.device))

            loss = compute_infoceLoss(pred, args.tao, args.device)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            loss = loss.item()

            if batch_idx % args.display_interval == 0:
                logging.info(f"batch_idx: {batch_idx}, loss: {loss:>10f}")

        save_path = model_out / "0824_bert_epoch_{0}-batch_{1}-loss_{2:.6f}".format(epoch_idx, batch_idx, loss)
        model.save_pretrained(
            save_path)
        tokenizer.save_vocabulary(
            save_path)
        conf.architectures =  ["BertForMaskedLM"]
        conf.save_pretrained(save_path)

    conf = BertConfig.from_pretrained(save_path)
    simcse = SimCSE(conf,save_path)

def main():
    args = parse_args()
    print('args',args)
    train(args)

if __name__ == "__main__":
    log_fmt = "%(asctime)s|%(name)s|%(levelname)s|%(message)s"
    logging.basicConfig(level=logging.INFO, format=log_fmt)
    main()


# -*-coding:utf-8
-*-
__author__ = 'inctrl'
from display import *
import datetime
import calendar
import random
import math
import os.path
import numpy as np
from numpy.linalg import inv

from display import *

class ThisStudy:
    def __init__(self):

        self.mode = 0  # Study mode variable
        self.group_index_out = 0  # Group index
        self.dic_index_out = 0  # Dictionary index within a group
        self.filename = ''  # Dictionary index within a group

        # The number of blank
        self.min_level = 2
        self.max_level = 7
        self.min_sent_length = 20
        self.max_sent_length = 120

        pass

    def lin_reg_out(self, in_len):  # Linear regression function to set the level of sentence reconstruction.

        Y = np.array([[self.min_level],
                      [self.max_level]])
        A = np.array([[1, self.min_sent_length],
                      [1, self.max_sent_length]])

        coeff = np.dot(inv(A), Y)  # Linear regression

        out_level = coeff[1] * in_len + coeff[0]

        return out_level

    def sentence_eraser(self, list_eng, degree):
        '''
        The module that blanks the sentence.

        :param list_eng:
        :param degree:
        :return erased_sent:
        :return key_words:

        '''
        split_sent = list_eng.split()  # Split the sentence with regard to spacing.
        len_sent = len(split_sent)  # Length of the splited sentence
        num_of_erase = int(math.floor(len_sent * degree))  # the number of blanks to be erased.
        ones_zeros = [1] * num_of_erase + [0] * (len_sent - num_of_erase)
        random.shuffle(ones_zeros)
        key_words = []

        for idx, words in enumerate(split_sent):
            if ones_zeros[idx] == 1:
                key_words.append(split_sent[idx])  # append the splited words.
                split_sent[idx] = '_' * len(split_sent[idx])  # Blank the word with the same length of underlines.

        erased_sent = ' '.join(split_sent)  # Put together all the erased splited words.
        key_words = ' '.join(key_words)  # Put together all the splited keywords.

        return erased_sent, key_words

    def word_classifier(self, mytext):
        pass

    def sentence_shuffler(self, list_eng):

        split_sent = list_eng.split()
        random.shuffle(split_sent)
        shuffle_out = ' | '.join(split_sent)
        shuffle_out = '| ' + shuffle_out + ' |'
        return shuffle_out

    def read_dic_table(self, my_date):
        '''
        inctrl:
        This function contains list of dictionary names.
        Modify 'pyEng_dic' if you want to change the dictionary.

        :param my_date: The day of the week which has an index range of 0 to 6.
        :return pyEng_dic[my_date]: Return a dictionary name.
        '''

        # The keys for dictionaries.
        pyEng_dic = [['paper_ex1',
                      'wiki_edu1',
                      'esst2_file',
                      'wiki_tec1',
                      'brain_drain',
                      'capitalism',
                      'esst3_file'],
                     ['article1',
                      'short_rand1',
                      'short_rand2',
                      'short_rand3',
                      'short_rand4',
                      'short_rand5',
                      'short_rand6']]

        pyEng_folder = ['textdata0',
                        'textdata2']

        return pyEng_dic, pyEng_folder

    def date_func(self):
        '''
        inctrl:
        Call datetime function and module to return date information.
        :return the day of the week, date info (all of them), index of the day of the week:
        '''

        my_date = datetime.datetime.today().weekday()
        today = str(datetime.datetime.today())
        weekofday = calendar.day_name[my_date]

        return weekofday, today, my_date

    def read_file(self):
        '''
        inctrl:
        Read files from 'textdata' folder.
        :return 0:
        '''

        weekofday, today, my_date = self.date_func()

        pyEng_dic, pyEng_folder = self.read_dic_table(self.dic_index_out)

        self.filename = pyEng_dic[self.group_index_out - 1][
            self.dic_index_out - 1]  # index - 1 to fit into python list index
        total_path = str(os.getcwd()) + '/' + pyEng_folder[
            self.group_index_out - 1] + '/' + self.filename + '.txt'  # index - 1 to fit into python list index
        total_path_kor = str(os.getcwd()) + '/' + pyEng_folder[
            self.group_index_out - 1] + '/' + self.filename + '_kor' + '.txt'  # index - 1 to fit into python list index
        total_path_csv = str(os.getcwd()) + '/' + pyEng_folder[
            self.group_index_out - 1] + '/' + self.filename + '_rec.csv'  # index - 1 to fit into python list index

        '''
        Open a Korean list and a English list.
        Save both lists into each variable.
        '''

        list_eng, list_kor = ['', '']

        print(os.path.isfile(total_path))
        print(os.path.isfile(total_path_kor))
        print(os.getcwd())

        try:
            with open(total_path, mode='r', encoding='cp949') as f_eng:
                list_eng = f_eng.readlines()
                f_eng.close()

        except IOError:
            print_screen('An error occurred trying to read the flie:', total_path)

        try:

            with open(total_path_kor, mode='r', encoding='cp949') as f_kor:
                list_kor = f_kor.readlines()
                f_kor.close()

        except IOError:
            print_screen('An error occurred trying to read the flie:', total_path_kor)

        return list_eng, list_kor,

    def question_with_sanity_check(self, bd):

        while 1:

            print_screen('Type an index.')
            dic_index = prompt_question()

            if dic_index in str(bd):

                idx_out = int(dic_index)
                break

            else:
                print_screen('You have entered an absurd number. Please do it again.')

        return idx_out

    def delete_blank(self, p_string):

        k = len(p_string) - 1  # Get rid of the last index.
        stc = ' '

        while stc == ' ':
            stc = p_string[k]
            k -= 1
            if k == 0:
                break

        return p_string[0:(k + 2)]  # Return a string without blank at the end of the sentence.

    def ask_the_group(self):
        '''
        Ask a question to user to select a DIC.
        :return dic_idx_out:
        '''

        print_screen('Choose the dictionary group.')

        # The possible range of dictionary indexes.
        dic_bd = list(range(1, 3))
        self.group_index_out = self.question_with_sanity_check(dic_bd)

        return self.group_index_out

    def ask_the_dic(self):
        '''
        Ask a question to user to select a DIC.
        :return dic_idx_out:
        '''

        clear_screen()

        # print_screen out the list of all dictionaries.

        weekofday, today, my_date = self.date_func()
        pyEng_dic, pyEng_folder = self.read_dic_table(my_date)

        idx = self.group_index_out
        self.display_dic_group(idx, pyEng_dic[idx - 1])

        # The possible range of dictionary indexes.

        print_screen('Choose the dictionary.')
        dic_bd = list(range(1, 8))
        self.dic_index_out = self.question_with_sanity_check(dic_bd)

        return self.dic_index_out

    def ask_the_mode(self):

        clear_screen()

        self.mode_str = ['[1] Standard', '[2] Review', '[3] Check mode']
        print_screen('Type a model of study loop...')
        print_screen(self.mode_str[0])
        print_screen(self.mode_str[1])

        mode_bd = list(range(1, 4))  # Set the range of mod

        self.mode = self.question_with_sanity_check(mode_bd)

        return self.mode

    def display_dic_group(self, group_num, pyEng_dic_part):

        print('Group:' + str(group_num))
        for idx, item in enumerate(pyEng_dic_part):
            wd = str(calendar.day_name[idx])
            print_screen('[ ' + str(idx + 1) + ' : ' + wd + ' ]: ' + item)

        print_screen('\n')

    def main_prompt_screen(self):
        '''
        inctrl:
        This function prints out the necessary commands for users:
        Displays the date of today and the dictionary according to the day

        :return message:
        '''

        clear_screen()

        # Call the data_func to get date information
        weekofday, today, my_date = self.date_func()
        pyEng_dic, pyEng_folder = self.read_dic_table(my_date)

        # print_screen out the list of all dictionaries.
        for idx, item in enumerate(pyEng_dic):
            self.display_dic_group(idx + 1, item)

        message = 'Today is ' + weekofday + ": " + today[:10]
        print_screen(message)

        return message

    def correct_sentence(self, list_eng, line_count, len_list):

        print_screen('\n[SYSTEM] The correct answer:')
        print_screen(list_eng[line_count][:-1])
        input_val = prompt_type(line_count, len_list)

    def study_loop_main(self, list_eng, list_kor):
        '''
        inctrl:
        :param steady_message: This message lingers on the top of the screen, while every time you type a sentences.
        :param list_eng: The list that contains English sentences.
        :param list_kor: The list that contains Korean sentences.
        :return: 0
        '''

        len_list = [len(list_eng), len(list_kor)]

        # assert (len_list[0]==len_list[1]), 'Error: The length of list_eng and list_kor must be the same.'

        flag = 1
        line_count = 0

        weekofday, today, my_date = self.date_func()
        # print_screen('Today is', weekofday[:]+'.')

        clear_screen()

        print_screen('[SYSTEM] Train Dictionary:[', self.filename, ']')
        print_screen('[SYSTEM] Mode of Loop:[', self.mode_str[self.mode - 1], ']')
        print_screen('[SYSTEM] Press ENTER to start.\n')
        input()

        clear_screen()

        while flag:

            '''
            inctrl:
            # The main studyloop #
            When the index of loop is in the loop,
            pop out question sentences.
            Firstly, pop out a Korean sentence (question).
            Secondly, display a model answer in English.
            '''

            if line_count < len_list[1]:  # Until we reach the end of the file.

                '''
                2-1.Display shuffled sentence.
                '''
                # mytext = nltk.word_tokenize(list_eng[line_count])
                # print_screen(str(nltk.pos_tag(mytext)))

                current_sentence = self.delete_blank(list_eng[line_count][:-1])
                len_sent = len(current_sentence)
                regressed_level = self.lin_reg_out(len_sent)
                wrong = 1

                regressed_er = np.arange(0.3, 1, (1 / regressed_level))  # The regressed level of blank.

                if self.mode == 1:
                    sent_er = regressed_er

                elif self.mode == 2:
                    sent_er = [0]

                elif self.mode == 3:
                    sent_er = regressed_er

                    print_screen(list_kor[line_count][:-1])  # Display the current Korean text (the current bullet)
                    # Display the index.
                    print_screen('[SYSTEM] This is bullet [' + str(line_count + 1) + '/' + str(len_list[1]) + '].')
                    print_screen('[SYSTEM] If you are certain, type in the sentence in English.')

                    input_val = prompt_type(line_count, len_list)
                    if input_val == current_sentence:
                        wrong = 0
                    else:
                        wrong = 1

                if self.mode == 1 or ( wrong == 1 and self.mode == 3):
                    for idx, degree in enumerate(sent_er):
                        clear_screen()  # Clear the screen

                        print_screen(list_kor[line_count][:-1])  # Display the current Korean text (the current bullet)
                        # Display the index.
                        print_screen('[SYSTEM] This is bullet [' + str(line_count + 1) + '/' + str(len_list[1]) + '].')
                        print_screen('[SYSTEM] Sentence blank level ' + str(idx + 1) + ' out of ' + str(len(sent_er)))
                        if self.mode == 3:
                            print_screen('[SYSTEM] Incorrect answer. Study mode has been activated.')

                        erased_sent, key_words = self.sentence_eraser(current_sentence,
                                                                      degree)  # Randomly blank the sentence.
                        shuffled_key_words = self.sentence_shuffler(key_words)  # Show shuffled words.

                        print_screen(shuffled_key_words + '\n')
                        print_screen(erased_sent + '\n')
                        input_val = prompt_type(line_count, len_list)

                '''
                3.Pop out the model answer in English if the last answer is wrong:
                '''
                if input_val != self.delete_blank(current_sentence):
                    self.correct_sentence(list_eng, line_count, len_list)

                clear_screen()
                line_count += 1  # Heading to the next sentence.

            '''
            inctrl:
            # End of the studyloop #
            When the index of loop reaches the end,
            pop out a message that asks an intend to restart the loop.
            And I put some lines to prevent error cases.
            '''

            if line_count >= len_list[1]:

                print_screen('This is end of the bullet. Do you want to repeat the training loop?')
                yon = input()

                if yon in ['y', 'Y', 'yes', 'Yes', 'YES']:
                    line_count = 0
                    clear_screen()

                elif yon in ['n', 'N', 'No', 'no', 'NO']:
                    print_screen('Abort pyEng.')
                    print_screen('Good bye! \n')
                    flag = 0

                else:
                    pass

        return 0


# -*- coding: utf-8 -*-
import json
from chaoslib.types import Configuration, Secrets
from kubernetes import client

from logzero import logger
from chaosk8s_wix import create_k8s_api_client
from . import get_active_nodes, load_taint_list_from_dict, is_equal_V1Taint
import datetime
from chaosk8s_wix.slack.logger_handler import SlackHanlder

__all__ = ["get_nodes", "all_nodes_are_ok",
           "have_new_node", "check_min_nodes_exist"]

slack_handler = SlackHanlder()
slack_handler.attach(logger)

def get_nodes(label_selector: str = None,
              secrets: Secrets = None):
    """
    List all Kubernetes worker nodes in your cluster. You may filter nodes
    by specifying a label selector.
    """
    api = create_k8s_api_client(secrets)

    v1 = client.CoreV1Api(api)
    if label_selector:
        ret = v1.list_node(
            label_selector=label_selector, _preload_content=False)
    else:
        ret = v1.list_node(_preload_content=False)

    return json.loads(ret.read().decode('utf-8'))

def check_containers_for_node(client, nodename):
    """
       Helper function.
       Checks that all pods on specific container are in running state

    """
    pods = client.list_pod_for_all_namespaces(
        watch=False, field_selector="spec.nodeName=" + nodename)
    retval = True
    for i in pods.items:
        if i.status.container_statuses is not None:
            for status in i.status.container_statuses:
                if status.state.running is None:
                    logger.info("%s\t%s\t%s \t%s is not good" % (
                        nodename, i.metadata.namespace, i.metadata.name, i.status.container_statuses[0].state))
                    retval = False
    if not retval:
        logger.error("%s\tis NOT OK" % nodename)
    else:
        logger.info("%s\tis OK" % nodename)
    return retval

def all_nodes_are_ok(label_selector: str = None,
                     configuration: Configuration = None,
                     secrets: Secrets = None):
    """
    List all Kubernetes worker nodes in your cluster. You may filter nodes
    by specifying a label selector.
    """
    retval = True
    ignore_list = []
    if configuration is not None:
        ignore_list = load_taint_list_from_dict(
            configuration.get("taints-ignore-list", {}))
    resp, k8s_api_v1 = get_active_nodes(label_selector, ignore_list, secrets)

    for item in resp.items:
        localresult = True
        for condition in item.status.conditions:
            if condition.type == "Ready" and condition.status == "False":
                logger.debug("{p} Ready=False  ".format(
                    p=item.metadata.name))
                localresult = False
        if item.spec.unschedulable:
            logger.debug("{p} unschedulable ' ".format(
                p=item.metadata.name))
            localresult = False

        # if item.spec.taints and len(item.spec.taints) > 0:
        #     logger.debug("{p} Tainted node ' ".format(
        #         p=item.metadata.name))
        #     localresult = False

        if not localresult:
            logger.debug("{p} Is not healthy ' ".format(
                p=item.metadata.name))
        if localresult is False:
            retval = localresult

    return retval

def have_new_node(k8s_label_selector: str = None,
                  age_limit: int = 600,
                  configuration: Configuration = None,
                  secrets: Secrets = None):
    """
    Check that there is at least one new node created at time interval
    defined by age_limit, that matches k8s_label_selector
    :param k8s_label_selector: k8s label selector to filter nodes
    :param age_limit: seconds, time interval to check for new nodes. If node was created
    at time that was before current time minus age_limit, it will be filtered out
    :param configuration: chaostoolkit will inject configuration
    :param secrets: chaostoolkit will inject secrets
    :return: True if at least one node was created, False otherwise
    """

    resp, k8s_api_v1 = get_active_nodes(k8s_label_selector, None, secrets)
    new_nodes = []
    for node in resp.items:
        now = datetime.datetime.now(tz=node.metadata.creation_timestamp.tzinfo)
        begining_of_time = now - datetime.timedelta(seconds=age_limit)
        if node.metadata.creation_timestamp > begining_of_time:
            logger.debug("New node found :" + node.metadata.name)
            new_nodes.append(node)
    return len(new_nodes) > 0

def check_min_nodes_exist(k8s_label_selector: str = None,
                          min_limit: int = 2,
                          configuration: Configuration = None,
                          secrets: Secrets = None):
    """
    Check that there are least min_limit new node that matches k8s_label_selector
    :param k8s_label_selector: k8s label selector to filter nodes
    :param min_limit: minimum amount of nodes that have matching k8s_label_selector
    :param configuration: chaostoolkit will inject configuration
    :param secrets: chaostoolkit will inject secrets
    :return: True if there are at least min_limit nodes exists, False otherwise
    """

    resp, k8s_api_v1 = get_active_nodes(k8s_label_selector, None, secrets)

    return len(resp.items) >= min_limit

def get_tainted_nodes(key: str = None, value: str = None, effect: str = None,
                      secrets: Secrets = None):
    """
    Get nodes that are tainted with specific taint.
    :param key: Taint key
    :param value: Taint value
    :param effect: Taint effect
    :param secrets: chaostoolkit will inject secrets
    :return: array of nodes with specific taint
    """
    all_nodes, k8s_api_v1 = get_active_nodes(
        label_selector="", taints_ignore_list=None, secrets=secrets)
    taint_to_find = client.V1Taint(
        effect=effect, key=key, time_added=None, value=value)
    retval = []
    for node in all_nodes.items:
        if node.spec is not None and node.spec.taints is not None:
            for taint in node.spec.taints:
                if is_equal_V1Taint(taint, taint_to_find):
                    retval.append(node)
    return retval


import os
import tarfile
from six.moves import urllib
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.model_selection import train_test_split
import numpy as np
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import OneHotEncoder, LabelEncoder, LabelBinarizer
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedShuffleSplit
from pandas.plotting import scatter_matrix
import hashlib

DOWNLOAD_ROOT = "https://github.com/ageron/handson-ml/tree/master/"
HOUSING_PATH = "datasets/housing"
HOUSING_URL = DOWNLOAD_ROOT + HOUSING_PATH + "/housing.tgz"

def fetch_housing_data(housing_url=HOUSING_URL, housing_path=HOUSING_PATH):
    if not os.path.isdir(housing_path):
        os.makedirs(housing_path)
    tgz_path = os.path.join(housing_path, "housing.tgz")
    urllib.request.urlretrieve(housing_url, tgz_path)
    housing_tgz = tarfile.open(tgz_path)
    housing_tgz.extractall(path=housing_path)
    housing_tgz.close()

def load_housing_data(housing_path=HOUSING_PATH):           # Pandas
    csv_path = os.path.join(housing_path, "housing.csv")
    return pd.read_csv(csv_path)

# fetch_housing_data()                                    Not called (Error in fetching details)
housing = load_housing_data()
print(housing.head())                                     # Print is necessary
print(housing.info())
print(housing["ocean_proximity"].value_counts())
# print(housing.describe()) : Return statistical information of the attribute (Count, mean, std, min, max, 25%)

housing.hist(bins=50, figsize=(20, 15))                      # Matplotlib
plt.show()

"""
def split_train_test(data, test_ratio):
    shuffled_indices = np.random.permutation(len(data))
    test_set_size = int(len(data) * test_ratio)
    test_indices = shuffled_indices[:test_set_size]
    train_indices = shuffled_indices[test_set_size:]
    return data.iloc[train_indices], data.iloc[test_indices]

    >>> train_set, test_set = split_train_test(housing, 0.2)
    >>> print(len(train_set), "train +", len(test_set), "test"
    16512 train + 4128 test
    #This works but not perfect, it will generate different test set over time MLA will get to see whole dataset
    #Set RNG seed (eg np.random.seed(42)) before calling np.random.permutation(), (Generate same shuffled indices)
    #Again it will break if you fetch an updated dataset

# import hashlib
def test_set_check(identifier, test_ratio, hash):
    return hash(np.int64(identifier)).digest()[-1] < 256 * test_ratio

def split_train_test_by_id(data, test_ratio, id_column, hash = hashlib.md5):
    ids = data[id_column]
    in_test_set = ids.apply(lambda id_: test_set_check(id_, test_ratio, hash))
    return data.loc[~in_test_set], data.loc[in_test_set]
# Unfortunately, it doesn't have identifier column; so use row index as id

housing_with_id = housing.reset_index()                                         # adds an 'index' column
train_set, test_set = split_train_test_by_id(housing_with_id, 0.2, "index")
# Make sure new data appended at end. Else...

housing_with_id["id"] = housing["longitude"] * 1000 + housing["latitude"]
train_set, test_set = split_train_test_by_id(housing_with_id, 0.2, "id")
# Stable for a few million year. Else Scikit-Learn provide split_train_test
"""
# from sklearn.model_selection train_test_split
train_set, test_set = train_test_split(housing, test_size=0.2, random_state=42)
# Same as split_train_test defined earlier with direct random_state parameter and you can pass multiple dataset with
# identical number of rows, and it will split them on the same indices

housing["income_cat"] = np.ceil(housing["median_income"] / 1.5)
housing["income_cat"].where(housing["income_cat"] < 5, 5.0, inplace=True)
# ceil to have discrete category (also merging all categories greater than 5 into category 5)

# This one is pretty straight forward and different from text
strat_train_set, strat_test_set = train_test_split(housing, test_size=0.2, random_state=42, stratify=housing["income_cat"])

print(housing["income_cat"].value_counts() / len(housing))

"""
print(strat_test_set["income_cat"].value_counts() / len(strat_test_set))
print(strat_test_set.head())
Test set generated using stratified sampling has income category proportion identical to full dataset
Removing the income_cat to make it original
"""

for sets in (strat_train_set, strat_test_set):
    sets.drop(["income_cat"], axis=1, inplace=True)
# Making copy without harming thr training set

housing = strat_train_set.copy()

"""
U can remove alpha but it is much easier to visualize to see high density,
option s (radius of the circle), option c (color represent the price)
option cmap (Used predefined color map jet, which ranges from blue to red (high))
"""
housing.plot(kind="scatter", x="longitude", y="latitude", alpha=0.4, s=housing["population"]/100, label="population",
             c="median_house_value", cmap=plt.get_cmap("jet"), colorbar=True)
# plt.legend()   Check why this isn't showing
plt.show()
# Computing the standard correlation coefficient b/w every pair of attribute
corr_matrix = housing.corr()
print(corr_matrix["median_house_value"].sort_values(ascending=False))

# Another way to check correlation b/w attributes use pandas scatter_matrix function 11^2 too high
attributes = ["median_house_value", "median_income", "total_rooms", "housing_median_age"]
# scatter_matrix(housing[attributes], figsize=(12,8)) this will show all 16, main diagonal histogram
housing.plot(kind="scatter", x="median_income", y="median_house_value", alpha=0.1)
plt.show()

# Trying different meaningful combinations of the attribute to see useful result
housing["rooms_per_household"] = housing["total_rooms"] / housing["households"]
housing["bedrooms_per_room"] = housing["total_bedrooms"] / housing["total_rooms"]
housing["population_per_household"] = housing["population"] / housing["households"]

corr_matrix = housing.corr()
# The bedrooms_per_room is much more correlated than the total number of rooms or bedrooms.
print(corr_matrix["median_house_value"].sort_values(ascending=False))

# Prepare the data for ML Algorithm (Instead of doing manually always write function)
housing = strat_train_set.drop("median_house_value", axis=1)
housing_labels = strat_train_set["median_house_value"].copy()
# Separated predictors and labels as transformation is not common(Drop does not effect the set)--(next)-->Data Cleaning

""""
Remember, total_bedrooms attribute has some missing value, Option: Remove corresponding district/whole attribute/Set the
value to some value(0/mean/median)
housing.dropna(subset=["total_bedrooms"])    #1
housing.drop("total_bedrooms", axis=1)       #2
median=housing["total_bedrooms"].median()
housing["total_bedrooms"].fillna(median)     #3
//Don't forget to save median value
Scikit-Learn provides a handy class SimpleImputer, Different from text it has been changed.
"""
imputer = SimpleImputer(strategy="median")
# Removing non numerical attribute
housing_num = housing.drop("ocean_proximity", axis=1)
# Fit imputer instance to training data using fit
imputer.fit(housing_num)
"""Computed the median value of each attribute and stored in statistics_ instance variable
print(imputer.statistics_)
print(housing_num.median().values)
Check both values are same
We can use this trained imputer to transform the training set by the learned medians
"""
X = imputer.transform(housing_num)
# This result is plain Numpy 2D array putting back in Pandas DataFrame
housing_tr = pd.DataFrame(X, columns=housing_num.columns)

# MLA prefers to work with numbers, we have left ocean_proximity beacause it was a text.

encoder = LabelEncoder()
housing_cat = housing["ocean_proximity"]
housing_cat_encoded = encoder.fit_transform(housing_cat)
print(housing_cat_encoded)
print(encoder.classes_)
"""
Issue MLA will assume two nearby values are similar than two distant value.
So ScikitLearn provide OneHotEncoder
"""
encoder = OneHotEncoder()
# Note fit_transform expects a 2D array, but housing_cat_encoded is 1D array
housing_cat_1hot = encoder.fit_transform(housing_cat_encoded.reshape(-1, 1))
# print(housing_cat_1hot) ; Result will not be as text but it will be a SciPy sparse matrix, instead of NumPy array, So
print(housing_cat_1hot.toarray())

"""
We can apply both the transformation in one shot (category to integer category then to one-hot-vector)
from sklearn.preprocessing import LabelBinarizer
encoder = LabelBinarizer()
housing_cat_1hot = encoder.fit_transform(housing_cat)
print(housing_cat_1hot)
# Notice this return dense NumPy array by default
"""
# Custom Transformers
# from sklearn.base import BaseEstimator, TransformerMixin
rooms_ix, bedrooms_ix, population_ix, household_ix = 3, 4, 5, 6
# To be used for slicing

class CombinedAttributesAdder(BaseEstimator, TransformerMixin):
    def __init__(self, add_bedrooms_per_room=True):                 # No *args or kargs
        self.add_bedrooms_per_room = add_bedrooms_per_room

    def fit(self, X, y=None):
        return self                                                 # Nothing else to do

    def transform(self, X, y=None):
        rooms_per_household = X[:, rooms_ix] / X[:, household_ix]
        population_per_household = X[:, population_ix] / X[:, household_ix]
        if self.add_bedrooms_per_room:
            bedrooms_per_room = X[:, bedrooms_ix] / X[:, rooms_ix]
            return np.c_[X, rooms_per_household, population_per_household, bedrooms_per_room]
        else:
            return np.c_[X, rooms_per_household, population_per_household]

attr_adder = CombinedAttributesAdder(add_bedrooms_per_room=False)
housing_extra_attribs = attr_adder.transform(housing.values)

"""
np.c_ is used for concatenating the array along the 1st axis. add_bedrooms_per_room set to true by default (sensible
default). Generally add hyper parameter for which you are not 100% sure. Automate it to find great combination.

### Feature Scaling (Two Common Way's)###
Min-Max: Subtracting min value dividing by max minus min (ScikitLearn's MinMaxScaler with feature_range).
Standardization: First subtracts the mean value (Standardized values always have 0 mean) then divide by hte variance
so that the resulting variance have the unit variance. (Less affected by outliers) (StanderedScaler)

### Transformation Pipelines ###
All but the last estimator must be transformers (i.e., they must have a fit_transform() method).
When you call the pipeline’s fit() method, it calls fit_transform() sequentially on all transformers.
Passing output of each as parameter to next call, until it reaches final estimator, for it just calls fit() method.

num_pipeline = Pipeline([
            ('imputer', Imputer(strategy="median")),
            ('attribs_adder', CombinedAttributesAdder()),
            ('std_scaler', StandardScaler()),
        ])
housing_num_tr = num_pipeline.fit_transform(housing_num)

In this example, the last estimator is a StandardScaler, which is a transformer, so the pipeline has a transform()
method that applies all the transforms to the data in sequence. This pipeline is for numerical values, for categorical
value ? SKL provide FeatureUnion class for this. You give a list of transformers (Which can be entire transformers
pipeline), and when transform() method is called it runs each transformer's transform() method in parallel, waits for
their output, and then concatenates them and return the result.
"""

class DataFrameSelector(BaseEstimator, TransformerMixin):
    def __init__(self, attribute_names):
        self.attribute_names = attribute_names

    def fit(self, X, y=None):
        return self

    def transform(self, X):
        return X[self.attribute_names].values

# There is nothing in SKL to handle Pandas DataFrame so we need to write a simple custome transformer

num_attribs = list(housing_num)
cat_attribs = ["ocean_proximity"]

num_pipeline = Pipeline([
    ('selector', DataFrameSelector(num_attribs)),
    ('imputer', SimpleImputer(strategy="median")),
    ('attribs_adder', CombinedAttributesAdder()),
    ('std_scaler', StandardScaler())
        ])

cat_pipeline = Pipeline([
    ('selector', DataFrameSelector(cat_attribs)),
    ('label_binarizer', LabelBinarizer())
])

full_pipeline = FeatureUnion(transformer_list=[
    ("num_pipeline", num_pipeline),
    ("cat_pipeline", cat_pipeline)
])

# We can run the whole pipeline simply:
housing_prepared = full_pipeline.fit_transform(housing_num)
print(housing_prepared)

"""
class LabelBinarizerPipelineFriendly(LabelBinarizer):
    def fit(self, X, y=None):
        # this would allow us to fit the model based on the X input.
        super(LabelBinarizerPipelineFriendly, self).fit(X)
    def transform(self, X, y=None):
        return super(LabelBinarizerPipelineFriendly, self).transform(X)

    def fit_transform(self, X, y=None):
        return super(LabelBinarizerPipelineFriendly, self).fit(X).transform(X)
"""


import FWCore.ParameterSet.Config as cms

from Configuration.Generator.PythiaUESettings_cfi import *
generator = cms.EDFilter("Pythia6GeneratorFilter",
    pythiaHepMCVerbosity = cms.untracked.bool(False),
    maxEventsToPrint = cms.untracked.int32(0),
    pythiaPylistVerbosity = cms.untracked.int32(0),
    filterEfficiency = cms.untracked.double(1.0),
    comEnergy = cms.double(10000.0),
    PythiaParameters = cms.PSet(
        pythiaUESettingsBlock,
        processParameters = cms.vstring('MSEL         = 11 ',
            'MDME( 174,1) = 0    !Z decay into d dbar',
            'MDME( 175,1) = 0    !Z decay into u ubar',
            'MDME( 176,1) = 0    !Z decay into s sbar',
            'MDME( 177,1) = 0    !Z decay into c cbar',
            'MDME( 178,1) = 0    !Z decay into b bbar',
            'MDME( 179,1) = 0    !Z decay into t tbar',
            'MDME( 182,1) = 0    !Z decay into e- e+',
            'MDME( 183,1) = 0    !Z decay into nu_e nu_ebar',
            'MDME( 184,1) = 0    !Z decay into mu- mu+',
            'MDME( 185,1) = 0    !Z decay into nu_mu nu_mubar',
            'MDME( 186,1) = 1    !Z decay into tau- tau+',
            'MDME( 187,1) = 0    !Z decay into nu_tau nu_taubar',
            'CKIN( 1)     = 40.  !(D=2. GeV)',
            'CKIN( 2)     = -1.  !(D=-1. GeV)',
            'MDME(89,1)   = 0    ! no tau->electron',
            'MDME(90,1)   = 0    ! no tau->muon'),
        # This is a vector of ParameterSet names to be read, in this order
        parameterSets = cms.vstring('pythiaUESettings',
            'processParameters')
    )
)


from typing import Dict, Any

import pytest

class TestBaseError:
    """BaseErrorのテスト"""

    def test___repr__(self):
        """__repr__のテスト"""
        from domain.errors import BaseError

        try:
            var = 1 / 0
            pytest.fail("not raised.")
        except ZeroDivisionError as e:
            message: str = "sample message."
            values: Dict[str, Any] = {"reference item": "reference value"}
            error: BaseError = BaseError(message, values, e)
            result: str = error.__repr__()
            module_name: str = error.__module__
            qualified_name: str = error.__class__.__qualname__
            assert result == f"{module_name}.{qualified_name}({message}, {values}, {e})"


log_file = open("um-server-01.txt") #this loads a file called 'um-server-01' to build sales information from

def sales_reports(log_file): # initializes script to process log into sales report
    for line in log_file: # loops through each item in the log
        line = line.rstrip() # applies a function to remove any whitespace in line of report
        day = line[0:3] # records the first three chars as the day of week sale was made

        if day == "Mon": # checks if the day matches the sales info wanted

        # changed 'Tue' to 'Mon'
            print(line) # prints out relevant sales data for desired day of week

sales_reports(log_file) # runs function to select relevant sales for a given week day


# -*- coding: utf-8 -*-
"""
CIERZO CTE DB SE-AE
Controlador de proyecto

Created on Thu Mar 2 13:45:00 2020

__author__ = Pedro Biel
__version__ = 1.0.0
__email__ = structural.eng.biel@gmail.com
"""

from dlg_proyecto.datos.iniciales_proyecto import DatosInicialesProyecto

class CntProyecto:
    """Controlador de los datos de proyecto."""

    def __init__(self, dialogo):
        self.d = dialogo

        # self.d.proyecto = DatosInicialesProyecto.PROYECTO

    def datos_iniciales_proyecto(self):

        proyecto = DatosInicialesProyecto.PROYECTO

        return proyecto

    def datos_proyecto(self):

        proyecto = self.d.lnt_proyecto.text()
        nombre = self.d.lnt_nombre.text()
        empresa = self.d.lnt_empresa.text()
        objeto = self.d.lnt_objeto.text()
        autor = self.d.lnt_autor.text()
        comentario = self.d.lnt_comentario.text()
        self.d.dcc_proyecto['proyecto'] = proyecto
        self.d.dcc_proyecto['nombre'] = nombre
        self.d.dcc_proyecto['empresa'] = empresa
        self.d.dcc_proyecto['objeto'] = objeto
        self.d.dcc_proyecto['autor'] = autor
        self.d.dcc_proyecto['comentario'] = comentario

        return self.d.dcc_proyecto


# -*- coding: utf-8 -*-
"""
Created on Mon Jun  5 03:32:30 2017

@author: SHUBHAM067
"""

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import lxml.html
from urllib.parse import urljoin
from selenium.common.exceptions import NoSuchElementException
import xlwings as xw

l1 = []
l2 = []
l3 = []
l4 = []
l5 = []
l6 = []
l7 = []
driver = webdriver.Chrome()
driver.get('https://www.yell.com/')
driver.implicitly_wait(10)
search_keyword = driver.find_element_by_id("search_keyword")
search_keyword.send_keys("Personal Trainers")
search_location = driver.find_element_by_id("search_location")
search_location.send_keys("UK")
search_location.send_keys(Keys.RETURN)
#driver.close()
url = driver.current_url
l1.append(url)
while True:
    try:
        driver.find_element_by_xpath("//div[@class='row pagination']//a[contains(text(), 'Next')]").click()
    except NoSuchElementException:
        break
    url = driver.current_url
    if url not in l1:
        l1.append(url)

for l in l1:
    driver.get(l)
    tree = lxml.html.fromstring(driver.page_source)
    urls = tree.xpath("//div[@class='row businessCapsule--title']//@href")
    u = 'https://www.yell.com'
    for url in urls:
        l2.append(urljoin(u,url))

for l in l2:
    driver.get(l)
    tree = lxml.html.fromstring(driver.page_source)
    Business_Name = tree.xpath('//div[@class=\"row businessCapsule--title\"]//h2//text()')
    Phone_Number = tree.xpath('//div[@class=\"col-sm-12 businessCapsule--telephone\"]/ul/li/strong/text()')
    Website = tree.xpath('//div[@class=\"col-sm-19 businessCapsule--callToAction\"]/a[1]/@href')
    Email = urljoin(u,tree.xpath('//div[@class=\"col-sm-19 businessCapsule--callToAction\"]/a[2]/@href'))
    Business_Overview = tree.xpath('//div[@id=\"aboutUs\"]/p/text()')
    if Business_Name not in l3:
        l3.append(Business_Name)
    if Phone_Number not in l4:
        l4.append(Phone_Number)
    if Website not in l5:
        l5.append(Website)
    if Email not in l6:
        l6.append(Email)
    if Business_Overview not in l7:
        l7.append(Business_Overview)
    wkb = xw.Book('Yell_Scrape_UK.xlsx')
    sht = wkb.sheets['Sheet1']
    sht.range('A2').value = l3
    sht.range('B2').value = l4
    sht.range('C2').value = l5
    sht.range('D2').value = l6
    sht.range('E2').value = l7


a = input()
i = 0
result = 0
while i < len(a):
    if i == len(a) - 1:
        break
    if a[i + 1] != a[i]:
        result += 1
    i += 1
if a[-1] == '0':
    result += 1
print(result)


import pygame
import os
from pygame.locals import *
import config
import game
import engine
import menu
from random import randint
import _fighter
from pygame_functions import *

class Scenario:

    def __init__(self, game, scenario):
        self.game = game
        self.scenario = scenario
        pygame.mixer.music.stop()
        music = engine.Music("mkt")
        music.play()
        music.volume(0.1)

    def setScenario(self, scenario):
        if scenario == 9:
            scenario = randint(1, 8)
        #self.scene = pygame.image.load('../res/Background/Scenario'+str(scenario)+'.png')
        #self.game.getDisplay().blit(self.scene, (0, 0))
        #pygame.display.update()
        #screenSize(800, 500,"pyKombat",None,None,True) # FullScreen
        screenSize(800, 500,"pyKombat") # Minimized
        setBackgroundImage('../res/Background/Scenario'+str(scenario)+'.png')
        self.judge(scenario)

    def judge(self,scenario):
        [player1,player2] = self.addFigther(scenario)
        player1.act()
        player2.act()
        nextFrame1 = clock()
        nextFrame2 = clock()
        hitCounter = 0
        dizzyCounter = 1
        specialCounter = 1
        while True:
            aux1 = player1.fight(clock(),nextFrame1)
            player1.life.render()
            nextFrame1 = aux1
            aux2 = player2.fight(clock(),nextFrame2)
            player2.life.render()
            nextFrame2 = aux2
            x1 = player1.getX()
            x2 = player2.getX()
            #print(x1, x2, x2-x1)

            if not player1.isAlive() or not player2.isAlive():
                if not player1.isAlive():
                    player1.takeHit("dizzy")
                    if dizzyCounter >= 100:
                        print(dizzyCounter)
                        player1.takeHit("dead")
                if not player2.isAlive():
                    player2.takeHit("dizzy")
                    if dizzyCounter >= 100:
                        player2.takeHit("dead")
                dizzyCounter += 1

            # caso encostem na tela
            if player1.getX() < 20:
                player1.setX(20)

            if player2.getX() < 20:
                player2.setX(20)

            if player1.getX() > (800-20):
                player1.setX(800-20)

            if player2.getX() > (800-20):
                player2.setX(800-20)

            if(collide(player1.currentSprite(),player2.currentSprite())):
                # caso só encostem
                if ( (player1.isWalking() or player1.isJumping()) and (player2.isDancing() or player2.isCrouching() or player2.isWalking()) ) or ((player2.isWalking() or player2.isJumping()) and (player1.isDancing() or player1.isCrouching() or player2.isWalking()) ) or (player1.isWalking() and player2.isWalking()) or (player1.isJumping() and player2.isJumping()) or (player1.isDancing() and player2.isDancing()) or (player2.isSpecialMove() and player1.ishitSpecial()):
                    player1.setX(x1-8)
                    if not player2.isSpecialMove() :player2.setX(x2+8)
                # caso houve soco fraco:
                if ( player1.isApunching() and (player2.isWalking() or player2.isDancing() or player2.isApunching() or player2.ishitSpecial()) ) or ( player2.isApunching() and (player1.isWalking() or player1.isDancing() or player1.isApunching()) ):
                    if player1.isApunching():
                        player2.takeHit("Apunching")
                    if player2.isApunching():
                        player1.takeHit("Apunching")
                    print("socofraco")
                    engine.Sound("Hit0").play()
                    if hitCounter == 0: engine.Sound().roundHit()
                    hitCounter = (hitCounter+1) % 5
                # caso houve soco forte:
                if ( player1.isBpunching() and (player2.isWalking() or player2.isDancing() or player2.isBpunching()) ) or ( player2.isBpunching() and (player1.isWalking() or player1.isDancing() or player1.isBpunching()) ):
                    if player1.isBpunching():
                        player2.takeHit("Bpunching")
                    if player2.isBpunching():
                        player1.takeHit("Bpunching")
                    print("socoforte")
                    engine.Sound("Hit0").play()
                    if hitCounter == 0: engine.Sound().roundHit()
                    hitCounter = (hitCounter+1) % 5
                # caso houve chute fraco:
                if ( player1.isAkicking() and (player2.isWalking() or player2.isDancing() or player2.isAkicking() or player2.isCrouching()) and not player2.isBblocking() ) or ( player2.isAkicking() and (player1.isWalking() or player1.isDancing() or player1.isAkicking() or player1.isCrouching() and not player1.isBblocking()) ):
                    if player1.isAkicking():
                        player2.takeHit("Akicking")
                    if player2.isAkicking():
                        player1.takeHit("Akicking")
                    print("chutefraco")
                    engine.Sound("Hit0").play()
                    if hitCounter == 0: engine.Sound().roundHit()
                    hitCounter = (hitCounter+1) % 5
                # caso houve chute forte:
                if ( player1.isBkicking() and (player2.isWalking() or player2.isDancing() or player2.isBkicking()) ) or ( player2.isBkicking() and (player1.isWalking() or player1.isDancing() or player1.isBkicking()) ):
                    if player1.isBkicking():
                        player2.takeHit("Bkicking")
                    if player2.isBkicking():
                        player1.takeHit("Bkicking")
                    print("chuteforte")
                    engine.Sound("Hit0").play()
                    if hitCounter == 0: engine.Sound().roundHit()
                    hitCounter = (hitCounter+1) % 5
                # caso houve bloqueio em pé:
                if ( (player1.isApunching() or player1.isBpunching() or player1.isDpunching() or player1.isAkicking() or player1.isBkicking() ) and player2.isAblocking() ) or ( (player2.isApunching() or player2.isBpunching() or player1.isDpunching() or player2.isAkicking() or player2.isBkicking() ) and player1.isAblocking() ):
                    if player1.isAblocking():
                        player1.takeHit("Ablocking")
                    if player2.isAblocking():
                        player2.takeHit("Ablocking")
                    engine.Sound("block").play()
                    player1.setX(x1-12)
                    player2.setX(x2+12)
                    print("ablock")
                # caso houve soco ou chute agachado fraco em alguém em pé:
                if ( ((player1.isCpunching() or player1.isCkicking() ) and not player2.isCrouching() and not player2.isBblocking() ) or ((player2.isCpunching() or player2.isCkicking() ) and not player1.isCrouching() and not player1.isBblocking() ) ): # falta adicionar o Bblock
                    if player1.isCpunching() or player1.isCkicking():
                        player2.takeHit("Cpunching")
                    if player2.isCpunching() or player2.isCkicking():
                        player1.takeHit("Cpunching")
                    print("socofraco!!!!!!!")
                    engine.Sound("Hit0").play()
                    if hitCounter == 0: engine.Sound().roundHit()
                    hitCounter = (hitCounter+1) % 5
                # caso houve soco agachado forte em alguém em pé:
                if ( (player1.isDpunching() and (not player2.isAblocking() and not player2.isBblocking())  )  or player2.isDpunching() and (not player1.isAblocking() and not player1.isBblocking()) ):
                    if player1.isDpunching():
                        player2.takeHit("Bkicking")
                    if player2.isDpunching():
                        player1.takeHit("Bkicking")
                    print("socofraco$#$")
                    engine.Sound("Hit0").play()
                    if hitCounter == 0: engine.Sound().roundHit()
                    hitCounter = (hitCounter+1) % 5
                # caso houve chute agachado forte em alguém em pé:
                if ( player1.isDkicking()  or player2.isDkicking() ):
                    if player1.isDkicking():
                        player2.takeHit("Dkicking")
                    if player2.isDkicking():
                        player1.takeHit("Dkicking")
                    print("socofraco")
                    engine.Sound("Hit0").play()
                    if hitCounter == 0: engine.Sound().roundHit()
                    hitCounter = (hitCounter+1) % 5
                # caso houve soco ou chute agachado fraco em alguém agachado:
                if ( ( (player1.isCpunching() or player1.isCkicking()) and player2.isCrouching() and not player2.isBblocking()  )  or ( (player2.isCpunching() or player2.isCkicking()) and player1.isCrouching() and not player1.isBblocking() ) ):
                    if player1.isCpunching() or player1.isCkicking():
                        player2.takeDownHit("Ehit")
                    if player2.isCpunching() or player2.isCkicking():
                        player1.takeDownHit("Ehit")
                    print("socofraco**")
                    engine.Sound("Hit0").play()
                    if hitCounter == 0: engine.Sound().roundHit()
                    hitCounter = (hitCounter+1) % 5
                # caso houve bloqueio agachado:
                if ( (player1.isCpunching() or player1.isDpunching() or player1.isAkicking() or player1.isCkicking() ) and player2.isBblocking() ) or ( (player2.isCpunching() or player2.isDpunching() or player2.isAkicking() or player2.isCkicking() ) and player1.isBblocking() ):
                    if player1.isBblocking():
                        player1.takeDownHit("Bblocking")
                    if player2.isBblocking():
                        player2.takeDownHit("Bblocking")
                    engine.Sound("block").play()
                    player1.setX(x1-12)
                    player2.setX(x2+12)
                    print("bblock")

            # caso houve special
            if ( player1.isSpecialMove() and (player2.isWalking() or player2.isDancing() or player2.isAblocking()) ) or ( player2.isSpecialMove() and (player1.isWalking() or player1.isDancing() or player1.isAblocking()) ):
                if player1.isSpecialMove() and collide(player1.getProjectile().getProjectileSprite(), player2.currentSprite()):   # and collide(projetil,player2)
                    player1.getProjectile().endProjectile()
                    if not player2.isAblocking() and specialCounter <= 100:   player2.takeHit("special")

                if player2.isSpecialMove() and collide(player2.getProjectile().getProjectileSprite(), player1.currentSprite()):   # and collide(projetil,player1)
                    if not player1.isAblocking():   player1.takeHit("special")
                print("special")

            for event in pygame.event.get():
                if event.type == QUIT:
                    pygame.quit()
            if keyPressed("backspace"):
                pygame.quit()
            if keyPressed("esc"):
                self.goBack(player1,player2)

    def addFigther(self,scenario):
        player1 = _fighter.Fighter(0,scenario) # 0: subzero
        player2 = _fighter.Fighter(1,scenario) # 1: scorpion
        return player1,player2

    def goBack(self,player1,player2):
        player1.killPlayer()
        player2.killPlayer()
        del(player1)
        del(player2)
        sound = engine.Sound("back")
        sound.play()
        pygame.mixer.music.stop()
        music = engine.Music("intro")
        music.play()
        music.volume(0.5)
        menu.ScenarioMenu()

def collide(sprite1,sprite2):
    return pygame.sprite.collide_mask(sprite1,sprite2)


import pandas as pd
import numpy as np
import networkx as nx
import networkx as nx
from bokeh.io import show, output_file
from bokeh.models import Plot, Range1d, MultiLine, Circle, HoverTool, TapTool, BoxSelectTool
from bokeh.models.graphs import from_networkx, NodesAndLinkedEdges, EdgesAndLinkedNodes
from bokeh.palettes import Spectral4
import matplotlib as plt
import Levenshtein
from datetime import datetime

class PPHStat:
    def __init__(self,dataloc = './'):
            self.dataloc = loc(dataloc)
            self.nodes = pd.read_csv(dataloc + "fidesz_nodes.csv")[
            ["ID","name", "address", "org_relations","person_relations"]]
            self.edges = pd.read_csv(dataloc + "fidesz_edges.csv")[
            ["Source","Target","id","name","node1_role", "node2_role","start_time","end_time"]]
    #tulajlista function

    def doshit(self):
        self.edges=self.edges[["Source","Target","id","name","node1_role", "node2_role","start_time","end_time"]]
        ownership(self.edges,self)[["Source","Target","id","name","node1_role", "node2_role"]].to_csv(self.dataloc + "P-O relations.csv", index=False)
        #Search for P-P relations

        self.friend_edges=friendship(self.edges)

    def doothershit(self):
        self.friend_ids=self.friend_edges["Source"],self.friend_edges["Target"]
        self.halo = pd.read_csv(self.dataloc + 'list_firms.csv',index_col=0,header=None)[1]
        self.kozbesz = pd.read_csv(self.dataloc + 'list_kozbesz.csv',index_col=0,header=None)[1]
        self.ck = self.kozbesz.apply(cleanfirm).unique()
        self.ch = self.halo.apply(cleanfirm).unique()

        self.df = split_lev(self.ch,self.ck,10,0.8)
        self.df[self.df['rate'] > 0.9].sort_values('rate').to_csv(self.dataloc + "mapping.csv")

        ##here is the main process
        self.part  = pd.read_csv(self.dataloc + 'part.csv')
        self.win_final = pd.read_csv(self.dataloc + 'winner_resolved_final.csv')
        self.p_o = pd.read_csv(self.dataloc + 'P-O relations.csv')
        self.fidesz_nodes = pd.read_csv(self.dataloc + 'fidesz_nodes.csv')
        self.mapping = pd.read_csv(self.dataloc + 'mapping.csv')

        self.final_kozbesz3 = preproc (self.part, self.win_final)

        self.fidesz_nodes = self.fidesz_nodes.rename(index=str, columns={'ID': 'Target'})
        self.p_o = self.p_o.drop(['name'], axis=1)
        self.merged_po = pd.merge(self.p_o, self.fidesz_nodes, how = 'inner', on = ['Target'])

        self.fidesz_nodes = self.fidesz_nodes.rename(index=str, columns={'Target': 'Source'})
        self.merged_po2 = pd.merge(self.merged_po, self.fidesz_nodes, how = 'inner', on = ['Source'])

        self.merged_po2 = self.merged_po2.reset_index(drop=True)

        self.merged_po2['name_x'] = self.merged_po2['name_x'].apply(cleanfirm)

        ####Innentől csináljuk meg az összekötést
        self.list_name_y = list(set(self.merged_po2['name_y']))

        self.dict_name_y = pd.DataFrame()

        for i in self.list_name_y:
            self.filtered_po = self.merged_po2[self.merged_po2['name_y'] == i]
            self.kuka=pd.DataFrame(list(set(self.filtered_po['name_x'])))
            self.kuka["id"]=[i for x in range(len(self.kuka))]
            self.dict_name_y=self.dict_name_y.append(self.kuka, ignore_index=True)
        del self.kuka

        #dict_name_y[dict_name_y[0].apply(lambda x: 'közgép' in x.lower())]

        #########
        #Joinoljuk a mappingel
        self.dict_name_y=pd.merge(self.dict_name_y,self.mapping, how="left", left_on=0 ,right_on='0')
        self.dict_name_y=self.dict_name_y[["id",0,"1"]].dropna()

        ############
        #Összehúzzuk a közbesz táblával
        self.final_kozbesz3['NAME'] = self.final_kozbesz3['NAME'].apply(cleanfirm)
        self.final_kozbesz3=self.final_kozbesz3[["UID","NOOFBIDDERS","VALUE","PUBLIDATE","NAME","CITY"]]

        self.dict_name_y=pd.merge(self.dict_name_y,self.final_kozbesz3, how="left", left_on='1', right_on="NAME")
        self.dict_name_y["PName"]=self.dict_name_y["id"]
        self.dict_name_y=self.dict_name_y.set_index(["id"])

        ############
        #print("to run the gecikereső use stats(df,start,end) function!")

def loc(dataloc = './'):
    dataloc=dataloc
    return dataloc
def ownership(df,self):
    df['felt'] = df['Source'].astype(str).str[0]
    df["felt2"] = df['Target'].astype(str).str[0]
    own1 = df[(df['felt'] == "P") & (df['felt2'] == "O")]
    own2 = df[(df['felt'] == "O") & (df['felt2'] == "P")]
    ownerships=pd.concat([own1,own2])
    #filter by ownership
    viszony=pd.read_csv(self.dataloc + "viszony.csv")
    viszony=viszony[viszony['tulaj']==1]
    ownerships=ownerships[(ownerships["node1_role"].isin(viszony['viszony']))|(ownerships["node2_role"].isin(viszony['viszony']))]

    print(len(ownerships)," relations found.")
    return ownerships

def friendship(df):
    df['felt'] = df['Source'].astype(str).str[0]
    df["felt2"] = df['Target'].astype(str).str[0]
    friendships = df[(df['felt'] == "P") & (df['felt2'] == "P")]
    print(len(friendships),"friendship relations found.")
    return friendships

def stats(df,start,end,merged_po2):
    df=df[(df['PUBLIDATE'] > start) & (df['PUBLIDATE'] < end)]
    stat = pd.DataFrame(columns = ['ID','Nev', 'Nyert','Ertek_ossz', 'Atlag_ertek', 'Atlag_palyazo', 'Min_palyazo', 'Max_palyazo'])
    stat['ID']=pd.Series(list(set(merged_po2['name_y'])))
    stat=stat.set_index(['ID'])
    stat['Nev'] = pd.Series(list(set(merged_po2['name_y'])))
    stat['Nyert']=df[['PName','UID']].groupby(['PName']).agg(['count'])
    stat['Ertek_ossz']=df[['PName','VALUE']].groupby(['PName']).agg(['sum'])
    stat['Atlag_ertek']=df[['PName','VALUE']].groupby(['PName']).agg(['mean'])
    stat['Atlag_palyazo']=df[['PName','NOOFBIDDERS']].groupby(['PName']).agg(['mean'])
    stat['Min_palyazo']=df[['PName','NOOFBIDDERS']].groupby(['PName']).agg(['min'])
    stat['Max_palyazo']=df[['PName','NOOFBIDDERS']].groupby(['PName']).agg(['max'])
    #df.groupby(['PName']).agg([count'])

    return stat
def keres (df, start1,end1, start2,end2,x):

    idoszak1 = stats (df,start1,end1).sort_values(by=['Ertek_ossz'], ascending = False)
    idoszak2 = stats (df, start2, end2).sort_values(by=['Ertek_ossz'], ascending = False)

    metszet1 = idoszak1[:x].index.values
    metszet2 = idoszak2[:x].index.values

    metszet = np.intersect1d(metszet1, metszet2)

    kulonbseg = np.append(np.setdiff1d(metszet1, metszet2), np.setdiff1d(metszet2, metszet1))

    return metszet, kulonbseg

def preproc (part, win_final):
    filtered_winfinal = win_final[win_final['TEXTSCORE']>0]

    ossz_kozbesz = pd.merge(part, filtered_winfinal, how = 'inner', on=['UID'])

    final_kozbesz = ossz_kozbesz[['UID', 'NOOFBIDDERS', 'SUBJECT', 'VALUE', 'CUR', 'VAT', 'VATRATE', 'PUBLIDATE', 'NAME', 'WHITELISTNAME', 'CITY']]

    final_kozbesz3 = final_kozbesz[(final_kozbesz['CUR'] == 'HUF') & final_kozbesz['NOOFBIDDERS']>0]

    final_kozbesz3 = final_kozbesz3.reset_index(drop=True)

    final_kozbesz3['VALUE'] = pd.to_numeric(final_kozbesz3['VALUE'])

    final_kozbesz3['VATRATE'] = pd.to_numeric(final_kozbesz3['VATRATE'], errors = 'coerce')

    return final_kozbesz3

def cleanfirm(firmname):
    """
    cleans common misspellings and abbreviations
    in firms names
    puts them in lowercase
    """
    return firmname.lower().replace(
    'korlátolt felelősségű társaság','kft').replace(
    'korlátolt felelősségü társaság','kft').replace(
    'korlátotl felelősségű társaság','kft').replace(
    'korlátolt felelısségő társaság','kft').replace(
    'korlátolt felelősség társaság','kft').replace(
    'korlátolt felelősségá társaság','kft').replace(
    'korlátolt felelősságű társaság','kft').replace(
    'korlátozott felelősségű társaság','kft').replace(
    'korlátolt felelőségű társaság','kft').replace(
    'kolrátolt felelősségű társaság','kft').replace(
    'korlátol felelősségű társaság','kft').replace(
    'korlát felelősségű társaság','kft').replace(
    'korlátolt felelősségű társaság','kft').replace(
    'korlátolt felelősségő társaság','kft').replace(
    'korlátolt felelősségűbtársaság','kft').replace(
    'korlátolt felelősségű táraság','kft').replace(
    'korlátolt felelősségű táűrsaság','kft').replace(
    'korlátolt felelősségű társasát','kft').replace(
    'korlátolt felelősségű társasád','kft').replace(
    'korlátolt felelősségű társasát','kft').replace(
    ' kereskedelmi és szolgáltató',' k & sz').replace(
    ' szolgáltató és kereskedelmi',' k & sz').replace(
    'tanácsadó és szolgáltató kft', 't & sz kft').replace(
    'kereskedelmi kft', 'kkft').replace(
    'építőipari kft', 'ékft').replace(
    'betéti társaság','bt').replace(
    'közhasznú társaság','kht').replace(
    'zártkörűen működő részvénytársaság','zrt').replace(
    'részvénytársaság','rt').replace(
    'kft.','kft').replace(
    'bt.','bt').replace(
    'rt.','rt').replace(
    'zrt.','zrt')

def split_lev(narray1,narray2,splitnum=3,limit=0.75):
    """
    takes 2 arrays, calcuates pairwise Levenshtein
    distance rates between them
    --
    splits the task to splitnum chunks to save memory
    returns pairs with similarity higyhar than limit
    """
    chunks = np.array_split(narray2,splitnum)
    dflist = []

    i = 1
    for chunk in chunks:
        print(i,' of ',splitnum)
        i += 1
        df = pd.DataFrame(np.array(pd.core.reshape.util.cartesian_product([narray1, chunk])).T)
        df['rate'] = df.apply(lambda x: Levenshtein.ratio(x[0],x[1]),axis=1)
        dflist.append(df[(df['rate'] >= limit)].copy())
    return pd.concat(dflist)