# -*- coding: utf-8 -*-
"""Matrix構造解析を行うモジュール.

二次元トラス構造の解析用.
"""
import csv
import numpy as np
import matplotlib.pyplot as plt

class Node:
    """節点データのクラス.

    Attributes:
        number (int): 節点の番号
        index (int): 計算用のindex, 0からスタート
        x (double): 全体座標系でのx座標
        y (double): 全体座標系でのy座標
        disp_cond_x (double): x方向変位境界条件.初期値は, None
        disp_cond_y (double): y方向変位境界条件.初期値は, None
        load_cond_x (double): x方向荷重境界条件.初期値は, None
        load_cond_y (double): y方向荷重境界条件.初期値は, None
    """
    def __init__(self, number_param, index_param, x_param, y_param, disp_cond_x_param, disp_cond_y_param, load_cond_x_param, load_cond_y_param):
        """節点のコンストラクタ.

        Args:
            number_param (int): 節点の番号
            index_param (int): 計算用のindex, 0からスタート
            x_param (double): 全体座標系でのx座標
            y_param (double): 全体座標系でのy座標
            disp_cond_x_param (float): 変位境界条件.何もないときは, None
            disp_cond_x_param (float): 変位境界条件.何もないときは, None
            load_cond_y_param (float): 荷重境界条件.何もないときは, None
            load_cond_y_param (float): 荷重境界条件.何もないときは, None
        """
        self.number = number_param
        self.index = index_param
        self.x = x_param
        self.y = y_param
        self.disp_cond_x = disp_cond_x_param
        self.disp_cond_y = disp_cond_y_param
        self.load_cond_x = load_cond_x_param
        self.load_cond_y = load_cond_y_param

    def __str__(self):
        expl = 'num: '+str(self.number)+', x: '+str(self.x)+', y: '+str(self.y)+', disp_cond_x: '+str(self.disp_cond_x)+', disp_cond_y: '+str(self.disp_cond_y)+', load_cond_x: '+str(self.load_cond_x)+', load_cond_y: '+str(self.load_cond_y)
        return expl

    def has_disp_cond(self):
        """変位境界条件を持っているかどうかを返す.

        Returns:
            disp_cond が存在するならTrue
        """
        if self.disp_cond_x is not None or self.disp_cond_y is not None:
            return True
        else:
            return False

    def has_load_cond(self):
        """荷重境界条件を持っているかどうかを返す.

        Returns:
            load_cond_cond が存在するならTrue
        """
        if self.load_cond_x is not None or self.load_cond_y is not None:
            return True
        else:
            return False

    def set_load_cond_x(self, Px):
        """x方向の荷重をセットする.

        Args:
            Px (double): x方向の荷重条件.
        """
        if self.has_disp_cond():
            raise Exception('one node can only have one condition')
        self.load_cond_x = Px

    def set_load_cond_y(self, Py):
        """y方向の荷重をセットする.

        Args:
            Py (double): y方向の荷重条件.
        """
        if self.has_disp_cond():
            raise Exception('one node can only have one condition')
        self.load_cond_y = Py

    def add_load_cond_x(self, Px):
        """x方向の荷重を, 現在の条件に加える.

        Args:
            Px (double): 追加するx方向荷重条件.
        """
        if self.has_disp_cond():
            raise Exception('one node can only have one condition')
        self.load_cond_x += Px

    def add_load_cond_y(self, Py):
        """y方向の荷重を, 現在の条件に加える.

        Args:
            Px (double): 追加するx方向荷重条件.
        """
        if self.has_disp_cond():
            raise Exception('one node can only have one condition')
        self.load_cond_y += Py

class Member:
    """部材データのクラス.

    Attributes:
        nodes (list(Node)): 関連する節点のリスト
        area (double): 断面積
        E (double): ヤング率
    """
    def __init__(self, nodes_param, area_param, E_param):
        """コンストラクタ.

        Args:
            nodes_param (list(int)): 関連する節点の番号のリスト
            area_param (double): 断面積
            E_param (double): ヤング率
        """
        self.nodes = nodes_param
        self.area = area_param
        self.E = E_param

    def __str__(self):
        expl = 'nodes: '+str(self.nodes)+', area: '+str(self.area)+', E: '+str(self.E)
        return expl

class Load:
    """荷重データのクラス.

    Attributes:
        node (int): 荷重がかかっている節点の番号
        x (double): x方向の荷重
        y (double): y方向の荷重
    """
    def __init__(self, node_num, Px, Py):
        """コンストラクタ.

        Args:
            node_num (int): 荷重がかかっている節点の番号
            Px (double): x方向の荷重.
            Py (double): y方向の荷重
        """
        self.node = node_num

        self.x = Px
        self.y = Py

    def __str__(self):
        expl = 'node: '+str(self.node)+', Px: '+str(self.x)+', Py: '+str(self.y)
        return expl

class System:
    """構造解析を行う対象のシステム.

    Attributes:
        nodes (dict(Node)): システム内に存在する節点の辞書.辞書のキーがnode番号と対応する.
        members (dict(Member)): システム内に存在する部材の辞書.辞書のキーが部材の番号に対応する.
        K_total (numpy.array): トータルの剛性マトリックス
        f_total (numpy.array): トータルの荷重ベクトル
        d_total (numpy.array): トータルの変位ベクトル
    """
    def __init__(self):
        """Attributesを初期化するコンストラクタ
        """
        self.nodes = dict()
        self.members = dict()
        self.K_total = None
        self.f_total = None
        self.d_total = None

    def load_nodes_csv(self, filename):
        """節点の条件をcsvファイルから読み取る.

        Note:
            csvファイルは, 一行目はヘッダー, 二行目以降にデータ(番号, x座標, y座標, x方向変位境界条件, y方向変位境界条件, x方向荷重条件, y方向荷重条件(いずれもない場合はNone))

        Args:
            filename (str): ファイル名
        """
        current_index = 0
        with open(filename, 'r') as f:
            reader = csv.reader(f)
            header = next(reader)

            for row in reader:
                if len(row) == 7:
                    try:
                        dx = float(row[3])
                    except ValueError:
                        dx = None
                    try:
                        dy = float(row[4])
                    except ValueError:
                        dy = None
                    try:
                        Px = float(row[5])
                    except ValueError:
                        Px = None
                    try:
                        Py = float(row[6])
                    except ValueError:
                        Py = None
                    self.nodes[int(row[0])] = Node(int(row[0]), current_index, float(row[1]), float(row[2]), dx, dy, Px, Py)
                    current_index += 1
                else:
                    raise Exception('invalid data input')

    def load_members_csv(self, filename):
        """ 節点の条件をcsvファイルから読み取る.

        Note:
            csvファイルは，一行目はヘッダー，二行目以降にデータ（番号，右端の節点番号，左端の節点番号，断面積，ヤング率)

        Args:
            filename (str): ファイル名
        """
        with open(filename, 'r') as f:
            reader = csv.reader(f)
            header = next(reader)

            for row in reader:
                if len(row) == 5:
                    self.members[int(row[0])] = Member([int(row[1]), int(row[2])], float(row[3]), float(row[4]))
                else:
                    raise Exception('invalid data input')

    def calculate_partial_stiffness_matrix(self):
        """各部材に対して, 全体座標系で表した剛性マトリックスを計算する."""
        for m in self.members.values():
            node = [self.nodes[m.nodes[0]], self.nodes[m.nodes[1]]]
            L = np.sqrt((node[1].x-node[0].x)**2 + (node[1].y-node[0].y)**2)
            c = (node[1].x - node[0].x) / L
            s = (node[1].y - node[0].y) / L
            m.K = np.array([[c*c, c*s, -c*c, -c*s],
                            [c*s, s*s, -c*s, -s*s],
                            [-c*c, -c*s, c*c, c*s],
                            [-c*s, -s*s, c*s, s*s]]) * m.E * m.area / L

    def calculate_total_stiffness_matrix(self):
        """全体の剛性マトリックスを計算する"""
        matrix_size = 2*len(self.nodes)
        self.K_total = np.zeros((matrix_size, matrix_size))
        for m in self.members.values():
        #for m in [self.members[1]]:
            node = [self.nodes[m.nodes[0]], self.nodes[m.nodes[1]]]
            self.K_total[node[0].index*2:node[0].index*2+2, node[0].index*2:node[0].index*2+2] += m.K[0:2, 0:2]
            self.K_total[node[0].index*2:node[0].index*2+2, node[1].index*2:node[1].index*2+2] += m.K[0:2, 2:4]
            self.K_total[node[1].index*2:node[1].index*2+2, node[0].index*2:node[0].index*2+2] += m.K[2:4, 0:2]
            self.K_total[node[1].index*2:node[1].index*2+2, node[1].index*2:node[1].index*2+2] += m.K[2:4, 2:4]

        self.f_total = np.zeros(matrix_size)
        self.d_total = np.zeros(matrix_size)
        for node in self.nodes.values():
            self.f_total[node.index*2] = node.load_cond_x
            self.f_total[node.index*2+1] = node.load_cond_y
            self.d_total[node.index*2] = node.disp_cond_x
            self.d_total[node.index*2+1] = node.disp_cond_y

    def solve_stiffness_equation(self):
        """剛性方程式を解く"""
        Kff_sf = self.K_total[np.where(np.logical_not(np.isnan(self.f_total))) ]
        Kff_size = Kff_sf.shape[0]
        Kff = Kff_sf[ :,  np.where(np.logical_not(np.isnan(self.f_total)))].reshape(Kff_size, Kff_size)
        Ksf = Kff_sf[:, np.where(np.isnan(self.f_total))].reshape(Kff_size, Kff_sf.shape[1]-Kff_size)

        Kfs_ss = self.K_total[np.where(np.logical_not(np.isnan(self.d_total)))]
        Kss_size = Kfs_ss.shape[0]
        Kfs = Kfs_ss[:, np.where(np.isnan(self.d_total))].reshape(Kss_size, Kfs_ss.shape[1]-Kss_size)
        Kss = Kfs_ss[:, np.where(np.logical_not(np.isnan(self.d_total)))].reshape(Kss_size, Kss_size)

        ff = self.f_total[np.where(np.logical_not(np.isnan(self.f_total)))]
        fs = self.f_total[np.where(np.isnan(self.f_total)) ]

        df = self.d_total[np.where(np.isnan(self.d_total)) ]
        ds = self.d_total[np.where(np.logical_not(np.isnan(self.d_total))) ]

        df = np.linalg.inv(Kff).dot(ff - Ksf.dot(ds))
        fs = Kfs.dot(df) + Kss.dot(ds)

        self.f_total[np.where(np.isnan(self.f_total)) ] = fs
        self.d_total[np.where(np.isnan(self.d_total)) ] = df

    def solve(self):
        """剛性方程式をつくり, 解を求める"""
        self.calculate_partial_stiffness_matrix()
        self.calculate_total_stiffness_matrix()
        self.solve_stiffness_equation()

    def draw_displacement(self, disp_scale=10, x_margin_scale=0.1, y_margin_scale=0.1):
        """トラスの変形の様子を描画する

        Args:
            disp_scale (int): 変位の拡大倍率. default=10
            x_margin_scale (int): x方向の余白の拡大倍率. default=0.1
            y_margin_scale (int): y方向の余白の拡大倍率. default=0.1
        """
        # 元のsystemを表示
        for n in self.nodes.values():
            plt.plot(n.x, n.y, 'ko')
        for member in self.members.values():
            node = [[self.nodes[member.nodes[0]].x, self.nodes[member.nodes[1]].x],
                    [self.nodes[member.nodes[0]].y, self.nodes[member.nodes[1]].y]]
            plt.plot(node[0], node[1], 'k-', lw=2)

        # 変形後のシステムを表示
        sorted_node = sorted(self.nodes.values(), key=lambda n: n.index)
        node_xy = []
        for n in sorted_node:
            node_xy.append(n.x)
            node_xy.append(n.y)
        node_xy = np.array(node_xy)
        node_xy += self.d_total*disp_scale
        for i in range(0, node_xy.size, 2):
            plt.plot(node_xy[i], node_xy[i+1], 'bo')
        for member in self.members.values():
            node = [[self.nodes[member.nodes[0]].x+(self.d_total*disp_scale)[self.nodes[member.nodes[0]].index*2],
                    self.nodes[member.nodes[1]].x+(self.d_total*disp_scale)[self.nodes[member.nodes[1]].index*2] ],
                    [self.nodes[member.nodes[0]].y+(self.d_total*disp_scale)[self.nodes[member.nodes[0]].index*2+1],
                    self.nodes[member.nodes[1]].y+(self.d_total*disp_scale)[self.nodes[member.nodes[1]].index*2+1] ]]
            plt.plot(node[0], node[1], 'b-', lw=2)

        #荷重を矢印で表示
        node_x = node_xy[0::2]
        node_y = node_xy[1::2]
        f_x = self.f_total[0::2]
        f_y = self.f_total[1::2]
        plt.quiver(node_x, node_y, f_x, f_y)

        #整形して表示
        plt.axis('scaled')
        xmin = min([n.x for n in self.nodes.values()])
        xmax = max([n.x for n in self.nodes.values()])
        xmargin = (xmax-xmin)*x_margin_scale
        plt.xlim(xmin-xmargin, xmax+xmargin)
        ymin = min([n.y for n in self.nodes.values()])
        ymax = max([n.y for n in self.nodes.values()])
        ymargin = (ymax-ymin)*y_margin_scale
        plt.ylim(ymin-ymargin, ymax+ymargin)
        plt.show()

    def calculate_stress(self):
        """各部材の応力を計算する

        Returns:
            stress (dict(float)): 部材の番号をキーとした各部材の応力の辞書
        """
        stress = dict()
        for i, m in zip(self.members.keys(),self.members.values()):
            node = [self.nodes[m.nodes[0]], self.nodes[m.nodes[1]]]
            L = np.sqrt((node[1].x-node[0].x)**2 + (node[1].y-node[0].y)**2)
            dL = np.sqrt((node[1].x+self.d_total[node[1].index*2]-node[0].x-self.d_total[node[0].index*2])**2 +
                    (node[1].y+self.d_total[node[1].index*2+1]-node[0].y-self.d_total[node[0].index*2+1])**2 ) -L
            stress[i] =  m.E* dL/L
        return stress

    def print_result(self):
        """計算結果を表示する"""
        sorted_node = sorted(self.nodes.values(), key=lambda n: n.index)
        print('各ノードにかかる荷重')
        for n in sorted_node:
            i=n.index
            print('node no. '+str(n.number)+ ' : '+ str([self.f_total[i], self.f_total[i+1]]))

        print('各ノードの変位')
        for n in sorted_node:
            i=n.index
            print('node no. '+str(n.number)+ ' : '+ str([self.d_total[i], self.d_total[i+1]]))

        print('各部材内部の応力')
        stress = self.calculate_stress()
        for i, s in zip(stress, stress.values()):
            print('member no. '+str(i)+' : '+str(s))

        print('荷重f')
        print(self.f_total)
        print('全体剛性マトリックス')
        print(self.K_total)
        print('変位d')
        print(self.d_total)

        print('f と Kd の計算誤差')
        print(self.f_total-self.K_total.dot(self.d_total))


#!/usr/bin/env python

from math import atan2, sin, cos, pow, sqrt

from dataclasses import dataclass
@dataclass
class EndEffectorWorkingSpace:
    # Pos of The P2, which is the one we resolved for
    P2_x: float
    P2_y: float
    # Orientation of the X axis respect frame 0, which is regulated by Chi
    chi: float

class ComputeIk():

    def __init__(self, DH_parameters):

        # DH parameters
        self.DH_parameters_ = DH_parameters

    def get_dh_param(self, name):

        if name in self.DH_parameters_:
            return self.DH_parameters_[name]
        else:
            assert False, "Asked for Non existen param DH name ="+str(name)

    def compute_ik(self, end_effector_pose, elbow_configuration = "down"):

        # Initialization
        p2_x = end_effector_pose.P2_x
        p2_y = end_effector_pose.P2_y

        # The Angle that Xee makes with frames 0 x axis
        chi = end_effector_pose.chi

        # We get all the DH parameters
        r1 = self.get_dh_param("r1")
        r2 = self.get_dh_param("r2")
        r3 = self.get_dh_param("r3")

        print("Input Data===== ELBOW P1 CONFIG = "+str(elbow_configuration))
        print("p2_x = "+str(p2_x))
        print("p2_y = "+str(p2_y))
        print("chi = "+str(chi))
        print("r1 = "+str(r1))
        print("r2 = "+str(r2))

        # We declare all the equations for theta1, theta2, theta3 and auxiliary
        #########################################################################
        # theta_2
        G = ( pow(p2_x,2) + pow(p2_y,2) - pow(r1,2) - pow(r2,2)) / (2.0 * r1 * r2)

        print(G)

        ## WE HAVE TO CHECK THAT ITS POSSIBLE
        ## -1 <= G <= 1
        possible_solution = False
        if (G <= 1 ) and ( G>= -1):
            possible_solution = True
        else:
            pass

        theta_array = []

        if possible_solution:
            # We have to decide which solution we want
            if elbow_configuration == "down":
                # Positive
                numerator_1 = sqrt(1-pow(G,2))
            else:
                numerator_1 = -1.0 * sqrt(1-pow(G,2))

            denominator_1 = G

            theta_2 = atan2(numerator_1, denominator_1)
            #########################################################################

            #########################################################################
            # theta1
            theta_1 = atan2(p2_y,p2_x) - atan2(r2*sin(theta_2),(r1 + r2*cos(theta_2)))

            #########################################################################

            #########################################################################
            # theta3
            #
            theta_3 = chi - theta_1 - theta_2
            #########################################################################

            theta_array = [theta_1, theta_2, theta_3]

        return theta_array, possible_solution

def calculate_ik(P2_x, P2_y, chi, DH_parameters, elbow_config = "down"):
    ik = ComputeIk(DH_parameters = DH_parameters)
    end_effector_pose = EndEffectorWorkingSpace(P2_x = P2_x,
                                                P2_y = P2_y,
                                                chi = chi)

    thetas, possible_solution = ik.compute_ik(  end_effector_pose=end_effector_pose,
                                                elbow_configuration = elbow_config)

    print("Angles thetas solved ="+str(thetas))
    print("possible_solution = "+str(possible_solution))

if __name__ == '__main__':

    r1 = 1.0
    r2 = 1.0
    r3 = 1.0

    # theta_i here are valriables of the joints
    # We only fill the ones we use in the equations, the others were already
    # replaced in the Homogeneous matrix
    DH_parameters={"r1":r1,
                    "r2":r2,
                    "r3":r3}

    P2_x = 1.0
    P2_y = 1.0
    chi = 0.7

    calculate_ik(P2_x=P2_x, P2_y=P2_y, chi=chi,DH_parameters=DH_parameters, elbow_config = "down")
    calculate_ik(P2_x=P2_x, P2_y=P2_y, chi=chi,DH_parameters=DH_parameters, elbow_config = "up")


import os
import cv2
import numpy as np
import json
import copy
import pycocotools
from pycocotools.coco import COCO
from .dataset import DetDataset
from ppdet.core.workspace import register, serializable

@serializable
class Pose3DDataset(DetDataset):
    """Pose3D Dataset class.

    Args:
        dataset_dir (str): Root path to the dataset.
        anno_list (list of str): each of the element is a relative path to the annotation file.
        image_dirs (list of str): each of path is a relative path where images are held.
        transform (composed(operators)): A sequence of data transforms.
        test_mode (bool): Store True when building test or
            validation dataset. Default: False.
        24 joints order:
        0-2: 'R_Ankle', 'R_Knee', 'R_Hip',
        3-5:'L_Hip', 'L_Knee', 'L_Ankle',
        6-8:'R_Wrist', 'R_Elbow', 'R_Shoulder',
        9-11:'L_Shoulder','L_Elbow','L_Wrist',
        12-14:'Neck','Top_of_Head','Pelvis',
        15-18:'Thorax','Spine','Jaw','Head',
        19-23:'Nose','L_Eye','R_Eye','L_Ear','R_Ear'
    """

    def __init__(self,
                 dataset_dir,
                 image_dirs,
                 anno_list,
                 transform=[],
                 num_joints=24,
                 test_mode=False):
        super().__init__(dataset_dir, image_dirs, anno_list)
        self.image_info = {}
        self.ann_info = {}
        self.num_joints = num_joints

        self.transform = transform
        self.test_mode = test_mode

        self.img_ids = []
        self.dataset_dir = dataset_dir
        self.image_dirs = image_dirs
        self.anno_list = anno_list

    def get_mask(self, mvm_percent=0.3):
        num_joints = self.num_joints
        mjm_mask = np.ones((num_joints, 1)).astype(np.float)
        if self.test_mode == False:
            pb = np.random.random_sample()
            masked_num = int(
                pb * mvm_percent *
                num_joints)  # at most x% of the joints could be masked
            indices = np.random.choice(
                np.arange(num_joints), replace=False, size=masked_num)
            mjm_mask[indices, :] = 0.0

        mvm_mask = np.ones((10, 1)).astype(np.float)
        if self.test_mode == False:
            num_vertices = 10
            pb = np.random.random_sample()
            masked_num = int(
                pb * mvm_percent *
                num_vertices)  # at most x% of the vertices could be masked
            indices = np.random.choice(
                np.arange(num_vertices), replace=False, size=masked_num)
            mvm_mask[indices, :] = 0.0

        mjm_mask = np.concatenate([mjm_mask, mvm_mask], axis=0)
        return mjm_mask

    def filterjoints(self, x):
        if self.num_joints == 24:
            return x
        elif self.num_joints == 14:
            return x[[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 18], :]
        elif self.num_joints == 17:
            return x[
                [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 14, 15, 18, 19], :]
        else:
            raise ValueError(
                "unsupported joint numbers, only [24 or 17 or 14] is supported!")

    def parse_dataset(self):
        print("Loading annotations..., please wait")
        self.annos = []
        im_id = 0
        for idx, annof in enumerate(self.anno_list):
            img_prefix = os.path.join(self.dataset_dir, self.image_dirs[idx])
            dataf = os.path.join(self.dataset_dir, annof)
            with open(dataf, 'r') as rf:
                anno_data = json.load(rf)
                annos = anno_data['data']
                new_annos = []
                print("{} has annos numbers: {}".format(dataf, len(annos)))
                for anno in annos:
                    new_anno = {}
                    new_anno['im_id'] = im_id
                    im_id += 1
                    imagename = anno['imageName']
                    if imagename.startswith("COCO_train2014_"):
                        imagename = imagename[len("COCO_train2014_"):]
                    elif imagename.startswith("COCO_val2014_"):
                        imagename = imagename[len("COCO_val2014_"):]
                    imagename = os.path.join(img_prefix, imagename)
                    if not os.path.exists(imagename):
                        if "train2017" in imagename:
                            imagename = imagename.replace("train2017",
                                                          "val2017")
                            if not os.path.exists(imagename):
                                print("cannot find imagepath:{}".format(
                                    imagename))
                                continue
                        else:
                            print("cannot find imagepath:{}".format(imagename))
                            continue
                    new_anno['imageName'] = imagename
                    new_anno['bbox_center'] = anno['bbox_center']
                    new_anno['bbox_scale'] = anno['bbox_scale']
                    new_anno['joints_2d'] = np.array(anno[
                        'gt_keypoint_2d']).astype(np.float32)
                    if new_anno['joints_2d'].shape[0] == 49:
                        #if the joints_2d is in SPIN format(which generated by eft), choose the last 24 public joints
                        #for detail please refer: https://github.com/nkolot/SPIN/blob/master/constants.py
                        new_anno['joints_2d'] = new_anno['joints_2d'][25:]
                    new_anno['joints_3d'] = np.array(anno[
                        'pose3d'])[:, :3].astype(np.float32)
                    new_anno['mjm_mask'] = self.get_mask()
                    if not 'has_3d_joints' in anno:
                        new_anno['has_3d_joints'] = int(1)
                        new_anno['has_2d_joints'] = int(1)
                    else:
                        new_anno['has_3d_joints'] = int(anno['has_3d_joints'])
                        new_anno['has_2d_joints'] = int(anno['has_2d_joints'])
                    new_anno['joints_2d'] = self.filterjoints(new_anno[
                        'joints_2d'])
                    self.annos.append(new_anno)
                del annos

    def __len__(self):
        """Get dataset length."""
        return len(self.annos)

    def _get_imganno(self, idx):
        """Get anno for a single image."""
        return self.annos[idx]

    def __getitem__(self, idx):
        """Prepare image for training given the index."""
        records = copy.deepcopy(self._get_imganno(idx))
        imgpath = records['imageName']
        assert os.path.exists(imgpath), "cannot find image {}".format(imgpath)
        records['image'] = cv2.imread(imgpath)
        records['image'] = cv2.cvtColor(records['image'], cv2.COLOR_BGR2RGB)
        records = self.transform(records)
        return records

    def check_or_download_dataset(self):
        alldatafind = True
        for image_dir in self.image_dirs:
            image_dir = os.path.join(self.dataset_dir, image_dir)
            if not os.path.isdir(image_dir):
                print("dataset [{}] is not found".format(image_dir))
                alldatafind = False
        if not alldatafind:
            raise ValueError(
                "Some dataset is not valid and cannot download automatically now, please prepare the dataset first"
            )


import io

def file_to_stream(file_object):
    """Ability to get file from stream instead of hard disk

    :param file_object:
    :return: BytesIO stream
    """
    buffer = io.BytesIO(file_object)
    stream = buffer.getvalue()
    buffer.close()
    return stream


from dateutil.parser import parse
from ods.kktix_ticket_orders.udfs import kktix_api, klaviyo_loader

def main(**context):
    """
    Extract user info from kktix api and load to mailer
    """
    schedule_interval = context["dag"].schedule_interval
    # If we change the schedule_interval, we need to update the logic in condition_filter_callback
    assert schedule_interval == "0 * * * *"  # nosec
    ts_datetime_obj = parse(context["ts"])
    year = ts_datetime_obj.year
    timestamp = ts_datetime_obj.timestamp()
    event_raw_data_array = kktix_api._extract(year=year, timestamp=timestamp,)
    # load name and email to mailer before data has been hashed
    klaviyo_loader.load(event_raw_data_array)
    print(f"Batch load {len(event_raw_data_array)} data to downstream task")


import typer

def main(delete: bool = typer.Option(False, help="Delete found files"), extension: str = typer.Argument("txt", help="File extension")):
    """
    Display files find by extension
    """

    print(delete)

    typer.echo(f"Recherche de fichiers avec l'extension { extension }.")

    ext = typer.prompt("Quelle extension souhaitez-vous rechercher ?")
    print(ext)
    if delete:
        do_delete = typer.confirm("Voulez-vous vraiment supprimer les fichiers trouvés ?")
        # do_delete = typer.confirm("Voulez-vous vraiment supprimer les fichiers trouvés ?", abort=True)
        if not do_delete:
            typer.echo("Annulation de l'opération.")
            raise typer.Abort()
        typer.echo("Suppression des fichiers trouvés.")

typer.run(main)


import json

data = '''
[
    { "id" : "001",
    "x" : "2",
    "name" : "Chuck"
    },
    { "id" : "009",
    "x" : "7",
    "name" : "Chuck"
    }
]'''

# json.loads() processes dictionaries in java format
# using len() will count total dictionaries
info = json.loads(data)
print('User count:' , len(info))

for item in info:
    print('Name:' , item['name'])
    print('ID:' , item['id'])
    print('Attribute:' , item['x'])


import socket
import gevent

from gevent import monkey

monkey.patch_all(thread=False)  # 如果使用vscode启动调试器，则必须加上thread=False,否则报错
# monkey.patch_all()

hosts = ['www.baidu.com', 'www.taobao.com',
         'www.jd.com', 'www.bilibili.com']
jobs = [gevent.spawn(socket.gethostbyname, host) for host in hosts]
gevent.joinall(jobs, timeout=5)
for job in zip(hosts, jobs):
    print(job[0], '->', job[1].value)


"""
Based on http://www.logarithmic.net/pfh/blog/01164790008
"""
import deprecation
import heapq

from ..dists import Dist
from .._version import __version__

class _Node:
    def __init__(self):
        self.lower_bounds = []
        self.upper_bounds = []
        self.children = []
        self.vantage = None

    def minimum_distance(self, distances):
        minimum = 0.0
        for i in range(len(distances)):
            if distances[i] < self.lower_bounds[i]:
                minimum = max(minimum, self.lower_bounds[i] - distances[i])
            elif distances[i] > self.upper_bounds[i]:
                minimum = max(minimum, distances[i] - self.upper_bounds[i])
        return minimum

    def help_find(self, item, distances, heap, distance):
        d = distance(self.vantage, item)
        new_distances = distances + (d,)

        heapq.heappush(heap, (d, 0, self.vantage))

        for child in self.children:
            heapq.heappush(heap, (child.minimum_distance(new_distances), 1, child, new_distances))

    def __lt__(self, other):
        return len(self.children) < len(other.children)

    def __gt__(self, other):
        return len(self.children) > len(other.children)

class VP_tree(object):
    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def __init__(self, distance, max_children=2):
        """ items        : list of items to make tree out of
            distance     : function that returns the distance between two items
            max_children : maximum number of children for each node

            Using larger max_children will reduce the time needed to construct the tree,
            but may make queries less efficient.
        """
        self.distance = distance
        self.max_children = max_children

    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def fit(self, points) -> None:
        self.root = self._make_node([(item, ()) for item in points])

    def _make_node(self, items):
        if not items:
            return None

        node = _Node()
        for i in range(len(items[0][1])):
            distance_list = [item[1][i] for item in items]
            node.lower_bounds.append(min(distance_list))
            node.upper_bounds.append(max(distance_list))

        item = items.pop(0)
        node.vantage = item[0]

        if not items:
            return node

        items = self._fill_items(node, items)

        distances = {}
        for item in items:
            distances[item[1][-1]] = True
        distance_list = list(distances.keys())
        distance_list.sort()
        n_children = self._get_n_children(distance_list)
        split_points = [-1]
        for i in range(n_children):
            split_points.append(distance_list[(i + 1) * (len(distance_list) - 1) // n_children])

        for i in range(n_children):
            child_items = [item for item in items if split_points[i] < item[1][-1] <= split_points[i + 1]]
            child = self._make_node(child_items)
            if child:
                node.children.append(child)

        return node

    def _fill_items(self, node, items):
        return [(item[0], item[1] + (self.distance(node.vantage, item[0]),)) for item in items]

    def _get_n_children(self, distance_list):
        return min(self.max_children, len(distance_list))

    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def find(self, item):
        """ Return iterator yielding items in tree in order of distance from supplied item.
        """
        if not self.root:
            return

        heap = [(0, 1, self.root, ())]

        while heap:
            top = heapq.heappop(heap)
            if top[1]:
                top[2].help_find(item, top[3], heap, self.distance)
            else:
                yield top[2], top[0]

class CustomVPTree(VP_tree):
    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def __init__(self, dist: Dist):
        super().__init__(dist.dist)
        self.dist_batch_fn = dist.batch_dist
        self.max_children = 2
        self.leaf_size = 5

    @deprecation.deprecated(deprecated_in="0.6", removed_in="1.0", current_version=__version__,
                            details="Use jvptree.VPTree instead")
    def set_dist(self, dist: Dist):
        self.distance = dist.dist
        self.dist_batch_fn = dist.batch_dist

    def __getstate__(self):
        """Return state values to be pickled."""
        return self.root, self.max_children, self.leaf_size

    def __setstate__(self, state):
        """Restore state from the unpickled state values."""
        self.root, self.max_children, self.leaf_size = state

    def _fill_items(self, node, items):
        dists = self.dist_batch_fn([(node.vantage, item[0]) for item in items])
        return [(item[0], item[1] + (dists[i],)) for i, item in enumerate(items)]

    def _get_n_children(self, distance_list):
        if len(distance_list) > self.leaf_size:
            return self.max_children
        else:
            return len(distance_list)


#!/usr/bin/env python
# coding: utf-8

# # Facebook Prophet
# March 1st 2022

# Facebook Prophet is an **general additive model** that includes a number of highly advanced, intelligent forecasting methods, incluiding changepoint analysis.
# _y = g(t) + s(t) + h(t) + $\epsilon_t$_
# Here g(t) => the **trend** function which models non-periodic changes, s(t) => **periodic** changes (e.g. weekly and yearly seasonality), and h(t) => represents the effects of **holidays** (which occur on potentially irregular scheadules over one or more days)

# Prophet was optimized with the **business forecast tasks**, typically following the next features:
# * We need a bit of data, at least a year. With hourly, daily, or weekly observations
# * Strong multiple human scale seasonality (e.g. week day)
# * Important holidays that occur at irregular intervals
# * A reasonable number of missing observations or large outliers
# * Historical changes may not persist and die out
# * Trends that are non-linear growth curves

# In[133]:

import sys
print(sys.executable)
import numpy as np
import pandas as pd
import os
print(os.getcwd())
from IPython.display import display
import seaborn as sns
import matplotlib.pyplot as plt
from datetime import datetime
from prophet import Prophet
from prophet.diagnostics import cross_validation, performance_metrics

# In[134]:

# Custom functions:
import src.colorsetup

# ## Input data
# Peyton Manning (football player) data

# In[135]:

data_path = 'https://raw.githubusercontent.com/PinkWink/DataScience/master/data/07.%20example_wp_peyton_manning.csv'
peyton = pd.read_csv(data_path)
print("Data shape: {}".format(peyton.shape))
peyton.head()

# In[136]:

peyton.plot()
plt.title("Wikipedia searches for Peyton")
plt.grid()
plt.show()

# In[137]:

# Let's do a log transformation
peyton['y']= np.log(peyton['y'])
display(peyton.head())
peyton.ds.max()

# In[138]:

peyton.plot()
plt.grid()
plt.show()

# # 1) Fitting a vanilla model

# In[139]:

m= Prophet()
m.fit(peyton)

# In[140]:

# Forecast 365 days into future
# prophet requires a blank df to input predictions
future= m.make_future_dataframe(periods= 365)
# display(peyton)
# display(future.head())
# display(future.tail())

# In[141]:

# Populate forecast
forecast= m.predict(future)
display(forecast.head())
print(forecast.columns)

# In[142]:

display(forecast[['ds', 'yhat', 'yhat_lower', 'yhat_upper']].tail())
forecast[forecast.ds >= '2016-01-20'][['ds', 'yhat', 'yhat_lower', 'yhat_upper']].head()

# ## Cross validate

# In[ ]:

# Period specifiy the cutoffs, every 180 days
df_cv= cross_validation(m, horizon= '365 days', period= '180 days', initial= '730 days')

# In[144]:

display(peyton.head())
df_cv.head()

# In[145]:

# We predicted a year into the future
first_cut= df_cv[df_cv.cutoff == datetime(2010,2,15)]
last_cut= df_cv[df_cv.cutoff == datetime(2015,1,20)]
display(first_cut.tail())
print("CV shape: {}, First cutoff: {}, Number of cutoffs: {}".format(df_cv.shape, first_cut.shape, len(df_cv.cutoff.unique())))

# In[189]:

metrics= performance_metrics(df_cv)
display(metrics[['horizon', 'rmse', 'mape']])
print("Mean-RMSE: {:.4f} and Mean-MAPE: {:.2%}".format(np.mean(metrics['rmse']), np.mean(metrics['mape'])))
print("Horizon of 365 days RMSE: {:.4f} and MAPE: {:.2%}".format(metrics.iloc[metrics.shape[0]-1]['rmse'], metrics.iloc[metrics.shape[0]-1]['mape']))

# In[202]:

h_1_mape= metrics.iloc[metrics.shape[0]-1]['mape']
h_1_rmse= metrics.iloc[metrics.shape[0]-1]['rmse']

image_dir= '../../../plots/time_series_analysis/'
plot_name= 'initial.prophet.model.png'

forecast_plot= m.plot(forecast, figsize= (17,7), xlabel= "Time", ylabel= "Wikipedia searches for Peyton (log)")
ax = forecast_plot.gca()
ax.set_title("Initial model= Prophet(), MAPE: {:.2%} and RMSE: {:.3f}".format(h_1_mape, h_1_rmse))
ax.set_xlabel("Time")
ax.set_ylabel("Wikipedia searches for Peyton (log)")

forecast_plot.savefig(image_dir+plot_name)

# In[190]:

plot_name= 'initial.prophet.model.cross.validation.png'

fig, axes= plt.subplots(1,2, sharey= True)
fig.set_figwidth(20)

axes[0].plot(first_cut.ds, first_cut.y, color= "black", label= "True")
axes[0].plot(first_cut.ds, first_cut.yhat, color= "orange", label= "Predicted")
axes[0].fill_between(first_cut.ds, first_cut.yhat_lower, first_cut.yhat_upper, color= "orange", alpha= 0.2)
axes[0].set_title("First fold")
axes[0].legend(loc= "upper left")

axes[1].plot(last_cut.ds, last_cut.y, color= "black", label= "True")
axes[1].plot(last_cut.ds, last_cut.yhat, color= "orange", label= "Predicted")
axes[1].fill_between(last_cut.ds, last_cut.yhat_lower, last_cut.yhat_upper, color= "orange", alpha= 0.2)
axes[1].set_title("Last fold (11th)")
axes[1].legend(loc= "upper left")

axes[0].grid(alpha= 0.3)
axes[1].grid(alpha= 0.3)

plt.tight_layout()
fig.savefig(image_dir+plot_name, transparent= True)

# In[191]:

forecast_components= m.plot_components(forecast)

# # 2) Add Holidays information
# In our case playoffs and superbowls dates

# In[192]:

playoffs = pd.DataFrame({
  'holiday': 'playoff',
  'ds': pd.to_datetime(['2008-01-13', '2009-01-03', '2010-01-16',
                        '2010-01-24', '2010-02-07', '2011-01-08',
                        '2013-01-12', '2014-01-12', '2014-01-19',
                        '2014-02-02', '2015-01-11', '2016-01-17',
                        '2016-01-24', '2016-02-07']),
  'lower_window': 0, # these help us specify spillover into previous and future days which will be treated as own holidays
  'upper_window': 1,
})

# display(playoffs)

superbowls = pd.DataFrame({
  'holiday': 'superbowl',
  'ds': pd.to_datetime(['2010-02-07', '2014-02-02', '2016-02-07']),
  'lower_window': 0,
  'upper_window': 1,
})

# display(superbowls)
holidays= pd.concat((playoffs, superbowls))
holidays.head()

# In[193]:

# fit and predict
m_holidays= Prophet(holidays= holidays)
forecast_holidays = m_holidays.fit(peyton).predict(future)

# In[194]:

forecast_holidays.columns

# In[195]:

# We want to see the holiday effects:
forecast_holidays[(forecast_holidays['playoff'] + forecast_holidays['superbowl']).abs() > 0][
        ['ds', 'yhat','playoff', 'superbowl']].tail()

# In[196]:

forecast_holidays_components= m_holidays.plot_components(forecast_holidays)

# Peyton won Superbowls: 2016. While losing 2010 and 2014.

# In[ ]:

df2_cv= cross_validation(m_holidays, horizon= '365 days', period= '180 days', initial= '730 days')

# In[203]:

metrics_2= performance_metrics(df2_cv)
display(metrics_2[['horizon', 'rmse', 'mape']])
print("Adding Holidays: Mean RMSE: {:.4f} Mean MAPE: {:.2%}".format(np.mean(metrics_2['rmse']), np.mean(metrics_2['mape'])))
print("Horizon of 365 days RMSE: {:.3f} and MAPE: {:.2%}".format(metrics_2.iloc[metrics_2.shape[0]-1]['rmse'], metrics_2.iloc[metrics_2.shape[0]-1]['mape']))

# In[256]:

h_1_mape= metrics_2.iloc[metrics_2.shape[0]-1]['mape']
h_1_rmse= metrics_2.iloc[metrics_2.shape[0]-1]['rmse']

plot_name= 'prophet.holidays.model.png'

forecast_holidays_plot= m_holidays.plot(forecast_holidays, figsize= (17,7))

ax = forecast_holidays_plot.gca()
ax.set_title("Model= Prophet(holidays), MAPE: {:.2%} and RMSE: {:.3f}".format(h_1_mape, h_1_rmse))
ax.set_xlabel("Time")
ax.set_ylabel("Wikipedia searches for Peyton (log)")

plt.show()
forecast_holidays_plot.savefig(image_dir+plot_name)

# # 3) Add another seasonality
# Besides yearly, weekly, and daily. Let's add monthly seasonality

# In[209]:

m_holidays_extra_s= Prophet(holidays= holidays)
# Increasing the number of Fourier components allows the seasonality to change more quickly (at risk of overfitting). Default values for yearly
# and weekly seasonalities are 10 and 3 respectively.
m_holidays_extra_s.add_seasonality(name= 'monthly', period= 30.5, fourier_order= 5)
m_holidays_extra_s.fit(peyton)

# In[210]:

fcst_month = m_holidays_extra_s.predict(future)

# In[213]:

monthly_components= m_holidays_extra_s.plot_components(fcst_month)

# In[ ]:

df3_cv= cross_validation(m_holidays_extra_s, horizon= '365 days', period= '180 days', initial= '730 days')

# In[217]:

metrics_3= performance_metrics(df3_cv)
display(metrics_3[['horizon', 'rmse', 'mape']])
print("Adding Holidays: Mean RMSE: {:.4f} Mean MAPE: {:.2%}".format(np.mean(metrics_3['rmse']), np.mean(metrics_3['mape'])))
print("Horizon of 365 days RMSE: {:.3f} and MAPE: {:.2%}".format(metrics_3.iloc[metrics_3.shape[0]-1]['rmse'], metrics_3.iloc[metrics_3.shape[0]-1]['mape']))

# In[225]:

h_1_mape= metrics_3.iloc[metrics_3.shape[0]-1]['mape']
h_1_rmse= metrics_3.iloc[metrics_3.shape[0]-1]['rmse']

# plot_name= 'initial.prophet.model.png'

extra_seasonality_plot= m_holidays_extra_s.plot(fcst_month, figsize= (17,7))

ax = extra_seasonality_plot.gca()
ax.set_title("Initial model= Prophet(holidays).add_seasonality, MAPE: {:.2%} and RMSE: {:.3f}".format(h_1_mape, h_1_rmse))
ax.set_xlabel("Time")
ax.set_ylabel("Wikipedia searches for Peyton (log)")

plt.show()

# # 4) Adding a regressor

# In[226]:

# creating an indicator variable for NFL sundays
def nfl_sundays(ds):
    """
    NFL season: August, September, October, November, December, January, February
    Sunday => 6 | range: 0-6 => Monday-Sunday
    """
    date= pd.to_datetime(ds)
    if date.weekday() == 6 and (date.month > 8 or date.month < 2):
        return 1
    else:
        return 0

# In[227]:

peyton['nfl_sunday']= peyton['ds'].apply(nfl_sundays)
peyton.head()

# In[228]:

m= Prophet()
m.add_regressor('nfl_sunday')
m.fit(peyton)

# In[231]:

# Regressor must also be available in future df
future['nfl_sunday'] = future['ds'].apply(nfl_sundays)
forecast = m.predict(future)
fig = m.plot_components(forecast)

# In[ ]:

df4_cv= cross_validation(m, horizon= '365 days', period= '180 days', initial= '730 days')

# In[244]:

metrics_4= performance_metrics(df4_cv)
display(metrics_4[['horizon', 'rmse', 'mape']])
print("Adding Holidays: Mean RMSE: {:.4f} Mean MAPE: {:.2%}".format(np.mean(metrics_4['rmse']), np.mean(metrics_4['mape'])))
print("Horizon of 365 days RMSE: {:.3f} and MAPE: {:.2%}".format(metrics_4.iloc[metrics_4.shape[0]-1]['rmse'], metrics_4.iloc[metrics_4.shape[0]-1]['mape']))

# In[245]:

h_1_mape= metrics_4.iloc[metrics_4.shape[0]-1]['mape']
h_1_rmse= metrics_4.iloc[metrics_4.shape[0]-1]['rmse']

# plot_name= 'initial.prophet.model.png'

regressor_plot= m.plot(forecast, figsize= (17,7))

ax = regressor_plot.gca()
ax.set_title("Initial model= Prophet() plus regressor, MAPE: {:.2%} and RMSE: {:.3f}".format(h_1_mape, h_1_rmse))
ax.set_xlabel("Time")
ax.set_ylabel("Wikipedia searches for Peyton (log)")

plt.show()

# ## We can specifiy if there's some change taking place
# For instance COVID

# In[25]:

# These are points where trend has changed
print('originally: ',m.changepoints[:5])

# you can specify changepoints if you want trend to only be allowed at certain points
m_c = Prophet(changepoints=['2014-01-01'])

print('\nnow: ',m_c.changepoints[:5])

# # 5) Combine Holidays and regressor

# In[249]:

m_holidays_regressor= Prophet(holidays= holidays)
fcst_holidays_regressor=  m_holidays_regressor.add_regressor('nfl_sunday').fit(peyton).predict(future)

# In[250]:

holidays_regressor= m_holidays_regressor.plot_components(fcst_holidays_regressor)

# In[ ]:

df5_cv= cross_validation(m_holidays_regressor, horizon= '365 days', period= '180 days', initial= '730 days')

# In[254]:

metrics_5= performance_metrics(df5_cv)
display(metrics_5[['horizon', 'rmse', 'mape']])
print("Adding Holidays: Mean RMSE: {:.4f} Mean MAPE: {:.2%}".format(np.mean(metrics_5['rmse']), np.mean(metrics_5['mape'])))
print("Horizon of 365 days RMSE: {:.3f} and MAPE: {:.2%}".format(metrics_5.iloc[metrics_5.shape[0]-1]['rmse'], metrics_5.iloc[metrics_5.shape[0]-1]['mape']))

# In[257]:

import session_info
session_info.show()

# In[ ]:


from Products.CMFCore.utils import UniqueObject
from OFS.SimpleItem import SimpleItem
from AccessControl import ClassSecurityInfo
from Products.CMFCore.utils import getToolByName
from Globals import InitializeClass
import Missing
from lxml import etree

try:
    from zope.app.component.hooks import getSite
except ImportError:
    from zope.component.hooks import getSite

class SolrCatalogTool(UniqueObject, SimpleItem):

    id = 'solr_catalog'
    meta_type = 'Solr Catalog Tool'

    security = ClassSecurityInfo()

    @property
    def hello(self):
        return "hello"

    def getProperty(self, property_name, default=''):
        site = getSite()

        ptool = getToolByName(site, 'portal_properties')
        props = ptool.get("agsci_solr_properties")

        if props:
            return props.getProperty(property_name, default)
        else:
            return default

    def getSolrURL(self):
        protocol = self.getProperty('solr_protocol', 'http')
        host = self.getProperty('solr_host', '127.0.0.1')
        port = self.getProperty('solr_port', '8080')
        base_url = self.getProperty('solr_base_url', '/solr')

        return "%s://%s:%s%s" % (protocol, host, port, base_url)

    @property
    def siteId(self):
        site = getSite()
        return site.getId()

    def indexObject(self, context):
        metadata = self.getMetaDataFor(context)
        indexdata = self.getIndexDataFor(context)
        return False

    @property
    def portal_catalog(self):
        return getToolByName(getSite(), 'portal_catalog')

    def getMetadataFor(self, context):
        m = self.portal_catalog.getMetadataForUID("/".join(context.getPhysicalPath()))
        for k in m.keys():
            if m[k] == Missing.Value:
                m[k] = ''
        return m

    def getIndexDataFor(self, context):
        return self.portal_catalog.getIndexDataForUID("/".join(context.getPhysicalPath()))

    def getCatalogIndexes(self):
        return self.portal_catalog.indexes()

    def getCatalogMetadata(self):
        return self.portal_catalog.schema()

    def exportSchemaXML(self):
        # <schema name="example" version="1.5">
        schema = etree.Element("schema", name="example", version="1.5")

        # Create unique key (UID)
        unique_key = etree.Element('uniqueKey')
        unique_key.text = 'UID'
        schema.append(unique_key)

        # Get indexes and metadata
        indexes = self.getCatalogIndexes()
        metadata = self.getCatalogMetadata()

        # Add a <field> tag for all (most) indexed fields
        fields = list(set(indexes).union(set(metadata)))

        for f in fields:
            p = self.getIndexProperties(f, indexes=indexes, metadata=metadata)

            if not p.get('skip'):
                el = etree.Element("field",
                                    name=p['name'], type=p['type'],
                                    indexed=repr(p['indexed']).lower(), stored=repr(p['stored']).lower(),
                                    multiValued=repr(p['multiValued']).lower(),
                                    required=repr(p['required']).lower()
                                    )
                schema.append(el)

        # Return XML
        return etree.tostring(schema, pretty_print=True, xml_declaration=True, encoding='UTF-8')

    def getIndexProperties(self, index, indexes=[], metadata=[]):

        required = ['getId', 'UID', 'id']

        p = {
                'skip' : False,
                'type' : 'string',
                'name' : index,
                'indexed' : False,
                'stored' : False,
                'multiValued' : False,
                'required' : False,
        }

        i_obj = self.portal_catalog.Indexes.get(index)

        index_config = {
                        'DateIndex' : ('date', False),
                        'FieldIndex' : ('string', False),
                        'ZCTextIndex' : ('text', False),
                        'BooleanIndex' : ('boolean', False),
                        'KeywordIndex' : ('string', True),
                        'UUIDIndex' : ('string', True),
        }

        if index in ('Tags', ):
            import pdb; pdb.set_trace()

        if i_obj:
            (index_type, multiValued) = index_config.get(i_obj.meta_type, (None,False))

            if not index_type:
                p['skip'] = True
            else:
                p['type'] = index_type
                p['multiValued'] = multiValued
                if index in indexes:
                    p['indexed'] = True
                if index in metadata:
                    p['stored'] = True
                if index in required:
                    p['required'] = True
                if hasattr(i_obj, 'indexed_attrs') and i_obj.indexed_attrs:
                    p['name'] = i_obj.indexed_attrs[0]

        return p

InitializeClass(SolrCatalogTool)


import keras
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D, Flatten, Dense
from keras.preprocessing.image import ImageDataGenerator

# Define the paths to your image repositories
train_path = 'files/train'
validation_path = 'files/validation'
test_path = 'files/test'

# Define the image size and number of classes
img_size = (512, 512)
img_shape = (*img_size, 3)
num_classes = 2

# Create an ImageDataGenerator for data loading and preprocessing
datagen = ImageDataGenerator(rescale=1./255)

# Load and preprocess the training data
train_generator = datagen.flow_from_directory(
    train_path,
    target_size=img_size,
    batch_size=32,
    class_mode='categorical',
    shuffle=True
)

# Load and preprocess the validation data
validation_generator = datagen.flow_from_directory(
    validation_path,
    target_size=img_size,
    batch_size=32,
    class_mode='categorical',
    shuffle=True
)

# Load and preprocess the testing data
test_generator = datagen.flow_from_directory(
    test_path,
    target_size=img_size,
    batch_size=32,
    class_mode='categorical',
    shuffle=True
)

# Define the model
model = Sequential()
model.add(Conv2D(32, (3, 3), activation='relu', input_shape=img_shape))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

model.add(Conv2D(128, (3, 3), activation='relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))

# Flatten the output from previous layers
model.add(Flatten())

# Add a fully connected layer
model.add(Dense(128, activation='relu'))

# Add the output layer with softmax activation for classification
model.add(Dense(num_classes, activation='softmax'))

# Compile the model
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy', keras.metrics.Recall()])

# Train the mode
model.fit(train_generator, epochs=1, validation_data=validation_generator)

# Evaluate the model
print(model.evaluate(test_generator))
loss, accuracy, sensitivity = model.evaluate(test_generator)
print('Test Loss:', loss)
print('Test Accuracy:', accuracy)
print('Test Sensivity:', sensitivity)


#!/usr/bin/env python3
import sys
import os
import time
import argparse
import requests
from datetime import datetime
from urllib.parse import urlparse
from termcolor import colored

def append_http(url, port=None):
    if "https://" in url:
        url = f"{url}"
    elif "http://" not in url:
        o = urlparse(f"http://{url}")
        if o.port:
            url = f"http://{url}"
        else:
            url = f"http://{url}:{port}"
    return url

def get_loopchain_state(ipaddr="localhost", port=os.environ.get('RPC_PORT', 9000)):
    url = append_http(ipaddr, port) + "/admin/chain"
    return_result = {}
    try:
        session = requests.Session()
        r = session.get(url, verify=False, timeout=10)
        return_result = r.json()[0]
        return_result['prev_time'] = time.time()
        return_result['url'] = url
    except Exception as e:
        print(f"Error while connecting server... {url}: {str(e)}")
        return_result = {
            "block_height": 0
        }
    return return_result

def disable_ssl_warnings():
    from urllib3.exceptions import InsecureRequestWarning
    requests.packages.urllib3.disable_warnings(InsecureRequestWarning)

def print_status(count, block_diff, sync_speed, finish_block_height, finish_time):
    status = f"({count})   BlockDiff [{block_diff}]    SyncSpeed [{sync_speed:.0f}]    FinishHeight [{finish_block_height:.0f}]    FinishTime [{finish_time}]"
    if sync_speed == 0:
        return colored(status, 'red')
    return colored(status, 'green')

if __name__ == '__main__':
    disable_ssl_warnings()
    parser = argparse.ArgumentParser(prog='checker')
    parser.add_argument('url', nargs='?', default="localhost:9000")
    parser.add_argument('-n', '--network', type=str, help='parent network type (mainnet|berlin|sejong|lisbon)', default="mainnet")
    parser.add_argument('-v', '--verbose', action='count', help='verbose mode. view level', default=0)
    parser.add_argument('-c', '--count', type=int, help='check count(sec).', default=0)

    args = parser.parse_args()

    network_info = {
        "mainnet": "http://52.196.159.184:9000",
        "berlin": "https://berlin.net.solidwallet.io",
        "sejong": "https://sejong.net.solidwallet.io",
        "lisbon": "https://lisbon.net.solidwallet.io",
        "havah": "https://ctz.havah.io",
        "vega": "https://ctz.vega.havah.io",
    }

    if os.environ.get('SERVICE', None):
        args.network = os.environ['SERVICE']
    elif os.environ.get('NETWORK_ENV', None):
        args.network = os.environ['NETWORK_ENV']

    if args.network in network_info.keys():
        parent_network_url = network_info.get(args.network)
    else:
        parent_network_url = args.network

    block_diff = 0
    block_diff2 = 0
    network_block_height = 0
    first_block_height = 0
    second_block_height = 0
    finish_block_height = 0
    prev_time = 0
    sync_speed = 0
    height = 0
    count = 0

    print(f"Target peer: {args.url}")
    if args.verbose > 0:
        print(f"Get parent status from {parent_network_url}")

    first_block_height, first_prevtime = get_loopchain_state(args.url)["height"], get_loopchain_state(args.url)["prev_time"]

    try:
        while True:
            count += 1
            second_block_height, second_prevtime = get_loopchain_state(args.url)["height"], get_loopchain_state(args.url)["prev_time"]
            network_block_height, prevtime = get_loopchain_state(parent_network_url)["height"], get_loopchain_state(parent_network_url)["prev_time"]

            block_diff = network_block_height - second_block_height
            block_diff2 = second_block_height - first_block_height
            sync_speed = block_diff2 / (second_prevtime - first_prevtime) if first_prevtime != 0 and second_prevtime != first_prevtime else 0
            finish_block_height = (block_diff / sync_speed / 2 + network_block_height) if sync_speed != 0 else float('inf')
            finish_time = datetime.fromtimestamp(int(((finish_block_height - second_block_height) / sync_speed) + time.time())) if sync_speed != 0 and finish_block_height != float('inf') else 'N/A'

            status = print_status(count, block_diff, sync_speed, finish_block_height, finish_time)
            sys.stdout.write('\r' + status)
            sys.stdout.flush()

            if count == args.count:
                break

            time.sleep(1)
    except KeyboardInterrupt:
        print("\nProgram interrupted. Exiting...")
        sys.exit(0)


import requests
from colorama import Fore
from .colorizer import colorize_error
from .exceptions import *

class Requester:
    def __init__(self) -> None:
        pass

    def request(
        self,
        url: str,
        params: dict = None,
        files: dict = None,
        proxies: dict = None,
        method: str = 'GET',
        data: dict = None,
        headers: dict = None,
        timeout: float = None,
        auth: tuple = None,
        cookies: dict = None,
        verify: bool = True,
        cert: str = None,
        allow_redirects: bool = True,
        nocolor=False,
    ) -> requests.Response:
        try:
            response = requests.request(
                method,
                url,
                params=params,
                files=files,
                proxies=proxies,
                data=data,
                headers=headers,
                timeout=timeout,
                auth=auth,
                cookies=cookies,
                verify=verify,
                cert=cert,
                allow_redirects=allow_redirects
            )

            response.raise_for_status()

            return response

        except requests.exceptions.HTTPError as errh:
            err_text = f"HTTP Error: {errh}"
            raise HttpRequestError(colorize_error(err_text) if not nocolor else err_text) from errh
        except requests.exceptions.ConnectionError as errc:
            err_text = f"Error Connecting: {errc}"
            raise CliRequestsError(colorize_error(err_text) if not nocolor else err_text) from errc
        except requests.exceptions.Timeout as errt:
            err_text = f"Timeout Error: {errt}"
            raise CliRequestsError(colorize_error(err_text) if not nocolor else err_text) from errt
        except requests.exceptions.RequestException as err:
            err_text = f"Error: {err}"
            raise CliRequestsError(colorize_error(err_text) if not nocolor else err_text) from err


from torch.utils.data import Dataset
import numpy as np

class AdvImgDataset(Dataset):
    def __init__(self, path, transform=None):
        self.transform = transform

        self.data = np.load(path + "/advimg_images.npy")
        self.labels = np.load(path + "/advimg_labels.npy")

    def __len__(self):
        return self.labels.size

    def __getitem__(self, idx):
        image = self.data[idx, :, :, :]
        label = self.labels[idx]

        if self.transform is not None:
            image = self.transform(image)

        return image, label


import os
from pyrogram import Client, filters
from pyrogram.types import *

from DewmiBot.config import get_str_key
from DewmiBot import pbot

REPO_TEXT = "**A Powerful and Simple Group Manager Bot To Manage Your Groups Easily ! \n\n↼ Øwñêr ⇀ : 『 PSJ 』\n╭──────────────\n┣─ » Python ~ 3.8.6\n┣─ » Update ~ Resently\n╰──────────────\n\n»»» @Aurora_x_Bot «««"

BUTTONS = InlineKeyboardMarkup(
      [[
        InlineKeyboardButton("Repository", url=f"https://github.com/ParkseoJoon2005/Aurora"),
        InlineKeyboardButton("Video info ", url=f"https://www.youtube.com/channel/UCvYfJcTr8RY72dIapzMqFQA"),
      ],[
        InlineKeyboardButton("𝑺𝒍 𝑩𝒐𝒕 𝒁𝒐𝒏𝒆 ", url="https://t.me/Hermione_Updates"),
        InlineKeyboardButton("𝓢𝓛 𝓑𝓸𝓽 𝓒𝓱𝓪𝓽", url="https://t.me/dihan_official"),
      ],[
        InlineKeyboardButton("Aurora Bot update info", url="https://t.me/HermioneSupport_Official"),
        InlineKeyboardButton("Developer", url="https://t.me/Boy_Alone_In_Universe"),
      ]]
    )

@pbot.on_message(filters.command(["repo"]))
async def repo(pbot, update):
    await update.reply_text(
        text=REPO_TEXT,
        reply_markup=BUTTONS,
        disable_web_page_preview=True,
        quote=True
    )


import re
import argparse

parser = argparse.ArgumentParser(description='Markdown TOC automated generation')
parser.add_argument('--input_path', '-i',
                    type=str,
                    required=True,
                    help='the markdown input file path')

parser.add_argument('--output_path', '-o',
                    type=str,
                    required=False,
                    help='the output markdown file path')

parser.add_argument('--update_md', '-u',
                    action='store_const',
                    const=True,
                    default=False,
                    required=False,
                    help='if used, updates the original markdown file')

parser.add_argument('--output_screen', '-p',
                    action='store_const',
                    const=True,
                    default=False,
                    required=False,
                    help='if used, outputs the TOC on the terminal')

parser.add_argument('--encoding', '-e',
                    type=str,
                    default='utf-8',
                    required=False,
                    help='encoding to be used for read and write files')

parser.add_argument('--toc_header',
                    type=str,
                    default='# Table of Contents',
                    required=False,
                    help='header to be displayed on the TOC')

args = parser.parse_args()
HEADINGS_REGEX = r'^(#+)\s+(.*)$'

def read_file(input_path, encoding):
    with open(file=input_path, mode='r', encoding=encoding) as f:
        text = f.read()
    return text

def write_file(output_path, content, encoding):
    with open(file=output_path, mode='w', encoding=encoding) as f:
        f.write(content)

def build_toc(headings):
    toc = f'{args.toc_header}\n'
    for level, heading in headings:
        identation = '  '*(len(level)-1) + '-'

        heading_index = heading.lower()
        heading_index = heading_index.replace(' ', '-') \
                                     .replace('.', '') \
                                     .replace(',', '') \
                                     .replace(':', '') \
                                     .replace(';', '') \
                                     .replace('!', '') \
                                     .replace('?', '')

        toc += f'{identation} [{heading}](#{heading_index})\n'
    return toc

if __name__ == '__main__':
    print(args)

    text = read_file(args.input_path, args.encoding)
    headings = re.findall(HEADINGS_REGEX, text, flags=re.MULTILINE)
    toc = build_toc(headings)

    output_options = {
        args.output_path: {
            'output_path': args.output_path,
            'content': toc,
            'encoding': args.encoding
        },
        args.update_md: {
            'output_path': args.input_path,
            'content': toc+'\n'+text,
            'encoding': args.encoding
        }
    }

    if args.output_path:
        write_file(**output_options.get(args.output_path))

    if args.update_md:
        write_file(**output_options.get(args.update_md))

    if args.output_screen:
        print('\n' + toc)


import requests
import hashlib
import csv
import sqlite3
import datetime
import time
import io

DATABASE = 'data.db'
LINKLIST = 'link_list.csv'

def get_url_list(file_path):
    with open(file_path, 'r', encoding='utf-8') as file:
        url_list = csv.reader(file, delimiter=',')
        return [row for row in url_list]

def get_url_content(url):
    return requests.get(url,).text.replace("'", "''") # удваиваю одинарную кавычку, что бы можно было вставить в БД

def apply_sql_query(query):
    connection = sqlite3.connect(DATABASE).cursor().executescript(query)
    connection.close()

def create_db():
    db_create_query = '''CREATE TABLE url(url text, content text, date text)'''
    apply_sql_query(db_create_query)

def add_content_to_db(url_list):
    for url in url_list:
        url_content = get_url_content(url[0])
        time.sleep(1)
        content_inser_query = '''insert into content_tab(url, content, date) VALUES('{0}', '{1}', '{2}')'''.format(
            str(url[0]), url_content, str(int(datetime.datetime.now().timestamp())))
        apply_sql_query(content_inser_query)

add_content_to_db(get_url_list(LINKLIST))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Apr 29 20:19:18 2020
Example to run survival model.
@author: wouterpotters
"""
import configparser
import matplotlib
import sys
import os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), './../'))
from covid19_ICU_admission import load_data, preprocess, prepare_for_learning
from get_feature_set import get_1_premorbid, \
                            get_2_clinical_presentation, \
                            get_3_laboratory_radiology_findings, \
                            get_5_premorbid_clin_rep_lab_rad

import matplotlib.pyplot as plt
from lifelines import *
import pandas as pd
import numpy as np
from sklearn.experimental import enable_iterative_imputer  # Required to enable experimental iterative imputer
from sklearn.impute import SimpleImputer, IterativeImputer
from sklearn.preprocessing import StandardScaler, RobustScaler

plt.rcParams["font.family"] = "Times New Roman"
matplotlib.rcParams.update({'font.size': 14})

c1 = ( 57/255, 106/255, 177/255)  # blue
c2 = (218/255, 124/255,  48/255)  # orange
c3 = ( 83/255,  81/255,  84/255)  # black
c4 = ( 62/255, 150/255,  81/255)  # green
c5 = (204/255,  37/255,  41/255)  # red

englishvar_to_dutchnames = {'age_yrs':'Leeftijd (jaren)','gender_male':'Geslacht (man)','gender':'Geslacht (man)','hypertension':'Hypertensie (ja)','Coagulation_disorder_1_1':'Stollingsstoornis (ja)','autoimm_disorder_1':'Auto-immuunziekte (ja)','rheuma_disorder':'Reumatologische ziekte (ja)','organ_transplant_1':'Orgaan donatie ondergaan (ja)',
                            'diabetes_without_complications':'DM (zonder complicaties) (ja)','diabetes_complications':'DM (met complicaties) (ja)','ccd':'Chronische hartziekte (ja)','ckd':'Chronische nierziekte (ja)','cnd':'Chronische neurologische aandoening (ja)','cpd':'chronische longziekte (ja)','asthma':'Asthma (ja)','Dementia':'dementie (ja)',
                            'mld':'Milde leveraandoening (ja)','live_disease':'Matig-ernstige leveraandoening (ja)','mneoplasm':'Maligniteit (ja)','chd':'Chronische hematologische ziekte (ja)','alcohol':'Alcohol abusus (ja)','Smoking in History or Active':'Roken actief of in verleden (ja)','obesity':'Obesitas (BMI > 30) (ja)',
                            'home_medication':'Thuismedicatie (ja)','immune_sup':'Immunosupressiva (ja)','ace_i_spec_1':'Gebruik ACE inhibitor (ja)','Corticosteroid_type_1':'Corticosteroiden gebruik (ja)','sys_bp':'Systolische bloeddruk (mmHg)','dias_bp':'Diastolische bloeddruk (mmHg)','HtR':'hart frequentie (1/min)','rtr':'Ademhalingsfrequentie (1/min)',
                            'oxygen_saturation':'Saturatie (%)','Temperature':'Temperatuur (ºC)','Antiviral_agent_1':'Behandeling met antivirale middelen (ja)','Oxygen_therapy_1':'zuurstof therapie (ja)','Non_invasive_ventilation_1':'non-invasieve beademing (ja)','Invasive_ventilation_1':'Invasieve beademing (ja)','resucitate':'Niet reanimeren (ja)',
                            'intubate_yesno':'Niet intuberen (ja)','CT_thorax_performed':'CT thorax verricht op SEH (ja)','corads_admission':'CO-RAD score  (1-5)','pcr_pos':'Corona virus PCR + op SEH (ja)','Coagulation_disorder_1_1':'Stollingsstoornis (ja)',
                            'Acute_renal_injury_Acute_renal_failure_1_1':'Nierfunctie stoornis waarvoor dialyse (ja)','diabetes_complications':'Diabetes met complicaties (ja)','cpd':'chronische longziekte (ja)','Smoking':'Actieve roker (ja)','obesity':'Obesitas (ja)','immune_sup':'immunosupressiva (ja)','ace_i_spec_1':'Gebruik ACE remmer (ja)',
                            'sys_bp':'Systolische bloeddruk (mmHg)','dias_bp':'Diastolische bloeddruk (mmHg)','HtR':'hart frequentie (1/min)','capillary_refill':'Verlengde capillaire refill (ja)','rtr':'Ademhalingsfrequentie (1/min)','oxygen_saturation':'Zuurstof saturatie (%)','SaO2_1':'SaO2 (%)','PaO2_1_Arterial':'PaO2 arteriëel (kPa)',
                            'PaO2_1_Venous':'PaO2 veneus (kPa)', 'PaO2_1_Capillary':'PaO2 capillair (kPa)', 'PaO2_1_nan':'PaO2 onbekend (kPa)', 'PCO2_1':'pCO2 (kPa)','fio2_1':'FiO2 (%)','Temperature':'Temperatuur (ºC)','eye':'EMV','Antiviral_agent_1':'Anti-virale middelen (ja)','Antibiotic_1':'Antibiotica (ja)',
                            'Corticosteroid_1':'Corticosteroiden voor ARDS (ja)','Corticosteroid_type_1':'gebruik corticosteroiden (ja)','Oxygen_therapy_1':'Zuurstof behandeling (ja)',
                            'Non_invasive_ventilation_1':'Niet-invasieve beademing (ja)','Invasive_ventilation_1':'Invasieve beademing (ja)','resucitate':'Niet reanimeren (ja)','intubate_yesno':'Niet intuberen (ja)','auxiliary_breathing_muscles':'Gebruik van extra ademhalingsspieren (ja)','fever':'Koorts gehad sinds eerste klachten (ja)',
                            'Anosmia':'Anosmie (ja)','Rhinorrhoea':'Rhinorrhoea (ja)','Sore_throat':'Keelpijn (ja)','cough_sputum':'Hoest met sputum (ja)','cough_sputum_haemoptysis':'Hoest met bloederig slijm/ haemoptoë (ja)','Arthralgia':'Arthralgie (ja)','Myalgia':'Spierpijn (ja)','Fatigue_Malaise':'Vermoeidheid/ algehele malaise (ja)',
                            'Abdominal_pain':'Buikpijn (ja)','Vomiting_Nausea':'Misselijkheid/ overgeven (ja)','Diarrhoea':'diarree (ja)','Dyspnea':'Dyspneu (ja)','Wheezing':'Piepende ademhaling (ja)','Chest_pain':'Pijn op de borst (ja)','ear_pain':'oorpijn (ja)','Bleeding_Haemorrhage':'bloeding (ja)','Headache':'Hoofdpijn (ja)',
                            'confusion':'Veranderd bewustzijn (ja)','Seizures':'Insulten (ja)','infiltrates_2':'Infiltraat op X-thorax (ja)','corads_admission':'CO-RADS (1-5)','Glucose_unit_1_1':'glucose (mmol/L)','Sodium_1_1':'Natrium (mmol/L)','Potassium_1_1':'Kalium (mmol/L)','Blood_Urea_Nitrogen_value_1':'Ureum (mmol/L)',
                            'Creatinine_value_1':'Creatinine (µmol/L)','Calcium_adm':'Ca (totaal) (mmol/L)','ferritine_admin_spec':'ferritine (mg/L)','creatininekinase':'CK (U/L)','d_dimer':'D-Dimeer (nmol/L)','AST_SGOT_1_1':'ASAT (U/L)','ALT_SGPT_1_1':'ALAT (U/L)','Total_Bilirubin_2_1':'totaal Bilirubine (IE)','LDH':'LDH (U/L)',
                            'PH_value_1':'Zuurgraad (pH)','Lactate_2_1':'Lactaat (mmol/L)','crp_1_1':'CRP (mg/L)','Haemoglobin_value_1':'Hb (mmol/L)','Platelets_value_1':'trombocyten (x10^9/L)','WBC_2_1':'Leukocyten (x10^3/µL)','Lymphocyte_1_1':'Lymfocyten (x10^9/L)','Neutrophil_unit_1':'Neutrofielen (x10^9/L)','INR_1_1':'INR',
                            'pt_spec':'PT (sec)','fibrinogen_admin':'Fibrinogeen (g/L)','pcr_pos':'Corona PCR + (ja)','Adenovirus':'Adenovirus (ja)','RSV_':'RS virus (ja)','Influenza':'Influenza virus (ja)','Bacteria':'Sputumkweek (ja)','days_untreated':'Tijd sinds eerste klachten (dagen)','irregular':'Onregelmatige hartslag (ja)',
                            'DNR_yesno':' (ja)','healthcare_worker':'Zorgmedewerker (ja)','infec_resp_diagnosis':'Andere infectieuze ademhalingsdiagnose','Blood_albumin_value_1':'Albumine (g/L)','culture':'Bloedkweek (positief)','APT_APTR_1_1':'APTT (sec)','uses_n_medicine':'Aantal thuismedicamenten'}

class StandardScaler_min2cols(StandardScaler):
    def __init__(self,columns,copy=True, with_mean=False, with_std=True):
        self.scaler = StandardScaler(copy, with_mean=with_mean, with_std=with_std)
        self.columns = columns

    def fit(self, X, y=None):
        self.scaler.fit(X[self.columns], y)
        return self

    def transform(self, X, y=None, copy=None):
        init_col_order = X.columns
        X_scaled = pd.DataFrame(self.scaler.transform(X[self.columns]), columns=self.columns)
        X_not_scaled = X.iloc[:, ~X.columns.isin(self.columns)]
        return pd.concat([X_not_scaled, X_scaled], axis=1)[init_col_order]

class RobustScaler_min2cols(RobustScaler):
    def __init__(self, columns, copy=True, with_centering=True,
                 with_scaling=True, quantile_range=(25.0, 75.0)):
        self.scaler = RobustScaler(with_centering=with_centering,
                                   with_scaling=with_scaling,
                                   quantile_range=quantile_range,
                                   copy=copy)
        self.columns = columns

    def fit(self, X, y=None):
        self.scaler.fit(X[self.columns], y)
        return self

    def transform(self, X, y=None, copy=None):
        init_col_order = X.columns
        X_scaled = pd.DataFrame(self.scaler.transform(X[self.columns]), columns=self.columns)
        X_not_scaled = X.iloc[:, ~X.columns.isin(self.columns)]
        return pd.concat([X_not_scaled, X_scaled], axis=1)[init_col_order]

def simple_estimates(features):
    T = features['Time to event']
    E = features['Event death']
    fig, axes = plt.subplots(1, 1)

    kmf = KaplanMeierFitter().fit(T, E, label='KaplanMeierFitter')
    kmf.plot_survival_function()
    axes.set_xlim(0, 21)
    axes.set_ylim(0, 1)

    axes.set_xlabel('Tijd in dagen')
    axes.set_ylabel('Cumulatieve survival')
    plt.savefig('simple_estimate.png')
    plt.show()

def KM_age_groups(features):
    T = features['Time to event']
    E = features['Event death']

    #age_70_plus = features['age_yrs'] >= 70.
    age_70_plus = features['Leeftijd (jaren)'] >= 70.
    print('Alle patiënten: n=', len(age_70_plus))
    print('Leeftijd ≥ 70: n=', sum(age_70_plus))
    print('Leeftijd < 70: n=', sum(~age_70_plus))

    was_not_on_icu = data[['Levend ontslagen en niet heropgenomen - waarvan niet opgenomen geweest op IC',
                           'Levend dag 21 maar nog in het ziekenhuis - niet op IC geweest',
                           'Dood op dag 21 - niet op IC geweest']].any(axis=1)
    was_on_icu = data[['Levend ontslagen en niet heropgenomen - waarvan opgenomen geweest op IC',
                       'Levend dag 21 maar nog in het ziekenhuis - op IC geweest',
                       'Levend dag 21 maar nog in het ziekenhuis - waarvan nu nog op IC',
                       'Dood op dag 21 - op IC geweest']].any(axis=1)
    print('Alle patiënten: n=', (np.nansum(was_on_icu)+np.nansum(was_not_on_icu)))
    print('ICU yes: n=', np.nansum(was_on_icu))
    print('ICU no: n=', np.nansum(was_not_on_icu))

    icu = False
    special = False
    fig, axes = plt.subplots(1, 1)

    if special:
        kmf1 = KaplanMeierFitter().fit(T[features['Leeftijd (jaren)'] <= 47],
                                       E[features['Leeftijd (jaren)'] <= 47], label='Leeftijd <= 47 jaar (n=260)')
        kmf2 = KaplanMeierFitter().fit(T[np.logical_and(features['Leeftijd (jaren)'] > 47, features['Leeftijd (jaren)'] <= 61)],
                                       E[np.logical_and(features['Leeftijd (jaren)'] > 47, features['Leeftijd (jaren)'] <= 61)], label='Leeftijd > 47 jaar, <= 61 (n=260)')
        kmf3 = KaplanMeierFitter().fit(T[features['Leeftijd (jaren)'] > 61],
                                       E[features['Leeftijd (jaren)'] > 61], label='Leeftijd > 61 jaar (n=876)')
        kmf1.plot_survival_function(color=c5)
        kmf2.plot_survival_function(color=c4)
        kmf3.plot_survival_function(color=c3)
    else:
        if icu:
            kmf1 = KaplanMeierFitter().fit(T[was_on_icu], E[was_on_icu], label='Wel op ICU geweest')
            kmf2 = KaplanMeierFitter().fit(T[was_not_on_icu], E[was_not_on_icu], label='Niet op ICU geweest')
        else: # age 70
            kmf1 = KaplanMeierFitter().fit(T[age_70_plus], E[age_70_plus], label='Leeftijd ≥ 70 jaar')
            kmf2 = KaplanMeierFitter().fit(T[~age_70_plus], E[~age_70_plus], label='Leeftijd < 70 jaar')

        kmf3 = KaplanMeierFitter().fit(T, E, label='Alle patienten')
        if icu:
            kmf1.plot_survival_function(color=c4)
            kmf2.plot_survival_function(color=c5)
            kmf3.plot_survival_function(color=c3)
        else:
            kmf1.plot_survival_function(color=c1)
            kmf2.plot_survival_function(color=c2)
            kmf3.plot_survival_function(color=c3)

    axes.set_xticks([1, 5, 9, 13, 17, 21])
    axes.set_xticklabels(['1','5','9','13','17','21'])
    axes.set_xlabel('Aantal dagen sinds opnamedag')
    axes.set_ylabel('Proportie overlevend')

    axes.set_xlim(0, 21)
    axes.set_ylim(0, 1)
    titledict = {'fontsize': 18,
                 'fontweight': 'bold',
                 'verticalalignment': 'baseline',
                 'horizontalalignment': 'center'}
    # plt.title('COVID-PREDICT survival functie tot t = 21 dagen (n='+str(len(T))+')',fontdict=titledict)
    plt.tight_layout()
    if icu:
        plt.savefig('KM_survival_curve_ICU.png', format='png', dpi=300, figsize=(20,20), pad_inches=0, bbox_inches='tight')
    else:
        plt.savefig('KM_survival_curve_DEATH.png', format='png', dpi=300, figsize=(20,20), pad_inches=0, bbox_inches='tight')
    plt.show()

    return kmf1, kmf2, kmf3

def cox_ph_loho(features, hazard_ratios=False,
                imputation_method='median',
                minimal_n=10,
                pvalue=0.05):
    cindex = []
    n = []
    for test_hospital in features['hospital'].unique():
        print(test_hospital)
        train_set = features[features['hospital'] != test_hospital]
        test_set = features[features['hospital'] == test_hospital]

        if len(test_set) >= minimal_n:
            test_set = test_set.drop(columns=['hospital'])

            cph, p, imputer, scaler = cox_ph(train_set, hazard_ratios=False,
                                             imputation_method=imputation_method,
                                             return_scaler_imputer=True)

            test_set = pd.DataFrame(imputer.transform(test_set), columns=test_set.columns)
            test_set = scaler.transform(test_set)

            hazard_ratios_sum = cph.summary
            print(hazard_ratios_sum)
            sel_corr = hazard_ratios_sum[hazard_ratios_sum['p'] <= pvalue]
            c = cph.score(test_set, scoring_method="concordance_index")
            print('test hospital:', test_hospital, 'test c-index', c)
            print(sel_corr)
            cindex += [c]
            n += [len(test_set)]
        else:
            print('test hospital:', test_hospital, ' has <= ',minimal_n,' records. Cannot determine concordance.')
    return cindex, n

def cox_ph(features, hazard_ratios=False,
           imputation_method='median', return_scaler_imputer=False,
           show_progress=True,
           penalizer=0.,
           l1_ratio=0.):
    train_set = features.drop(columns=['hospital']) # NOTE aids_hiv is an outlier for plotting coefs

    # impute nan with median; note that time to event and event death cannot contain nan
    if imputation_method == 'iterative':
        imputer = IterativeImputer(missing_values=np.nan,
                                   initial_strategy='median')
    else:
        imputer = SimpleImputer(missing_values=np.nan, strategy=imputation_method)
    train_set = pd.DataFrame(imputer.fit_transform(train_set), columns=train_set.columns)

    # scale data using zscore
    scaler = RobustScaler_min2cols(columns=train_set.columns)
    train_set = scaler.fit_transform(train_set)

    cph = CoxPHFitter(penalizer=penalizer, l1_ratio=l1_ratio)
    cph.fit(train_set, duration_col='Time to event', event_col='Event death', show_progress=show_progress, step_size=0.1)

    if True:
        cph.print_summary()

        fig, ax = plt.subplots(figsize=(5, 25))
        p = cph.plot(hazard_ratios=hazard_ratios)
    else:
        p = None

    if return_scaler_imputer:
        return cph, p, imputer, scaler
    else:
        return cph, p

def cox_plot_pvalue_only(cph, pvalue, hazard_ratios=False):
    hazard_ratios_sum = cph.summary
    hazard_ratios_sum.sort_values('coef', ascending=False, inplace=True)
    sel_corr = hazard_ratios_sum[hazard_ratios_sum['p'] <= pvalue]
    cphplt = cph.plot(sel_corr.index.to_list(), hazard_ratios=hazard_ratios)
    return cphplt

if __name__ == '__main__':
    config = configparser.ConfigParser()
    config.read('../user_settings.ini')
    path_creds = config['CastorCredentials']['local_private_path']

    # %% STEP 1: GET THE DATA USING MAARTEN'S FUNCTIONS
    # For more info: please check covid19_ICU_util.py:select_x_y()
    goal = ['survival', 'mortality_all']

    # Add all 'Field Variable Name' from data_struct to
    # INCLUDE variables from analysis
    #  NOTE: See get_feature_set.py for preset selections
    variables_to_include = {
        'Form Collection Name': [],  # groups
        'Form Name':            [],  # variable subgroups
        'Field Variable Name': get_5_premorbid_clin_rep_lab_rad() # single variables
    }

    # Add all 'Field Variable Name' from data_struct to
    # EXCLUDE variables from analysis, because <10 yes or no answers are present.
    variables_to_exclude = ['Cardiac_consultation','Meningitis_Encephalitis_1_1','CAR_MH_AF','susp_pe','FH_Cor','CP_oedema','Seizure_1_1','MH_cardiacdevice','ear_pain','MH_SVT_AF_1','CAR_T0_PALP','Inhaled_Nitric_Oxide_1','MH_SVT_AFL','Outcome_7d_hosp','Endocarditis_Myocarditis_Pericarditis_1_1','MH_SVT_AF','Adenovirus','VAP','pregyn_rptestcd','carmed_ami','CAR_CM_heartfailure','CP_ortho','vasc_lipidlowering','discharge_AKI','MH_CABG','carmed_cipro','microbiology_worker','discharge_ards','post_partum','MH_HF','CAR_PM','cvrisk_famhist','dx_his_acq_LQT_1','CAR_MH_aLQT','MH_ACS','discharge_CT','vasc_creat','same_id','Med_New_QT_Prol','carmed_arb_chronic','CP_sync','Adm_CT','MH_SVT_AFL_1','CRF_QT','echo_FU','MH_PCI','CAR_T0_PE','ECG_adm','aids_hiv','discharge_swap','CP_PE','MH_ischemia_det','vasc_antifxa_measured','ECG_adm_ST','carmed_cita','corona_ieorres','Pregnancy']
    variables_to_exclude += ['Coronavirus']  # duplicate of PCR information
    data, data_struct = load_data(path_creds)
    data, data_struct = preprocess(data, data_struct)
    # %%
    features, outcomes, data, hospital, record_id, days_until_death = prepare_for_learning(data, data_struct,
                                                                          variables_to_include,
                                                                          variables_to_exclude,
                                                                          goal,
                                                                          remove_records_threshold_above=1.0,
                                                                          remove_features_threshold_above=0.5,
                                                                          pcr_corona_confirmed_only=False)
    del outcomes  # not relevant in this form. Use features and hospital.

    # %% STEP 2: CALCULATE and CORRECT ALL TIMINGS.
    times = data['Days until death'].copy()  # TIME THAT PEOPLE DIED
    events = data['Dood - totaal'].copy()  # patient that died. TRUE FALSE

    # now add the people that are ALIVE DISCHARGED. At 21 days.
    times.loc[data['Levend ontslagen en niet heropgenomen - totaal']] = 21.  # or 365   #  discharged alive - ALL 3
    events.loc[data['Levend ontslagen en niet heropgenomen - totaal']] = False  # censor at t=21 days

    times.loc[data['Levend dag 21 maar nog in het ziekenhuis - totaal']] = 21.
    events.loc[data['Levend dag 21 maar nog in het ziekenhuis - totaal']] = False

    # now add the people that are UNKNOWN at 21 days.
    times.loc[data['Onbekend (alle patiënten zonder outcome)']] = data['days_since_admission_current_hosp'][data['Onbekend (alle patiënten zonder outcome)']]  # Onbekend (alle patiënten zonder outcome) == UNKNOWN outcome
    events.loc[data['Onbekend (alle patiënten zonder outcome)']] = False  # censor at time of event.

    # now mark the people that died after 21 days as alive at 21 days.
    times.loc[data['Days until death'] > 21.] = 21.
    events.loc[data['Days until death'] > 21.] = False  # censor at time of event.

    # fix remaining issues with events
    sel = np.logical_and(times.isna(),data['Outcome_cat_4'] == 1) # transfer
    events[sel] = np.nan
    sel = np.logical_and(times.isna(),data['Outcome_cat_8'] == 1) # death
    events[sel] = False  # no information on timing present, hence state as alive at t=0
    times[sel] = 0.
    sel = np.logical_and(times.isna(),data['Outcome_cat_10'] == 1) # discharged readmitted
    events[sel] = np.nan

    # remaining data: does not have any outcome or follow-up: we only now these patients are alive at t=0.
    times[times.isna()] = 0.0
    times[events.isna()] = np.nan

    features = pd.concat([features,
                          pd.Series(name='Time to event',
                                    data=times),
                          pd.Series(name='Event death',
                                    data=events),
                         pd.Series(name='hospital',
                                    data=hospital)], axis=1)

    # pd.Series(name='Record Id',data=record_id)[[d not in ['VieCuri'] for d in data['hospital']]]

    # drop hospitals without enough data
    # data.drop(index=features.index[[d in ['VieCuri'] for d in features['hospital']]], inplace=True)
    # features.drop(index=features.index[[d in ['VieCuri'] for d in features['hospital']]], inplace=True)

    # now drop the people that have no information regarding durations (very new records).
    # now drop n/a's in time to event; these are incomplete records that cannot be used.
    data.drop(index=features.index[features['Time to event'].isna()], inplace=True)
    features.drop(index=features.index[features['Time to event'].isna()], inplace=True)

    # translate feature columns
    features.rename(columns=englishvar_to_dutchnames, inplace=True)

    # do not model data with > 50% missing data
    drop_vars = features.columns[np.sum(features.isnull())/len(features) > 0.5]
    features = features.drop(columns=drop_vars)

    # %% STEP 3: SHOW KM CURVE
    # simple_estimates(features)  # events: True = death, False = censor, None = ?
    kmf =  KM_age_groups(features)

    # %% step 5: valdiate concordance with LOHO
    imputation_method = 'iterative'
    hazard_ratios = False
    cindex, testsetn = cox_ph_loho(features, hazard_ratios=hazard_ratios,
                         imputation_method=imputation_method,
                         minimal_n=25)
    print('concordances (mean: {}) = {}'.format(round(np.mean(cindex),3), [round(c,3) for c in cindex]))

    # %% STEP 4: COX REGRESSION.
    # USE HERE: features, hospital, times, events
    cph, plot = cox_ph(features, hazard_ratios=True,
                       imputation_method=imputation_method)
    plot.figure.savefig('Cox_coef_all.png', format='png', dpi=300, figsize=(3,23), pad_inches=0, bbox_inches='tight')

    # %%
    cphplt = cox_plot_pvalue_only(cph, 0.05, hazard_ratios=True)

    # run on command line:
    cphplt.figure.set_figheight(6)
    cphplt.figure.savefig('Cox_coef_pvalue_.05.png', format='png', dpi=300, figsize=(7,9), pad_inches=0, bbox_inches='tight')


import cv2 as open_cv
import numpy as np
import logging
from drawing_utils import draw_contours
from colors import COLOR_GREEN, COLOR_WHITE, COLOR_BLUE

class MotionDetector:
    LAPLACIAN = 2
    DETECT_DELAY = 1
    # Initializing object variables
    def __init__(self, video, coordinates, start_frame):
        self.video = video
        self.coordinates_data = coordinates
        self.start_frame = start_frame
        self.contours = []
        self.bounds = []
        self.mask = []

    def detect_motion(self):
        capture = open_cv.VideoCapture(self.video)
        capture.set(open_cv.CAP_PROP_POS_FRAMES, self.start_frame)

        coordinates_data = self.coordinates_data
        logging.debug("coordinates data: %s", coordinates_data)

        # Printing the coordinates on video
        for p in coordinates_data:
            coordinates = self._coordinates(p)   # getting coordinates sent to the object
            logging.debug("coordinates: %s", coordinates)

            rect = open_cv.boundingRect(coordinates)
            logging.debug("rect: %s", rect)

            new_coordinates = coordinates.copy()
            new_coordinates[:, 0] = coordinates[:, 0] - rect[0]  # x coordinate(column) of all points(rows)
            new_coordinates[:, 1] = coordinates[:, 1] - rect[1]  # y coordinate(column) of all points(rows)
            logging.debug("new_coordinates: %s", new_coordinates)

            self.contours.append(coordinates)
            self.bounds.append(rect)   # saving coordinates of rectangle around the car

            # Making Image for the zero matrix for selected car parking area
            mask = open_cv.drawContours(
                np.zeros((rect[3], rect[2]), dtype=np.uint16),  #
                [new_coordinates],
                contourIdx=-1,  # Drawing all contours
                color=255,      # Line styling below this
                thickness=-1,
                lineType=open_cv.LINE_8)

            mask = mask == 255
            self.mask.append(mask)
            logging.debug("mask: %s", self.mask)

        statuses = [False] * len(coordinates_data)   # Initially setting status of all parking to empty
        times = [None] * len(coordinates_data)     # Array to store timeframes for each frame

        # Detecting the car parking
        while capture.isOpened():
            result, frame = capture.read()
            if frame is None:     # Checking if the vidieo feed is still on
                break

            if not result:
                raise CaptureReadError("Error reading video capture on frame %s" % str(frame))

            # Bluring the image and Greying
            blurred = open_cv.GaussianBlur(frame.copy(), (5, 5), 3)
            grayed = open_cv.cvtColor(blurred, open_cv.COLOR_BGR2GRAY)
            # edges = open_cv.Canny(grayed,threshold1=50,threshold2=150,apertureSize=3)

            new_frame = frame.copy()
            logging.debug("new_frame: %s", new_frame)

            position_in_seconds = capture.get(open_cv.CAP_PROP_POS_MSEC) / 1000.0

            for index, c in enumerate(coordinates_data):   # loops (index, coordinate) on the coor. data
                status = self.__apply(grayed, index, c)    # Finding if car is in parking

                # If nothing is changed don't add timestamp
                if times[index] is not None and self.same_status(statuses, index, status):
                    times[index] = None
                    continue

                # Check if status has changed after last change in status
                if times[index] is not None and self.status_changed(statuses, index, status):
                    if position_in_seconds - times[index] >= MotionDetector.DETECT_DELAY:
                        statuses[index] = status
                        times[index] = None
                    continue

                # Add time when the status of parking is changed
                if times[index] is None and self.status_changed(statuses, index, status):
                    times[index] = position_in_seconds

            # Coloring the contours
            for index, p in enumerate(coordinates_data):
                coordinates = self._coordinates(p)

                color = COLOR_GREEN if statuses[index] else COLOR_BLUE
                draw_contours(new_frame, coordinates, str(p["id"] + 1), COLOR_WHITE, color)

            open_cv.imshow(str(self.video), new_frame)

            # Checking for user input to Exit the program
            k = open_cv.waitKey(1)
            if k == ord("q"):
                break
        capture.release()
        open_cv.destroyAllWindows()

    # Finds if the car has entered the zone
    def __apply(self, grayed, index, p):
        coordinates = self._coordinates(p)
        logging.debug("points: %s", coordinates)

        rect = self.bounds[index]
        logging.debug("rect: %s", rect)

        roi_gray = grayed[rect[1]:(rect[1] + rect[3]), rect[0]:(rect[0] + rect[2])]  # Slicing img to get Region of Intrest
        laplacian = open_cv.Laplacian(roi_gray, open_cv.CV_64F)     # Applying laplacian to img
        logging.debug("laplacian: %s", laplacian)

        # Getting coordinates of the rectangle/parking area
        coordinates[:, 0] = coordinates[:, 0] - rect[0]
        coordinates[:, 1] = coordinates[:, 1] - rect[1]

        # Finding Average pixels in an parking lot
        status = np.mean(np.abs(laplacian * self.mask[index])) < MotionDetector.LAPLACIAN  # Gives true/ false
        # print(np.mean(np.abs(laplacian * self.mask[index])))
        logging.debug("status: %s", status)

        return status

    @staticmethod
    def _coordinates(p):
        return np.array(p["coordinates"])

    # Checking if the new status of parking is changed
    @staticmethod
    def same_status(coordinates_status, index, status):
        return status == coordinates_status[index]

    @staticmethod
    def status_changed(coordinates_status, index, status):
        return status != coordinates_status[index]

class CaptureReadError(Exception):
    pass