"""TODO: Docstring."""
import pytest
import uclasm
from uclasm import Graph
from uclasm.matching.matching_utils import GlobalCostsArray
import numpy as np
from scipy.sparse import csr_matrix
import pandas as pd
from laptools import clap

@pytest.fixture
def graph():
    """Create a graph with some nodes and edges."""
    adj0 = csr_matrix([[0, 0, 0],
                       [1, 0, 0],
                       [0, 0, 0]])
    adj1 = csr_matrix([[0, 0, 0],
                       [0, 0, 0],
                       [0, 1, 0]])
    nodelist = pd.DataFrame(['a', 'b', 'c'], columns=[Graph.node_col])
    edgelist = pd.DataFrame([['b', 'a'],
                             ['c', 'b']], columns=[Graph.source_col,
                                                   Graph.target_col])
    g = Graph([adj0, adj1], ['c1', 'c2'], nodelist, edgelist)
    return g

@pytest.fixture
def node_subgraph():
    """Create a subgraph of the graph created by the graph fixture."""
    adj0 = csr_matrix([[0, 0],
                       [0, 0]])
    adj1 = csr_matrix([[0, 0],
                       [1, 0]])
    nodelist = pd.DataFrame(['b', 'c'], columns=[Graph.node_col])
    edgelist = pd.DataFrame([['c', 'b']], columns=[Graph.source_col,
                                                   Graph.target_col])
    subg = Graph([adj0, adj1], ['c1', 'c2'], nodelist, edgelist)
    return subg

class TestGraph:
    """Tests related to the Graph class and its methods/functions."""

    def test_graph_adj_mats(self, graph):
        """Check that the various derived matrices are computed."""
        adj0 = graph.adjs[0]
        adj1 = graph.adjs[1]

        comp = adj0 + adj1
        sym_comp = comp + comp.T
        is_nbr = sym_comp > 0

        # Check the various composite adjacency and related matrices.
        assert (graph.composite_adj != comp).nnz == 0
        assert (graph.sym_composite_adj != sym_comp).nnz == 0
        assert (graph.is_nbr != is_nbr).nnz == 0

    def test_node_subgraph(self, graph, node_subgraph):
        """Check the subgraph of `graph` against the expected subgraph."""
        node_bools = graph.nodelist[Graph.node_col]\
                     .isin(node_subgraph.nodelist[Graph.node_col])
        node_idxs = node_bools[node_bools].index
        actual_subgraph = graph.node_subgraph(node_idxs)

        # Check the adjacency matrices of the subgraph against expected
        for act_adj, exp_adj in zip(actual_subgraph.adjs, node_subgraph.adjs):
            assert (act_adj != exp_adj).nnz == 0
            assert (act_adj != exp_adj).nnz == 0
        assert (actual_subgraph.nodelist == node_subgraph.nodelist).all(axis=None)
        assert (actual_subgraph.edgelist == node_subgraph.edgelist).all(axis=None)
        for i in range(actual_subgraph.n_nodes):
            assert actual_subgraph.orig_idxs[i] == node_idxs[i]

    def test_node_cover(self, graph):
        """Check that the node cover is minimal for the example graph."""
        cover = graph.node_cover()
        assert len(cover) == 1
        assert cover[0] == 1

    def test_edge_src_idxs(self, graph):
        """Test the function that retrieves source indices of the edges"""
        src_idxs = graph.edge_src_idxs
        for edge_idx, edge in graph.edgelist.iterrows():
            assert src_idxs[edge_idx] == graph.node_idxs[edge[graph.source_col]]

    def test_edge_dst_idxs(self, graph):
        """Test the function that retrieves destination indices of the edges"""
        dst_idxs = graph.edge_dst_idxs
        for edge_idx, edge in graph.edgelist.iterrows():
            assert dst_idxs[edge_idx] == graph.node_idxs[edge[graph.target_col]]

class TestGlobalCostArray:
    """Tests related to the global costs array"""
    def test_global_costs_array(self):
        test_array = np.zeros((5,5))
        global_costs_array = test_array.view(GlobalCostsArray)
        assert np.all(global_costs_array.candidates)
        global_costs_array[3,3] = 2
        assert np.sum(global_costs_array.candidates) == 5*5-1
        assert not global_costs_array.candidates[3,3]
        global_costs_array[2,:] = 5
        assert not np.any(global_costs_array.candidates[2,:])

    def test_set_global_cost_threshold(self):
        test_array = np.zeros((5,5))
        test_array[1,:] = 3
        test_array[2,:] = 4
        candidates = test_array <= 3
        candidates[4,:] = False
        global_costs_array = GlobalCostsArray(test_array,
                                              global_cost_threshold=3,
                                              candidates=candidates)
        assert np.all(global_costs_array.candidates[1,:])
        assert not np.any(global_costs_array.candidates[2,:])
        assert not np.any(global_costs_array.candidates[4,:])
        global_costs_array.set_global_cost_threshold(2)
        assert not np.any(global_costs_array.candidates[1,:])
        assert not np.any(global_costs_array.candidates[4,:])
        global_costs_array[4,:] = 2
        assert not np.any(global_costs_array.candidates[4,:])
        assert global_costs_array.candidates[0,2]
        global_costs_array[0,2] = 3
        assert not global_costs_array.candidates[0,2]

    def test_indexing(self):
        test_array = np.zeros((5,5))
        global_costs_array = GlobalCostsArray(test_array,
                                             global_cost_threshold=3)
        global_costs_array[2,:] = 1
        sliced_array = global_costs_array[:, 2:3]
        assert isinstance(sliced_array, GlobalCostsArray)
        assert sliced_array.global_cost_threshold == 3
        assert sliced_array.shape == sliced_array.candidates.shape


# -*- coding: utf-8 -*-
"""
Module containing logic for table with Payment plan

"""

__author__ = 'Samir Adrik'
__email__ = 'samir.adrik@gmail.com'

import pandas as pd

from PyQt5.QtCore import QObject, pyqtSlot, Qt
from source.util import Assertor

from .table_model import TableModel
from .model import Model

class PaymentModel(Model):
    """
    Implementation of model for Payment plan

    """
    payment_keys = ["start_dato_annuitet", "slutt_dato_annuitet", "total_termin_annuitet",
                    "aar_annuitet", "termin_aar_annuitet", "laan_annuitet", "rente_annuitet",
                    "total_rente_annuitet", "total_belop_annuitet", "start_dato_serie",
                    "slutt_dato_serie", "total_termin_serie", "aar_serie",
                    "termin_aar_serie", "laan_serie", "rente_serie",
                    "total_rente_serie", "total_belop_serie", "netto_likviditet_plan_serie",
                    "netto_likviditet_mnd_serie", "netto_likviditet_plan_annuitet",
                    "netto_likviditet_mnd_annuitet"]

    def __init__(self, parent: QObject):
        """
        Constructor / Instantiation of class

        Parameters
        ----------
        parent      : QObject
                      Parent view for which the model in to be linked

        """
        Assertor.assert_data_types([parent], [QObject])
        super().__init__(parent)

    @pyqtSlot()
    def generate_payment_plan(self):
        """
        method for payment plan to front end

        """
        analysis_model = self.parent.parent.analysis_model
        restructure_model = self.parent.parent.restructure_model

        if all(val in restructure_model.data.keys() for val in self.payment_keys):
            if "nedbetalingsplan_annuitet" in restructure_model.data.keys():
                payment_data_model_fixed = TableModel(
                    pd.DataFrame(restructure_model.data["nedbetalingsplan_annuitet"]),
                    alignment=Qt.AlignCenter)
                self.parent.ui_form.table_view_annuitet.setModel(payment_data_model_fixed)
            if "nedbetalingsplan_serie" in restructure_model.data.keys():
                payment_data_model_serie = TableModel(
                    pd.DataFrame(restructure_model.data["nedbetalingsplan_serie"]),
                    alignment=Qt.AlignCenter)
                self.parent.ui_form.table_view_serie.setModel(payment_data_model_serie)

            self.set_line_edits("", line_edits=self.payment_keys, data=restructure_model.data)

        elif all(val in analysis_model.data.keys() for val in self.payment_keys):
            if "nedbetalingsplan_annuitet" in analysis_model.data.keys():
                payment_data_model_fixed = TableModel(
                    pd.DataFrame(analysis_model.data["nedbetalingsplan_annuitet"]),
                    alignment=Qt.AlignCenter)
                self.parent.ui_form.table_view_annuitet.setModel(payment_data_model_fixed)
            if "nedbetalingsplan_serie" in analysis_model.data.keys():
                payment_data_model_serie = TableModel(
                    pd.DataFrame(analysis_model.data["nedbetalingsplan_serie"]),
                    alignment=Qt.AlignCenter)
                self.parent.ui_form.table_view_serie.setModel(payment_data_model_serie)

            self.set_line_edits("", line_edits=self.payment_keys, data=analysis_model.data)

    def clear_all(self):
        """
        method for clearing data in model

        """
        self.parent.ui_form.table_view_annuitet.setModel(None)
        self.parent.ui_form.table_view_annuitet.clearSpans()
        self.parent.ui_form.table_view_serie.setModel(None)
        self.parent.ui_form.table_view_serie.clearSpans()
        self.clear_line_edits(self.payment_keys)


import functions
import PySimpleGUI as sg
import time
import os

TIME_FORMAT = "%b %d, %Y %H:%M:%S"
THEME = "DarkGreen1"

if not os.path.exists(functions.FILEPATH):
    with open(functions.FILEPATH, "w") as f:
        pass

sg.theme(THEME)
clock = sg.Text("", key='clock')
label = sg.Text("Type in a to-do")
input_box = sg.InputText(tooltip='Enter to-do', key='todo')
add_button = sg.Button("Add")
list_box = sg.Listbox(values=functions.get_todos(), key='current_todos',
                      enable_events=True, size=[45, 10])
edit_button = sg.Button("Edit")
complete_button = sg.Button("Complete")
exit_button = sg.Button("Exit")

window = sg.Window('My To-Do App',
                   layout=[[clock],
                           [label],
                           [input_box, add_button],
                           [list_box, edit_button, complete_button],
                           [exit_button]],
                   font=('Helvetica', + 20))

while True:
    event, values = window.read(timeout=200)
    window["clock"].update(value=time.strftime(TIME_FORMAT))
    print(event)
    print(values)
    if event == "Add":
        todos = functions.get_todos()
        new_todo = values['todo'] + "\n"
        todos.append(new_todo)
        functions.write_todos(todos)
        window['current_todos'].update(values=todos)

    elif event == "current_todos":
        if not values['current_todos']:
            continue
        else:
            window['todo'].update(value=values['current_todos'][0])

    elif event == "Edit":
        try:
            todo_to_edit = values['current_todos'][0]
            new_todo = values['todo'] + '\n'

            todos = functions.get_todos()
            index = todos.index(todo_to_edit)
            todos[index] = new_todo

            functions.write_todos(todos)
            window['current_todos'].update(values=todos)
        except IndexError:
            sg.popup("Please select an item first.", font=("Helvetica", 20))

    elif event == "Complete":
        try:
            todo_to_complete = values['current_todos'][0]
            todos = functions.get_todos()
            todos.remove(todo_to_complete)
            functions.write_todos(todos)
            window['current_todos'].update(values=todos)
            window['todo'].update(value='')
        except IndexError:
            sg.popup("Please select an item first.", font=("Helvetica", 20))

    elif event == sg.WIN_CLOSED or event == "Exit":
        break

window.close()

# install pyinstaller and run command to create executable for windows:
# pyinstaller --onefile --windowed --clean gui.py


import os
import cv2

dirname = os.path.dirname(__file__)                         #현재 파일의 위치
filename = os.path.join(dirname, '../images/trump.jpg')     #현재 위치로 부터 원하는 파일의 [상대적 위치]
img = cv2.imread(filename)                                  #이미지 열어서 img에 저장
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
print(len(gray[0][0]))

while True:                                                 #무조건 반복
    cv2.imshow('window', gray)                                #img를 window창에 표시
    if cv2.waitKey(1) == ord('q'):                          #키보드 q가 눌리면
        break                                               #반복 중단
cv2.destroyAllWindows()                                     #모든 cv창 종료


#!/usr/bin python3

# Script by Daniel Leon-Perinan at CABD
# MIT License, 2020
# Version 1.0.5

from pycudadecon import RLContext, rl_decon, TemporaryOTF
from contextlib import contextmanager
from tqdm import tqdm
import numpy as np
import tifffile
import mrc
import os
import argparse
import sys
import time

class Image:
    def load_image(self, route):
        extension = os.path.splitext(route)[1]
        self.wavelengths, size, shape, image = opener[extension](route)
        try:
            self.nt, self.nz, self.nw, self.nx, self.ny = shape
        except (Exception):
            os.system(Exception)
            exit(1)
        self.image = image
        self.pxx, self.pxy, self.pxz = size

    def __init__(self, route):
        self.load_image(route)
        super().__init__()

    def frame(self, t, w):
        if self.nt <= 1 and self.nw > 1:
            im = self.image[0 : self.nz, w, :, :]
        elif self.nw <= 1:  # Fixed for only 1 channel!
            if len(self.image.shape) == 5:
                im = self.image[t, 0 : self.nz, w, :, :]
            elif self.nt > 1:
                im = self.image[t, 0 : self.nz, :, :]
            else:
                im = self.image[0 : self.nz, :, :]
                # More than one z is mandatory!
        else:
            if self.image.shape[1] == self.nz:
                im = self.image[t, 0 : self.nz, w, :, :]
            elif self.image.shape[1] == self.nw:
                im = self.image[t, w, 0 : self.nz, :, :]
        return im

    # TODO implement a duplication function

def open_tif(route):
    """[summary]

    Args:
        route (string): route for the .tiff file being processed

    Raises:
        ValueError: when wavelengths (even if one) are not provided
        ValueError: when a 4D stack is not provided

    Returns:
        tuple: complying the order in the Image class, returns the tuple
        of dimensions (TZWXY), as well as pixel size and other features
    """
    # Check that wavelengths have been provided through arguments
    if args.waves == None:
        raise ValueError("No wavelengths were provided. Please refer to --help")

    # Get pixel and stack size from arguments (defaults are set if none)
    pxx, pxz = args.xyimage, args.zimage

    # Read the image using tifffile
    image = tifffile.imread(route)

    # Check that the image is a 4D stack
    if len(image.shape) < 5:
        raise ValueError("Dimensions are wrong. You must provide a 4D stack")

    # Detuple the image properties
    nt, nz, nw, nx, ny = image.shape

    return args.waves, [pxx, pxx, pxz], [nt, nz, nw, nx, ny], image

def open_dv(route):
    """Opens a DeltaVision .dv image stack, and generates a tuple of
    settings using mrc

    Args:
        route (string): route for the .dv file being processed

    Returns:
        tuple: complying the order in the Image class, returns the tuple
        of dimensions (TZWXY), as well as pixel size and other features
    """
    # Load the .dv file into memory with mrc
    image = mrc.imread(route)

    # Parse the header
    header = image.Mrc.hdr

    # Parse the number of timepoints and wavelengths
    nt, nw = header.NumTimes, header.NumWaves

    # Parse the number of XY pixels (image resolution)
    nx, ny, nsecs = header.Num

    # Parse the number of z planes
    nz = int(nsecs / nt / nw)

    # Parse the pixel and stack sizes in microns
    pxx, pxy, pxz = header.d[0:3]
    return header.wave[0:4], [pxx, pxy, pxz], [nt, nz, nw, nx, ny], image

def join_spinning(route, channels):
    """Joins a set of Spinning-Disk folder
    structure to a .tif file consisting of a Z-Stack.

    You must provide the route as a foldername, it is
    mandatory that inside the folder, channel names (subfolders)
    must be provided. This set of subfolders contains the Z-stacks
    in individual files for each timepoint. They will be joined in a
    two step process:
        1. A stack for a single channel is reconstructed in memory
        2. The stack is merged into a single file
    This process takes place in RAM: please, make sure enough memory is
    available!

    Args:
        route ([string]): folder name containing files to reconstruct a movie
        channels ([list]): list of channel names (same as subsequent folder names)

    Returns:
        [string]: The generated 4D stack file route
    """
    # Creates an empty list that will contain the final stack
    movieStack = []

    # Iterates over the user-provided list of channels
    for channel in channels:  # argument provided as spinning

        # Creates an empty list that will contain the channel stack
        channelStack = []

        # Route for the channel stack (individual timepoitns)
        channelPath = os.path.join(route, channel)

        # Individual files are parsed. Timepoint order in filename (..._t1, ..._tn)
        files = [
            os.path.join(channelPath, f)
            for f in os.listdir(channelPath)
            if os.path.isfile(os.path.join(channelPath, f))
        ]

        # Select timepoints based on *_{tN}.TIF notation
        frames = [
            int("".join(os.path.basename(os.path.splitext(f)[0]).split("_")[-1][1:]))
            for f in files
        ]
        # Sort the timepoints
        frames.sort()

        # Select a basename to iterate over timepoints
        filenameBase = "_".join(os.path.splitext(files[0])[0].split("_")[:-1])

        # Store the exact file extension
        extension = os.path.splitext(files[1])[1]

        # Iterating over files, construct the channelStack list
        for frame in frames:
            image = tifffile.imread("{}_t{}{}".format(filenameBase, frame, extension))
            channelStack.append(image)

        # Stores the channel inside the final movie
        movieStack.append(channelStack)

    # Transposes the array to maintain the classical TCZXY ordering (4D stack)
    # TODO: this is a bit wanky, will be fixed
    movieStack = np.array(movieStack)
    movieStack = np.swapaxes(movieStack, 0, 1)
    movieStack = np.swapaxes(movieStack, 1, 2)

    # Stores the file with a new, derived name
    filename = "{}_stack.tif".format(route)
    tifffile.imsave("{}".format(filename), movieStack)

    # Returns the generated stack image fileroute
    return filename

# https://stackoverflow.com/questions/5081657/how-do-i-prevent-a-c-shared-library-to-print-on-stdout-in-python/17954769#17954769
@contextmanager
def stdout_redirected(to=os.devnull):
    """Redirects the CUDAdecon output to the provided route, avoiding displaying it
    in the display (iterations, lambda values...). Instead, it is better to store it
    in a log.

    Args:
        to (string, optional): text file to write into. Defaults to os.devnull.
    """
    fd = sys.stdout.fileno()

    def _redirect_stdout(to):
        sys.stdout.close()
        os.dup2(to.fileno(), fd)
        sys.stdout = os.fdopen(fd, "w")

    with os.fdopen(os.dup(fd), "w") as old_stdout:
        # Changed to append
        with open(to, "a") as file:
            _redirect_stdout(to=file)
        try:
            yield
        finally:
            _redirect_stdout(to=old_stdout)

def log(message, fname=None, show=True):
    """Logging function used throughout the program

    Args:
        message (string): what is to be logged
        fname (string, optional): route of the log file, if not provided, no file will be written. Defaults to None.
        show (bool, optional): to be displayed onscreen. Defaults to True.
    """

    # Check function arguments
    if show:
        print("{}".format(message))
    if fname is not None:
        # Append the message content into the file if it exists, or create it from scratch
        with open(fname, "a" if os.path.exists(fname) else "w") as f:
            f.write("{}\n".format(message))

def getGPUinfo():
    """Gets GPU information with logging purposes. Please note that
    CUDA library must be installed, with nvidia-smi available.

    Returns:
        stirng: formatted output of the nvidia-smi command
    """
    # Call a new nvidia-smi process and read its output
    smiOutput = os.popen("nvidia-smi").read()

    # Clean output contents
    smiSplitted = smiOutput.splitlines()

    return "\n".join(smiSplitted[1:10])

def max_projection(im, minimum=0, maximum=-1):
    """Returns the Z maximum projection of an image, between the
    provided minimum and maximum slices

    Args:
        im (np.array): image data as numpy array, any format. 4D stack!
        minimum (int, optional): minimum slice in Z projection. Defaults to 0.
        maximum (int, optional): maximum slice in Z projection. Defaults to -1.

    Returns:
        np.array: z maximum projected image
    """
    if im.shape == 5 and np.size(im, axis=0) <= 1:
        if len(im.shape) == 4:
            result = np.max(im[0, minimum:maximum, :, :], axis=0)
        else:
            result = np.max(im[minimum:maximum, :, :], axis=0)
    else:
        result = np.max(im[:, minimum:maximum, :, :], axis=1)

    return result

def deconvolve_all():
    """Main deconvolution function, where all logging, file exploration and
    processing is wrapped. Continuous references to external libraries...
    Many thanks to tlambert01 for the libs mrc and pyCUDAdecon
    """

    log("# Starting deconvolution")
    log("# GPU information")
    log(getGPUinfo())

    # Get the paths for each filter to be used, in the same order
    filter_paths = args.psf

    # Iterate over the list of files provided
    # Note that queuing can be done using a simple .sh script to call the
    # application with different parameters
    for f in args.source:

        # Log file naming
        logfile = "{}.log".format(f)

        # Start a timer for profiling time elapsed
        start = time.time()

        # Check whether source provided is SpinningDisk-like
        if os.path.isdir(f):
            channels = args.spinning
            log(
                "# Reconstructing and saving .tif file from folder structure with channels {}".format(
                    channels
                ),
                logfile,
            )
            f = join_spinning(f, channels)

        # Store the file extension to save in the same format
        extension = os.path.splitext(f)[1]

        # Generate an output name based on the source name
        outname = os.path.basename(os.path.splitext(f)[0])

        # Load the image
        log("# Loading the file {} and creating log".format(f), logfile)
        im = Image(f)

        # Calculate an appropriate number of wavelengths to process
        nw = min(len(filter_paths), im.nw)

        # Create the header for the current file
        log("\n\n# Processing file {}".format(f))
        log(
            "# Starting deconvolution at {}".format(
                time.strftime("%b %d %Y %H:%M:%S", time.localtime(start))
            )
        )
        log("####################################################", logfile, show=False)
        log("deCU v1.0.5, by DLP (CABD), 2020", logfile, show=False)
        log("####################################################", logfile, show=False)
        log("\n# Deconvolution settings", logfile, show=False)
        log("## Projection {}".format(args.project), logfile, show=False)
        log("## Channels {}".format(im.nw), logfile, show=False)
        log("## File {}".format(f), logfile, show=False)
        log(
            "## XY pixel size {}".format(round(im.pxx, 3)), logfile, show=False,
        )
        log(
            "## Z-stack depth {}".format(round(im.pxz, 3)), logfile, show=False,
        )
        log("## GPU information", logfile, show=False)
        log(getGPUinfo(), logfile, show=False)
        log(
            "## Time start {}".format(
                time.strftime("%b %d %Y %H:%M:%S", time.localtime(start)),
            ),
            logfile,
            show=False,
        )
        log("\n# Deconvolution process steps", logfile, show=False)

        # Iterate over the range of wavelengths
        for w in range(0, nw):
            # Skip wavelengths marked as null
            if filter_paths[w] == "_":
                continue

            log(
                "\n## Deconvolving channel {} with PSF {}. {} frame{}".format(
                    w, filter_paths[w], im.nt, "s" if im.nt > 1 else ""
                ),
                logfile,
            )

            # Provisional list to store multiframe images
            stack = []

            # Loading all libraries for the deconvolution process
            with TemporaryOTF(filter_paths[w]) as otf:
                with stdout_redirected(to=logfile):
                    with RLContext([im.nz, im.nx, im.ny], otf.path) as ctx:
                        # Iterate over all frames in the image
                        for t in tqdm(range(0, im.nt)):

                            # Using echo to save into the logfile the frame no.
                            os.system("echo ## Frame {}".format(t))

                            # Retrieve current frame
                            image = im.frame(t, w)

                            # Perform pyCUDAdecon deconvolution
                            result = rl_decon(
                                image,
                                output_shape=ctx.out_shape,
                                dxdata=im.pxx,
                                dxpsf=args.xyfilter,
                                dzdata=im.pxz,
                                dzpsf=args.zfilter,
                                wavelength=im.wavelengths[w],
                                n_iters=args.i,
                                na=args.na,
                                nimm=args.refractive,
                            )

                            # Convert the result to uint16 to preserve format
                            result = result.astype(np.uint16)

                            # Append the timepoint to the stack
                            stack.append(result)

                        # Generate a np array from the stack
                        imResult = np.array(stack)

            # Generate a definitive saving name
            savename = ("{0}_{1}_D3D{2}").format(
                outname, str(im.wavelengths[w]), extension
            )

            log("## Saving file as {}".format(savename), logfile)
            saveformat[extension](
                savename, imResult,
            )

            # Check whether max-projection has to occur
            # TODO implement more types of projection
            if args.project:
                log("## Creating maximum projection", logfile)
                imProjected = max_projection(
                    imResult, minimum=args.planes[0], maximum=args.planes[1]
                )
                savename = ("{0}_{1}_PRJ_D3D{2}").format(
                    outname, str(im.wavelengths[w]), extension
                )
                log("## Saving maximum projection as {}".format(savename), logfile)
                saveformat[extension](
                    savename, imProjected,
                )

        # Stop profiling timer and calculate time elapsed
        end = time.time()
        elapsed = end - start

        log(
            "# Deconvolution ended at {}, elapsed {}".format(
                time.strftime("%b %d %Y %H:%M:%S", time.localtime(end)),
                "{} m : {} s".format(int(elapsed / 60), int(elapsed % 60)),
            ),
            logfile,
        )

def cmdline_args():
    """Parses the command line arguments as program parameters

    Returns:
        argparse.args: comprehensive list of arguments
    """
    p = argparse.ArgumentParser(
        description="""
        CUDA deconvolution of (multi)frame .dv or .tif(f) files
        """,
        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
    )
    p.add_argument(
        "--source",
        nargs="+",
        required=True,
        help="Source file(s) to deconvolve (or folder if SpinningDisk)",
    )
    p.add_argument(
        "--psf",
        nargs="+",
        required=True,
        help="PSF file(s) (one per wavelength). Provide _ for channel negative selection",
    )
    p.add_argument(
        "--zfilter",
        default=0.1,
        help="Z-depth of the given filter(s) (PSF) in microns",
    )
    p.add_argument(
        "--xyfilter",
        default=0.1,
        help="XY-pixel size of the given filter(s) (PSF) in microns",
    )
    p.add_argument(
        "--zimage",
        default=0.1,
        type=float,
        help="Z-depth of the given image stack in microns",
    )
    p.add_argument(
        "--xyimage",
        default=0.1,
        type=float,
        help="XY-pixel size of the given image in microns",
    )
    p.add_argument(
        "--na", default=1.4, help="Numerical aperture units for selected objective",
    )
    p.add_argument(
        "--refractive", default=1.5, help="Refractive index of interface media"
    )
    p.add_argument(
        "--spinning",
        nargs="+",
        type=str,
        help="Folder names for each channel in a SpinningDisk type file structure",
    )
    p.add_argument("-i", default=15, help="Iterations of the Richardson-Lucy algorithm")
    p.add_argument(
        "-p", "--project", action="store_true", help="Project the z-stack in a new file"
    )
    p.add_argument(
        "--planes",
        type=int,
        nargs=2,
        default=[0, -1],
        help="Select range for z-stack projection, two numbers, space delimited",
    )
    p.add_argument("-w", "--waves", nargs="+", help="Select wavelengths to process")

    return p.parse_args()

# Dictionary for opening functions
opener = {".tif": open_tif, ".tiff": open_tif, ".dv": open_dv}

# Dictionary for saving functions
saveformat = {
    ".tif": tifffile.imsave,
    ".TIF": tifffile.imsave,
    ".tiff": tifffile.imsave,
    ".TIFF": tifffile.imsave,
    ".dv": mrc.imsave,
    ".DV": mrc.imsave,
}

if __name__ == "__main__":
    args = cmdline_args()
    deconvolve_all()


import pandas as pd
import networkx as nx
import matplotlib.pyplot as plt
import os
import codecs

df = pd.read_csv("docs/links.txt")

# list_actor = []
# for i in df["actor1"]:
#     if i not in list_actor:
#         list_actor.append(i)
# for i in df["actor2"]:
#     if i not in list_actor:
#         list_actor.append(i)
# list_actor.append(0)
# #trier actor en ordre croissant
# list_actor.sort()

top_500_actors = pd.read_csv("top_500/links.txt")
list_500_actors = []
for i in top_500_actors["actor1"]:
    if i not in list_500_actors:
        list_500_actors.append(i)
for i in top_500_actors["actor2"]:
    if i not in list_500_actors:
        list_500_actors.append(i)

# représenter un résaux d'acteur avec un graphe ou les poids sont le number_of_commune_movies entre deux acteurs
def transform_to_graph(df,list_actor):
    """Transform the df dataframe in a graph with the weight between two actor as the number of film in commun"""
    g = nx.Graph()
    for actor in list_actor:
        g.add_node(actor)
    for index, row in df.iterrows():
        if row["number_of_common_movies"] > 0:
            g.add_edge(row["actor1"], row["actor2"], weight = row["number_of_common_movies"])
    return g

# x = transform_to_graph(df,list_actor)
# nx.write_gml(x, "graph/graph.gml")
#
# df1 = pd.read_csv("text_analysis/cosine_similarity_TF_IDF.txt")
#
# y = transform_to_graph(top_500_actors,list_500_actors)
#
# nx.write_gml(y, "graph/top_500_actors.gml")

top_500_tmdb_actor = pd.read_csv("top_500_tmdb/links.txt")
print(top_500_tmdb_actor.head())
list_500_tmdb = []
for i in top_500_tmdb_actor["actor1"]:
    if i not in list_500_tmdb:
        list_500_tmdb.append(i)
for i in top_500_tmdb_actor["actor2"]:
    if i not in list_500_tmdb:
        list_500_tmdb.append(i)
g = transform_to_graph(top_500_tmdb_actor,list_500_tmdb)
nx.write_gml(g, "graph/top_500_tmdb_actors.gml")


from flask import jsonify

from austin import client

def create_album(name):
    client.create_bucket(Bucket=name)

def list_albums():
    return client.list_buckets()['Buckets']

def list_images(album):
    objects = []
    for key in client.list_objects(Bucket=album)['Contents']:
        objects.append(key['Key'])

    return objects

def add_image(permisson, album, file_object, filename):
    client.put_object(ACL=permisson, Body=file_object, Bucket=album, Key=filename, ContentType='image/jpeg')

def get_URL(file_name):
    return client.generate_presigned_post(Bucket='austin', Key=file_name)

def get_albums():
    try:

        result = client.list_objects(Bucket='austin', Prefix='albums/', Delimiter='/')
        album_names = []
        for object in result.get('CommonPrefixes'):
            album_names.append(object['Prefix'][6:].strip('/'))

        albums = []
        for album_name in album_names:
            album = {
                'Name': album_name,
                'images': get_images(album_name)
            }
            albums.append(album)

        return albums

    except Exception as e:
        print(e)
        return jsonify({"error": "There was a problem with the data you provided."})

def get_images(album):
    try:

        prefix = 'albums/' + str(album) + '/'
        result = client.list_objects(Bucket='austin', Prefix=prefix, Delimiter='/')

        image_urls = []
        skipthedir = 0  # becuase the directory itself is also retrived we want to skip it
        for object in result.get('Contents'):
            if skipthedir > 0:
                url = get_URL(object.get('Key'))
                image_urls.append(url.get('url') + '/' + url.get('fields')['key'])
            else:
                skipthedir += 1

        return image_urls

    except Exception as e:
        print(e)
        return jsonify({"error": "There was a problem with the data you provided."})


import argparse
import os
import json
import numpy as np
import mido
import matplotlib.pyplot as plt

import numpy as np
import mir_eval
import matplotlib.pyplot as plt

from data.data_utils import remove_audio_filename_extension, remove_midi_filename_extension, maestro_piano_roll_to_midi
from data.data_utils import maestro_piano_roll_to_note_list, maestro_onset_offset_pitch_list_to_piano_roll
from data.data_utils import maestro_get_tick_per_second, maestro_midi_to_note_list
from data.data_utils import decompose_note_list, maestro_read_midi_to_piano_roll, amp2velocity_appox

from inference_main import transcribe_audio, prepare_model_elements
from utils import midi_to_hz
from hyperparameters import SUPPORTED_MIDI_TYPES, SUPPORTED_AUDIO_TYPES
from hyperparameters import MAESTRO_TICK_PER_SEC, LOWEST_PITCH
from hyperparameters import TEMPLATE_PATH, TRNASDUCTION_NN_CKPT_PATH, NOTE_TRANSITION_DICT_PATH
from hyperparameters import MODEL_TYPE_BASELINE, MODEL_TYPE_UNROLLEDNET

def clipped_translation(vec, k):
    len_vec = len(vec)
    out_vec = np.zeros(len_vec)
    if k >= 0:
        out_vec[k:] = vec[:len_vec-k]
    else:
        out_vec[:len_vec+k] = vec[-k:]
    return out_vec

ROUND1_SEARCH_WINDOW = 2 * MAESTRO_TICK_PER_SEC
ROUND1_SEARCH_STEP = 8
ROUND2_SEARCH_WINDOW = int(0.5 * MAESTRO_TICK_PER_SEC)
ROUND2_SEARCH_STEP = 2
ROUND3_SEARCH_WINDOW = int(0.05 * MAESTRO_TICK_PER_SEC)
ROUND3_SEARCH_STEP = 1
def calculate_transcription_note_metric_note_list(gt_note_list, est_note_list,
    tick_per_sec = MAESTRO_TICK_PER_SEC, lowest_pitch = 21, auto_time_align = False,
    onset_tolerance = 0.05, pitch_tolerance=50.0, offset_ratio=0.2, offset_min_tolerance=0.05, strict=False, beta=1.0,
    plot = False
):
    gt_pitch_list, _, gt_onset_list, gt_offset_list = decompose_note_list(gt_note_list)
    est_pitch_list, _, est_onset_list, est_offset_list = decompose_note_list(est_note_list)

    gt_intervals = np.array([gt_onset_list, gt_offset_list]).T
    est_intervals = np.array([est_onset_list, est_offset_list]).T

    # this is an algorithm to align the est and gt, when there is a shift of onset/offset in est
    if auto_time_align:
        if np.min(est_intervals) < 2 and (np.min(gt_intervals) - np.min(est_intervals)) > 0.05 * tick_per_sec:
            est_intervals = est_intervals + np.min(gt_intervals)

        max_time = max(np.max(gt_intervals), np.max(est_intervals))
        gt_time_roll = np.zeros(max_time)
        est_time_roll = np.zeros(max_time)
        gt_time_roll[gt_intervals[:,0]] = 1
        est_time_roll[est_intervals[:,0]] = 1
        gt_time_roll = np.convolve(gt_time_roll, np.hanning(9), mode="same")
        est_time_roll = np.convolve(est_time_roll, np.hanning(9), mode="same")
        L = len(gt_time_roll)

        if len(gt_time_roll) < 2*ROUND1_SEARCH_WINDOW: # this method has very high runtime when length of time_roll is high
            correlate_roll = np.correlate(est_time_roll, gt_time_roll, mode="same") # pad_on_est = [L/2,...,0,...,-L/2], id = [0,...,L/2,...,L]
            max_corr_id = np.argmax(correlate_roll)
            pad = int(L/2 - max_corr_id)
        else: # this method has 3 round search for best match
            correlate_roll = np.zeros(L) # pad_on_est = [L/2,...,0,...,-L/2], id = [0,...,L/2,...,L]
            # round 1:
            for k in range(-ROUND1_SEARCH_WINDOW, ROUND1_SEARCH_WINDOW, ROUND1_SEARCH_STEP): # zeros to pad
                corr_roll_k = int(L/2 - k)
                correlate_roll[corr_roll_k] = np.inner(clipped_translation(est_time_roll, k), gt_time_roll)
            round1_max = np.argmax(correlate_roll)
            round1_pad = int(L/2 - round1_max)
            # round 2:
            for k in range(round1_pad-ROUND2_SEARCH_WINDOW, round1_pad+ROUND2_SEARCH_WINDOW, ROUND2_SEARCH_STEP): # zeros to pad
                corr_roll_k = int(L/2 - k)
                if correlate_roll[corr_roll_k] == 0:
                    correlate_roll[corr_roll_k] = np.inner(clipped_translation(est_time_roll, k), gt_time_roll)
            round2_max = np.argmax(correlate_roll)
            round2_pad = int(L/2 - round2_max)
            # round 3:
            for k in range(round2_pad-ROUND3_SEARCH_WINDOW, round2_pad+ROUND3_SEARCH_WINDOW, ROUND3_SEARCH_STEP): # zeros to pad
                corr_roll_k = int(L/2 - k)
                if correlate_roll[corr_roll_k] == 0:
                    correlate_roll[corr_roll_k] = np.inner(clipped_translation(est_time_roll, k), gt_time_roll)
            round3_max = np.argmax(correlate_roll)
            pad = int(L/2 - round3_max)

        est_intervals = est_intervals[:] + pad

    gt_intervals = gt_intervals / tick_per_sec
    est_intervals = est_intervals / tick_per_sec

    gt_pitches = midi_to_hz(np.array(gt_pitch_list))
    est_pitches = midi_to_hz(np.array(est_pitch_list))

    if plot:
        gt_roll_bool = maestro_onset_offset_pitch_list_to_piano_roll(
            gt_onset_list, gt_offset_list, gt_pitch_list, 88, lowest_pitch
        )
        est_roll_bool = maestro_onset_offset_pitch_list_to_piano_roll(
            est_onset_list, est_offset_list, est_pitch_list, 88, lowest_pitch
        )
        fig, ax = plt.subplots(2,1, figsize=(12,6))
        show_len = min(len(gt_roll_bool), len(est_roll_bool))
        show_len = min(24000, show_len)
        ax[0].matshow(gt_roll_bool[:show_len:48].T, origin = "lower")
        ax[0].set_title("ground-truth pitch piano roll")
        ax[1].matshow(est_roll_bool[:show_len:48].T, origin = "lower")
        ax[1].set_title("estimated pitch piano roll")

    (precision, recall, F1, _) = mir_eval.transcription.precision_recall_f1_overlap(
        gt_intervals, gt_pitches, est_intervals, est_pitches,
        onset_tolerance, pitch_tolerance, offset_ratio, offset_min_tolerance, strict, beta
    )
    return precision, recall, F1

def calculate_transcription_note_metric_piano_roll(gt_piano_roll, est_piano_roll,
    tick_per_sec = MAESTRO_TICK_PER_SEC, lowest_pitch = 21, auto_time_align = False,
    onset_tolerance = 0.05, pitch_tolerance=50.0, offset_ratio=0.2, offset_min_tolerance=0.05, strict=False, beta=1.0,
    plot = False
):
    gt_note_list = maestro_piano_roll_to_note_list(gt_piano_roll, lowest_pitch)
    est_note_list = maestro_piano_roll_to_note_list(est_piano_roll, lowest_pitch)

    return calculate_transcription_note_metric_note_list(gt_note_list, est_note_list,
        tick_per_sec = tick_per_sec, lowest_pitch = lowest_pitch, auto_time_align = auto_time_align,
        onset_tolerance = onset_tolerance, pitch_tolerance=pitch_tolerance,
        offset_ratio=offset_ratio, offset_min_tolerance=offset_min_tolerance,
        strict=strict, beta=beta, plot = plot
    )

def calculate_transcription_note_metric_midi(gt_midi_path, est_midi_path,
    tick_per_sec = MAESTRO_TICK_PER_SEC, lowest_pitch = 21, auto_time_align = False,
    onset_tolerance = 0.05, pitch_tolerance=50.0, offset_ratio=0.2, offset_min_tolerance=0.05, strict=False, beta=1.0,
    plot = False
):
    gt_midi_obj = mido.MidiFile(gt_midi_path)
    est_midi_obj = mido.MidiFile(est_midi_path)

    gt_tick_per_sec = maestro_get_tick_per_second(gt_midi_obj)
    est_tick_per_sec = maestro_get_tick_per_second(est_midi_obj)
    assert gt_tick_per_sec == est_tick_per_sec, "tick per sec of the two midi should be the same"

    gt_note_list = maestro_midi_to_note_list(gt_midi_obj)
    est_note_list = maestro_midi_to_note_list(est_midi_obj)

    return calculate_transcription_note_metric_note_list(gt_note_list, est_note_list,
        tick_per_sec = tick_per_sec, lowest_pitch = lowest_pitch, auto_time_align = auto_time_align,
        onset_tolerance = onset_tolerance, pitch_tolerance=pitch_tolerance,
        offset_ratio=offset_ratio, offset_min_tolerance=offset_min_tolerance,
        strict=strict, beta=beta, plot = plot
    )

def compare_single_chords(gt_pitch_roll, est_pitch_roll):
    num_tp = np.sum(np.logical_and(gt_pitch_roll!=0, est_pitch_roll!=0).astype(int))
    num_fp = np.sum(np.logical_and(gt_pitch_roll==0, est_pitch_roll!=0).astype(int))
    num_fn = np.sum(np.logical_and(gt_pitch_roll!=0, est_pitch_roll==0).astype(int))
    return num_tp, num_fp, num_fn

def calculate_P_R_F1_given_counts(TP, FP, FN):
    if TP != 0:
        P = 1.0*TP / (TP + FP)
        R = 1.0*TP / (TP + FN)
        F1 = 2*P*R / (P+R)
    else:
        P = 0
        R = 0
        F1 = 0
    return P,R,F1

def calculate_single_chord_metric(gt_pitch_rolls, est_pitch_rolls, calculate_var_n_chord = 0):
    assert len(gt_pitch_rolls) == len(est_pitch_rolls), "input pitch roll lists should have the same length"
    N_chords = len(gt_pitch_rolls)
    num_tp_all = 0
    num_fp_all = 0
    num_fn_all = 0
    for i_roll in range(N_chords):
        num_tp, num_fp, num_fn = compare_single_chords(gt_pitch_rolls[i_roll], est_pitch_rolls[i_roll])
        num_tp_all += num_tp
        num_fp_all += num_fp
        num_fn_all += num_fn

    P,R,F1 = calculate_P_R_F1_given_counts(num_tp_all, num_fp_all, num_fn_all)

    if calculate_var_n_chord > 0:
        num_tp_temp = 0
        num_fp_temp = 0
        num_fn_temp = 0
        P_batch = []
        R_batch = []
        F1_batch = []
        for i_roll in range(N_chords):
            num_tp, num_fp, num_fn = compare_single_chords(gt_pitch_rolls[i_roll], est_pitch_rolls[i_roll])
            num_tp_temp += num_tp
            num_fp_temp += num_fp
            num_fn_temp += num_fn
            if i_roll % calculate_var_n_chord == 0:
                P_temp,R_temp,F1_temp = calculate_P_R_F1_given_counts(num_tp_temp, num_fp_temp, num_fn_temp)
                P_batch.append(P_temp)
                R_batch.append(R_temp)
                F1_batch.append(F1_temp)
                num_tp_temp = 0
                num_fp_temp = 0
                num_fn_temp = 0

        P_batch = np.array(P_batch)
        R_batch = np.array(R_batch)
        F1_batch = np.array(F1_batch)
        return P,R,F1,P_batch.std(),R_batch.std(),F1_batch.std()
    else:
        return P,R,F1

def batch_AMT_with_eval(audio_dir, save_dir, model_type, gt_midi_dir = None, eval_piece_names = None,
    segment_with_gt = False, gt_acoustic = False, no_transition = False, no_hmm = False, gt_smoothing = False,
    onset_tolerance = 0.05, pitch_tolerance=50.0, offset_ratio=0.2, offset_min_tolerance=0.05,
    verbose = False
):
    # do preparations
    cisoid_dict, transduction_nn, note_transition_dict = prepare_model_elements(model_type, verbose)

    # parse path information and detect running mode
    audio_names = os.listdir(audio_dir)
    piece_names_by_audio = [remove_audio_filename_extension(filename) for filename in audio_names]

    if gt_midi_dir is not None and os.path.exists(gt_midi_dir):
        evaluate_flag = True
        midi_names = os.listdir(gt_midi_dir)
        piece_names_by_midi = [remove_midi_filename_extension(filename) for filename in midi_names]
        if verbose:
            print("Groud-truth midi directory detected. Evaluate model switched on.")
    else:
        evaluate_flag = False
        midi_names = []
        piece_names_by_midi = []
        if verbose:
            print("Groud-truth midi directory not detected. Evaluate model switched off.")

    valid_piece_names = []
    valid_audio_names = []
    valid_midi_names = []
    for i_file,piece_name in enumerate(piece_names_by_audio):
        if evaluate_flag:
            run_this_piece_flag = (piece_name is not None) and (piece_name in piece_names_by_midi)
            if type(eval_piece_names) == list:
                if not piece_name in eval_piece_names:
                    run_this_piece_flag = False
        else:
            run_this_piece_flag = (piece_name is not None)

        if run_this_piece_flag:
            valid_piece_names.append(piece_name)
            for possible_ext in SUPPORTED_AUDIO_TYPES:
                if piece_name + possible_ext in audio_names:
                    valid_audio_names.append(piece_name + possible_ext)
                    break
            if evaluate_flag:
                for possible_ext in SUPPORTED_MIDI_TYPES:
                    if piece_name + possible_ext in midi_names:
                        valid_midi_names.append(piece_name + possible_ext)
                        break

    # do transcription and evaluation
    eval_results = {}
    eval_save_path = os.path.join(save_dir, "mir_eval_notes.json")
    for i_file_valid, piece_name in enumerate(valid_piece_names):
        # get the file paths
        audio_name = valid_audio_names[i_file_valid]
        if evaluate_flag:
            midi_name = valid_midi_names[i_file_valid]
            midi_path = os.path.join(gt_midi_dir, midi_name)
            if not os.path.exists(midi_path):
                midi_path = None
        else:
            midi_path = None
        audio_path = os.path.join(audio_dir, audio_name)
        save_path = os.path.join(save_dir, piece_name+".midi")

        # run the transcription model
        print("Transcription "+str(i_file_valid)+": "+piece_name+"...")
        est_piano_roll = transcribe_audio(
            audio_path, model_type, cisoid_dict, transduction_nn, note_transition_dict, amp2vel_mapping = amp2velocity_appox,
            vel_roll_path = None, segment_with_gt = segment_with_gt, gt_acoustic = gt_acoustic,
            no_transition = no_transition, no_hmm = no_hmm, gt_smoothing = gt_smoothing, verbose = False
        )

        # save the piano roll to midi
        maestro_piano_roll_to_midi(est_piano_roll, save_path,
            lowest_pitch = LOWEST_PITCH, amp2vel_mapping = None
        )

        # run evaluation
        if midi_path is not None:
            gt_piano_roll, tick_per_sec = maestro_read_midi_to_piano_roll(midi_path)
            P,R,F1 = calculate_transcription_note_metric_piano_roll(gt_piano_roll, est_piano_roll, auto_time_align = False,
                onset_tolerance = onset_tolerance, pitch_tolerance = pitch_tolerance,
                offset_ratio = offset_ratio, offset_min_tolerance = offset_min_tolerance,
                strict = False, beta = 1.0, plot = False
            )
            eval_results[piece_name] = (P,R,F1)
            print("Transcription eval result (P,R,F1):", (P,R,F1))
            with open(eval_save_path, "w") as outfile:
                json.dump(eval_results, outfile)

    N_eval = len(eval_results)
    PRF1_results = np.array(list(eval_results.values()))
    ave_results = PRF1_results.mean(axis=0)
    ave_std = PRF1_results.std(axis=0)
    print(f"Average P,R,F1: {ave_results[0]:.5f}±{ave_std[0]:.5f}, {ave_results[1]:.5f}±{ave_std[1]:.5f}, {ave_results[2]:.5f}±{ave_std[2]:.5f}")
    print("done!")

# example running:
# python evaluate_main.py data/synth/2004 results/baseline_2004 --gt_midi_dir data/midi_no_cc/2004 -v
if __name__ == '__main__':
    parser = argparse.ArgumentParser(
        description="Give audio directory to transcribe and the saving directory; specify the ground-truth directory for mid_eval evaluation."
    )
    parser.add_argument("audio_dir", type=str,
        help="the directory of the audio files (relative to the code dir)"
    )
    parser.add_argument("save_dir", type=str,
        help="the directory to save the midis as transcription results"
    )
    parser.add_argument("model_type", type=str,
        help="choose model_type from ['baseline', 'unrollednet']"
    )
    parser.add_argument("--gt_midi_dir", type=str, required=True,
        help="the directory of the ground-truth midis (should have same names as audios before .mid or .midi)"
    )
    parser.add_argument("-v", "--verbose", action="store_true", help="activate verbose mode with '-v'")
    args = parser.parse_args()

    batch_AMT_with_eval(args.audio_dir, args.save_dir, args.model_type, gt_midi_dir = args.gt_midi_dir, verbose = args.verbose)


import lightgbm as lgb
import csv
import pandas as pd
import time
import json
# -*- coding: utf-8 -*-
from joblib import load, dump
from train import Train
from tqdm import tqdm
import re
from utils.features_ents import feature_ents
from utils.features_emos import feature_emos
from utils.nerdict import jieba_ner
import numpy as np
from sklearn.preprocessing import normalize
from sklearn.metrics import f1_score, recall_score, precision_score
# from utils.model_lgb import model_lgb
import pandas as pd
import numpy as np
import xgboost as xgb
from xgboost.sklearn import XGBClassifier
from sklearn.model_selection import GridSearchCV   #Perforing grid search
from sklearn.model_selection import train_test_split
from collections import Counter
from imblearn.over_sampling import SMOTE

all_Y_ents = []
test_file = '../data/coreEntityEmotion_train.txt'
test_file = open(test_file)
for news in tqdm(test_file):
    news = json.loads(news)
    Y_ents_emos = news['coreEntityEmotions']
    Y_ents = []
    for y in Y_ents_emos:
        Y_ents.append(y['entity'])
    all_Y_ents.append(Y_ents)

def loadData(path):
    res = []
    file = open(path)
    for f in file:
        res.append(json.loads(f))
    return res

count = 0
# ners_appear_times = {}   #自己的分词出现的次数
ners_select_times = {}   #实体被当做真实实体的次数
ners_select_per = {}     #实体被当做真实实体的概率
for Y_ents in all_Y_ents:
    count+=1
    for ent in Y_ents:
        if ners_appear_times[ent] == 0:
            print(ent)
            continue
        if ent in ners_select_times:
            ners_select_times[ent] += 1
            ners_select_per[ent] = ners_select_times[ent]/ners_appear_times[ent]
        else:
            ners_select_times[ent] = 1
            ners_select_per[ent] = 1/ners_appear_times[ent]

dump(ners_select_per,'features/all_ners_select_per.joblib')
dump(ners_select_times,'features/all_ners_select_times.joblib')
print("done")

import time
import scipy.special as special
class BayesianSmoothing(object):
    def __init__(self, alpha, beta):
        self.alpha = alpha
        self.beta = beta

    def sample(self, alpha, beta, num, imp_upperbound):
        sample = numpy.random.beta(alpha, beta, num)
#         print(sample)
        I = []
        C = []
        for clk_rt in sample:
            imp = imp_upperbound
            clk = imp * clk_rt
            I.append(imp)
            C.append(clk)
        return I, C

    def update(self, imps, clks, iter_num, epsilon):

        for i in range(iter_num):
            new_alpha, new_beta = self.__fixed_point_iteration(imps, clks, self.alpha, self.beta)
            if abs(new_alpha - self.alpha) < epsilon and abs(new_beta - self.beta) < epsilon:
                break
            self.alpha = new_alpha
            self.beta = new_beta

    def __fixed_point_iteration(self, imps, clks, alpha, beta):
        numerator_alpha = 0.0
        numerator_beta = 0.0
        denominator = 0.0
#         print(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())))
        for i in range(0, len(imps), 1):    # 步长复用时去掉
            numerator_alpha += (special.digamma(clks[i] + alpha) - special.digamma(alpha))
            numerator_beta += (special.digamma(imps[i] - clks[i] + beta) - special.digamma(beta))
            denominator += (special.digamma(imps[i] + alpha + beta) - special.digamma(alpha + beta))
        return alpha * (numerator_alpha / denominator), beta * (numerator_beta / denominator)
def bys(I, C):
    bs = BayesianSmoothing(1, 1)
    # I, C = bs.sample(500, 500, 10, 1000)
    bs.update(I, C, 1000, 0.00001)   #
#     print(bs.alpha, bs.beta)
    ctr = []
    for i in range(len(I)):
        ctr.append((C[i] + bs.alpha) / (I[i] + bs.alpha + bs.beta))
    return ctr

bys_ctr = {}
for ent in tqdm(ners_select_times):
    if ners_select_times[ent] > 9:
        bys_ctr[ent] = bys([ners_select_times[ent]],[ners_select_per[ent]])

dump(bys_ctr,'features/bys_ctr_10.joblib')


import pandas as pd
from nrclex import NRCLex
import numpy as np
import os
import matplotlib as mpl
from matplotlib import cm
from nltk import word_tokenize
import spacy
nlp = spacy.load('en_core_web_sm')
import re

filepath = os.path.abspath('')
output_path = os.path.abspath('dict')
#READ .dic
def _parse_categories(lines):
    """
    Read (category_id, category_name) pairs from the categories section.
    Each line consists of an integer followed a tab and then the category name.
    This section is separated from the lexicon by a line consisting of a single "%".
    """
    for line in lines:
        line = line.strip()
        if line == "%":
            return
        # ignore non-matching groups of categories
        if "\t" in line:
            category_id, category_name = line.split("\t", 1)
            yield category_id, category_name

def _parse_lexicon(lines, category_mapping):
    """
    Read (match_expression, category_names) pairs from the lexicon section.
    Each line consists of a match expression followed by a tab and then one or more
    tab-separated integers, which are mapped to category names using `category_mapping`.
    """
    for line in lines:
        line = line.strip()
        parts = line.split("\t")
        yield parts[0], [category_mapping[category_id] for category_id in parts[1:]]

def read_dic(filepath):
    """
    Reads a LIWC lexicon from a file in the .dic format, returning a tuple of
    (lexicon, category_names), where:
    * `lexicon` is a dict mapping string patterns to lists of category names
    * `category_names` is a list of category names (as strings)
    """
    with open(filepath) as lines:
        # read up to first "%" (should be very first line of file)
        for line in lines:
            if line.strip() == "%":
                break
        # read categories (a mapping from integer string to category name)
        category_mapping = dict(_parse_categories(lines))
        # read lexicon (a mapping from matching string to a list of category names)
        lexicon = dict(_parse_lexicon(lines, category_mapping))
    return lexicon, list(category_mapping.values())
    #return dizionario
#####################################
#Create emotion lists
def create_emotion_lists(df):
    emotion_lists = []
    for i in df:
        emotion_lists.append(df.loc[df[i]!=0][i].reset_index()['index'].tolist())
    return emotion_lists
####################################
# Funzione per lemmanization

def space(tweet):
    doc = nlp(tweet)
    return " ".join([token.lemma_ for token in doc])
###########################################
#CREATE SENTIMENT LIST
def sentiment_lists(df):
    sentiment_list= []
    for i in df:
        sentiment_list.append(df.loc[df[i]!=0][i].reset_index()['index'].tolist())
    return sentiment_list

########################################################

#Return df with the count of es. positive words present in the tweet
def text_emotion0(df_tweet, column, df_dic, emotions):
    #df : contiene i tweet
    #df3 : vocabolario
    '''
    INPUT: DataFrame, string
    OUTPUT: the original DataFrame with ten new columns for each emotion
    '''
    df = df_tweet.copy()
    #Set index as column and call it 'word'
    df_dic = df_dic.reset_index().rename(columns = {'index':'word'})
    #emotions = df_dic.columns.drop('word')
    emo_df = pd.DataFrame(0, index=df.index, columns=emotions)
    for i,row in df.iterrows():
        l=[]
        tweet = word_tokenize(df.loc[i][column])
        for wt in tweet:
            for word in df_dic['word']:
                l.append(bool(re.findall(r"\b"+word,wt)))
            s = pd.Series(l,name = 'word')
            emo_score = df_dic[s]
            l=[]
            if not emo_score.empty:
                for emotion in emotions:
                    emo_df.at[i, emotion] += emo_score[emotion]
    df = pd.concat([df, emo_df], axis=1)
    #Remove i tweet che non hanno associato nessun sentimento
    df = df[df[emotions].any(axis=1)]

    return df

def text_emotion(df_tweet, column, df_dic):
    '''
    INPUT: DataFrame, string
    OUTPUT: the original DataFrame with ten new columns for each emotion
    '''
    df = df_tweet.copy()
    #Set index as column and call it 'word'
    #df_dic = df_dic.reset_index().rename(columns = {'index':'word'})
    emotions = df_dic.columns.drop('word')
    emo_df = pd.DataFrame(0, index=df.index, columns=emotions)
    for i,row in df.iterrows():
        l=[]
        tweet = word_tokenize(df.loc[i][column])
        for wt in tweet:
            for word in df_dic['word']:
                l.append(bool(re.findall(r"\b"+word,wt)))
            s = pd.Series(l,name = 'word')
            emo_score = df_dic[s]
            if len(emo_score) > 1:
                emo_score = emo_score.reset_index().drop(['index'],axis=1).head(1)
            l=[]
            if not emo_score.empty:
                for emotion in emotions:
                    emo_df.at[i, emotion] += emo_score[emotion]
    df = pd.concat([df, emo_df], axis=1)
    #Remove i tweet che non hanno associato nessun sentimento
    df = df[df[emotions].any(axis=1)]
    return df

#######################################################################
#########################ANALYSIS########################

#Mi prende il df del parco che mi interessa e mi calcola la statistica
def df_byparks(park,df,emotions):
    df_park = df[df['text1'].str.contains(park)] #cerco i tweet che hanno quel parco all'interno del testo
    ds = df_park[emotions].describe()
    #inserire questa linea se voglio il mean sentiment counting di ogni parco
    #df_park= pd.DataFrame({park: aggr}, index = emotions)
    #emotional dataframe sorted with most common words
    #df_em_mc = pd.DataFrame({'emotion': sentiment_list, 'aggregation': aggr}).sort_values(by=['aggregation'],ascending=False)
    return df_park, ds

#Find words in tweet which are in sentiment_lists
def find_values(x,sent_list):
    results = []
    for value in sent_list:
        for word in x.split():
            if word == value:
                results.append(word)
    return results

def explode(df,sent_list):
    #inserisco nella colonna result le parole dei tweet che si trovano nella lista
    df['result'] = df['text1'].apply(lambda x: find_values(x,sent_list))
    #elimino le righe in cui la colonna è vuota (perchè non ha trovato niente)
    df_match_list = df[df['result'].map(lambda d: len(d)) > 0]
    df_result = df_match_list.explode("result").groupby(by="result")["result"].count().sort_values(ascending=False)
    return df_result.reset_index(name="count"), df_match_list

def colors(sent):
    # import the desired colormap from matplotlib
    if sent == 'pos':
        cmap = mpl.cm.Blues(np.linspace(0,1,20))
        # the darker part of the matrix is selected for readability
        cmap = mpl.colors.ListedColormap(cmap[-10:,:-1])
        color = 'lightblue'
    else:
        cmap = mpl.cm.Reds(np.linspace(0,1,20))
        cmap = mpl.colors.ListedColormap(cmap[-10:,:-1])
        color = 'red'
    return cmap, color


from typing import List

import aiofiles
from fastapi import APIRouter, UploadFile, File
from fastapi.responses import HTMLResponse
from pathlib import Path

from config import IMAGE_EXTENSIONS, WEBSITE_PATH

router = APIRouter(
    prefix='/admin',
)

@router.get('/upload')
async def page_for_uploading():
    content = f"""

        <form action="{WEBSITE_PATH}/admin/upload-files" enctype="multipart/form-data" method="post">
            <input name="in_files" type="file" multiple>
            <input type="submit">
        </form>

    """
    return HTMLResponse(content=content)

@router.post('/upload-files')
async def upload_files(in_files: List[UploadFile] = File(...)):
    page = ""

    for in_file in in_files:
        filename = in_file.filename.lower()
        async with aiofiles.open(f'originals/{filename}', 'wb') as out_file:
            content = await in_file.read()
            await out_file.write(content)

        link_to_file = f'{WEBSITE_PATH}/{filename}'
        page += f"""{link_to_file}
"""

    page += ""
    return HTMLResponse(content=page)

@router.get('/images')
async def images_list():
    images = []
    directory = Path('originals')

    for extension in IMAGE_EXTENSIONS:
        for image_path in directory.glob(f'*.{extension}'):
            images.append(image_path.name)

    page = "
Images:

"
    for image in images:
        page += f"""
{image}
"""
    page += "
"

    return HTMLResponse(content=page)


#!/usr/bin/env python3

import sys
import pathlib

tokens = [
    "EBD"   , "FOR"   , "NEXT"  , "DATA"  , "VPOKE" , "INPUT" , "DIM"   , "READ"  ,
    "LET"   , "GOTO"  , "RUN"   , "IF"    , "RESTOR", "GOSUB" , "RETURN", "REM"   ,

    "STOP"  , "ON"    , "WAIT"  , "LOAD"  , "SAVE"  , "VERIFY", "DEF"   , "POKE"  ,
    "PRINT#", "PRINT" , "CONT"  , "LIST"  , "CLS"   , "CMD"   , "SYS"   , "OPEN"  ,

    "CLOSE" , "GET"   , "NEW"   , "TAB("  , "TO"    , "FN"    , "SPC("  , "THEN"  ,
    "NOT"   , "STEP"  , "+"     , "-"     , "*"     , "/"     , "???"   , "AND"   ,

    "OR"    , ">"     , "="     , "<"     , "SGN"   , "INT"   , "ABS"   , "USR"   ,
    "FRE"   , "POS"   , "SQR"   , "RND"   , "LOG"   , "EXP"   , "COS"   , "SIN"   ,

    "TAN"   , "ATN"   , "PEEK"  , "LEN"   , "STR$"  , "VAL"   , "ASC"   , "CHR$"  ,
    "LEFT$" , "RIGHT$", "MID$"  , "GO"    ,
    ]

if len(sys.argv) < 2:
    sys.exit(-1)

file_name = sys.argv[1]

state = 0
ptr   = 0x0801
addr  = 0

for byte in pathlib.Path(file_name).read_bytes():
    if state == 0:
        if byte != 1:
            print("ERROR");
        state = 1
    elif state == 1:
        if byte != 8:
            print("ERROR");
        addr = 0x0800
        state = 2
    elif state == 2:
        if addr != ptr:
            print("ERROR");
        ptr = byte
        state = 3
    elif state == 3:
        ptr = ptr + (byte << 8)
        state = 4
    elif state == 4:
        val = byte
        state = 5
    elif state == 5:
        val = val + (byte << 8)
        print(val, end='')
        print(' ', end='')
        state = 6
    elif state == 6:
        if byte == 0:
            print()
            state = 2
        elif byte < 0x80:
            print(chr(byte), end='')
        elif byte >= 0x80 and byte <= 0xcb:
            print(tokens[byte-0x80], end='')
    addr += 1


from django.urls import path

from internaladmin.views import AdminStatisticsView, AdminBuildMirrorStatusView

urlpatterns = [
    path("statistics/", AdminStatisticsView.as_view(), name="admin_statistics"),
    path(
        "mirror_status/",
        AdminBuildMirrorStatusView.as_view(),
        name="admin_build_mirror_status",
    ),
]


import json

import nonebot
from apscheduler.schedulers.asyncio import AsyncIOScheduler
from nonebot import require
from .config import Config
from .data_source import get_leetcode_question_everyday

global_config = nonebot.get_driver().config
nonebot.logger.info("global_config:{}".format(global_config))
plugin_config = Config(**global_config.dict())

if hasattr(plugin_config, 'leetcode_qq_friends') and hasattr(plugin_config,'leetcode_qq_groups') and hasattr(plugin_config,'leetcode_inform_time'):
    nonebot.logger.success("plugin_config:{}".format(plugin_config))
else:
    nonebot.logger.critical("plugin_config:{}".format(plugin_config))
    raise Exception("leetcode config error, please check env file")

scheduler = require("nonebot_plugin_apscheduler").scheduler  # type:AsyncIOScheduler

#
# @scheduler.scheduled_job("cron", hour="*/2", id="xxx", args=[1], kwargs={"arg2": 2})
# async def run_every_2_hour(arg1, arg2):
#     pass

async def send_leetcode_everyday():
    question = get_leetcode_question_everyday()
    # 转化成json格式
    jsonText = json.loads(question)
    # 题目题号
    no = jsonText.get('questionFrontendId')
    # 题名（中文）
    leetcodeTitle = jsonText.get('translatedTitle')
    # 提名 (英文)
    titleSlug = jsonText.get('titleSlug')
    # 题目难度级别
    level = jsonText.get('difficulty')
    # 题目内容
    context = jsonText.get('translatedContent')
    # 题目链接
    link = "https://leetcode-cn.com/problems/{}/".format(titleSlug)
    message = "no:{}\ntitle:{}\nlevel:{}\n".format(no, leetcodeTitle, level)
    message += "link:{}\n".format(link)

    # 给配置的列表里的qq好友发leetcode通知
    for qq in plugin_config.leetcode_qq_friends:
        await nonebot.get_bot().send_private_msg(user_id=qq, message=message)
    # 给群发送leetcode通知
    for qq_group in plugin_config.leetcode_qq_groups:
        await nonebot.get_bot().send_group_msg(group_id=qq_group, message="[CQ:at,qq={}]{}".format("all", message))

#调试用，可以每秒看到函数调用情况
scheduler.add_job(send_leetcode_everyday, "interval", seconds=1, id="114514")

# 根据配置的参数，注册定时任务,每天发送
for index, time in enumerate(plugin_config.leetcode_inform_time):
    nonebot.logger.info("id:{},time:{}".format(index, time))
    scheduler.add_job(send_leetcode_everyday, "cron", hour=time.hour, minute=time.minute, id=str(index))


#!/usr/bin/env python3

import matplotlib.pyplot as plt
import numpy as np

theta = np.linspace(0, 2*np.pi, 500)
r = 2*np.sin(3*theta)

ax = plt.subplot(111, polar=True)

ax.plot(theta, r, 'b-', theta, theta/(np.pi), 'g-')
ax.grid(True)

ax.set_title(r"Rose Plot $r = 2\sin(2\theta)$ and Archimedean Spiral $r =\theta/2\pi$")
plt.show()


from setuptools import find_packages, setup
from pathlib import Path

here = Path(__file__).parent
readme = (here / "README.md").read_text()

setup(
    name='BerkeleyPhotonicsGenerator',
    version='0.8.5',
    author='Pavan Bhargava and Sidney Buchbinder',
    author_email='pvnbhargava@berkeley.edu',
    description='A Python framework for generating masks and simulating integrated optical systems',
    long_description=readme,
    long_description_content_type="text/markdown",
    install_requires=[
        'setuptools>=18.5',
        'PyYAML>=5.1',
        'numpy>=1.10',
        'pytest>=4',
        'matplotlib>=3',
        'gdspy @ https://github.com/heitzmann/gdspy/archive/900ec9efecd066e3440e502bd8a046789bbab552.zip',
        'scipy>=1.1.0',
        'memory_profiler>=0.54.0',
        'Jinja2>=2.10.1',
    ],
    url='https://github.com/BerkeleyPhotonicsGenerator/BPG',
    license='BSD-3-Clause',
    packages=find_packages(exclude=("tests", "docs")),
    scripts=['BPG/bpg'],
    include_package_data=True,
)


from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.model_selection import train_test_split
import streamlit as st
import pickle
import pandas as pd

# Load the CSV file

X = pd.read_csv("genres.csv")

y = pd.read_csv("popularity.csv")

# Split the data into training and testing sets
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

st.title("Evaluating KNN Performance in Movie Recommender System")

# Get user input for number of neighbors
n_neighbors = st.slider("Number of neighbors:", 2, 30, 10, 1)

# Fit a KNN model to the training data
knn = KNeighborsClassifier(n_neighbors=n_neighbors)
knn.fit(X_train, y_train)

# Make predictions on the testing data
y_pred = knn.predict(X_test)

# Calculate the evaluation metrics
accuracy = accuracy_score(y_test, y_pred)
precision = precision_score(y_test, y_pred)
recall = recall_score(y_test, y_pred)
f1 = f1_score(y_test, y_pred)

# Print the evaluation metrics
st.write("Evaluation Metrics for k =", n_neighbors)
st.write("Accuracy:", accuracy)
st.write("Precision:", precision)
st.write("Recall:", recall)
st.write("F1-score:", f1)


try:
    import ConfigParser as configparser
except ImportError:
    import configparser
import os
import pytest

def pytest_addoption(parser):
    parser.addoption('--config', action='store', default='3.0-nginx')
    parser.addoption('--output', action='store', default='output.csv')

def pytest_sessionfinish(session, exitstatus):
    # print(session.config.option.__dict__)
    summary = f'''

    [+] Summary result:

    Test: {session.config.option.output}
    Total: {session.testscollected}
    Number Test Pass: {session.testscollected - session.testsfailed}
    Number Test Failed: {session.testsfailed}
    '''

    print(summary)

@pytest.fixture(scope='session')
def config(request):
    cp = configparser.RawConfigParser()
    cp.read(os.path.join(os.path.dirname(__file__), 'config.ini'))
    return dict(cp.items(request.config.getoption('--config')))


from .models import Product,ProductAttribute
from django.db.models import Min,Max
def get_filters(request):
	cats=Product.objects.distinct().values('Price_range__title','Price_range__id')
	brands=Product.objects.distinct().values('brand__title','brand__id')
	colors=ProductAttribute.objects.distinct().values('color__title','color__id','color__color_code')
	sizes=ProductAttribute.objects.distinct().values('Ram_and_memory__title','Ram_and_memory__id')
	minMaxPrice=ProductAttribute.objects.aggregate(Min('price'),Max('price'))
	data={
		'cats':cats,
		'brands':brands,
		'colors':colors,
		'sizes':sizes,
		'minMaxPrice':minMaxPrice,
	}
	return data


"""CLI to convert a single-image checkpoint to a tracking checkpoint."""

import argparse
import logging
import os

import torch

import openpifpaf

LOG = logging.getLogger(__name__)

def cli():
    parser = argparse.ArgumentParser(description=__doc__)

    openpifpaf.plugin.register()
    openpifpaf.logger.cli(parser)
    openpifpaf.network.Factory.cli(parser)

    parser.add_argument('-o', '--output', default=None)
    args = parser.parse_args()

    openpifpaf.logger.configure(args, LOG)
    openpifpaf.network.Factory.configure(args)

    assert args.checkpoint, 'have to specify a checkpoint as input'
    if args.output is None:
        basename = 't' + os.path.basename(args.checkpoint)
        if not basename.endswith('.pkl'):
            basename += '-{}.pkl'.format(openpifpaf.__version__)
        args.output = os.path.join('outputs', basename)

    return args

def main():
    args = cli()
    model, _ = openpifpaf.network.Factory().factory()
    model.base_net = openpifpaf.network.TrackingBase(model.base_net)
    # model = openpifpaf.network.batchrenorm.BatchRenorm2d.convert_to(model)

    LOG.info('saving %s', args.output)
    torch.save({
        'model': model,
        'epoch': 0,
        'meta': {
            'image-source': args.checkpoint,
        },
    }, args.output)

if __name__ == '__main__':
    main()


# -*- coding: utf-8 -*-
# @Author: ZhuZefeng
# @Date:   2018-12-20 17:56:56
# @Last Modified by:   ZhuZefeng
# @Last Modified time: 2018-12-27 11:46:35

"""
为应用程序accounting定义URL模式
"""

from django.urls import path, re_path
from .           import views

app_name = 'accounting'

urlpatterns = [
               #accounting程序的主页
               path('', views.index, name='index'),

               #显示所有销售订单的页面
               path('orders', views.orders, name='orders'),

               #新增销售订单的页面
               path('new_order', views.new_order, name='new_order'),

               #显示所有客户信息的页面
               path('clients', views.clients, name='clients'),

               #新增客户的页面
               path('new_client', views.new_client, name='new_client'),

               #显示所有产品信息的页面
               path('products', views.products, name='products'),

               #新增产品的页面
               #path('new_product', views. new_product, name=' new_product'),

              ]


from threading import Thread
import socket
import json
import time

class GameSocket(Thread):
    def __init__(self, host, port, accept_num=2):
        super().__init__()

        self.client_list = []
        self.flag = True
        self.server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
        self.server_socket.bind((host, port))
        self.server_socket.listen(accept_num)

    def terminate(self):
        self.flag = False

    def serve(self):
        self.start()

    def run(self):
        while self.flag:
            connection, address = self.server_socket.accept()
            self.client_list.append((address, connection))

            print("{} connected".format(address))

            self.new_connection(connection=connection,
                                address=address)
            if(len(self.client_list)==2):
                break
    def new_connection(self, connection, address):
        Thread(target=self.receive_message_from_client,
               kwargs={
                   "connection": connection,
                   "address": address}, daemon=True).start()

    def receive_message_from_client(self, connection, address):
        keep_going = True
        while keep_going:
            try:
                message = connection.recv(1024).strip().decode()
            except:
                print(message)
                print('[GameSocket EXCEPTION]')
                keep_going = False
            else:
                if not message:
                    break

                print('[TEST] {}'.format(message))
                message = json.loads(message)

                print('  server received: {} form {}'.format(message, address))

                if message['command'] == "close":
                    break
                else:
                    self.client_list[0][1].send(json.dumps(message).encode())
                    self.client_list[1][1].send(json.dumps(message).encode())

        connection.close()
        print("close connection")

    # def find_other(self, address):
    #     if(address[1] == self.client_list[0][0][1]):
    #         return 1
    #     return 0


#!/usr/bin/env python3

"""Generators for the `gKmg` Clustering Datasets

Copyleft (C) 2018-2022, Marek Gagolewski <https://www.gagolewski.com>

A generalised version of SIPU's G2 sets
(see https://cs.joensuu.fi/sipu/datasets/G2.txt),
but with a correction for dimensionality.

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
"""

#%%
import numpy as np
import scipy.spatial.distance

#%%
def generate_gKmg(d, n, mu, s, random_state=None):
    """Generates K=len(n) groups of points in R^d together with their
    corresponding labels.

    The i-th group, i=1,...,K, consists of n[i-1] points
    that are sampled from the Gaussian distribution with mean mu[i-1,:]
    and covariance matrix diag(s[i]).
    """
    assert mu.shape[0] == n.shape[0] == s.shape[0]
    assert mu.shape[1] == d
    assert (s>0).all()
    assert (n>0).all()

    K = mu.shape[0] # number of groups

    if random_state is None:
        random_state = np.random.randint(0, 2**32)

    # Each point group is generated separately,
    # with different (yet predictable) random_state,
    # so that changing n[i] generates the same points

    X = []
    for i in range(K):
        rand = np.random.RandomState((random_state+i) % (2**32))
        X.append(rand.randn(n[i], d)*s[i] + mu[i,:])

    X = np.vstack(X)

    labels0 = np.repeat(np.arange(1, K+1), n) #[1,1,...,1,2,...,2,...,K,...,K]

    labels1 = np.argmax(scipy.spatial.distance.cdist(X, mu), axis=1)+1

    return X, labels0, labels1

#%%
# import matplotlib.pyplot as plt
# import genieclust
# random_state = 666
# mu1 = 500  # cluster1 center
# mu2 = 600  # cluster2 center
# d = 2
# s = 10
# s_cor = s*d

# for n  in [30, 50]:
#     X, labels = generate_gKmg(
#         d,
#         np.r_[n, n],
#         np.array([ [mu1]*d, [mu2]*d ]),
#         np.r_[s_cor, s_cor],
#         random_state)
#     genieclust.plots.plot_scatter(X, labels+n-30, alpha=0.5)

# plt.show()

#%%
if __name__ == "__main__":

    # This is how we generate the (perfectly balanced) g2mg sets
    # included in the Benchmark Suite.

    random_state = 123 # one of few unsuspicious RNG seeds :) [*]
    mu1 = 500  # cluster1 centre
    mu2 = 600  # cluster2 centre
    n   = 1024 # number of points in each cluster
    Ds = [2**k for k in range(0, 8)] # [1, 2, 4, ..., 128]
    Ss = list(range(10, 100, 10))     # [10, 20, ..., 90]

    for d in Ds:
        for s in Ss:
            base_name = "g2mg/g2mg_%d_%s"%(d,s)
            print(base_name)
            s_cor = s*np.sqrt(d/2)
            X, labels0, labels1 = generate_gKmg(
                d,
                np.r_[n, n],
                np.array([ [mu1]*d, [mu2]*d ]),
                np.r_[s_cor, s_cor],
                random_state)

            data_file = base_name+".data.gz"
            # round -> no "." -> reduced file size
            np.savetxt(data_file, np.round(X, 0), fmt="%d")

            labels0_file = base_name+".labels0.gz"
            np.savetxt(labels0_file, labels0, fmt="%d")

            labels1_file = base_name+".labels1.gz"
            np.savetxt(labels1_file, labels1, fmt="%d")

            readme_file = base_name+".txt"
            readme = """\
g2mg clustering benchmark
Two %d-dimensional clusters, each with %d points that have
i.i.d. Gaussian components with standard deviation %d*sqrt(%d/2).

Author: Marek Gagolewski (http://www.gagolewski.com)
Copyleft 2020
Licensed under the Creative Commons Attribution 4.0 International License

`labels0` gives reference labels based on which probability distribution's
mixture component was used to generate the corresponding points.
`labels1` gives reference labels based on the distance to
the true cluster centre (i.e., arg max_ p_i(x)).
There is no noise class.
""" % (d, n, s, d)
            with open(readme_file, "w") as f:
                f.write(readme)

    # [*] the other ones being 42, 666, 1234, 12345, and 123456.
    # EOF.


#dbot
#Mathieu Dombrock

import discord
import re
import random
import string
import requests
import json
import pickle
import glob
from pathlib import Path

client = discord.Client()

@client.event
async def on_message(message):
	global banned_words
	bot_command = "?"
	memories = []
	memories_file = 'memories.pk'
	help_list = []
	tut_dir = 'tutorials/'
	ftype = ".txt"

	msgin_original = message.content #some functions need case sensitive input
	msgin = message.content.lower()
	msgin = msgin.split(" ")
	msgin_original = msgin_original.split(" ")
	print(msgin)
	tag = '{0.author.mention} : \n'.format(message)

	print("chan: "+str(message.channel))
	if message.author == client.user:#make sure we are not talking to ourselves
		return

	for word in banned_words:
		if str(word).lower() in str(msgin_original).lower():
			msgout = tag+"**WOAH THERE!**\n\n**RULES:**\n```4.) NO HATE SPEECH OF ANY KIND (SEE RULE #1 & #3)```\n"
			msgout += "This incident has been logged.\nIf you feel like this happened in error, please contact a helper and let us know what happened."
			log_path = "deletion-log.txt"
			with open(log_path, 'r') as file:
				old = file.read()
			with open(log_path, 'w') as file:
				file.write(str(message.author)+": "+str(msgin_original)+"\n"+"OFFENDING WORD: "+str(word)+"\n-------\n"+old)
			await client.delete_message(message)
			await client.send_message(message.channel, msgout)
			return

	if str(message.channel)=="welcome":
		tutorial_location = 'tutorials/welcome-message.txt'
		file = open(tutorial_location, "r")
		msgout = file.read()
		await client.send_message(message.channel, msgout)
		return

	if random.randrange(50)<2:
		msgout = tag+"Thanks for being here!"
		await client.send_message(message.channel, msgout)

	help_list.append(["tutorial <topic>","Display a preset tutorial message."])
	help_list.append(["tutorial list","List available tutorial topics."])
	if (bot_command+'tutorial') in msgin[0] or (bot_command+'tut') in msgin[0]:
		if ('list') in msgin[1]:
			print("printing tut list")
			tut_list = (glob.glob(tut_dir+"*.txt"))
			msgout = "**TUTORIALS AVAILABLE:**"
			for title in tut_list:
				title = title.replace(tut_dir,"")
				title = title.replace(ftype,"")
				msgout += "\n**"+title+"**"
		else:
			tutorial_location = 'tutorials/'+msgin[1]+'.txt'
			print(tutorial_location)
			#tutorail_file = Path(tutorial_location)
			if Path(tutorial_location).is_file():
				file = open(tutorial_location, "r")
				msgout = file.read()
				#msgout += "\n \n[If you have suggestions for a new tutorial or concerns about this one, please let us know in #suggestions-and-meta]"
			else:
				msgout = tag+"I can't find any saved tutorials on "+msgin[1]+"... sorry :/"
		await client.send_message(message.channel, msgout)

	help_list.append(["test","See if I am working. Or more specifically, still responding to messages."])
	if ((bot_command+'test') in msgin[0]):
		msgout = "Yeah, yeah I, oh, I'm still alive."
		await client.send_message(message.channel, msgout)

	help_list.append(["roll <x>d<y>","Example: roll 1d3"])
	if ((bot_command+'roll') in msgin[0]):
		msgin = msgin[1].split("d")
		result = ""
		results = ""
		for x in range(int(msgin[0])):
			result = random.randrange( 0, int(msgin[1]) ) +1
			results += str(result)+"\n"
		msgout = tag+results
		await client.send_message(message.channel, msgout)

	help_list.append(["choose <choice1> <choice2> <choice3> <...>","Helps you make a choice. \nExample: choose Debian Ubuntu CentOS\nOutput: Ubuntu"])
	if ((bot_command+'choose') in msgin[0]):
		print("choose")
		msgin = msgin[1:]
		print(msgin)
		msgout = tag+"Choosing from: "+', '.join(msgin)+"\n"+random.choice(msgin)
		await client.send_message(message.channel, msgout)

	help_list.append(["google <search-term>","I'm feeling lucky"])
	if ((bot_command+'google') in msgin[0]):
		r = requests.get("http://www.google.com/search?q="+' '.join(msgin[1:])+"&btnI");
		msgout = r.url
		msgout = tag+str(msgout)
		await client.send_message(message.channel, msgout)

	help_list.append(["wiki <search-term>","searches wikipedia"])
	if ((bot_command+'wiki') in msgin[0]):
		r = requests.get("https://en.wikipedia.org/w/api.php?action=opensearch&search="+' '.join(msgin[1:])+"&limit=1&namespace=0&format=json");
		msgout = json.loads(r.text)
		msgout = tag+str(msgout[3]).strip("[\"']")
		await client.send_message(message.channel, msgout)

	help_list.append(["remember <topic/key> <memory/value>","Remember the following\nExample: remember Dre hip-hop mastermind"])
	if ((bot_command+'remember') in msgin[0]):
		subject = msgin[1]
		memory = ' '.join(msgin_original[2:])
		msgout = tag+'Remembered '+subject+" as "+memory
		memories.append([subject, memory])
		#open a pickle file
		with open(memories_file, 'wb') as fi:
		# dump your data into the file
			pickle.dump(memories, fi)
		await client.send_message(message.channel, msgout)

	help_list.append(["recall <topic/key>","recall a memory\nExample: recall Dre"])
	if ((bot_command+'recall') in msgin[0]):
		with open(memories_file, 'rb') as fi:
			memories = pickle.load(fi)
		found = False
		for x in memories:
			if x[0] == msgin[1]:
				found = True
				subject = x[0]
				memory = x[1]
				msgout = tag+"I Recall "+subject+" as "+memory
		if found == False:
			msgout = tag+"I don't recall that."
		await client.send_message(message.channel, msgout)

	#help must be last to ensure the help list is loaded
	help_list.append(["help","Displays Help Message."])
	if ((bot_command+'help') in msgin[0]):
		msgout = "```CSS\nDiscover-Bot```"
		for x in sorted(help_list):
			msgout += "```\n"
			msgout += bot_command+'\n'.join(x).replace("\n", "\n--")+"\n\n"
			msgout += "\n```"
		msgout += "\nhttps://github.com/matdombrock/Discover-Bot\n"
		await client.send_message(message.channel, msgout)

@client.event
async def on_ready():
    print('Logged in as')
    print(client.user.name)
    print(client.user.id)
    print('------')

def load_banned_words(location):
	print("Loading Baned Words List")
	banned_words = []
	with open(location) as f:
		banned_words = f.read().splitlines()
	return banned_words

banned_words = load_banned_words('banned_words.txt')
client.run('change-me')


import time
import datetime
from unittest import TestCase
from models import ShortUrls

class TestCreateShortUrl(TestCase):
    def test_create_short_url(self):
        short_url = ShortUrls.create_short_url(
            "https://www.canada.ca", "test1234", "actor"
        )
        assert short_url == "test1234"

    def test_create_short_url_with_existing_url__returns_same_shorten_url(self):
        url_a = ShortUrls.create_short_url("https://www.canada.ca", "test1234", "actor")
        url_b = ShortUrls.create_short_url("https://www.canada.ca", "test1234", "actor")
        assert url_a == url_b

    def test_get_short_url(self):
        ShortUrls.create_short_url("https://www.canada.ca", "test", "actor")
        short_url = ShortUrls.get_short_url("test")
        assert short_url["original_url"]["S"] == "https://www.canada.ca"
        assert short_url["created_by"]["S"] == "actor"
        assert short_url["click_count"]["N"] == "0"
        short_url = ShortUrls.get_short_url("test")
        assert short_url["click_count"]["N"] == "1"
        short_url = ShortUrls.get_short_url("test")
        assert short_url["click_count"]["N"] == "2"

    def test_update_last_access_at(self):
        """Test that we are setting the last_access_date properly and it is being updated each time the shortned url is retrieved"""
        ShortUrls.create_short_url("https://www.canada.ca", "xyz", "actor")
        time.sleep(1)
        short_url = ShortUrls.get_short_url("xyz")
        assert short_url["last_access_date"]["N"] == short_url["created_at"]["N"]
        short_url = ShortUrls.get_short_url("xyz")
        time.sleep(1)
        assert short_url["last_access_date"]["N"] != short_url["created_at"]["N"]
        previous_last_access_date = int(short_url["last_access_date"]["N"])
        short_url = ShortUrls.get_short_url("xyz")
        assert short_url["last_access_date"]["N"] != previous_last_access_date

    def test_update_ttl_time(self):
        """Test that we are setting the ttl properly and it is being updated each time the shortned url is retrieved"""
        ShortUrls.create_short_url("https://www.canada.ca", "abc", "actor")
        time.sleep(1)
        short_url = ShortUrls.get_short_url("abc")
        previous_ttl = int(short_url["ttl"]["N"])
        short_url = ShortUrls.get_short_url("abc")
        assert short_url["ttl"]["N"] != str(previous_ttl)

    def test_ttl_is_valid(self):
        short_url = ShortUrls.get_short_url("test")
        epoch_time_now = int(time.time())
        assert int(short_url["ttl"]["N"]) > epoch_time_now

    def test_ttl_is_invalid(self):
        future_epoch_time = int(
            time.mktime(
                (
                    datetime.datetime.today() + datetime.timedelta(days=(365 * 5))
                ).timetuple()
            )
        )
        short_url = ShortUrls.get_short_url("test")
        assert short_url["ttl"]["N"] < str(future_epoch_time)

    def test_create_short_url_with_different_url_but_same_hash__raises_value_error_for_collision(
        self,
    ):
        with self.assertRaises(ValueError):
            ShortUrls.create_short_url("https://www.canada.ca", "test1234", "actor")
            ShortUrls.create_short_url("https://nrc.gc.ca", "test1234", "actor")


#Christopher Marotta
#November 13, 2019
#Lab Assignmnet - Hi-Lo Game
from tkinter import *
from PIL import ImageTk
import random

#setup game
window = Tk()
frame = Frame(window)
cardBackground = ImageTk.PhotoImage(file = "image/card/b1fv.gif")
firstCard = None
secondCard = None
cardOne = IntVar()
cardTwo = IntVar()
cardOne.set(0)
cardTwo.set(0)
selectVar = IntVar()
selectVar.set(0)
#returns a file location of a random card
def randomImage(cardNumVar):
    #global cardDrawn
    cardString = "image/card/"
    cardNum = random.randint(1,52)
    cardString = cardString + str(cardNum)
    cardString = cardString + ".gif"
    cardDrawn = ImageTk.PhotoImage(file = cardString)
    calcCardValue(cardNumVar, cardNum)
    return cardDrawn

#function to be run to calculate the "value" of both cards
def calcCardValue(cardNumVar, cardDrawn):
    cardValue = cardDrawn % 13
    if (cardValue == 0):
        cardValue = 13
    cardNumVar.set(cardValue)
    return

#function to be run when the left button is pressed
def drawFirstCard():
    global firstCard
    firstCard = randomImage(cardOne)
    leftButton['image'] = firstCard
    leftButton['state'] = "disabled"
    beginGameLabel.grid_remove()
    promptFrame.grid(row = 0, column = 1)
    rightButton.grid(row = 0, column = 2)
    return

#function to be run when the user wants to draw the second card
def drawSecondCard():
    global secondCard
    secondCard = randomImage(cardTwo)
    rightButton['image'] = secondCard
    rightButton['state'] = "disabled"
    promptFrame.grid_remove()
    endGameLabelString = checkWin()
    endGameLabel['text'] = endGameLabelString
    endGameLabel.grid(row = 0, column = 1)
    endGameButton.grid(row = 1, column = 1)
    return

#Resets the game and allows the player to play again
def playAgain():
    cardOne.set(0)
    cardTwo.set(0)
    selectVar.set(0)
    leftButton['image'] = cardBackground
    leftButton['state'] = "active"
    rightButton['image'] = cardBackground
    rightButton['state'] = "active"
    endGameButton.grid_remove()
    endGameLabel.grid_remove()
    beginGameLabel.grid(row = 0, column = 1)
    rightButton.grid_remove()

    return

#after the second card is drawn this will compare the two cards and determine if it was a win, lose, or tie.
def checkWin():
    returnString = ""
    higherSelected = None
    if (selectVar.get() == 1):
        higherSelected = False
    elif (selectVar.get() == 2):
        higherSelected = True
    else:
        higherSelected = False

    if (cardOne.get() > cardTwo.get()) and (not higherSelected):
        returnString = "Yay! You WIN!"
    elif (cardOne.get() > cardTwo.get()) and (higherSelected):
        returnString = "Sorry! You LOOSE!"
    elif (cardOne.get() < cardTwo.get()) and (higherSelected):
        returnString = "Yay! You WIN!"
    elif (cardOne.get() < cardTwo.get()) and (not higherSelected):
        returnString = "Sorry! You LOOSE!"
    elif cardOne.get() == cardTwo.get():
        returnString = "Sorry, you TIE!"
    else:
        returnString = "There has been a problem,\n so you WIN!"
    return returnString

#setup buttons
leftButton = Button(window, image = cardBackground, command = drawFirstCard)
rightButton = Button(window, image = cardBackground)

#Sets up and grids the label with instructions on how to begin the game
beginGameLabel = Label(window, text="Click the card to draw target.")
beginGameLabel.grid(row = 0, column = 1)

#Grids the left button, aka the first card.
leftButton.grid(row = 0, column = 0)

#Setup prompt frame for better formatting
promptFrame = Frame(window)

#Sets up the end game button and label
endGameLabel = Label(window, text = "PlaceHolder")
endGameButton = Button(window, text = "Draw", command = playAgain)

#prepares the prompt labels and button inputs.
promptLabel = Label(promptFrame, text = "Select Higher or Lower")
selectHigher = Radiobutton(promptFrame, text = "Higher", variable = selectVar, value = "2", command = drawSecondCard)
selectLower = Radiobutton(promptFrame, text = "Lower", variable = selectVar, value = "1", command = drawSecondCard)
promptLabel.grid(row = 0, column = 0)
selectHigher.grid(row = 1, column = 0)
selectLower.grid(row = 2, column = 0)

window.mainloop()