""" General functions for data_tables and data_table_manager
    We are using a class here just to make it easier to pass around
"""

import logging
import pprint
import subprocess
from pathlib import Path
import re
from typing import Union

import matplotlib.pyplot as mpl
import numpy as np
import pandas as pd
from pylibrary.plotting import plothelpers as PH
from pylibrary.tools import cprint
from pyqtgraph.Qt import QtGui

import ephys.datareaders as DR
from ephys.ephys_analysis import spike_analysis
from ephys.tools import utilities
import ephys

UTIL = utilities.Utility()
CP = cprint.cprint

class CustomFormatter(logging.Formatter):
    grey = "\x1b[38;21m"
    yellow = "\x1b[33;21m"
    red = "\x1b[31;21m"
    bold_red = "\x1b[31;1m"
    white = "\x1b[37m"
    reset = "\x1b[0m"
    lineformat = "%(asctime)s - %(levelname)s - (%(filename)s:%(lineno)d) %(message)s "

    FORMATS = {
        logging.DEBUG: grey + lineformat + reset,
        logging.INFO: white + lineformat + reset,
        logging.WARNING: yellow + lineformat + reset,
        logging.ERROR: red + lineformat + reset,
        logging.CRITICAL: bold_red + lineformat + reset,
    }

    def format(self, record):
        log_fmt = self.FORMATS.get(record.levelno)
        formatter = logging.Formatter(log_fmt)
        return formatter.format(record)

def get_git_hashes():
    process = subprocess.Popen(["git", "rev-parse", "HEAD"], shell=False, stdout=subprocess.PIPE)
    git_head_hash = process.communicate()[0].strip()

    ephyspath = Path(ephys.__file__).parent
    process = subprocess.Popen(
        ["git", "-C", str(ephyspath), "rev-parse", "HEAD"],
        shell=False,
        stdout=subprocess.PIPE,
    )
    ephys_git_hash = process.communicate()[0].strip()
    return {"project": git_head_hash, "ephys": ephys_git_hash}

def create_logger(
    log_name: str = "Log Name",
    log_file: str = "log_file.log",
    log_message: str = "Starting Logging",
):
    logging.getLogger("fontTools.subset").disabled = True
    Logger = logging.getLogger(log_name)
    level = logging.DEBUG
    Logger.setLevel(level)
    # create file handler which logs even debug messages
    logging_fh = logging.FileHandler(filename=log_file)
    logging_fh.setLevel(level)
    logging_sh = logging.StreamHandler()
    logging_sh.setLevel(level)
    log_formatter = logging.Formatter(
        "%(asctime)s - %(name)s - %(levelname)s  (%(filename)s:%(lineno)d) - %(message)s "
    )
    logging_fh.setFormatter(log_formatter)
    logging_sh.setFormatter(CustomFormatter())  # log_formatter)
    Logger.addHandler(logging_fh)
    Logger.addHandler(logging_sh)
    Logger.info(log_message)
    return Logger

Logger = create_logger(
    log_name="Spike Analysis",
    log_file="spike_analysis.log",
    log_message="Starting Process Spike Analysis",
)

PrettyPrinter = pprint.PrettyPrinter

datacols = [
    "holding",
    "RMP",
    "RMP_SD",
    "Rin",
    "taum",
    "dvdt_rising",
    "dvdt_falling",
    "current",
    "AP_thr_V",
    "AP_HW",
    "AP15Rate",
    "AdaptRatio",
    "AHP_trough_V",
    "AHP_depth_V",
    "tauh",
    "Gh",
    "FiringRate",
]

iv_keys: list = [
    "holding",
    "WCComp",
    "CCComp",
    "BridgeAdjust",
    "RMP",
    "RMP_SD",
    "RMPs",
    "Irmp",
    "taum",
    "taupars",
    "taufunc",
    "Rin",
    "Rin_peak",
    "tauh_tau",
    "tauh_bovera",
    "tauh_Gh",
    "tauh_vss",
]

spike_keys: list = [
    "FI_Growth",
    "AdaptRatio",
    "FI_Curve",
    "FiringRate",
    "AP1_Latency",
    "AP1_HalfWidth",
    "AP1_HalfWidth_interpolated",
    "AP2_Latency",
    "AP2_HalfWidth",
    "AP2_HalfWidth_interpolated",
    "FiringRate_1p5T",
    "AHP_Depth",
    "AHP_Trough",
    "spikes",
    "iHold",
    "pulseDuration",
    "baseline_spikes",
    "poststimulus_spikes",
]

# map spike measurements to top level keys
mapper: dict = {
    "AP1_HalfWidth": "halfwidth",
    "AP1_HalfWidth_interpolated": "halfwidth_interpolated",
    "AHP_trough_V": "trough_V",
    "AHP_Trough": "trough_T",
    "AHP_depth_V": "trough_V",
    "AP1_Latency": "AP_latency",
    "AP_thr_V": "AP_begin_V",
    "AP_HW": "halfwidth",
    "dvdt_rising": "dvdt_rising",
    "dvdt_falling": "dvdt_falling",
}
# map summary/not individual spike data to top level keys
mapper1: dict = {
    "AP15Rate": "FiringRate_1p5T",
    "AdaptRatio": "AdaptRatio",
}

iv_mapper: dict = {
    "tauh": "tauh_tau",
    "Gh": "tauh_Gh",
    "taum": "taum",
    "Rin": "Rin",
    "RMP": "RMP",
}

def print_spike_keys(row):
    if pd.isnull(row.IV):
        return row
    # print(row.IV)
    return row

class Functions:
    def __init__(self):
        self.textbox = None
        pass

    def get_row_selection(self, table_manager):
        """
        Find the selected rows in the currently managed table, and if there is a valid selection,
        return the index to the first row and the data from that row
        """
        self.selected_index_rows = table_manager.table.selectionModel().selectedRows()
        if self.selected_index_rows is None:
            return None, None
        else:
            index_row = self.selected_index_rows[0]
            selected = table_manager.get_table_data(index_row)  # table_data[index_row]
            if selected is None:
                return None, None
            else:
                return index_row, selected

    def get_multiple_row_selection(self, table_manager):
            """
            Find the selected rows in the currently managed table, and if there is a valid selection,
            return a list of indexs from the selected rows.
            """
            self.selected_index_rows = table_manager.table.selectionModel().selectedRows()
            if self.selected_index_rows is None:
                return None, None
            else:
                return self.selected_index_rows

    def get_datasummary_protocols(self, datasummary):
        """
        Print a configuration file-like text of all the datasummary protocols, as categorized here.

        """
        data_complete = datasummary["data_complete"].values
        print("# of datasummary entries: ", len(data_complete))
        protocols = []
        for i, prots in enumerate(data_complete):
            prots = prots.split(",")
            for prot in prots:
                protocols.append(prot[:-4].strip(" "))  # remove trailing "_000" etc

        allprots = sorted(list(set(protocols)))
        print("# of unique protocols: ", len(allprots))
        # print(allprots)

        # make a little table for config dict:
        txt = "protocols:\n"
        txt += "    CCIV:"
        ncciv = 0
        prots_used = []
        for i, prot in enumerate(allprots):
            if "CCIV".casefold() in prot.casefold():
                computes = "['RmTau', 'IV', 'Spikes', 'FI']"
                if "posonly".casefold() in prot.casefold():  # cannot compute rmtau for posonly
                    computes = "['IV', 'Spikes', 'FI']"
                txt += f"\n        {prot:s}: {computes:s}"
                prots_used.append(i)
                ncciv += 1
        if ncciv == 0:
            txt += " None"
        txt += "\n    VCIV:"
        nvciv = 0
        for i, prot in enumerate(allprots):
            if "VCIV".casefold() in prot.casefold():
                computes = "['VC']"
                txt += f"\n        {prot:s}: {computes:s}"
                nvciv += 1
                prots_used.append(i)
        if nvciv == 0:
            txt += " None"
        txt += "\n    Maps:"
        nmaps = 0
        for i, prot in enumerate(allprots):
            if "Map".casefold() in prot.casefold():
                computes = "['Maps']"
                txt += f"\n        {prot:s}: {computes:s}"
                nmaps += 1
                prots_used.append(i)
        if nmaps == 0:
            txt += " None"
        txt += "\n    Minis:"
        nminis = 0
        for i, prot in enumerate(allprots):
            cprot = prot.casefold()
            if "Mini".casefold() in cprot or "VC_Spont".casefold() in cprot:
                computes = "['Mini']"
                txt += f"\n        {prot:s}: {computes:s}"
                nminis += 1
                prots_used.append(i)
        if nminis == 0:
            txt += " None"
        txt += "\n    PSCs:"
        npsc = 0
        for i, prot in enumerate(allprots):
            if "PSC".casefold() in prot.casefold():
                computes = "['PSC']"
                txt += f"\n        {prot:s}: {computes:s}"
                npsc += 1
                prots_used.append(i)
        if npsc == 0:
            txt += " None"
        txt += "\n    Uncategorized:"
        allprots = [prot for i, prot in enumerate(allprots) if i not in prots_used]
        nother = 0
        for i, prot in enumerate(allprots):
            if len(prot) == 0 or prot == " ":
                prot = "No Name"
            computes = "None"
            txt += f"\n        {prot:s}: {computes:s}"
            nother += 1
        if nother == 0:
            txt += "\n        None"
        print(f"\n{txt:s}\n")
        # this print should be pasted into the configuration file (watch indentation)

    def moving_average(self, data, window_size):
        """moving_average Compute a triangular moving average on the data over a window

        Parameters
        ----------
        data : _type_
            _description_
        window_size : _type_
            _description_

        Returns
        -------
        _type_
            _description_
        """
        window = np.bartlett(window_size)
        # Normalize the window
        window /= window.sum()

        return np.convolve(data, window, "valid") / window_size

    def get_slope(self, y, x, index, window_size):
        """get_slope get slope of a smoothed curve at a given index

        Parameters
        ----------
        y : _type_
            _description_
        x : _type_
            _description_
        index : _type_
            _description_
        window_size : _type_
            _description_

        Returns
        -------
        _type_
            _description_
        """
        # Smooth the data
        y_smooth = self.moving_average(y, window_size)
        x_smooth = self.moving_average(x, window_size)

        # Adjust the index for the reduced size of the smoothed data
        index -= window_size // 2

        if index < 1 or index >= len(y_smooth) - 1:
            # Can't calculate slope at the start or end
            return None
        else:
            dy = y_smooth[index + 1] - y_smooth[index - 1]
            dx = x_smooth[index + 1] - x_smooth[index - 1]
            return dy / dx

    def draw_orthogonal_line(self, x, y, index, slope, length, color, ax):
        # Calculate the slope of the orthogonal line
        orthogonal_slope = -1.0 / slope

        # Calculate the start and end points of the orthogonal line
        x_start = x[index] - length / 2
        x_end = x[index] + length / 2
        y_start = y[index] + orthogonal_slope * (x_start - x[index])
        y_end = y[index] + orthogonal_slope * (x_end - x[index])

        # Plot the orthogonal line
        ax.plot([x_start, x_end], [y_start, y_end], color=color)

    def get_selected_cell_data_spikes(self, experiment, table_manager, assembleddata):
        self.get_row_selection(table_manager)
        if self.selected_index_rows is not None:
            for nplots, index_row in enumerate(self.selected_index_rows):
                selected = table_manager.get_table_data(index_row)
                day = selected.date[:-4]
                slicecell = selected.cell_id[-4:]
                cell_df, cell_df_tmp = self.get_cell(experiment, assembleddata, cell=selected.cell_id)
                protocols = list(cell_df["Spikes"].keys())
                min_index = None
                min_current = 1
                V = None
                min_protocol = None
                spike = None
                for ip, protocol in enumerate(protocols):
                    min_current_index, current, trace = self.find_lowest_current_trace(
                        cell_df["Spikes"][protocol]
                    )
                    if current < min_current:
                        I = current
                        V = trace
                        min_index = min_current_index
                        min_protocol = ip
                        min_current = current
                        spike = cell_df["Spikes"][protocol]
                pp = PrettyPrinter(indent=4)
                print("spike keys: ", spike["spikes"].keys())
                print(
                    "min I : ",
                    I,
                    "min V: ",
                    V,
                    "min index: ",
                    min_index,
                    "min_current: ",
                    min_current,
                )
                pp.pprint(spike["spikes"][V][min_index])
                low_spike = spike["spikes"][V][min_index]
                if nplots == 0:
                    import matplotlib.pyplot as mpl

                    f, ax = mpl.subplots(1, 2, figsize=(10, 5))
                vtime = (low_spike.Vtime - low_spike.peak_T) * 1e3
                ax[0].plot(vtime, low_spike.V * 1e3)
                ax[1].plot(low_spike.V * 1e3, low_spike.dvdt)
                dvdt_ticks = np.arange(-4, 2.01, 0.1)
                t_indices = np.array([np.abs(vtime - point).argmin() for point in dvdt_ticks])
                thr_index = np.abs(vtime - (low_spike.AP_latency - low_spike.peak_T) * 1e3).argmin()
                # Create a colormap
                cmap = mpl.get_cmap("tab10")
                # Create an array of colors based on the index of each point
                colors = cmap(np.linspace(0, 1, len(t_indices)))
                # for i in range(len(t_indices)):
                #     local_slope = self.get_slope(
                #         low_spike.V * 1e3, low_spike.dvdt, t_indices[i], 7,
                #     )
                #     if local_slope is not None:
                #         self.draw_orthogonal_line(
                #             low_spike.V * 1e3,
                #             low_spike.dvdt,
                #             index=t_indices[i],
                #             slope=local_slope,
                #             length=5.0,
                #             color=colors[i],
                #             ax=ax[1],
                #         )

                #     ax[1].scatter(
                #     low_spike.V[t_indices[i]] * 1e3,
                #     low_spike.dvdt[t_indices[i]],
                #     s=12,
                #     marker='|',
                #     color=colors[i],
                #     zorder = 10
                # )
                # Plot each point with a different color
                # ax[1].scatter(
                #     low_spike.V[t_indices] * 1e3,
                #     low_spike.dvdt[t_indices],
                #     s=12,
                #     marker='|',
                #     color=colors,
                #     zorder = 10
                # )
                ax[1].scatter(
                    low_spike.V[thr_index] * 1e3,
                    low_spike.dvdt[thr_index],
                    s=12,
                    marker="o",
                    color="r",
                    zorder=12,
                )

                latency = (low_spike.AP_latency - low_spike.peak_T) * 1e3  # in msec
                ax[0].plot(
                    latency,
                    low_spike.AP_begin_V * 1e3,
                    "ro",
                    markersize=2.5,
                    zorder=10,
                )
                ax[0].plot(
                    [
                        (low_spike.left_halfwidth_T - low_spike.peak_T - 0.0001) * 1e3,
                        (low_spike.right_halfwidth_T - low_spike.peak_T + 0.0001) * 1e3,
                    ],
                    [  # in msec
                        low_spike.halfwidth_V * 1e3,
                        low_spike.halfwidth_V * 1e3,
                    ],
                    "g-",
                    zorder=10,
                )
                # ax[0].plot(
                #     (low_spike.right_halfwidth_T - low_spike.peak_T)
                #     * 1e3,  # in msec
                #     low_spike.halfwidth_V * 1e3,
                #     "co",
                # )

                if nplots == 0:  # annotate
                    ax[0].set_xlabel("Time (msec), re Peak")
                    ax[0].set_ylabel("V (mV)")
                    ax[1].set_xlabel("V (mV)")
                    ax[1].set_ylabel("dV/dt (mV/ms)")
                    PH.nice_plot(ax[0])
                    PH.nice_plot(ax[1])
                    PH.talbotTicks(ax[0])
                    PH.talbotTicks(ax[1])

                nplots += 1

            if nplots > 0:
                mpl.show()

            return cell_df
        else:
            return None

    def get_selected_cell_data_FI(self, experiment, table_manager, assembleddata):
        self.get_row_selection(table_manager)
        pp = PrettyPrinter(indent=4, width=120)
        if self.selected_index_rows is not None:
            for nplots, index_row in enumerate(self.selected_index_rows):
                selected = table_manager.get_table_data(index_row)
                day = selected.date[:-4]
                slicecell = selected.cell_id[-4:]
                # cell_df, _ = self.get_cell(
                #     experiment, assembleddata, cell=selected.cell_id
                # )
                fig, ax = mpl.subplots(1, 1)
                self.compute_FI_Fits(
                    experiment, assembleddata, selected.cell_id, plot_fits=True, ax=ax
                )

            if nplots > 0:
                mpl.show()
            return self.selected_index_rows
        else:
            return None

    def average_FI(self, FI_Data_I_, FI_Data_FR_, max_current: float = 1.0e-9):
        if len(FI_Data_I_) > 0:
            try:
                FI_Data_I, FI_Data_FR = zip(*sorted(zip(FI_Data_I_, FI_Data_FR_)))
            except:
                raise ValueError("couldn't zip the data sets: ")
            if len(FI_Data_I) > 0:  # has data...
                print("averaging FI data")
                FI_Data_I_ = np.array(FI_Data_I)
                FI_Data_FR_ = np.array(FI_Data_FR)
                f1_index = np.where((FI_Data_I_ >= 0.0) & (FI_Data_I_ <= max_current))[
                    0
                ]  # limit to 1 nA, regardless
                FI_Data_I, FI_Data_FR, FI_Data_FR_Std, FI_Data_N = self.avg_group(
                    FI_Data_I_[f1_index], FI_Data_FR_[f1_index], ndim=FI_Data_I_.shape
                )
        return FI_Data_I, FI_Data_FR, FI_Data_FR_Std, FI_Data_N

    def avg_group(self, x, y, ndim=2):
        if ndim == 2:
            x = np.array([a for b in x for a in b])
            y = np.array([a for b in y for a in b])
        else:
            x = np.array(x)
            y = np.array(y)
        # x = np.ravel(x) # np.array(x)
        # y = np.array(y)
        xa, ind, counts = np.unique(
            x, return_index=True, return_counts=True
        )  # find unique values in x
        ya = y[ind]

        ystd = np.zeros_like(ya)
        yn = np.ones_like(ya)
        for dupe in xa[counts > 1]:  # for each duplicate value, replace with mean
            # print("dupe: ", dupe)
            # print(np.where(x==dupe), np.where(xa==dupe))
            ya[np.where(xa == dupe)] = np.nanmean(y[np.where(x == dupe)])
            ystd[np.where(xa == dupe)] = np.nanstd(y[np.where(x == dupe)])
            yn[np.where(xa == dupe)] = np.count_nonzero(~np.isnan(y[np.where(x == dupe)]))
        return xa, ya, ystd, yn

    # get maximum slope from fit.
    def hill_deriv(self, x: float, y0: float, ymax: float, m: float, n: float):
        """hill_deriv
        analyztical solution computed from SageMath

        Parameters
        ----------
        x : float
            current
        y0 : float
            baseline
        ymax : float
            maximum y value
        m : float
            growth rate
        n : float
            growth power
        """
        hd = m * n * ymax
        hd *= np.power(m / x, n - 1)
        hd /= (x * x) * np.power((np.power(m / x, n) + 1.0), 2.0)
        return hd

    def fit_FI_Hill(
        self,
        FI_Data_I,
        FI_Data_FR,
        FI_Data_FR_Std,
        FI_Data_I_,
        FI_Data_FR_,
        FI_Data_N,
        hill_max_derivs,
        hill_i_max_derivs,
        FI_fits,
        linfits,
        cell: str,
        celltype: str,
        plot_fits=False,
        ax: Union[mpl.Axes, None] = None,
    ):
        plot_raw = False  # only to plot the unaveraged points.
        spanalyzer = spike_analysis.SpikeAnalysis()
        spanalyzer.fitOne(
            i_inj=FI_Data_I,
            spike_count=FI_Data_FR,
            pulse_duration=None,  # protodurs[ivname],
            info="",
            function="Hill",
            fixNonMonotonic=True,
            excludeNonMonotonic=False,
            max_current=None,
        )

        try:
            fitpars = spanalyzer.analysis_summary["FI_Growth"][0]["parameters"][0]
        except:
            CP(
                "r",
                f"fitpars has no solution? : {cell!s}, {celltype:s}, {spanalyzer.analysis_summary['FI_Growth']!s}",
            )
            return (
                hill_max_derivs,
                hill_i_max_derivs,
                FI_fits,
                linfits,
            )  # no fit, return without appending a new fit
        # raise ValueError("couldn't get fitpars: no solution?")
        y0 = fitpars[0]
        ymax = fitpars[1]
        m = fitpars[2]
        n = fitpars[3]
        xyfit = spanalyzer.analysis_summary["FI_Growth"][0]["fit"]
        i_range = np.linspace(1e-12, np.max(xyfit[0]), 1000)
        # print(f"fitpars: y0={y0:.3f}, ymax={ymax:.3f}, m={m*1e9:.3f}, n={n:.3f}")
        deriv_hill = [self.hill_deriv(x=x, y0=y0, ymax=ymax, m=m, n=n) for x in i_range]
        deriv_hill = np.array(deriv_hill) * 1e-9  # convert to sp/nA
        max_deriv = np.max(deriv_hill)
        arg_max_deriv = np.argmax(deriv_hill)
        i_max_deriv = i_range[arg_max_deriv] * (1e12)
        hill_max_derivs.append(max_deriv)
        hill_i_max_derivs.append(i_max_deriv)
        # print(f"max deriv: {max_deriv:.3f} sp/nA at {i_max_deriv:.1f} pA")
        # print(xyfit[1])
        if len(spanalyzer.analysis_summary["FI_Growth"]) > 0:
            FI_fits["fits"].append(spanalyzer.analysis_summary["FI_Growth"][0]["fit"])
            FI_fits["pars"].append(spanalyzer.analysis_summary["FI_Growth"][0]["parameters"])
        linfit = spanalyzer.getFISlope(
            i_inj=FI_Data_I,
            spike_count=FI_Data_FR,
            pulse_duration=None,  # FR is already duration
            min_current=0e-12,
            max_current=300e-12,
        )
        linfits.append(linfit)
        linx = np.arange(0, 300e-12, 10e-12)
        liny = linfit.slope * linx + linfit.intercept

        if plot_fits:
            if ax is None:
                fig, ax = mpl.subplots(1, 1)
                fig.suptitle(f"{celltype:s} {cell:s}")

            line_FI = ax.errorbar(
                np.array(FI_Data_I) * 1e9,
                FI_Data_FR,
                yerr=FI_Data_FR_Std,
                marker="o",
                color="k",
                linestyle=None,
            )
            # ax[1].plot(FI_Data_I * 1e12, FI_Data_N, marker="s")
            if plot_raw:
                for i, d in enumerate(FI_Data_I_):  # plot the raw points before combining
                    ax.plot(np.array(FI_Data_I_[i]) * 1e9, FI_Data_FR_[i], "x", color="k")
            # print("fit x * 1e9: ", spanalyzer.analysis_summary['FI_Growth'][0]['fit'][0]*1e9)
            # print("fit y * 1: ", spanalyzer.analysis_summary['FI_Growth'][0]['fit'][1])

            # ax[0].plot(linx * 1e12, liny, color="c", linestyle="dashdot")
            celln = Path(cell).name

            if len(spanalyzer.analysis_summary["FI_Growth"]) >= 0:
                line_fit = ax.plot(
                    spanalyzer.analysis_summary["FI_Growth"][0]["fit"][0][0] * 1e9,
                    spanalyzer.analysis_summary["FI_Growth"][0]["fit"][1][0],
                    color="r",
                    linestyle="-",
                    zorder=100,
                )
                # derivative (in blue)
                line_deriv = ax.plot(
                    i_range * 1e9, deriv_hill, color="b", linestyle="--", zorder=100
                )
                d_max = np.argmax(deriv_hill)
                ax2 = ax.twinx()
                ax2.set_ylim(0, 500)
                ax2.set_ylabel("Firing Rate Slope (sp/s/nA)")
                line_drop = ax2.plot(
                    [i_range[d_max] * 1e9, i_range[d_max] * 1e9],
                    [0, 1.1 * deriv_hill[d_max]],
                    color="b",
                    zorder=100,
                )
                ax.set_xlabel("Current (nA)")
                ax.set_ylabel("Firing Rate (sp/s)")
                # turn off top box
                for loc, spine in ax.spines.items():
                    if loc in ["left", "bottom"]:
                        spine.set_visible(True)
                    elif loc in ["right", "top"]:
                        spine.set_visible(False)
                for loc, spine in ax2.spines.items():
                    if loc in ["right", "bottom"]:
                        spine.set_visible(True)
                    elif loc in ["left", "top"]:
                        spine.set_visible(False)
                # spine.set_color('none')
                # do not draw the spine
                # spine.set_color('none')
                # do not draw the spine
                PH.talbotTicks(ax, density=[2.0, 2.0])
                PH.talbotTicks(ax2, density=[2.0, 2.0])
                ax.legend(
                    [line_FI, line_fit[0], line_deriv[0], line_drop[0]],
                    ["Firing Rate", "Hill Fit", "Derivative", "Max Derivative"],
                    loc="best",
                    frameon=False,
                )

            mpl.show()

        return hill_max_derivs, hill_i_max_derivs, FI_fits, linfits

    def check_excluded_dataset(self, day_slice_cell, experiment, protocol):
        exclude_flag = day_slice_cell in experiment["excludeIVs"]
        print("    IV is in exclusion table: ", exclude_flag)
        if exclude_flag:
            exclude_table = experiment["excludeIVs"][day_slice_cell]
            print("    excluded table data: ", exclude_table)
            print("    testing protocol: ", protocol)
            proto = Path(protocol).name  # passed protocol has day/slice/cell/protocol
            if proto in exclude_table["protocols"] or exclude_table["protocols"] == ["all"]:
                CP(
                    "y",
                    f"Excluded cell/protocol: {day_slice_cell:s}, {proto:s} because: {exclude_table['reason']:s}",
                )
                Logger.info(
                    f"Excluded cell: {day_slice_cell:s}, {proto:s} because: {exclude_table['reason']:s}"
                )
                return True
        print("    Protocol passed: ", protocol)
        return False

    def compute_FI_Fits(
        self,
        experiment,
        df: pd.DataFrame,
        cell: str,
        protodurs: list = [1.0],
        plot_fits: bool = False,
        ax: Union[mpl.Axes, None] = None,
    ):
        CP("g", f"\n{'='*80:s}\nCell: {cell!s}, {df[df.cell_id==cell].cell_type.values[0]:s}")

        df_cell, df_tmp = self.get_cell(experiment, df, cell)
        if df_cell is None:
            return None
        print("    df_tmp group>>: ", df_tmp.Group.values)
        print("    df_cell group>>: ", df_cell.keys())
        protocols = list(df_cell.Spikes.keys())
        spike_keys = list(df_cell.Spikes[protocols[0]].keys())
        iv_keys = list(df_cell.IV[protocols[0]].keys())

        srs = {}
        dur = {}
        important = {}
        # for each CCIV type of protocol that was run:
        for nprot, protocol in enumerate(protocols):
            if protocol.endswith("0000"):  # bad protocol name
                continue
            day_slice_cell = str(Path(df_cell.date, df_cell.slice_slice, df_cell.cell_cell))
            CP("m", f"day_slice_cell: {day_slice_cell:s}, protocol: {protocol:s}")
            if self.check_excluded_dataset(day_slice_cell, experiment, protocol):
                continue
            fullpath = Path(experiment["rawdatapath"], experiment["directory"], protocol)
            with DR.acq4_reader.acq4_reader(fullpath, "MultiClamp1.ma") as AR:
                try:
                    AR.getData(fullpath)
                    sample_rate = AR.sample_rate[0]
                    duration = AR.tend - AR.tstart
                    srs[protocol] = sample_rate
                    dur[protocol] = duration
                    important[protocol] = AR.checkProtocolImportant(fullpath)
                    CP("g", f"    Protocol {protocol:s} has sample rate of {sample_rate:e}")
                except ValueError:
                    CP("r", f"Acq4Read failed to read data file: {str(fullpath):s}")
                    raise ValueError(f"Acq4Read failed to read data file: {str(fullpath):s}")

        protocols = list(srs.keys())  # only count valid protocols
        CP("c", f"Valid Protocols: {protocols!s}")
        if len(protocols) > 1:
            protname = "combined"
        elif len(protocols) == 1:
            protname = protocols[0]
        else:
            return None
        # parse group correctly.
        # the first point in the Group column is likely a nan.
        # if it is, then use the next point.
        print("Group: ", df_tmp.Group, "protoname: ", protname)
        group = df_tmp.Group.values[0]

        datadict = {
            "ID": str(df_tmp.cell_id.values[0]),
            "Subject": str(df_tmp.cell_id.values[0]),
            "cell_id": cell,
            "Group": group,
            "Date": str(df_tmp.Date.values[0]),
            "age": str(df_tmp.age.values[0]),
            "weight": str(df_tmp.weight.values[0]),
            "sex": str(df_tmp.sex.values[0]),
            "cell_type": df_tmp.cell_type.values[0],
            "protocol": protname,
            "important": important,
            "protocols": list(df_cell.IV),
            "sample_rate": srs,
            "duration": dur,
        }

        # get the measures for the fixed values from the measure list
        for measure in datacols:
            datadict = self.get_measure(df_cell, measure, datadict, protocols, threshold_slope=experiment["AP_threshold_dvdt"])
        # now combine the FI data across protocols for this cell
        FI_Data_I1_:list_ = []
        FI_Data_FR1_:list_ = []  # firing rate
        FI_Data_I4_:list_ = []
        FI_Data_FR4_:list_ = []  # firing rate
        FI_fits:dict = {"fits": [], "pars": [], "names": []}
        linfits:list = []
        hill_max_derivs:list = []
        hill_i_max_derivs:list = []
        protofails = 0
        for protocol in protocols:
            if protocol.endswith("0000"):  # bad protocol name
                continue
            # check if duration is acceptable:
            if protodurs is not None:
                durflag = False
                for d in protodurs:
                    if not np.isclose(dur[protocol], d):
                        durflag = True
                if durflag:
                    CP("y", f"    >>>> Protocol {protocol:s} has duration of {dur[protocol]:e}")
                    CP("y", f"               This is not in accepted limits of: {protodurs!s}")
                    continue
                else:
                    CP("g", f"    >>>> Protocol {protocol:s} has acceptable duration of {dur[protocol]:e}")
            # print("protocol: ", protocol, "spikes: ", df_cell.Spikes[protocol]['spikes'])
            if len(df_cell.Spikes[protocol]["spikes"]) == 0:
                CP("y", f"    >>>> Skipping protocol with no spikes:  {protocol:s}")
                continue
            else:
                CP("g", f"   >>>> Analyzing FI for protocol: {protocol:s}")
            try:
                fidata = df_cell.Spikes[protocol]["FI_Curve"]
            except KeyError:
                print("FI curve not found for protocol: ", protocol, "for cell: ", cell)
                # print(df_cell.Spikes[protocol])
                protofails += 1
                if protofails > 4:
                    raise ValueError(
                        "FI curve data not found for protocol: ",
                        protocol,
                        "for cell: ",
                        cell,
                    )
                else:
                    continue
            if np.max(fidata[0]) > 1.01e-9:  # accumulate high-current protocols
                FI_Data_I4_.extend(fidata[0])
                FI_Data_FR4_.extend(fidata[1] / dur[protocol])
            else:  # accumulate other protocols <= 1 nA
                FI_Data_I1_.extend(fidata[0])
                FI_Data_FR1_.extend(fidata[1] / dur[protocol])

        FI_Data_I1 = []
        FI_Data_FR1 = []
        FI_Data_I4 = []
        FI_Data_FR4 = []
        if len(FI_Data_I1_) > 0:
            FI_Data_I1, FI_Data_FR1, FI_Data_FR1_Std, FI_Data_N1 = self.average_FI(
                FI_Data_I1_, FI_Data_FR1_, 1e-9
            )
        if len(FI_Data_I4_) > 0:
            FI_Data_I4, FI_Data_FR4, FI_Data_FR4_Std, FI_Data_N1 = self.average_FI(
                FI_Data_I4_, FI_Data_FR4_, 4e-9
            )
        if len(FI_Data_I1) > 0:
            # do a curve fit on the first 1 nA of the protocol
            hill_max_derivs, hill_i_max_derivs, FI_fits, linfits = self.fit_FI_Hill(
                FI_Data_I=FI_Data_I1,
                FI_Data_FR=FI_Data_FR1,
                FI_Data_I_=FI_Data_I1_,
                FI_Data_FR_=FI_Data_FR1_,
                FI_Data_FR_Std=FI_Data_FR1_Std,
                FI_Data_N=FI_Data_N1,
                hill_max_derivs=hill_max_derivs,
                hill_i_max_derivs=hill_i_max_derivs,
                FI_fits=FI_fits,
                linfits=linfits,
                cell=cell,
                celltype=df_tmp.cell_type.values[0],
                plot_fits=plot_fits,
                ax=ax,
            )

        # save the results
        datadict["FI_Curve"] = [FI_Data_I1, FI_Data_FR1]
        datadict["FI_Curve4"] = [FI_Data_I4, FI_Data_FR4]
        datadict["current"] = FI_Data_I1
        datadict["spsec"] = FI_Data_FR1
        # datadict["Subject"] = df_tmp.cell_id.values[0]
        # datadict["Group"] = df_tmp.Group.values[0]
        # datadict["sex"] = df_tmp.sex.values[0]
        # datadict["celltype"] = df_tmp.cell_type.values[0]
        datadict["pars"] = [FI_fits["pars"]]
        datadict["names"] = []
        datadict["fit"] = [FI_fits["fits"]]
        datadict["F1amp"] = np.nan
        datadict["F2amp"] = np.nan
        datadict["Irate"] = np.nan
        datadict["maxHillSlope"] = np.nan
        datadict["maxHillSlope_SD"] = np.nan
        datadict["I_maxHillSlope"] = np.nan
        datadict["I_maxHillSlope_SD"] = np.nan
        if len(linfits) > 0:
            datadict["FISlope"] = np.mean([s.slope for s in linfits])
        else:
            datadict["FISlope"] = np.nan
        if len(hill_max_derivs) > 0:
            datadict["maxHillSlope"] = np.mean(hill_max_derivs)
            datadict["maxHillSlope_SD"] = np.std(hill_max_derivs)
            datadict["I_maxHillSlope"] = np.mean(hill_i_max_derivs)
            datadict["I_maxHillSlope_SD"] = np.std(hill_i_max_derivs)
        if len(FI_Data_I1) > 0:
            i_one = np.where(FI_Data_I1 <= 1.01e-9)[0]
            datadict["FIMax_1"] = np.nanmax(FI_Data_FR1[i_one])
        if len(FI_Data_I4) > 0:
            i_four = np.where(FI_Data_I4 <= 4.01e-9)[0]
            datadict["FIMax_4"] = np.nanmax(FI_Data_FR4[i_four])
        return datadict

    def get_cell(self, experiment, df: pd.DataFrame, cell: str):
        df_tmp = df[df.cell_id == cell] # df.copy() # .dropna(subset=["Date"])
        print("\nGet_cell:: df_tmp head: \n", "Groups: ", df_tmp["Group"].unique(), "\n len df_tmp: ", len(df_tmp))

        if len(df_tmp) == 0:
            return None, None
        try:
            celltype = df_tmp.cell_type.values[0]
        except ValueError:
            celltype = df_tmp.cell_type
        celltype = str(celltype).replace("\n", "")
        if celltype == " ":  # no cell type
            celltype = "unknown"
        CP("m", f"get cell: df_tmp cell type: {celltype:s}")
        # look for original PKL file for cell in the dataset
        # if it exists, use it to get the FI curve
        # base_cellname = str(Path(cell)).split("_")
        # print("base_cellname: ", base_cellname)
        # sn = int(base_cellname[-1][1])
        # cn = int(base_cellname[-1][3])
        # different way from cell_id:
        # The cell name may be a path, or just the cell name.
        # we have to handle both cases.

        parent = Path(cell).parent
        if parent == ".":  # just cell, not path
            cell_parts = str(cell).split("_")
            re_parse = re.compile("([Ss]{1})(\d{1,3})([Cc]{1})(\d{1,3})")
            cnp = re_parse.match(cell_parts[-1]).group(2)
            cn = int(cnp)
            snp = re_parse.match(cell_parts[-1]).group(4)
            sn = int(snp)
            cell_day_name = cell_parts[-3].split("_")[0]
            dir_path = None
        else:
            cell = Path(cell).name  # just get the name here
            cell_parts = cell.split("_")
            re_parse = re.compile("([Ss]{1})(\d{1,3})([Cc]{1})(\d{1,3})")
            # print("cell_parts: ", cell_parts[-1])
            snp = re_parse.match(cell_parts[-1]).group(2)
            sn = int(snp)
            cnp = re_parse.match(cell_parts[-1]).group(4)
            cn = int(cnp)
            cell_day_name = cell_parts[0]
            dir_path = parent

        # print("Cell name, slice, cell: ", cell_parts, sn, cn)
        # if cell_parts != ['2019.02.22', '000', 'S0C0']:
        #     return None, None
        cname2 = f"{cell_day_name.replace('.', '_'):s}_S{snp:s}C{cnp:s}_{celltype:s}_IVs.pkl"
        datapath2 = Path(experiment["analyzeddatapath"], experiment["directory"], celltype, cname2)

        # cname2 = f"{cell_day_name.replace('.', '_'):s}_S{sn:02d}C{cn:02d}_{celltype:s}_IVs.pkl"
        # datapath2 = Path(experiment["analyzeddatapath"], experiment["directory"], celltype, cname2)
        # cname1 = f"{cell_day_name.replace('.', '_'):s}_S{sn:01d}C{cn:01d}_{celltype:s}_IVs.pkl"
        # datapath1 = Path(experiment["analyzeddatapath"], experiment["directory"], celltype, cname1)
        # print(datapath)
        # if datapath1.is_file():
        #     CP("c", f"... {datapath1!s} is OK")
        #     datapath = datapath1
        if datapath2.is_file():
            CP("c", f"...  {datapath2!s} is OK")
            datapath = datapath2
        else:
            CP("r", f"no file: matching: {datapath2!s}, \n") #    or: {datapath2!s}\n")
            print("cell type: ", celltype)
            raise ValueError
            return None, None
        try:
            df_cell = pd.read_pickle(datapath, compression="gzip")
        except ValueError:
            try:
                df_cell = pd.read_pickle(datapath)  # try with no compression
            except ValueError:
                CP("r", f"Could not read {datapath!s}")
                raise ValueError("Failed to read compressed pickle file")
        if "Spikes" not in df_cell.keys() or df_cell.Spikes is None:
            CP(
                "r",
                f"df_cell: {df_cell.age!s}, {df_cell.cell_type!s}, No spike protos:",
            )
            return None, None
        # print(
        #     "df_cell: ",
        #     df_cell.age,
        #     df_cell.cell_type,
        #     "N spike protos: ",
        #     len(df_cell.Spikes),
        #     "\n",
        #     df_tmp['Group'],
        # )
        return df_cell, df_tmp

    def get_lowest_current_spike(self, row, SP):
        measured_first_spike = False
        dvdts = []
        for tr in SP.spikeShapes:  # for each trace
            if len(SP.spikeShapes[tr]) > 1:  # if there is a spike
                spk = SP.spikeShapes[tr][0]  # get the first spike in the trace
                dvdts.append(spk)  # accumulate first spike info

        if len(dvdts) > 0:
            currents = []
            for d in dvdts:  # for each first spike, make a list of the currents
                currents.append(d.current)
            min_current = np.argmin(currents)  # find spike elicited by the minimum current
            row.dvdt_rising = dvdts[min_current].dvdt_rising
            row.dvdt_falling = dvdts[min_current].dvdt_falling
            row.dvdt_current = currents[min_current] * 1e12  # put in pA
            row.AP_thr_V = 1e3 * dvdts[min_current].AP_begin_V
            if dvdts[min_current].halfwidth_interpolated is not None:
                row.AP_HW = dvdts[min_current].halfwidth_interpolated * 1e3
            row.AP_begin_V = 1e3 * dvdts[min_current].AP_begin_V
            CP(
                "y",
                f"I={currents[min_current]*1e12:6.1f} pA, dvdtRise={row.dvdt_rising:6.1f}, dvdtFall={row.dvdt_falling:6.1f}, APthr={row.AP_thr_V:6.1f} mV, HW={row.AP_HW*1e3:6.1f} usec",
            )
        return row

    def find_lowest_current_trace(self, spikes):
        current = []
        trace = []
        for sweep in spikes["spikes"]:
            for spike in spikes["spikes"][sweep]:
                this_spike = spikes["spikes"][sweep][spike]
                current.append(this_spike.current)
                trace.append(this_spike.trace)
                break  # only get the first one
            # now find the index of the lowest current
        if len(current) == 0:
            return np.nan, np.nan, np.nan
        min_current_index = np.argmin(current)
        # print("current: ", current, "traces: ", trace)
        # print(current[min_current_index], trace[min_current_index])
        return min_current_index, current[min_current_index], trace[min_current_index]

    def convert_FI_array(self, FI_values):
        """convert_FI_array Take a potential string representing the FI_data,
        and convert it to a numpy array

        Parameters
        ----------
        FI_values : str or numpy array
            data to be converted

        Returns
        -------
        numpy array
            converted data from FI_values
        """
        if isinstance(FI_values, str):
            fistring = FI_values.replace("[", "").replace("]", "").replace("\n", "")
            fistring = fistring.split(" ")
            FI_data = np.array([float(s) for s in fistring if len(s) > 0])
            FI_data = FI_data.reshape(2, int(FI_data.shape[0] / 2))
        else:
            FI_data = FI_values
        FI_data = np.array(FI_data)
        return FI_data

    def get_measure(self, df_cell, measure, datadict, protocols, threshold_slope:float=20.0):
        """get_measure : for the giveen cell, get the measure from the protocols

        Parameters
        ----------
        df_cell : _type_
            _description_
        measure : _type_
            _description_
        datadict : _type_
            _description_
        protocols : _type_
            _description_

        Returns
        -------
        _type_
            _description_
        """
        m = []
        if measure in iv_keys:
            for protocol in protocols:
                if measure in df_cell.IV[protocol].keys():
                    m.append(df_cell.IV[protocol][measure])

        elif measure in iv_mapper.keys() and iv_mapper[measure] in iv_keys:
            for protocol in protocols:
                if iv_mapper[measure] in df_cell.IV[protocol].keys():
                    m.append(df_cell.IV[protocol][iv_mapper[measure]])
        elif measure in spike_keys:
            maxadapt = 0
            for protocol in protocols:
                # print("p: ", p)
                if measure == "AdaptRatio":
                    if df_cell.Spikes[protocol][mapper1[measure]] > 8.0:
                        continue
                        # print("\nprot, measure: ", protocol, measure, df_cell.Spikes[protocol][mapper1[measure]])
                        # print(df_cell.Spikes[protocol].keys())
                        # maxadapt = np.max([maxadapt, df_cell.Spikes[protocol][mapper1['AdaptRatio']]])

                if measure in df_cell.Spikes[protocol].keys():
                    m.append(df_cell.Spikes[protocol][measure])
            # if maxadapt > 8:
            #     exit()

        elif measure in mapper1.keys() and mapper1[measure] in spike_keys:
            for protocol in protocols:
                if mapper1[measure] in df_cell.Spikes[protocol].keys():
                    m.append(df_cell.Spikes[protocol][mapper1[measure]])
        elif measure == "current":
            for protocol in protocols:  # for all protocols with spike analysis data for this cell
                if "spikes" not in df_cell.Spikes[protocol].keys():
                    continue
                # we need to get the first spike evoked by the lowest current level ...
                min_current_index, current, trace = self.find_lowest_current_trace(
                    df_cell.Spikes[protocol]
                )
                if not np.isnan(min_current_index):
                    m.append(current)
                else:
                    m.append(np.nan)

        else:
            for protocol in protocols:  # for all protocols with spike analysis data for this cell
                # we need to get the first spike evoked by the lowest current level ...
                prot_spike_count = 0
                if "spikes" not in df_cell.Spikes[protocol].keys():
                    continue
                spike_data = df_cell.Spikes[protocol]["spikes"]
                if measure in [
                    "dvdt_rising",
                    "dvdt_falling",
                    "AP_HW",
                    "AHP_trough_V",
                    "AHP_depth_V",
                ]:  # use lowest current spike
                    min_current_index, current, trace = self.find_lowest_current_trace(
                        df_cell.Spikes[protocol]
                    )
                    if not np.isnan(min_current_index):
                        spike_data = df_cell.Spikes[protocol]["spikes"][trace][0].__dict__
                        # print("spike data ", spike_data['dvdt_rising'])
                        m.append(spike_data[mapper[measure]])
                    else:
                        m.append(np.nan)
                    # print("spike data: ", spike_data.keys())

                elif measure == "AP_thr_V":  # have to try two variants. Note that threshold slope is defined in config file
                    min_current_index, current, trace = self.find_lowest_current_trace(
                        df_cell.Spikes[protocol]
                    )
                    if not np.isnan(min_current_index):
                        spike_data = df_cell.Spikes[protocol]["spikes"][trace][0].__dict__
                        # CP("c", "Check AP_thr_V")

                        Vthr, Vthr_time = UTIL.find_threshold(
                            spike_data["V"],
                            np.mean(np.diff(spike_data["Vtime"])),
                            threshold_slope=threshold_slope,
                        )
                        m.append(Vthr)
                    else:
                        m.append(np.nan)

                elif (
                    measure in mapper.keys() and mapper[measure] in spike_data.keys()
                ):  # if the measure exists for this sweep
                    m.append(spike_data[mapper[measure]])
                else:
                    # print(measure in mapper.keys())
                    # print(spike_data.keys())
                    CP(
                        "r",
                        f"measure not found in spike_data, either: <{measure:s}>, {mapper.keys()!s}",
                    )
                    CP(
                        "r",
                        f"\n   or mapped in {mapper[measure]!s} to {spike_data.keys()!s}",
                    )
                    raise ValueError()
                    exit()
                prot_spike_count += 1

        # CP("c", f"measure: {measure!s}  : {m!s}")
        # else:
        #     print(
        #         f"measure {measure:s} not found in either IV or Spikes keys. Skipping"
        #     )
        #     raise ValueError(f"measure {measure:s} not found in either IV or Spikes keys. Skipping")
        for i, xm in enumerate(m):
            if xm is None:
                m[i] = np.nan
            # m = [u for u in m if u is not None else np.nan] # sanitize data
        N = np.count_nonzero(~np.isnan(m))
        # print("N: ", N)
        if N > 0:
            datadict[measure] = np.nanmean(m)
        else:
            datadict[measure] = np.nan
        return datadict

    def textbox_setup(self, textbox):
        self.textbox = textbox

    def textclear(self):
        if self.textbox is None:
            raise ValueError("datatables - functions - textbox has not been set up")

        if self is None:  #  or self.in_Parallel:
            return
        else:
            self.textbox.clear()

    def text_get(self):
        if self.textbox is None:
            raise ValueError("datatables - functions - textbox has not been set up")
        return self.textbox.toPlainText()

    def textappend(self, text, color="white"):
        if self.textbox is None:
            raise ValueError("datatables - functions - textbox has not been set up")

        colormap = {
            "[31m": "red",
            "[48:5:208:0m": "orange",
            "[33m": "yellow",
            "[32m": "limegreen",
            "[34m": "pink",
            "[35m": "magenta",
            "[36m": "cyan",
            "[30m": "black",
            "[37m": "white",
            "[0m": "white",
            "[100m": "backgray",
        }
        if self is None:
            CP(color, text)  # just go straight to the terminal
        else:
            text = "".join(text)
            text = text.split("\n")

            for textl in text:
                # print(f"text: <{textl:s}>")

                if len(textl) > 0 and textl[0] == "\x1b":
                    textl = textl[1:]  # clip the escape sequence
                    for k in colormap.keys():
                        if textl.startswith(k):  # skip the escape sequence
                            textl = textl[len(k) :]
                            textl = textl.replace("[0m", "")
                            color = colormap[k]
                            self.textbox.setTextColor(QtGui.QColor(color))
                            break
                textl = textl.replace("[0m", "")
                self.textbox.append(textl)
                self.textbox.setTextColor(QtGui.QColor("white"))


import random

def structDataSampling(**kwargs):
    """
       :param num:
       :param struct:
       :return:
    """
    result = []
    for index in range(0, kwargs['num']):
        element = []
        for key, value in kwargs['struct'].items():
            if value['datatype'] == "int":
                it = iter(value['datarange'])
                tmp = random.randint(next(it), next(it))
            elif value['datatype'] == "float":
                it = iter(value['datarange'])
                tmp = random.uniform(next(it), next(it))
            elif value['datatype'] == "str":
                tmp = ''.join(random.SystemRandom().choice(value['datarange']) for _ in range(value['len']))
            else:
                break
            element.append(tmp)
        result.append(element)
    return result

def structDataSamplinga(**kwargs):
    """
       :param num:
       :param struct:
       :return:
    """
    result = list()
    for index in range(0, kwargs['num']):
        element = list()
        for key, value in kwargs['struct'].items():
            if key == "int":
                it = iter(value['datarange'])
                tmp = random.randint(next(it), next(it))
            elif key == "float":
                it = iter(value['datarange'])
                tmp = random.uniform(next(it), next(it))
            elif key == "str":
                tmp = ''.join(random.SystemRandom().choice(value['datarange']) for _ in range(value['len']))
            else:
                break;
            element.append(tmp)
        result.append(element)
    return result

def out_1():
    para = {"num": 4, "struct": {"int": {"datarange": (0, 100)},
                                 "float": {"datarange": (0, 1000)}}}
    t = structDataSamplinga(**para)
    print(t)
    parb = {"num": 4, "struct": {"int": {"datarange": (0, 90)}, "str": {"datarange": "ABCD", "len": 2},
                                 "float": {"datarange": (0, 100)}}}
    m = structDataSamplinga(**parb)
    print(m)

    parc = {"num": 3, "struct": {
        "field1": {"datatype": 'int', "datarange": [100, 140]},
        "field2": {"datatype": 'float', "datarange": [100, 150]},
        "field3": {"datatype": 'str', "datarange": "ABCDEFG", "len": 3}
    }}
    n = structDataSampling(**parc)
    print(n)

if __name__ == '__main__':
    out_1()


#Dilation and Erosion

import cv2
import matplotlib.pyplot as plt
import numpy as np

#-----------------------------------Dilation------------------------------
# Reads in a binary image
img = cv2.imread('j.png',0)

# Create a 5x5 kernel of ones
Kernel = np.ones((5,5), np.uint8)

'''
To dilate an image in OpenCV, you can use the dilate function and three inputs:
an original binary image, a kernel that determines the size of the dilation (None will
result in a default size), and a number of iterations to perform the dilation (typically = 1).
In the below example, we have a 5x5 kernel of ones, which move over an image, like a filter,
and turn a pixel white if any of its surrounding pixels are white in a 5x5 window! We’ll
use a simple image of the cursive letter “j” as an example.
'''

dilation = cv2.dilate(img, Kernel, iterations = 1)
plt.imshow(dilation, cmap = 'gray')

#-----------------------------------Erosion--------------------------------
erosion = cv2.erode(img, Kernel, iterations = 1)
plt.imshow(erosion, cmap = 'gray')

#----------------------------------Opening------------------------------

'''
As mentioned, above, these operations are often combined for desired results! One such combination
 is called opening, which is erosion followed by dilation. This is useful in noise reduction
 in which erosion first gets rid of noise (and shrinks the object) then dilation enlarges the
 object again, but the noise will have disappeared from the previous erosion!

To implement this in OpenCV, we use the function morphologyEx with our original image,
 the operation we want to perform, and our kernel passed in.
 '''

opening = cv2.morphologyEx(img, cv2.MORPH_OPEN, Kernel)
plt.imshow(opening, cmap = 'gray')

#----------------------------------Closing------------------------------

'''
Closing is the reverse combination of opening; it’s dilation followed by erosion,
 which is useful in closing small holes or dark areas within an object.

Closing is reverse of Opening, Dilation followed by Erosion. It is useful in
 closing small holes inside the foreground objects, or small black points on the object.
'''

closing = cv2.morphologyEx(img, cv2.MORPH_CLOSE, Kernel)
plt.imshow(closing, cmap = 'gray')

Types of features and Image segmentation/Dilation- Erosion- Opeining and Closing/test.py


from pathlib import Path
from zoneinfo import ZoneInfo
import datetime
import sys

TIME_ZONE = ZoneInfo('US/Eastern')

def main():

    station_name = sys.argv[1]
    dir_path = Path(sys.argv[2])

    file_paths = sorted(dir_path.glob('*.WAV'))

    for file_path in file_paths:
        move_file(file_path, station_name)

def move_file(file_path, station_name):

    recorder_name = file_path.parent.parent.name
    file_name = file_path.name

    start_time = parse_file_name(file_name)
    night = get_night(start_time)

    night_dir_name = night.strftime('%Y-%m-%d')
    start_time_string = start_time.strftime('%Y-%m-%d_%H.%M.%S_Z')
    new_file_name = f'{station_name}_{recorder_name}_{start_time_string}.wav'

    night_dir_path = file_path.parent / night_dir_name
    night_dir_path.mkdir(mode=0o755, parents=True, exist_ok=True)

    new_file_path = night_dir_path / new_file_name
    file_path.rename(new_file_path)

    print(f'{start_time} {night_dir_path} {new_file_path}')

def parse_file_name(file_name):
    start_time = datetime.datetime.strptime(file_name, '%Y%m%d_%H%M%S.WAV')
    return start_time.replace(tzinfo=ZoneInfo('UTC'))

def get_night(dt):

    dt = dt.astimezone(TIME_ZONE)

    date = dt.date()
    hour = dt.hour

    if hour >= 12:
        return date

    else:
        return datetime.date.fromordinal(dt.toordinal() - 1)

if __name__ == '__main__':
    main()


from sqlalchemy import Column, INTEGER, Identity, String

from src.data_access.database.models.base_entity import InoversityLibraryBase

__all__ = [
    "StaffEntity"
]

class StaffEntity(InoversityLibraryBase):
    user_id = Column("id", INTEGER, Identity(), primary_key=True, index=True)
    role_level = Column("roleLevel", String(256), nullable=False, index=True)
    staff_number = Column("staffNumber", String(20), nullable=False, unique=True)
    department = Column("department", String(100), nullable=False)
    job_title = Column("jobTitle", String(100), nullable=False)


from ai_chatbot.scripts import REDataHeader as Header
import dateparser
import datetime

def printData(data):
    print('Station from : {0}'.format(data[Header.STATIONFROM]))
    print('Station to : {0}'.format(data[Header.STATIONTO]))
    print('departure date : {0}'.format(data[Header.DEPARTDATE]))
    print('departure time  : {0}'.format(data[Header.DEPARTTIME]))

def datecheck(date):
    now = datetime.datetime.now()
    dateobject = dateparser.parse(date)
    if dateobject > now:
        return 0
    else:
        return 1

def returndatecheck(date, returndate):
    dateobject = dateparser.parse(date)
    returndateobject = dateparser.parse(returndate)
    if dateobject < returndateobject:
        return 0
    else:
        return 1

def timecheck(date, time):
    now = datetime.datetime.now()
    dateobject = dateparser.parse(date)
    timeobject = dateparser.parse(time)
    fullobject = datetime.datetime.combine(dateobject.date(), timeobject.time())
    if fullobject > now:
        return 0
    else:
        return 1

def missingDataCheck(data):
    if data[Header.STATIONFROM] == '':
        return Header.STATIONFROM
    elif data[Header.STATIONTO] == '':
        return Header.STATIONTO
    elif data[Header.DEPARTDATE] == '':
        return Header.DEPARTDATE
    elif datecheck(data[Header.DEPARTDATE]) == 1:
        data[Header.DEPARTDATE] = ''
        return Header.BADDATE
    elif data[Header.DEPARTTIME] == '':
        return Header.DEPARTTIME
    elif timecheck(data[Header.DEPARTDATE], data[Header.DEPARTTIME]) == 1:
        data[Header.DEPARTDATE] = ''
        data[Header.DEPARTTIME] = ''
        return Header.BADTIME
    elif data[Header.SINGLERETURN].lower() == '':
        return Header.SINGLERETURN
    elif data[Header.SINGLERETURN].lower() == 'return':
        if data[Header.RETURNDATE] == '':
            return Header.RETURNDATE
        elif returndatecheck(data[Header.DEPARTDATE], data[Header.RETURNDATE]) == 1:
            data[Header.RETURNDATE] = ''
            return Header.BADDATE
        elif data[Header.RETURNTIME] == '':
            return Header.RETURNTIME
        return 0
    else:
        return 0

def verificationCheck(data):
    if data[Header.CONFIRMED] == 'true':
        return 0
    return 1

def getURL(data):
    # call function in scraper/scraper.py
    print('Getting URL for...')
    print('\t {0} --> {1}'.format(data.stationFrom, data.stationTo))
    print('\t Departure date : {0}'.format(data.DepDate))
    print('\t Departure time : {0}'.format(data.DepTime))


import matplotlib.pyplot as plt
from matplotlib.lines import Line2D
import matplotlib.animation as animation
import matplotlib.patches as mpatches
from matplotlib import ticker
from matplotlib import cm
from matplotlib.ticker import FuncFormatter
import numpy as np

from utils.occ_map_utils import load_map, display_occ_map, plot_grid_map_hmm, show_traj, \
    black_cm, green_cm, red_cm, blue_cm, greens_cm, greys_cm
from utils.occ_map_utils import show_map
from utils.plot_utils import plot_4d_tensor

# key press to update figure for the next tracking setp
def on_press(event, animation):
    if event.key == ' ':
        if animation.pause:
            animation.event_source.stop()
        else:
            animation.event_source.start()
        animation.pause ^= True

def onclick(event, anim):
    models = anim.models
    ix, iy = event.xdata, event.ydata
    coords = np.floor(np.array([ix, iy]) / models[0].map_res).astype(int)
    print("Click at coordinates: {}".format(coords))

    all_axes = [plot.axes for plot in anim.plots]

    for i, ax in enumerate(all_axes):
        # For infomation, print which axes the click was in
        if ax == event.inaxes:
            #print "Click is at filter {}".format(anim.models[i].name)
            break

    clicked = np.zeros_like(models[0].map)
    x, y = coords[0], coords[1]
    clicked[x, y] = 1
    for plot in anim.plots:
        plot.set_axes_data("occupancy_axes", clicked)
    anim.fig.canvas.draw()

    accessories_figures = [anim.nn_output_fig, anim.kernel_fig]
    for fig in accessories_figures:
        if fig is not None:
            fig.clear()
    anim.nn_output_fig.suptitle('Network Output')
    anim.kernel_fig.suptitle('Motion pattern')

    if models[i].kernels.ndim == 6:
        kernel = models[i].kernels[x, y]
        condi_prob = models[i].nn_probs[x, y]
    else:
        kernel = models[i].kernels
        condi_prob = None

    plot_4d_tensor(kernel, fig=anim.kernel_fig)
    if condi_prob is not None:
        plot_4d_tensor(condi_prob, fig=anim.nn_output_fig)

    anim.set_axis_ticks(models[i].extent)
    anim.accessories_plots['ma_plot'].set_axes_data("occupancy_axes", np.ones_like(models[i].ma_vel))
    anim.accessories_plots['vel_plot'].set_axes_data("occupancy_axes", models[i].P_Vt_pred[x, y])
    anim.accessories_plots['merge_vel_plot'].set_axes_data("occupancy_axes", models[i].P_Vt_merged[x, y])
    anim.accessories_plots['final_vel_plot'].set_axes_data("occupancy_axes", models[i].P_Vt[x, y])

    accessories_figures += [anim.vel_fig]
    for fig in accessories_figures:
        if fig is not None:
            fig.canvas.draw()

    for k, plot in anim.accessories_plots.items():
        plot.refresh_colorbar()

    model_names = map(lambda model: model.name, models)
    occs = map(lambda model: model.P_Ot[x, y], models)
    for name, occ in zip(model_names, occs):
        print("loc ({}, {}) of model {} has occupancy of {}".format(x, y, name, occ))

class Plot(object):

    def __init__(self, axes, map, res, plot_map=True, plot_seen=False, show_text=True, colorbar_on=None, title=None):
        self.axes = axes
        self.map = map
        self.res = res
        self.plot_seen = plot_seen
        self.plot_map = plot_map
        self.map_axes = None
        self.occupancy_axes = None
        self.ground_truth_axes = None
        self.seen_axes = None
        self.colorbars = []
        self.show_text = show_text

        if title is None:
            title = 'Measurements'
        self.axes.set_title(title)

        if show_text:
            self.text = self.axes.text(0.92, 0.92, "", bbox={'facecolor': 'red', 'alpha': 0.5, 'pad': 5},
                        transform=self.axes.transAxes, ha="right", color='white', zorder=14)

        self.add_images()
        self.add_colorbar(colorbar_on)

    def add_images(self):
        """Add AxesImages for showing map, occupancy and seen."""
        occupancy = np.zeros(self.map.shape, dtype=float)
        self.occupancy_axes = show_map(occupancy, self.res, cmap=red_cm, ax=self.axes, zorder=11)
        # initialize plots with map
        map_ = self.map if self.plot_map else np.zeros_like(self.map)
        self.map_axes = show_map(map_, self.res, cmap=black_cm, ax=self.axes, zorder=12)
        if self.plot_seen:
            # add seen image
            self.seen_axes = show_map(occupancy, self.res, cmap=black_cm, alpha=0.2, ax=self.axes)

    def set_axes_data(self, axes_name, data, vmin=None, vmax=None):
        image_ax = getattr(self, axes_name)
        image_ax.set_data(np.rot90(data))
        vmin = vmin if vmin is not None else data.min()
        vmax = vmax if vmax is not None else data.max()
        image_ax.set_clim([vmin, vmax])

    def add_custom_image(self, axes_name, cmap=None, image=None, **kwargs):
        if image is None:
            image = np.zeros(self.map.shape, dtype=float)

        image_ax = show_map(image, self.res, cmap=cmap, ax=self.axes, **kwargs)
        setattr(self, axes_name, image_ax)

    def add_colorbar(self, colorbar_on):
        if colorbar_on is None:
            return

        image_axes = getattr(self, colorbar_on)

        if image_axes is not None:
            cb = plt.colorbar(image_axes, ax=self.axes, fraction=0.046, pad=0.04)
            tick_locator = ticker.MaxNLocator(nbins=5)
            cb.locator = tick_locator
            self.colorbars.append(cb)

    def set_ylabel(self, text='', **kwargs):
        self.axes.set_ylabel(text, **kwargs)

    def refresh_colorbar(self):
        for cb in self.colorbars:
            cb.update_ticks()

    def add_traj_line(self, num_targets=1):
        """ Add 2D Lines for showing trajectories."""
        colors = cm.Dark2(np.linspace(0, 1, num_targets))
        # add lines for showing trajectories
        self.lines = map(lambda _: self.axes.add_line(Line2D([], [], zorder=14, color='grey')), range(num_targets))

    def set_title(self, title):
        self.axes.set_title(title)

    def set_text(self, text):
        self.text.set_text(text)

class TrackingAnimation(animation.TimedAnimation):

    def __init__(self, models, num_steps, simulated_data, plot_seen=False, plot_map=True, show_text=True, accessories=None):
        self.num_models = len(models)
        self.models = models
        self.map = models[0].map
        self.res = models[0].map_res
        self.num_steps = num_steps
        self.simulated_data = simulated_data
        self.show_map = plot_map
        self.show_seen = plot_seen
        self.show_text = show_text
        self.nn_output_fig = None
        self.kernel_fig = None
        self.vel_fig = None
        self.accessories_plots = None
        self.accessories = accessories
        self.initialize_figure()
        self.initialize_models()
        self.initialize_accessories()
        print(self.accessories_plots)
        self.fig.canvas.mpl_connect('key_press_event', lambda event: on_press(event, self))
        self.fig.canvas.mpl_connect('button_press_event', lambda event: onclick(event, self))

        animation.TimedAnimation.__init__(self, self.fig, interval=500, blit=True, repeat=True, repeat_delay=1000)

    def initialize_figure(self):
        fig_size = (5 * self.num_models, 5)
        self.fig = plt.figure(figsize=fig_size)
        self.pause = True
        # bind key press event to pause animation
        self.plots = []
        for i in range(self.num_models):
            axes = self.fig.add_subplot(1, self.num_models, i + 1)
            title = self.models[i].name
            colorbar_on = "occupancy_axes" if self.simulated_data else None
            plot = Plot(axes, self.map, self.res, self.show_map, self.show_seen, self.show_text, colorbar_on, title=title)
            self.add_custom_element(plot)
            self.plots.append(plot)
        self.fig_title_axes = self.fig.add_axes([.4, .9, .2, .05])
        self.fig_title_axes.set_axis_off()
        self.fig_title = self.fig.text(.49, .9, "", transform=self.fig_title_axes.transAxes, fontsize=15, color='r', ha='center')
        if not self.simulated_data:
            self.add_legend()

    def initialize_accessories(self):

        if "motion_pattern" in self.accessories:
            self.nn_output_fig = plt.figure(figsize=(5, 5))
            self.nn_output_fig.suptitle('Network Output')
            self.kernel_fig = plt.figure(figsize=(5, 5))
            self.kernel_fig.suptitle('Motion pattern')

        if "velocities" in self.accessories:
            self.vel_fig = plt.figure(figsize=(12, 3))
            ma_ax = self.vel_fig.add_subplot(141)
            ma_plot = Plot(ma_ax, self.models[0].ma_vel, 1, False, False, False, colorbar_on=None, title=r'$P(V_{ma})$')
            vel_ax = self.vel_fig.add_subplot(142)
            vel_plot = Plot(vel_ax, self.models[0].ma_vel, 1, False, False, False, colorbar_on=None, title=r'$P(V_{pred})$')
            merge_vel_ax = self.vel_fig.add_subplot(143)
            merge_vel_plot = Plot(merge_vel_ax, self.models[0].ma_vel, 1, False, False, False, colorbar_on=None, title=r'$P(V_{merge})$')
            final_vel_ax = self.vel_fig.add_subplot(144)
            final_vel_plot = Plot(final_vel_ax, self.models[0].ma_vel, 1, False, False, False, colorbar_on=None, title='$P(V)$')
            self.accessories_plots = dict(ma_plot=ma_plot, vel_plot=vel_plot,
                                          merge_vel_plot=merge_vel_plot, final_vel_plot=final_vel_plot)

    def set_axis_ticks(self, extent):

        if self.vel_fig is not None:

            xlabels = (np.arange(extent) + np.array([-(extent // 2)])).tolist()
            ylabels = xlabels

            def format_fn_x(tick_val, tick_pos):
                if int(tick_val) in range(7):
                    return xlabels[int(tick_val)]
                else:
                    return ''

            def format_fn_y(tick_val, tick_pos):
                if int(tick_val) in range(7):
                    return ylabels[int(tick_val)]
                else:
                    return ''

            ax = self.vel_fig.get_axes()[0]
            max_extent = float(extent)
            ax.set_xticks(np.arange(.5, max_extent, 1.0))
            ax.set_yticks(np.arange(0.5, max_extent, 1.0))
            ax.xaxis.set_major_formatter(FuncFormatter(format_fn_x))
            ax.yaxis.set_major_formatter(FuncFormatter(format_fn_y))

            ylabel = ax.set_ylabel(r'$V_y$', color='darkred', fontsize=12)
            ylabel.set_rotation(0)
            ax.yaxis.set_label_coords(-0.06, .95)
            ax.set_xlabel(r'$V_x$', color='darkred', fontsize=12)
            ax.xaxis.set_label_coords(1.05, -0.025)

            for ax in self.vel_fig.get_axes()[1:]:
                ax.set_xticks([])
                ax.set_yticks([])

    def add_custom_element(self, plot):
        """Add extra elements to plot. This method has to be overwritten by subclasses. """
        pass

    def update_custom_element(self, idx):
        """ Update custom elements on animation. This method has to be overwritten by subclasses. """

    def initialize_models(self):
        """Initialize BOFUM models. This method has to be overwritten by subclasses."""
        pass

    def _draw_frame(self, framedata):

        t = self.models[0].t
        t_count = "frame = " + str(t)
        print(t_count)
        self.fig_title.set_text(t_count)

        measurement = self.models[0].measurement_at()
        for model in self.models:
            model.tracking_step(measurement=measurement)

        # plot new occupancy
        Ot_max = max(map(lambda model: model.P_Ot.max(), self.models))
        Ot_min = min(map(lambda model: model.P_Ot.min(), self.models))
        for i, model in enumerate(self.models):
            self.plots[i].set_axes_data("occupancy_axes", model.P_Ot, Ot_min, Ot_max)
            if self.show_seen:
                seen = model.evaluate_loc_at(t)
                self.plots[i].set_axes_data("seen_axes", seen)
            if self.plots[i].show_text:
                x_ent = model.calc_cross_entropy()
                f1_score = model.calc_f1_score()
                average_precision = model.calc_average_precision()
                self.plots[i].text.set_text("x_ent: {:.3f}, f1: {:.3f}, ap: {:.3f}".format(x_ent, f1_score, average_precision))
            self.update_custom_element(i)
            # if i == self.num_models-1:
            #     self.add_legend()
            self.plots[i].refresh_colorbar()

        # repeat tracking
        if framedata == self.num_steps-1:
            for model in self.models:
                model.reset()

    def new_frame_seq(self):
        return iter(range(self.num_steps))

    def _init_draw(self):
        pass

class TrackingAnimSimulation(TrackingAnimation):

    def __init__(self, models, num_steps, num_targets=1, diagonal=False, plot_map=True, **kwargs):

        self.num_targets = num_targets
        self.diagonal = diagonal
        self.trajs = None
        self.distances = None
        super(TrackingAnimSimulation, self).__init__(models, num_steps, True, plot_map=plot_map, **kwargs)

    def add_custom_element(self, plot):
        plot.add_traj_line(self.num_targets)

    def update_custom_element(self, idx):
        # add trajectory lines
        truncated_trajs = self.models[0].traversed_traj_at()

        for idx_, line in enumerate(self.plots[idx].lines):
            xs, ys = truncated_trajs[idx_].T[0][-5:], truncated_trajs[idx_].T[1][-5:]
            line.set_data(xs, ys)

    def initialize_models(self):
        self.distances , self.trajs = self.models[0].initialize(self.num_targets, self.num_steps)
        init_model = lambda model: model.initialize(self.num_targets, self.num_steps,
            distances=self.distances, trajectories=self.trajs)
        map(init_model, self.models[1:])

class TrackingAnimRealdata(TrackingAnimation):

    def __init__(self, models, num_steps, scene, plot_map=True,plot_seen=False, simulated_scenes=False, **kwargs):

        self.scene = scene
        self.simulated_scenes = simulated_scenes
        super(TrackingAnimRealdata, self).__init__(models, num_steps, False,
                                                     plot_seen=plot_seen, plot_map=plot_map, **kwargs)

    def update(self, scene, update_num_steps=True):
        self.scene = scene
        update_map = lambda plot: plot.set_axes_data("map_axes", self.scene.static_map)
        map(update_map, self.plots)
        map(lambda model: model.update(scene), self.models)
        if update_num_steps:
            self.num_steps = len(scene.hits)
        self.frame_seq = self.new_frame_seq()

    def initialize_models(self):
        init_model = lambda model: model.initialize(self.scene, not self.simulated_scenes)
        map(init_model, self.models)

    def add_custom_element(self, plot):
        # add false negative axes
        # it shows locations where ground truth is occupied
        # but BOFUM fails to track
        plot.add_custom_image("fn_axes", blue_cm)
        # add true positive axes
        # it shows locations where ground truth is occupied
        # and BOFUM predicts occupancy prob higher than 0
        plot.add_custom_image("tp_axes", greens_cm)
        plot.add_colorbar("tp_axes")
        # add false positive axes
        plot.add_custom_image("fp_axes", red_cm)

    def add_legend(self):
        g_patch = mpatches.Patch(color='g', label='True positive')
        b_patch = mpatches.Patch(color='b', label='False negative')
        o_patch = mpatches.Patch(color='orange', label='False positive')
        plt.legend(handles=[g_patch, b_patch, o_patch], bbox_to_anchor=(1, 1),
           bbox_transform=self.fig.transFigure)

    def update_custom_element(self, idx):
        t = self.models[0].t - 1
        model = self.models[idx]
        plot = self.plots[idx]
        occupancy_prob = model.P_Ot
        h_max = occupancy_prob.max()/2
        #occupancy_prob = np.where(occupancy_prob>h_max, occupancy_prob, 0)
        ground_truth = model.ground_truth_at(t)
        overlap = np.logical_and(occupancy_prob, ground_truth)
        # if occupany on ground truth location is higher than 0.1,
        # it is not thought as a false negative
        occupancy_temp = np.where(overlap, occupancy_prob, 0)
        #predicted = np.where(occupancy_temp>0.1, 1, 0)
        false_negative = np.where(occupancy_temp>0.1, 0, ground_truth)
        # if model predicts occupancy higher than 0 on ground truth locations,
        # it is thought as a true positive
        true_positive = np.where(overlap, occupancy_prob, 0)
        # if model predicts occupancy higher than 0 on non-ground truth locations,
        # it is thought as a false positive
        false_positive = occupancy_prob.copy()
        false_positive[overlap] = 0
        # only show for occupancies higher than 1/2 highest occupancy
        h_max = false_positive.max() / 4
        false_positive = np.where(false_positive>h_max, false_positive, 0)

        Ot_max = max(map(lambda model: model.P_Ot.max(), self.models))
        plot.set_axes_data("fn_axes", false_negative, 0, 1)
        plot.set_axes_data("tp_axes", true_positive, 0, Ot_max)
        plot.set_axes_data("fp_axes", false_positive, 0, Ot_max)
        plot.set_axes_data("occupancy_axes", np.zeros_like(occupancy_prob))


import sys
import unittest
import psycopg2

sys.path.insert(0, '../src')

from src.utils import daily_reports_return_json, daily_reports_return_csv, time_series_return_csv, check_query_data_active, check_request
from src.config import connect_database
# import copy

class TestUtils(unittest.TestCase):
    def __init__(self, *args, **kwargs):
        super(TestUtils, self).__init__(*args, **kwargs)
        self.conn = connect_database()

    def test_daily_reports_return_json(self):
        try:
            # conn = connect_database()
            cur = self.conn.cursor()
            # create a test table in the database with the table format of a daily report
            cur.execute("DROP TABLE IF EXISTS test;")
            self.conn.commit()
            cur.execute("CREATE TABLE test (state VARCHAR(5), region VARCHAR(5), last_update VARCHAR(20), "
                        "confirmed INTEGER, deaths INTEGER, recovered INTEGER, active INTEGER, combined_key VARCHAR("
                        "5));")
            self.conn.commit()
            cur.execute("INSERT INTO test VALUES ('a', 'a', '2021-01-02 05:22:33', 10, 5, 0, 5, 'a, a'), "
                        "(null, 'b', '2021-01-02 05:22:33', 1, 0, 0, 1, 'b'), "
                        "('b', 'b', '2021-01-02 05:22:33', 4, 3, 0, 1, 'b, b');"
                        )
            self.conn.commit()

            date = "01/01/21"
            types = ["Confirmed", "Deaths", "Recovered", "Active"]
            locations = [{"Country/Region": "b"},
                         {"Country/Region": "a", "Province/State": "a", "Combined_Key": "a, a"}
                         ]
            result = daily_reports_return_json(cur, date, locations, types, 'test')

            expected = {
                "Date": date,
                "Reports": [
                    {
                        "Active": 2,
                        "Confirmed": 5,
                        "Country/Region": "b",
                        "Deaths": 3,
                        "Recovered": 0
                    },
                    {
                        "Active": 5,
                        "Confirmed": 10,
                        "Country/Region": "a",
                        "Deaths": 5,
                        "Province/State": "a",
                        "Combined_Key": "a, a",
                        "Recovered": 0
                    }
                ]
            }
            self.assertEqual(result, expected)
        except psycopg2.Error:
            assert False, "Database Error"

    def test_daily_reports_return_csv(self):
        json_data = {
            "Date": "01/01/21",
            "Reports": [
                {
                    "Active": 2,
                    "Confirmed": 5,
                    "Country/Region": "b",
                    "Deaths": 3,
                    "Recovered": 0
                },
                {
                    "Active": 5,
                    "Confirmed": 10,
                    "Country/Region": "a",
                    "Deaths": 5,
                    "Province/State": "a",
                    "Combined_Key": "a, a",
                    "Recovered": 0
                }
            ]
        }
        result = daily_reports_return_csv(json_data, ["Confirmed", "Deaths", "Recovered", "Active"])
        expected = "Date,Province/State,Country/Region,Combined_Key,Confirmed,Deaths,Recovered,Active" \
                   "\n01/01/21,,b,,5,3,0,2\n01/01/21,a,a,a, a,10,5,0,5"
        self.assertEqual(result, expected)

    def test_time_series_return_csv(self):
        json_data = {"01/26/20": [{"Active": 0, "Confirmed": 0, "Country/Region": "Albania"}]}
        expected = "Date,Province/State,Country/Region,Confirmed\n01/26/20,,Albania,0"
        result = time_series_return_csv(json_data, ["01/26/20"], ["Confirmed"])
        self.assertEqual(result, expected)

    def test_check_query_data_active(self):
        try:
            # conn = connect_database()
            cur = self.conn.cursor()
            # create a test table in the database with the table format of a daily report
            cur.execute("DROP TABLE IF EXISTS test;")
            self.conn.commit()
            self.assertEqual(check_query_data_active(cur, ["test"]), False)
        except psycopg2.Error:
            assert False, "Database Error"

    def test_check_request(self):
        result = check_request(['test'], {})
        self.assertEqual(result[0], 'test')

if __name__ == '__main__':
    unittest.main()


########################################################
# Rodrigo Leite - drigols                              #
# Last update: 17/12/2021                              #
########################################################

import pandas as pd
from matplotlib import pyplot as plt

df = pd.DataFrame(
  {
    'Name': ['Dan', 'Joann', 'Pedro', 'Rosie', 'Ethan', 'Vicky', 'Frederic'],
    'Salary':[50000, 54000, 50000, 189000, 55000, 40000, 59000],
    'Hours':[41, 40, 36, 17, 35, 39, 40],
    'Grade':[50, 50, 46, 95, 50, 5,57]
  }
)

# Utiliza o atributo showfliers = False - Ou seja, ignora dados discrepantes.
df['Salary'].plot(kind='box', title='Salary Distribution', figsize=(10,8), showfliers=False)
plt.savefig('../images/first-boxplot-03.png', format='png')
plt.show()


import kivy
from kivy.app import App
from kivy.uix.label import Label

# 2
from kivymd.app import MDApp
from kivymd.uix.label import MDLabel
from kivymd.uix.screen import Screen

kivy.require('2.1.0')

class MyFirstApp(App):
    def build(self):
        # lbl = Label(text='Hello World')
        # lbl = Label(text='Hello World and Good Morning', font_size='20sp', color=[0.41, 0.42, 0.74, 1])
        lbl = Label(text="[color=ff3333][b]'Hello World'[/b][/color]\n[color=3333ff]Good Morning[/color]",
                    font_size='20sp', markup=True)
        """
        [b][/b] → 太字を有効にする
        [i][/i] → イタリック体のテキストをアクティブにする
        [u][/u] → 下線テキスト
        [s][/s] →取り消し線付きテキスト
        [font=][/font] → フォントを変更する
        [サイズ=][/size]]です。→ フォントサイズを変更する
        [色=#][/color] → 文字色の変更
        [ref=][/ref] -> インタラクティブゾーンを追加します。参照＋参照内部のバウンディングボックスがLabel.refsで利用可能になります。
        [anchor=] -> テキストにアンカーを入れる。テキスト内のアンカーの位置はLabel.anchorsで取得できます。
        [sub][/sub] -> 前のテキストからの相対的な添え字の位置でテキストを表示します。
        [sup][/sup] -> 前のテキストと相対的な上付き文字の位置でテキストを表示します。
        """

        return lbl

class Demo(MDApp):
    def build(self):
        screen = Screen()

        l = MDLabel(text="Welcome", pos_hint={'center_x': 0.8, 'center_y': 0.8},
                    theme_text_color='Custom',
                    text_color=(0.5, 0, 0.5, 1),
                    font_style='Caption'
                    )

        l1 = MDLabel(text="Welcome", pos_hint={'center_x': 0.8, 'center_y': 0.5},
                     theme_text_color='Custom',
                     text_color=(0.5, 0, 0.5, 1),
                     font_style='H2'
                     )

        l2 = MDLabel(text="Welcome", pos_hint={'center_x': 0.8, 'center_y': 0.2},
                     theme_text_color='Custom',
                     text_color=(0.5, 0, 0.5, 1),
                     font_style='H1'
                     )

        screen.add_widget(l)
        screen.add_widget(l1)
        screen.add_widget(l2)
        return screen

if __name__ == '__main__':
    # MyFirstApp().run()
    Demo().run()


import gym

class SparseRewardWrapper(gym.Wrapper):
    def __init__(self, env, sparse_level=-1, timestep_limit=-1):
        super(SparseRewardWrapper, self).__init__(env)
        self.sparse_level = sparse_level
        self.timestep_limit = timestep_limit
        self.acc_reward = 0
        self.acc_t = 0

    def step(self, action):
        obs, rew, done, info = self.env.step(action)

        self.acc_t += 1

        if self.timestep_limit > 0 and (self.acc_t) >= self.timestep_limit:
            done = True

        if self.sparse_level == 0:
            return obs, rew, done, info

        self.acc_reward += rew
        ret_rew = 0
        if self.sparse_level != -1:
            if done or (self.acc_t > 0 and self.acc_t % self.sparse_level == 0):
                ret_rew = self.acc_reward
                self.acc_reward = 0
        else:
            if done:
                ret_rew = self.acc_reward
                self.acc_reward = 0

        return obs, ret_rew, done, info

    def reset(self, **kwargs):
        self.acc_t = 0
        self.acc_reward = 0
        return self.env.reset(**kwargs)


import streamlit as st
st.set_option('deprecation.showPyplotGlobalUse', False)

# for manipulation
import pandas as pd
import numpy as np

# for data visualization
import matplotlib.pyplot as plt
import seaborn as sns
sns.set(style="ticks")
plt.style.use("dark_background")
#sns.set_style('whitegrid')

# to filter warnings
import warnings
warnings.filterwarnings('ignore')

# for interactivity
from ipywidgets import interact

st.title("Agricultural Production Optimization Engine")

# Reading the dataset
data= pd.read_csv('data.csv')

x= data.drop(['label'], axis=1)
y= data['label']

# let's create training and testing sets for validation of results
from sklearn.model_selection import train_test_split
x_train, x_test, y_train, y_test= train_test_split(x,y, test_size=0.2, random_state=42)

# let's create predictive model
from sklearn.linear_model import LogisticRegression
LogReg = LogisticRegression()
LogReg.fit(x_train,y_train)

from sklearn import linear_model
from sklearn.ensemble import RandomForestClassifier
random_forest= RandomForestClassifier(n_estimators=100)
random_forest.fit(x_train, y_train)

from sklearn.tree import DecisionTreeClassifier
DecTree= DecisionTreeClassifier()
DecTree.fit(x_train,y_train)

from sklearn.neighbors import KNeighborsClassifier
KNN= KNeighborsClassifier()
KNN.fit(x_train, y_train)

from sklearn.naive_bayes import GaussianNB
NB= GaussianNB()
NB.fit(x_train, y_train)

from sklearn.svm import SVC
svm = SVC()
svm.fit(x_train, y_train)

Nv = st.sidebar.radio("Navigator", ["Home","Prediction","Contribute"])
if Nv== "Home":
    #st.write("### Home")
    st.image("app.png", width= 700)
    if st.checkbox("Show Dataset"):
        st.table(data)

    st.subheader("\nSoil Requirement of Each Crop")
    if st.checkbox("Show Soil Requirement Graphs"):
        condition = st.selectbox("Conditions",['Nitrogen Requirement','Phosphorous Requirement','Potassium Requirement','Temperature Requirement',
                                               'PH Requirement','Humidity Requirement','Rainfall Requirement'])
        if condition == "Nitrogen Requirement":
            plt.figure(figsize=(5, 3))
            sns.barplot(data['label'], data["N"])
            plt.xlabel('\nCrops', fontsize=14)
            plt.xticks(rotation=90)
            plt.ylabel("Nitrogen Requirement", fontsize=12)
            st.pyplot()
        if condition == "Phosphorous Requirement":
            plt.figure(figsize=(5, 3))
            sns.barplot(data['label'], data["P"])
            plt.xlabel('\nCrops', fontsize=14)
            plt.xticks(rotation=90)
            plt.ylabel("Phosphorous Requirement", fontsize=12)
            st.pyplot()
        if condition == "Potassium Requirement":
            plt.figure(figsize=(5, 3))
            sns.barplot(data['label'], data["K"])
            plt.xlabel('\nCrops', fontsize=14)
            plt.xticks(rotation=90)
            plt.ylabel("Potassium Requirement", fontsize=12)
            st.pyplot()
        if condition == "Temperature Requirement":
            plt.figure(figsize=(5, 3))
            sns.barplot(data['label'], data["temperature"])
            plt.xlabel('\nCrops', fontsize=14)
            plt.xticks(rotation=90)
            plt.ylabel("Temperature Requirement", fontsize=12)
            st.pyplot()
        if condition == "Humidity Requirement":
            plt.figure(figsize=(5, 3))
            sns.barplot(data['label'], data["humidity"])
            plt.xlabel('\nCrops', fontsize=14)
            plt.xticks(rotation=90)
            plt.ylabel("Humidity Requirement", fontsize=12)
            st.pyplot()
        if condition == "PH Requirement":
            plt.figure(figsize=(5, 3))
            sns.barplot(data['label'], data["ph"])
            plt.xlabel('\nCrops', fontsize=14)
            plt.xticks(rotation=90)
            plt.ylabel("PH Requirement", fontsize=12)
            st.pyplot()
        if condition == "Rainfall Requirement":
            plt.figure(figsize=(5, 3))
            sns.barplot(data['label'], data["rainfall"])
            plt.xlabel('\nCrops', fontsize=14)
            plt.xticks(rotation=90)
            plt.ylabel("Rainfall Requirement", fontsize=12)
            st.pyplot()

    st.subheader("\nDistribution of Agricultural Conditions")
    if st.checkbox("Show Distribution Graphs"):
        con = st.selectbox("Conditions",['N','P','K','Temperature','PH','Humidity','Rainfall'])
        if con == "N":
            plt.figure(figsize=(5, 3))
            sns.distplot(data["N"])
            plt.xlabel("\nNitrogen", fontsize=14)
            plt.ylabel('Density',fontsize=14)
            plt.axvline(data["N"].min(), color='y', label='Minimum')
            plt.axvline(data["N"].mean(), color='orange', label='Mean')
            plt.axvline(data["N"].max(), color='grey', label='Maximum')
            plt.legend()
            st.pyplot()
        if con == "P":
            plt.figure(figsize=(5, 3))
            sns.distplot(data["P"])
            plt.xlabel("\nPhosphourous", fontsize=14)
            plt.ylabel('Density',fontsize=14)
            plt.axvline(data["P"].min(), color='y', label='Minimum')
            plt.axvline(data["P"].mean(), color='orange', label='Mean')
            plt.axvline(data["P"].max(), color='grey', label='Maximum')
            plt.legend()
            st.pyplot()
        if con == "K":
            plt.figure(figsize=(5, 3))
            sns.distplot(data["K"])
            plt.xlabel("\nPotassium", fontsize=14)
            plt.ylabel('Density',fontsize=14)
            plt.axvline(data["K"].min(), color='y', label='Minimum')
            plt.axvline(data["K"].mean(), color='orange', label='Mean')
            plt.axvline(data["K"].max(), color='grey', label='Maximum')
            plt.legend()
            st.pyplot()
        if con == "Temperature":
            plt.figure(figsize=(5, 3))
            sns.distplot(data["temperature"])
            plt.xlabel("\nTemperature", fontsize=14)
            plt.ylabel('Density',fontsize=14)
            plt.axvline(data["temperature"].min(), color='y', label='Minimum')
            plt.axvline(data["temperature"].mean(), color='orange', label='Mean')
            plt.axvline(data["temperature"].max(), color='grey', label='Maximum')
            plt.legend()
            st.pyplot()
        if con == "PH":
            plt.figure(figsize=(5, 3))
            sns.distplot(data["ph"])
            plt.xlabel("\nPH", fontsize=14)
            plt.ylabel('Density',fontsize=14)
            plt.axvline(data["ph"].min(), color='y', label='Minimum')
            plt.axvline(data["ph"].mean(), color='orange', label='Mean')
            plt.axvline(data["ph"].max(), color='grey', label='Maximum')
            plt.legend()
            st.pyplot()
        if con == "Humidity":
            plt.figure(figsize=(5, 3))
            sns.distplot(data["humidity"])
            plt.xlabel("\nHumidity", fontsize=14)
            plt.ylabel('Density',fontsize=14)
            plt.axvline(data["humidity"].min(), color='y', label='Minimum')
            plt.axvline(data["humidity"].mean(), color='orange', label='Mean')
            plt.axvline(data["humidity"].max(), color='grey', label='Maximum')
            plt.legend()
            st.pyplot()
        if con == "Rainfall":
            plt.figure(figsize=(5, 3))
            sns.distplot(data["rainfall"])
            plt.xlabel("\nRainfall", fontsize=14)
            plt.ylabel('Density',fontsize=14)
            plt.axvline(data["rainfall"].min(), color='y', label='Minimum')
            plt.axvline(data["rainfall"].mean(), color='orange', label='Mean')
            plt.axvline(data["rainfall"].max(), color='grey', label='Maximum')
            plt.legend()
            st.pyplot()

if Nv == "Prediction":

    st.subheader("\nCrop Predictor\n")
    N = st.number_input("\nNitrogen Value: ",50.00, step=0.10)
    P = st.number_input("Phosphorous Value: ", 50.00 ,step=0.10)
    K = st.number_input("Potassium Value: ", 50.00 ,step=0.10)
    T = st.number_input("Tempreture: ", 25.00 ,step=0.10)
    H = st.number_input("Humidity: ", 50.00 ,step=0.10)
    PH = st.number_input("PH Value: ", 7.00 ,step=0.10)
    R = st.number_input("Rainfall: ", 200.00 ,step=0.10)

    st.write("\n\n\n")
    op=st.selectbox("Choose ML Algorithm",['Random Forest','Logistic Regression', 'Decision Tree','KNN', 'Naive Bayes', 'SVM'])

    st.write("\n\n\n")
    if st.button("Predict"):

        if op=="Logistic Regression":
            y_pred_LR= LogReg.predict([[N, P, K, T, H, PH, R]])
            st.subheader(f"\nPredicted Crop by using Logistic Regression is:")
            st.success(y_pred_LR)

        if op=="Random Forest":
            y_pred_RF= random_forest.predict([[N, P, K, T, H, PH, R]])
            st.subheader(f"\nPredicted Crop by using Random Forest is:")
            st.success(y_pred_RF)

        if op=="Decision Tree":

            y_pred_DT= DecTree.predict([[N, P, K, T, H, PH, R]])
            st.subheader(f"\nPredicted Crop by using Decision Tree is:")
            st.success(y_pred_DT)

        if op=="KNN":
            y_pred_KNN= DecTree.predict([[N, P, K, T, H, PH, R]])
            st.subheader(f"\nPredicted Crop by using KNN is:")
            st.success(y_pred_KNN)

        if op=="Naive Bayes":
            y_pred_NB= NB.predict([[N, P, K, T, H, PH, R]])
            st.subheader(f"\nPredicted Crop by using Naive Bayes is:")
            st.success(y_pred_NB)

        if op=="SVM":
            y_pred_SVM= svm.predict([[N, P, K, T, H, PH, R]])
            st.subheader(f"\nPredicted Crop by using SVM is:")
            st.success(y_pred_SVM)

if Nv == "Contribute":
    st.subheader("Contribute to our Dataset")
    N = st.number_input("Nitrogen Value: ", 0.00, 150.00, 50.00, step=0.5)
    P = st.number_input("Phosphorous Value: ", 0.00, 150.00, 50.00, step=0.5)
    K = st.number_input("Potassium Value: ", 0.00, 120.00, 50.00, step=0.5)
    T = st.number_input("Tempreture: ", 0.00, 60.00, 25.00, step=0.5)
    H = st.number_input("Humidity: ", 10.00, 100.00, 50.00, step=0.5)
    PH = st.number_input("PH Value: ", 0.00, 10.00, 7.00, step=0.5)
    R = st.number_input("Rainfall: ", 20.00, 300.00, 200.00, step=0.5)
    crop = st.text_input("Crop: ")

    if st.button("Contribute"):
        to_add= {"N":[N], "P":[P], "K":[K], "temperature":[T], "humidity":[H], "ph":[PH], "rainfall":[R], "label":[crop]}
        to_add= pd.DataFrame(to_add)
        to_add.to_csv("app.csv", mode='a', header=False, index=False)
        st.success("Thanks for Your Contribution")


import os
from flask import Flask, jsonify, request
from math import sqrt

app = Flask(__name__)

@app.route('/')
def nao_entre_em_panico():

    nmax = 50

    n1 = 0
    n2 = 1
    cont = 0
    fib = 0
    res = "Essa é sequencia dos 50 primeiros números da razão de Fibonacci:
 Desenvolvido por Jefferson Alves.

"

    while cont < nmax:
        fib = n1 + n2
        n1 = n2
        n2 = fib
        cont = cont + 1
        res = res + str(fib) + "
"
    return res

if __name__ == "__main__":
    port = int(os.environ.get("PORT", 5000))
    app.run(host='0.0.0.0', port=port)


import streamlit as st
from streamlit_option_menu import option_menu
import math
import datetime
from datetime import date
import calendar
from PIL import Image
from title_1 import *
from img import *
with open('final.css') as f:
        st.markdown(f"",unsafe_allow_html=True)
def average():
    image()
    st.markdown("
Application to Find the Average
", unsafe_allow_html=True)
    w1,col1,col2,w2=st.columns((1,2,2,1))
    us1,bc1,bc2,us2=st.columns((4,1.5,1.8,6))
    with col1:
        st.markdown("")
        st.write("# Enter the Date ")
    # ------------to create the function to clear the input-----------#
    with bc2:
        st.markdown("")
        st.markdown("")
        def clear_text():
            st.session_state["text"] = ""
        st.button("Clear", on_click=clear_text)
    with col2:
        vAR_input_num=st.text_input("",key="text")
        vAR_list=[]
    #----- Average -------#
    with bc1:
        st.markdown("")
        st.markdown("")
        if st.button("Submit"):
            with col2:
                if vAR_input_num != '':
                    vAR_input_data = vAR_input_num.split(",")
                    for i in vAR_input_data:
                        num=int(i)
                        vAR_list.append(num)
                    def Average(vAR_list):
                        vAR_avg= sum(vAR_list) / len(vAR_list)
                        vAR_avg=round(vAR_avg,4)
                        st.success(vAR_avg)
                    Average(vAR_list)
                else:
                    st.error("Error")
                with col1:
                    st.write("# Answer is ")