#!/usr/bin/env python
# -*- coding: cp1252 -*-

from __future__ import print_function, division, absolute_import

import numpy as np
import matplotlib.pyplot as plt
import argparse
import os
from matplotlib import rc

rc('font', size=17)
rc('legend', fontsize=15)
rc('xtick', labelsize=13)
rc('ytick', labelsize=13)
# rc('text', usetex=True)

def create_parser():
    def float_pair(arg):
        split_pair = [float(e) for e in arg.split(',')]
        if len(split_pair) != 2:
            raise ValueError
        return split_pair

    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--shot', '-s', dest='shot',
        required=True, type=str,
        help="Shot number"
    )

    parser.add_argument(
        '--channels-ignore', '-i', dest='channels_ignore',
        type=int, default={0}, nargs='+',
        help="Channels to ignore (default none)"
    )

    parser.add_argument(
        '--time-size', '-b', dest="time_n_bits",
        default='48', type=int, choices=(32, 48),
        help="Size of the timestamp (default 48b)"
    )

    parser.add_argument(
        '--freq-clock', '-f', dest='freq_clock',
        default=200, type=float,
        help="Clock frequency in MHz (default 200 MHz)"
    )

    parser.add_argument(
        '--time-delay', dest='time_delay',
        default=30, type=float,
        help="Time after trigger (default 30s)"
    )
    parser.add_argument(
        '--time-shot', dest='time_shot',
        default=[30, 100], type=float, nargs=2,
        metavar=('MIN_TIME', 'MAX_TIME'),
        help="Time span to analyse (default 30 to 100s)"
    )
    parser.add_argument(
        '--time-step', '-T', dest='time_step',
        default=1e-3, type=float,
        help="Time step for the histogram (default 1e-3 s)"
    )
    parser.add_argument(
        '--time-events', '-t', dest='events_limits',
        default=[], type=float_pair, nargs='+',
        metavar=('MIN1,MAX1', 'MIN2,MAX2'),
        help="Time of events to study (default none)"
    )

    parser.add_argument(
        '--time-background', '-B', dest='time_background',
        default=[1, -1], type=float, nargs=2,
        metavar=('TIME_START', 'TIME_END'),
        help="Time interval to consider as background (default no background)"
    )  # TODO: Grab background from another file

    parser.add_argument(
        '--adc-limits', '-a', dest='adc_limits',
        default=[0, 2 ** 15], type=int, nargs=2, metavar=('MIN_BIN', 'MAX_BIN'),
        help="Limits of the ADC (default 0 to 2^15)"
    )

    parser.add_argument(
        '--energy-windows', '-e', dest='energy_windows',
        default=[[250, 5000]], type=float_pair, nargs='+',
        metavar=('MIN1,MAX1', 'MIN2,MAX2'),
        help="Energy windows to count, in keV (default 250,5000keV)"
    )

    parser.add_argument(
        '--folder', '-F', dest='folder_name',
        default='.', type=str,
        help='Location of the bin files (default current folder)'
    )
    parser.add_argument(
        '--prefix', '-P', dest='prefix',
        default='', type=str,
        help="Prefix to prepend to output files (default none)"
    )
    parser.add_argument(
        '--suffix', '-S', dest='suffix',
        default='', type=str,
        help="Suffix to append to output files (default none)"
    )

    parser.add_argument(
        '--calibration-file', '-c', dest='calibration_file_name',
        default='calibration.csv', type=str,
        help="Calibration file (default 'calibration.csv')"
    )

    parser.add_argument(
        '--verbose', '-v', dest='verbose',
        action='store_true',
        help=""
    )

    parser.add_argument(
        '--plot-graphs', '-p', dest='plot_graphs',
        action='store_true',
        help="Show plots (by default they are not shown)"
    )
    parser.add_argument(
        '--energy-lines', '-E', dest='energy_lines',
        default=[], type=float, nargs='+', metavar='ENERGY',
        help="Values of vertical lines in energy spectrum (default none)"
    )
    parser.add_argument(
        '--energy-plot-limits', dest='energy_plot_limits',
        default=[0, 3000], type=float, nargs=2, metavar=('MIN', 'MAX'),
        help="Limits of the energy spectrum plot (default 0 to 3000 keV)"
    )

    return parser

fix_float_pair = lambda l: [[]] if len(l) == 0 else l if len(l[0]) == 2 else [l]

parser = create_parser()
args = parser.parse_args()

shot_str = args.shot
channels_to_use = sorted(set(range(1, 20)) - set(args.channels_ignore))

time_n_bits = args.time_n_bits
freq_clock = args.freq_clock * 1e6  # convert to MHz
time_delay = args.time_delay
min_time, max_time = args.time_shot
time_step = args.time_step
time_background = args.time_background
events_limits = args.events_limits

adc_limits = args.adc_limits

energy_windows = args.energy_windows
energy_lines = args.energy_lines

verbose = args.verbose
plot_graphs = args.plot_graphs
folder_name = args.folder_name
energy_plot_limits = args.energy_plot_limits

image_folder = os.path.join(folder_name, 'images')
try:
    os.makedirs(image_folder)
except OSError:
    if not os.path.isdir(image_folder):
        raise

out_folder = os.path.join(folder_name, 'out')
try:
    os.makedirs(out_folder)
except OSError:
    if not os.path.isdir(out_folder):
        raise

file_name_w_fix = lambda name: args.prefix + name + args.suffix

calibration_file_name = args.calibration_file_name
calibration_file_path = os.path.join(folder_name, calibration_file_name)

calibrations = np.loadtxt(
    fname=calibration_file_path,
    # comments='#', delimiter='\t',
    skiprows=1
)

with open(calibration_file_path, 'r') as calib_file:
    line = calib_file.readline().strip().split('#')[0]
    adc_n_bins = int(line)

counts_channels = []

# Channels selection
for channel in channels_to_use:
    filename = "shot_%s_ch%d.bin" % (shot_str, channel)
    file_path = os.path.join(folder_name, filename)

    word_64_type = np.dtype(
        [('energy', '<u2'), ('empty', '<u2'), ('time', '<u4')]
    )
    try:
        data = np.memmap(file_path, dtype=word_64_type, mode='r')
    except IOError:
        continue

    print('Reading from "' + filename + '".')

    energies = data['energy']

    if time_n_bits == 32:
        times = data['time']
    elif time_n_bits == 48:
        # Read entire 64b
        word_all = np.memmap(file_path, dtype=np.dtype('<u8'), mode='r')

        # Remove "garbage" data (energy)
        times = (word_all & 0xffffffffffff0000) >> 4 * 4
    else:
        raise ValueError("Unknown size of timestamp: '%d'.", time_n_bits)

    # Find last real measurement (time still increasing)
    num_pulses = np.argmax(times[1:] < times[:-1])

    # Remove garbage data
    times = times[:num_pulses]
    energies = energies[:num_pulses]

    # Convert to correct units
    times = times / freq_clock + time_delay  # time in seconds

    # min_time, max_time = times.min(), times.max()

    # Filter to wanted time span
    time_inds = (times >= min_time) & (times <= max_time)
    times = times[time_inds]
    energies = energies[time_inds]

    if verbose:
        # noinspection PyStringFormat
        print(' %d pulses found. %d pulses used (%.0d%%).' % (
            num_pulses, times.shape[0], times.shape[0]/num_pulses * 100
        ))

    # Find background
    bg_inds = (times >= time_background[0]) & (times <= time_background[1])
    times_bg = times[bg_inds]
    energies_bg = energies[bg_inds]

    # Find events
    times_events = []
    energies_events = []
    for event in events_limits:
        event_inds = (times >= event[0]) & (times <= event[1])
        times_events.append(times[event_inds])
        energies_events.append(energies[event_inds])

    # Calculate Time Histogram

    time_bins = np.arange(times[0], times[-1], time_step)
    time_hist, time_bins_edges = np.histogram(times, bins=time_bins)
    time_bins_centre = (time_bins_edges[1:] + time_bins_edges[:-1])/2

    time_bg_inds = (
        (time_bins_centre >= time_background[0]) &
        (time_bins_centre <= time_background[1])
    )

    # Calculate Energy Spectrum

    energy_hist, _ = np.histogram(
        energies, bins=adc_n_bins, range=adc_limits, density=False
    )

    energy_hist_bg, _ = np.histogram(
        energies_bg, bins=adc_n_bins, range=adc_limits, density=False
    )

    calib_channel = calibrations[channel - 1]

    energy_bins_center = np.arange(1, len(energy_hist) + 1, 1)
    energy_bins_center = energy_bins_center*calib_channel[0] + calib_channel[1]
    energy_bin_size = calib_channel[0]

    # TODO: Allow choice of normalization
    # norm_fun = lambda hist: max(np.max(hist), 1)
    # norm_fun = lambda hist: np.sqrt(np.sum(np.power(hist, 2.)))
    # norm_fun = lambda hist: np.sum(hist)
    norm_fun = lambda hist: np.trapz(hist, energy_bins_center)
    # norm_fun = lambda hist: np.sum(hist * energy_bin_size)

    norm_factor = norm_fun(energy_hist)
    norm_factor_bg = max(norm_fun(energy_hist_bg), 1)

    energy_hist = energy_hist / norm_factor
    energy_hist_bg = energy_hist_bg / norm_factor_bg

    energy_hist_events = []
    for energies_ev in energies_events:
        hist_temp, _ = np.histogram(
            energies_ev, bins=adc_n_bins, range=adc_limits, density=False
        )
        norm_temp = norm_fun(hist_temp)

        # TODO: Improve background subtraction
        out_temp = {
            'total': hist_temp,
            'no_bg': hist_temp - energy_hist_bg * norm_temp
        }

        energy_hist_events.append(out_temp)

    energy_hist *= norm_factor
    energy_hist_bg *= norm_factor_bg

    # ### FIGURE ###

    fig = plt.figure(figsize=(10, 8))

    bbox_anchor_limits = (1.03, 1)

    # ### Times ###
    # plt.subplot(4, 1, 1)
    # titlename = "%s%d" % ('CH', channel)
    # plt.title(titlename)
    #
    # plt.plot(times, '.b-')
    # plt.xlabel('Event #')
    # plt.ylabel('t (s)')

    # ### Events ###
    plt.subplot(3, 1, 1)

    plt.plot(times, energies, '.', ms=0.3, color='C0')
    plt.plot(times_bg, energies_bg, '.', ms=0.2, color='C1')
    for n, (times_ev, energies_ev) in enumerate(
            zip(times_events, energies_events)):
        plt.plot(times_ev, energies_ev, '.', ms=0.3, color='C' + str(n + 2))

    plt.xlim(min_time, max_time)

    plt.plot(np.nan, 'o', ms=5, color='C0', label='Shot')
    plt.plot(np.nan, 'o', ms=5, color='C1', label='Background')
    for n, _ in enumerate(times_events, start=1):
        plt.plot(np.nan, 'o', ms=5, color='C' + str(n + 1),
                 label='Event #%d' % n)
    plt.legend(bbox_to_anchor=bbox_anchor_limits, loc='upper left')

    plt.xlabel('t (s)')
    plt.ylabel('E (a.u.)')
    plt.title('Events')

    # ### TIME HISTOGRAM ###
    plt.subplot(3, 1, 2)

    plt.plot(time_bins_centre, time_hist, color='C0', label='Shot')
    plt.plot(time_bins_centre[time_bg_inds], time_hist[time_bg_inds],
             color='C1', label='Background')

    for n, event in enumerate(events_limits):
        inds = (time_bins_centre >= event[0]) & (time_bins_centre <= event[1])
        plt.plot(time_bins_centre[inds], time_hist[inds],
                 color='C' + str(n + 2), label='Event #' + str(n + 1))

    plt.xlim(min_time, max_time)
    plt.ylim(0, time_hist.max()*1.1)

    plt.legend(bbox_to_anchor=bbox_anchor_limits, loc='upper left')

    plt.xlabel('t (s)')
    plt.ylabel('Counts\n(every %.3g s)' % time_step)
    plt.title('Time Histogram')

    # #### ENERGY SPECTRUM ####

    plt.subplot(3, 1, 3)

    plt.axhline(0, ls='-', color='k', alpha=0.7)

    for line_energy in energy_lines:
        plt.axvline(line_energy, ls='--', color='k', alpha=0.7)
        plt.text(line_energy + 50, energy_hist.max() * 0.85,
                 '%.0f keV' % line_energy, alpha=0.7)

    plt.plot(energy_bins_center, energy_hist, color='C0', label='Entire Shot')
    plt.plot(energy_bins_center, energy_hist_bg, color='C1',
             label='Background')

    for n, energy_hist_ev in enumerate(energy_hist_events):
        plt.plot(energy_bins_center, energy_hist_ev['no_bg'],
                 '-', color='C' + str(n + 2),
                 label='Event #%d\n(background\n removed)' % (n + 1))

    plt.xlim(energy_plot_limits)
    if len(events_limits) > 0:
        plt.ylim(-10, 1.5*max(
            map(lambda f: f['no_bg'].max(), energy_hist_events)
        ))

    plt.legend(bbox_to_anchor=bbox_anchor_limits, loc='upper left')

    plt.xlabel('E (keV)')
    plt.ylabel('Counts')
    plt.title('Energy Spectrum')

    # ###### OUTPUT ######

    out_file_name = file_name_w_fix(filename[:-4])

    # ### Plots ###

    image_path = os.path.join(image_folder, out_file_name + '.png')

    plt.tight_layout(rect=[0, 0, 0.75, 1])

    plt.savefig(image_path, dpi=200)
    if verbose:
        print(' Plots saved to "' + image_path + '".')

    if plot_graphs:
        plt.show()
    else:
        plt.close(fig)

    # ### Events ###

    events_file = os.path.join(
        out_folder,
        file_name_w_fix(filename[:-4] + '_events')
    ) + '.dat'
    np.savetxt(
        events_file,
        np.vstack((times, energies)).T,
        fmt='%18g %18g',
        header='%16s %18s' % ('Time (s)', 'Energies (a.u.)')
    )
    if verbose:
        print(' Events saved to "' + events_file + '".')

    # ### Time Histogram ###

    times_file = os.path.join(
        out_folder,
        file_name_w_fix(filename[:-4] + '_times')
    ) + '.dat'
    np.savetxt(
        times_file,
        np.vstack((time_bins_centre, time_hist)).T,
        fmt='%18g %10d',
        header='Times are the centres of the bins\n'
               '%16s %10s' % ('Time (s)', 'Counts')
    )
    if verbose:
        print(' Time histogram saved to "' + times_file + '".')

    # ### Energy Histogram ###

    energies_file = os.path.join(
        out_folder,
        file_name_w_fix(filename[:-4] + '_energies')
    ) + '.dat'

    energy_table = [energy_bins_center, energy_hist, energy_hist_bg]

    for n_time, energy_hist_ev in enumerate(energy_hist_events):
        energy_table += [energy_hist_ev['total']]

    energy_table = np.vstack(energy_table).T

    format_str = '%18g %10d %10d ' + '%10d ' * len(energy_hist_events)
    format_str = format_str[:-1]

    header_str = 'Energies are the centres of the bins\n'
    header_str += '%16s %10s %10s ' % ('Energy (keV)', 'Shot', 'BG')
    for n, limits in enumerate(events_limits, start=1):
        header_str += '%10s ' % ('Event #%d' % n)
    header_str = header_str[:-1]
    header_str += '\n'

    np.savetxt(energies_file, energy_table, fmt=format_str, header=header_str)
    if verbose:
        print(' Energy spectrum saved to "' + energies_file + '".')

    # ### Energy Windows ###

    windows_counts = np.zeros(
        (len(energy_windows), 2 + 2*len(energy_hist_events))
    )

    for n_energy, energy_limits in enumerate(energy_windows):
        windows_counts[n_energy, 0:2] = energy_limits

        for n_time, energy_hist_ev in enumerate(energy_hist_events):
            # TODO (use edges)
            bins_to_use = (
                (energy_bins_center >= energy_limits[0])
                &
                (energy_bins_center <= energy_limits[1])
            )

            t_ind = 2 + 2*n_time

            windows_counts[n_energy, t_ind] = \
                np.sum(energy_hist_ev['total'][bins_to_use])
            windows_counts[n_energy, t_ind+1] = \
                np.sum(energy_hist_ev['no_bg'][bins_to_use])

    counts_channels.append(
        [channel, windows_counts[:, 2:]]
    )

    windows_file = os.path.join(
        out_folder,
        file_name_w_fix(filename[:-4] + '_counts')
    ) + '.dat'

    format_str = '%18g | %18g | ' + '%10d | %10d | '*len(energy_hist_events)
    format_str = format_str[:-3]

    header_str = 'Format = "%s"\n' % format_str
    header_str += '%16s | %18s | ' % ('Min Energy (keV)', 'Max Energy (keV)')
    for n, limits in enumerate(events_limits, start=1):
        header_str += '%23s | ' % (
                'Event #%d: %.2gs to %.2gs' % (n, limits[0], limits[1])
        )
    header_str = header_str[:-3]
    header_str += '\n'

    header_str += ' ' * 16 + ' | ' + ' ' * 18 + ' | '
    for _ in events_limits:
        header_str += '%10s | %10s | ' % ('Total', 'No BG')
    header_str = header_str[:-3]

    np.savetxt(windows_file, windows_counts, fmt=format_str, header=header_str)
    if verbose:
        print(' Energy windows saved to "' + windows_file + '".')

# Combined Counts File

combined_file_name = os.path.join(
    out_folder, file_name_w_fix('shot_' + shot_str + '_combined_counts')
) + '.dat'

format_str = '%4d | ' + '%10d | %10d | '*len(events_limits)
format_str = format_str[:-3]

header_str = '# Energy Window #%d: %.0f keV to %.0f keV\n'
header_str += '# ch | '
for n, limits in enumerate(events_limits, start=1):
    header_str += '%23s | ' % (
            'Event #%d: %.2gs to %.2gs' % (n, limits[0], limits[1])
    )
header_str = header_str[:-3]
header_str += '\n'

header_str += '#    | '
for _ in events_limits:
    header_str += '%10s | %10s | ' % ('Total', 'No BG')
header_str = header_str[:-3]
header_str += '\n'

with open(combined_file_name, 'w') as combined_file:
    combined_file.write('# Format = "' + format_str + '"\n')
    for n_energy, energy_limits in enumerate(energy_windows):
        combined_file.write(header_str % (
            n_energy+1, energy_limits[0], energy_limits[1]
        ))
        for channel, counts in counts_channels:
            line_values = tuple(np.hstack([channel, counts[n_energy, :]]))
            combined_file.write(format_str % line_values)
            combined_file.write('\n')

        combined_file.write('\n')

print('Counts from all channels saved to "' + combined_file_name + '".')


import subprocess
import argparse
import wandb
from fastai.vision.all import *
from fastai.callback.wandb import *
import matplotlib

def get_items(path):
	train_df = pd.read_csv(f'{PATH}/train.csv')
	return train_df.to_numpy().tolist()

def get_x(item):
	if len(item) == 785:
		x = np.array(item[1:]).reshape(28, 28).astype(np.uint8)
	else:
		x = np.array(item).reshape(28, 28).astype(np.uint8)
	return x

def get_y(item):
	return item[0]

def set_wandb(key, sha):
	wandb.login(key=key)
	ro = wandb.init(project='test.project')

	target_file = open('template/wandb.md')
	new_file_content = ""

	for line in target_file:
		stripped_line = line.strip()
		new_line = stripped_line.replace("t_wandb_link", ro.get_url())
		new_file_content += new_line +"\n"
	target_file.close()

	target_file = open("generated_template/wandb.md", "w")
	target_file.write(new_file_content)
	target_file.close()

	subprocess.run(f"cml-send-comment --commit-sha {sha} generated_template/wandb.md", shell=True)

if __name__ == "__main__":
	parser = argparse.ArgumentParser(description='Please specify wandb key if you want')
	parser.add_argument('--wandb_key', type=str)
	parser.add_argument('--sha', type=str)
	args = parser.parse_args()

	if args.wandb_key:
		set_wandb(args.wandb_key, args.sha)

	PATH = Path('data/')
	train = pd.read_csv(f'data/train.csv')

	db = DataBlock(blocks=(ImageBlock(cls=PILImageBW), CategoryBlock),
          	get_items=get_items,
          	get_x=get_x,
          	splitter=RandomSplitter(seed=42),
          	get_y=get_y,
		item_tfms=Resize(224))

	dls = DataLoaders.from_dblock(db, df=train, source=PATH, bs=32)
	dls.show_batch(max_n=5, ncols=5, nrows=1, figsize=(6,2))

	matplotlib.pyplot.savefig('show_batch.png')
	matplotlib.pyplot.clf()

	learn = cnn_learner(dls, resnet34, metrics=[accuracy, error_rate])
	if args.wandb_key:
		learn.fine_tune(5, cbs=WandbCallback())
	else:
		learn.fine_tune(5)

	matplotlib.pyplot.figure(figsize=(6,3))
	learn.recorder.plot_loss()
	matplotlib.pyplot.savefig('loss.png')
	matplotlib.pyplot.clf()

	interp = ClassificationInterpretation.from_learner(learn)
	interp.plot_confusion_matrix(figsize=(5,5))
	matplotlib.pyplot.savefig('confusion_matrix.png')
	matplotlib.pyplot.clf()


"""
2020-07-04 17:07

@author: Vito Vincenzo Covella
"""

import numpy as np
import dask.dataframe as dd
import dask.array as da
import pandas as pd
from numpy import random
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import f1_score
import sys
import matplotlib.pyplot as plt
import os

#######functions to count lines of a file, from https://stackoverflow.com/a/27518377/1262118
def _make_gen(reader):
    b = reader(1024 * 1024)
    while b:
        yield b
        b = reader(1024*1024)

def rawgencount(filename):
    f = open(filename, 'rb')
    f_gen = _make_gen(f.raw.read)
    return sum( buf.count(b'\n') for buf in f_gen )
########

def plot_bar_x(measureType, key, value):
    # this is for plotting purpose
    index = np.arange(len(key))
    plt.figure()
    for i, v in enumerate(value):
        v = np.round(v,2)
        color = 'red'
        if v < 0.8:
            color = 'chocolate'
        if v >=0.8:
            color = 'olive'
        if v >= 0.89:
            color = 'olivedrab'
        if v > 0.9:
            color = 'limegreen'

        if i == 0:
            plt.bar(index[i], v, width = 0.9, edgecolor = 'black', ls = '-', lw = 1.5, color = color)
            plt.text(index[i] - 0.1, 0.3, str('%.2f'%v), fontsize = 12, rotation = 90)
        else:
            plt.bar(index[i], v, width = 0.9, edgecolor = 'black', ls = '--', lw = 0.5, color = color)
            plt.text(index[i] - 0.1, 0.3, str('%.2f'%v), fontsize = 12, rotation = 90)
#    plt.xlabel('Genre', fontsize=5)
    plt.ylabel('F-score', fontsize=20)
    plt.ylim([0.0,1.0])
    plt.xticks(index, key, fontsize=15, rotation=270)
    #plt.title('%s'%measureType)
    fig1 = plt.gcf()
    plt.show()
    plt.draw()
    fig1.savefig('%s.png'%(measureType), bbox_inches='tight')

if __name__ == '__main__':

    random.seed(42)

    if len(sys.argv) != 3:
        print("Please insert a file path to analyze as argument and the number of important features to select")
        exit(1)

    numImportantFeatures = int(str(sys.argv[2]))

    input_file = str(sys.argv[1])
    input_name = input_file.split('.')[0]
    nodename = input_file.split('_')[0].split('/')[-1]
    fileN_mostImportant = input_name + "_result_decTreeFreature_" + str(numImportantFeatures) + "mostImportant.txt"
    featureFile = input_name + "_decTree_mostImportantFeatures.txt"

    len_features = rawgencount(input_file)
    #print(nodename)

    data = dd.read_csv(input_file, header=0)
    #all the columns except the label one
    metricKeys = list(data.columns)[:-1]

    #select the features (all the column except the label one)
    features = data[list(data.columns)[:-1]]
    features = features.to_dask_array(lengths=True)
    #select the labels
    labels = data[list(data.columns)[-1]]
    labels = labels.to_dask_array(lengths=True)
    #60-40 train-test split
    numTrain = int(0.6*len_features)
    trainData = features[:numTrain]
    trainLbl = labels[:numTrain].compute()
    #testData = features[numTrain:]
    #testLbl = labels[numTrain:]

    clf = DecisionTreeClassifier(random_state=42)

    with open(featureFile, 'w') as out:
        out.write('- Most important features:\n')

    clf.fit(trainData, trainLbl)

    importances = clf.feature_importances_
    indices = np.argsort(importances)[::-1]
    print('- Most important features:')
    with open(featureFile, 'a') as out:
       for idx in range(len(indices)):
           metric = indices[idx]
           print('---- %s: %s' % (metricKeys[metric], importances[metric]))
           out.write('---- %s: %s\n' % (metricKeys[metric], importances[metric]))

    #retrain with most important features
    mostImportantColumns = []
    for i in range(numImportantFeatures):
        metric = indices[i]
        mostImportantColumns.append(metricKeys[metric])

    #select features vectors using only the k most important features
    features = data[mostImportantColumns]
    features = features.to_dask_array(lengths=True)
    trainData = features[:numTrain]
    trainLbl = labels[:numTrain].compute()
    testData = features[numTrain:]
    testLbl = labels[numTrain:].compute()

    clf = RandomForestClassifier(n_estimators=30, max_depth=20, n_jobs=-1, random_state=42)

    with open(fileN_mostImportant, 'w') as out:
        out.write('- Classifier: %s\n' % clf.__class__.__name__)

    F = []

    clf.fit(trainData, trainLbl)
    pred = clf.predict(testData)
    print('- Classifier: %s' % clf.__class__.__name__)
    f1 = f1_score(testLbl, pred, average = 'weighted')
    F.append(f1)

    for l in np.unique(np.asarray(testLbl)):
        #select the correct labels from testLbl
        lab_tmp = testLbl[list(np.where(testLbl == l)[0])]
        #select the corresponding predicted lables
        pred_tmp = pred[list(np.where(testLbl == l)[0])]
        f1 = f1_score(lab_tmp, pred_tmp, average = 'micro')
        F.append(f1)
        print('Fault: %d,  F1: %f.\n'%(l,f1))
        with open(fileN_mostImportant, 'a') as out:
            out.write('Fault: %d,  F1: %f.\n'%(l,f1))

    keys = ['overall','healthy', 'memeater','memleak', 'membw', 'cpuoccupy','cachecopy','iometadata','iobandwidth']
    measureType = input_name + "_result_decTreeFeature" + str(numImportantFeatures) + "mostImportant"
    plot_bar_x(measureType, keys, F)


import json
import logging

from django.http import Http404
from django.http import HttpResponse
from django.http import JsonResponse
from rest_framework import status, exceptions
from rest_framework.exceptions import ValidationError
from rest_framework.utils import json
from rest_framework.views import exception_handler

from .error_codes_mapping import *

def custom_exception_handler(exc, context):
    """
    Custom exception handler for Django Rest Framework that adds
    the `status_code` to the response and renames the `detail` key to `error`.
    """
    response = exception_handler(exc, context)
    data = dict()
    if response:
        data = json.loads(json.dumps(response.data))
    status_code = status.HTTP_500_INTERNAL_SERVER_ERROR
    code = code_status_mapping[status_code]["code"]
    title = code_status_mapping[status_code]["title"]

    if response is not None:
        status_code = response.status_code
        code = code_status_mapping[status_code]["code"]
        title = code_status_mapping[status_code]["title"]
        custom_errors = []
        for key, value in response.data.items():
            if isinstance(exc, Http404):
                error = {"field": key, "message": str(exc)}
                del data[key]
                custom_errors.append(error)
            else:
                error = {"field": key, "message": value}
                del data[key]
                custom_errors.append(error)
        data["errors"] = custom_errors
        if isinstance(exc, ValidationError):
            data["exception"] = "Validation Error"

    else:
        logging.error(exc, exc_info=True)
        error = {"field": "detail", "exception": str(exc)}
        data["errors"] = [error]
        data["exception"] = "Internal Server Error"
    data["code"] = code
    data["title"] = title

    return HttpResponse(
        json.dumps(data), content_type="application/json", status=status_code
    )

def resource_not_found_404(request):
    return JsonResponse(
        {"status_code": 404, "error": "The resource was not found"},
        status=status.HTTP_404_NOT_FOUND,
    )

def record_not_found_error(message):
    """
    :param message: the message to be printed
    """
    raise exceptions.NotFound(message)


# -*- coding: utf-8 -*-
"""
Created on Sun Aug 15 21:35:46 2021

@author: Md. Sohrawordi
"""
from flask import Flask, render_template,request,send_file

def amino_acid_composition(dataset):
    import pandas as pd
    #A list of 21 Amino Acid
    amino_acid="ACDEFGHIKLMNPQRSTWYVX"
    colls=[]
    #sample = pd.read_csv(dataset)

    for i in range (len(amino_acid)):
        colls.append("Amino_acid_"+amino_acid[i])
    #print(colls)
    df=pd.DataFrame(columns=colls)
    for i in range(len(dataset['Sequence'])):
        #print(i)
        row={}
        for acid in amino_acid:
            #print(seq.count(acid))
            row["Amino_acid_"+acid]=dataset['Sequence'][i].count(acid)/len(dataset['Sequence'])
        df=df.append(row,ignore_index=True)

    return df

def binary_data(positive_sample):
    import pandas as pd
    amino_acid="ACDEFGHIKLMNPQRSTWYVX"
    binary_code={}
    colls=[]
    for i in range (len(amino_acid)*len(positive_sample['Sequence'][0])):
        colls.append("Binary"+str(i))

    for i in amino_acid:
        a=[]
        for j in amino_acid:
            if i==j:
                a.append(1)
            else:
                a.append(0)
        binary_code[i]=a

    pdf=pd.DataFrame(columns=colls)
    for k in range(len(positive_sample['Sequence'])):
        c=0
        row_value={}
        for i in positive_sample['Sequence'][k]:
            for j in binary_code[i]:
                row_value["Binary"+str(c)]=j
                c=c+1
        ##row_value["Class"]=1
        pdf=pdf.append(row_value,ignore_index=True)

    return pdf

def count(seq,a,b):
    fp=seq.find(a)
    fpb=seq.find(b)

    s=0
    if fp>=0 and fpb>=0:
        for j in range(len(seq)):
            if b==seq[j]:
                s=s+abs(j-fp)

        #ar.append(s)
        return s
    else:
        return -1
        #ar.append(-1)
    #return ar

def prim_rprim(dataset):
    import pandas as pd

    amino_acid="ACDEFGHIKLMNPQRSTWYVX"

    colls=[]
    for x in amino_acid:
        for y in amino_acid:
            colls.append("prim_"+x+"-->"+y)

    pdf=pd.DataFrame(columns=colls)

    for x in amino_acid:
        for y in amino_acid:
            colls.append("rv_prim_"+x+"-->"+y)

    colls=[]
    rdf=pd.DataFrame(columns=colls)

    #calculation of max length between resudues
    seq_length=len(dataset['Sequence'][0])
    max_distance=0
    for i in range(seq_length):
        max_distance=max_distance+i

    #print(max_distance)

    for i in range(len(dataset.index)):
        #print(dataset['Sequence'][i])
        seq=dataset['Sequence'][i]
        #seq="ADCDD"

        pr_row={}

        # for position relative incidence matrix
        for x in amino_acid:
            for y in amino_acid:
                li=count(seq,x,y)
                #print(x,"....>",y,li)
                pr_row["prim_"+x+"-->"+y]=li/max_distance

            #print()
            #print()

        # for reverse position relative incidence matrix
        rv_seq=seq[::-1]
        r_row={}
        for x in amino_acid:
            for y in amino_acid:
                li=count(rv_seq,x,y)
                #print(x,"....>",y,li)
                r_row["rv_prim_"+x+"-->"+y]=li /max_distance

        #print(dataset['Sequence'][i])

        pdf=pdf.append(pr_row,ignore_index=True)
        rdf=rdf.append(r_row,ignore_index=True)

    return pdf,rdf

# split each sequence into characters and create a dataframe with a column for each character

def propensity_matrix_generation(dataset,window_size):

    amino_acid="ACDEFGHIKLMNPQRSTWYVX"
    strim_len=int(window_size/2)

    p_matrix={}
    #calculate total element of a position in dataset
    n=len(dataset.index)

    for i in range (window_size):
        ar_list=[]
        if i!=strim_len:
            for j in range(len(dataset.index)):
                #print(dataset['Sequence'][j][i])
                ar_list.append(dataset['Sequence'][j][i])

            for acid in amino_acid:
                #print(acid,ar_list.count(acid))
                p_matrix[acid+"_"+str(i-strim_len)]=ar_list.count(acid)/n

    return p_matrix

def propensity_feature(dataset,window_size):
    import pickle
    matrix = pickle.load(open('propensity_matrix.pkl','rb'))

    import pandas as pd
    #amino_acid="ACDEFGHIKLMNPQRSTWYVX"
    strim_len=int(window_size/2)
    colls=[]
    #generate dataframe columns without central resudue
    for i in range (window_size):
        if i !=strim_len:
            colls.append("propensity_"+str(i-strim_len))
    #print(colls)
    df=pd.DataFrame(columns=colls)

    for i in range(len(dataset['Sequence'])):
        #print(dataset['Sequence'][i])
        seq=dataset['Sequence'][i]
        row={}

        #create a dataframe row without central resudue for each sample sequence
        for j in range(len(seq)):
            if j !=strim_len:
                row["propensity_"+str(j-strim_len)]=matrix[seq[j]+"_"+str(j-strim_len)]

        #add a row in dataframe
        df=df.append(row,ignore_index=True)

    return df

def featureset(data):
    import pickle
    import pandas as pd
    aac=amino_acid_composition(data)
    be=binary_data(data)
    prim,rprim=prim_rprim(data)
    psaap=propensity_feature(data,21)

    hybrid_sample=aac.copy()
    for col in be.columns:
        hybrid_sample[col]=be[col]
    for col in psaap.columns:
        hybrid_sample[col]=psaap[col]
    for col in prim.columns:
        hybrid_sample[col]=prim[col]

    for col in rprim.columns:
        hybrid_sample[col]=rprim[col]

    cols_index=pickle.load(open('optimal_feature_set_index.pkl','rb'))

    optimal_data=pd.DataFrame()
    for index in cols_index:
        optimal_data[index]=hybrid_sample[index]

    #print(optimal_data)
    return optimal_data

def prediction(seq,window):
    up=int(window/2)

    listk=[i for i, letter in enumerate(seq) if letter == 'K']
    import pandas as pd
    df=pd.DataFrame(columns=["Sequence","Position"])
    #print(listk)
    for i in listk:
        if i<up:
            #print(seq[0:i+up+1],i)
            raw={}
            tem=""
            for j in range(window-len(seq[0:i+up+1])):
                tem=tem+'X'
            newseq=tem+seq[0:i+up+1]
            raw["Sequence"]=newseq
            raw["Position"]=i+1
            df=df.append(raw,ignore_index=True)
            #print(newseq,len(newseq),i)

        else:
            raw={}
            newseq=seq[i-up:i+up+1]
            #print(newseq,len(newseq),i)
            raw["Sequence"]=newseq
            raw["Position"]=i+1
            df=df.append(raw,ignore_index=True)

    x=featureset(df)
    train=pd.read_csv('all_samples_optimal_features.csv')
    label=train.pop("Class")
    from sklearn.svm import SVC
    classifier =SVC(kernel='rbf', C=4,gamma=1,probability=True)
    model=classifier.fit(train, label)
    pred=model.predict(x)
    df['Class']=pred

    return df

app = Flask(__name__)

@app.route("/")
def home():

    return render_template('index.html')
@app.route('/about')
def about_me():
    return render_template('about.html')

@app.route('/download')
def download_file():
    file='raw_data.pdf'
    return send_file(file,as_attachment=True)

@app.route('/help')
def help():
    return render_template('help.html')

@app.route('/predict',methods=['POST'])
def predict():
    if request.method=="POST":

        seq=request.form["protein"]
        listk=[i for i, letter in enumerate(seq) if letter == 'K']
        if len(seq)>20 and len(listk)>0:
            #if seq[len(seq)-1]=='K':
            #seq=seq+"A"
            seq=seq+"XXXXXXXXXXXXXXXXXXXXXXXXX"
            result=prediction(seq,21)
            result['Class'].replace({0: "Non-Formylated Lysine", 1: "Formylated Lysine"}, inplace=True)

            return render_template('show.html',n=result)

        else:
            return "The length of the given protein may be less than 21
 You have entered blank space
   The protein does not contain any Lysine residue
 Please try again with another larger protein"

if __name__ == "__main__":
    app.run(debug=True)


"""
The training and testing loop.
"""

import logging
import multiprocessing as mp
import os

import tensorflow as tf
import wandb

import models.registry as models_registry
from plato.config import Config
from plato.trainers import base

class Trainer(base.Trainer):
    """A basic federated learning trainer for TensorFlow, used by both
    the client and the server.
    """
    def __init__(self, client_id=0, model=None):
        """Initializing the trainer with the provided model.

        Arguments:
        client_id: The ID of the client using this trainer (optional).
        model: The model to train.
        """
        super().__init__(client_id)

        if model is None:
            self.model = models_registry.get()

    def zeros(self, shape):
        """Returns a TensorFlow zero tensor with the given shape."""
        # This should only be called from a server
        assert self.client_id == 0
        return tf.zeros(shape)

    def save_model(self, filename=None):
        """Saving the model to a file."""
        model_name = Config().trainer.model_name
        model_dir = Config().params['model_dir']

        if not os.path.exists(model_dir):
            os.makedirs(model_dir)

        if filename is not None:
            model_path = f'{model_dir}{filename}'
        else:
            model_path = f'{model_dir}{model_name}.ckpt'

        self.model.save_weights(model_path)

        if self.client_id == 0:
            logging.info("[Server #%d] Model saved to %s.", os.getpid(),
                         model_path)
        else:
            logging.info("[Client #%d] Model saved to %s.", self.client_id,
                         model_path)

    def load_model(self, filename=None):
        """Loading pre-trained model weights from a file."""
        model_name = Config().trainer.model_name
        model_dir = Config().params['model_dir']

        if filename is not None:
            model_path = f'{model_dir}{filename}'
        else:
            model_path = f'{model_dir}{model_name}.ckpt'

        if self.client_id == 0:
            logging.info("[Server #%d] Loading a model from %s.", os.getpid(),
                         model_path)
        else:
            logging.info("[Client #%d] Loading a model from %s.",
                         self.client_id, model_path)

        self.model.load_weights(model_path)

    def train_process(self, config, trainset, sampler, cut_layer=None):
        """The main training loop in a federated learning workload, run in
          a separate process with a new CUDA context, so that CUDA memory
          can be released after the training completes.

        Arguments:
        config: a dictionary of configuration parameters.
        trainset: The training dataset.
        sampler: the sampler that extracts a partition for this client.
        cut_layer (optional): The layer which training should start from.
        """
        if hasattr(Config().trainer, 'use_wandb'):
            run = wandb.init(project="plato",
                             group=str(config['run_id']),
                             reinit=True)

        custom_train = getattr(self, "train_model", None)

        if callable(custom_train):
            self.train_model(config, trainset, sampler.get(), cut_layer)
        else:
            log_interval = 10
            batch_size = config['batch_size']

            logging.info("[Client #%d] Loading the dataset.", self.client_id)
            _train_loader = getattr(self, "train_loader", None)

            if callable(_train_loader):
                train_loader = self.train_loader(batch_size, trainset,
                                                 sampler.get(), cut_layer)
            else:
                train_loader = torch.utils.data.DataLoader(
                    dataset=trainset,
                    shuffle=False,
                    batch_size=batch_size,
                    sampler=sampler.get())

            iterations_per_epoch = np.ceil(len(trainset) /
                                           batch_size).astype(int)
            epochs = config['epochs']

            # Sending the model to the device used for training
            self.model.to(self.device)
            self.model.train()

            # Initializing the loss criterion
            _loss_criterion = getattr(self, "loss_criterion", None)
            if callable(_loss_criterion):
                loss_criterion = _loss_criterion(self.model)
            else:
                loss_criterion = nn.CrossEntropyLoss()

            # Initializing the optimizer
            get_optimizer = getattr(self, "get_optimizer",
                                    optimizers.get_optimizer)
            optimizer = get_optimizer(self.model)

            # Initializing the learning rate schedule, if necessary
            if hasattr(Config().trainer, 'lr_schedule'):
                lr_schedule = optimizers.get_lr_schedule(
                    optimizer, iterations_per_epoch, train_loader)
            else:
                lr_schedule = None

            for epoch in range(1, epochs + 1):
                for batch_id, (examples, labels) in enumerate(train_loader):
                    examples, labels = examples.to(self.device), labels.to(
                        self.device)
                    optimizer.zero_grad()

                    if cut_layer is None:
                        outputs = self.model(examples)
                    else:
                        outputs = self.model.forward_from(examples, cut_layer)

                    loss = loss_criterion(outputs, labels)

                    loss.backward()

                    optimizer.step()

                    if lr_schedule is not None:
                        lr_schedule.step()

                    if batch_id % log_interval == 0:
                        if self.client_id == 0:
                            logging.info(
                                "[Server #{}] Epoch: [{}/{}][{}/{}]\tLoss: {:.6f}"
                                .format(os.getpid(), epoch, epochs, batch_id,
                                        len(train_loader), loss.data.item()))
                        else:
                            if hasattr(Config().trainer, 'use_wandb'):
                                wandb.log({"batch loss": loss.data.item()})

                            logging.info(
                                "[Client #{}] Epoch: [{}/{}][{}/{}]\tLoss: {:.6f}"
                                .format(self.client_id, epoch, epochs,
                                        batch_id, len(train_loader),
                                        loss.data.item()))
                if hasattr(optimizer, "params_state_update"):
                    optimizer.params_state_update()

        self.model.cpu()

        model_type = Config().trainer.model_name
        filename = f"{model_type}_{self.client_id}_{config['run_id']}.pth"
        self.save_model(filename)

        if hasattr(Config().trainer, 'use_wandb'):
            run.finish()

    def train(self, trainset, sampler, cut_layer=None):
        """The main training loop in a federated learning workload.

        Arguments:
        trainset: The training dataset.
        """
        self.start_training()

        if mp.get_start_method(allow_none=True) != 'spawn':
            mp.set_start_method('spawn', force=True)

        config = Config().trainer._asdict()
        config['run_id'] = Config().params['run_id']

        proc = mp.Process(target=Trainer.train_process,
                          args=(
                              self,
                              config,
                              trainset,
                              sampler,
                              cut_layer,
                          ))
        proc.start()
        proc.join()

        model_name = Config().trainer.model_name
        filename = f"{model_name}_{self.client_id}_{Config().params['run_id']}.ckpt"
        self.load_model(filename)

        self.pause_training()

    def test(self, testset):
        """Testing the model using the provided test dataset.

        Arguments:
        testset: The test dataset.
        """
        self.start_training()

        if mp.get_start_method(allow_none=True) != 'spawn':
            mp.set_start_method('spawn', force=True)

        config = Config().trainer._asdict()
        config['run_id'] = Config().params['run_id']

        proc = mp.Process(target=Trainer.test_process,
                          args=(
                              self,
                              config,
                              testset,
                          ))
        proc.start()
        proc.join()

        model_name = Config().trainer.model_name
        filename = f"{model_name}_{self.client_id}_{Config().params['run_id']}.acc"
        accuracy = Trainer.load_accuracy(filename)

        self.pause_training()
        return accuracy

    """A custom trainer with custom training and testing loops. """

    def train_model(self, config, trainset, sampler, cut_layer=None):  # pylint: disable=unused-argument
        """A custom training loop. """
        criterion = nn.CrossEntropyLoss()
        optimizer = optim.Adam(self.model.parameters(), lr=0.02)
        train_loader = DataLoader(dataset=trainset,
                                  batch_size=config['batch_size'],
                                  sampler=sampler)

        # Training the model using Catalyst's SupervisedRunner
        runner = dl.SupervisedRunner()

        runner.train(model=self.model,
                     criterion=criterion,
                     optimizer=optimizer,
                     loaders={"train": train_loader},
                     num_epochs=1,
                     logdir="./logs",
                     verbose=True)

    def test_model(self, config, testset):  # pylint: disable=unused-argument
        """A custom testing loop. """
        test_loader = torch.utils.data.DataLoader(
            testset, batch_size=config['batch_size'], shuffle=False)

        # Using Catalyst's SupervisedRunner and AccuracyCallback to compute accuracies
        runner = dl.SupervisedRunner()
        runner.train(model=self.model,
                     num_epochs=1,
                     loaders={"valid": test_loader},
                     logdir="./logs",
                     verbose=True,
                     callbacks=[
                         dl.AccuracyCallback(input_key="logits",
                                             target_key="targets",
                                             num_classes=10)
                     ])

        # Retrieving the top-1 accuracy from SupervisedRunner
        accuracy = runner.epoch_metrics["valid"]["accuracy"]
        return accuracy


import networkx as nx
import itertools
from math import log2
from operator import itemgetter
import random
import numpy as np

from game.detective import Detective
import game.constants as const

class ExampleAIImplementationDetective(Detective):

    # static variables
    futureNodes = []  # List of lists of future nodes for every detective
    futureTransports = []  # List of lists of used transports for these nodes
    options = []  # List of lists of options for every detective
    living = []  # Gives first alive detective, needed in decide to make all choices when that detective plays

    def __init__(self, *args, **kwargs):
        self.trn = 1
        super().__init__(*args, **kwargs)

    def reset(self):
        super().reset()
        self.trn = 1
        self.emptyFutureLists()
        if self.options:

            for i in range(len(self.game.detectives)):
                del self.options[i][:]
        del self.living[:]
        self.living.extend([1 for _ in range(len(self.game.detectives))])

    def decide(self):
        # self.testMetroStartDists(3)

        # Determine turn
        self.trn = len(self.game.misterx.history)
        self.print_(f"Current Turn: {self.trn}")

        # Initialize static variables correctly
        if len(self.futureNodes) != len(self.game.detectives):
            self.futureNodes.extend([[] for _ in range(len(self.game.detectives))])
            self.futureTransports.extend([[] for _ in range(len(self.game.detectives))])
            self.living.extend([1 for _ in range(len(self.game.detectives))])

        disperseTurns = [1] # makes decission for turn 1 and 2 at the same time
        closeinTurns = [3, 4, 8, 13, 18, 24, 25]
        encircleTurns = [5, 6, 7, 9, 10, 11, 12, 14, 15, 16, 17, 19, 20, 21, 22, 23]
        randomTurns = []

        # Decide everything in first detective decide call of this turn
        if 1 in self.living and self.id == self.living.index(1):
            self.options = []
            # Generate all options for this turn

            for i, detective in enumerate(self.game.detectives):
                ops = self.game.board.getOptions(detective)
                if ops == []:
                   ops = [(None, None)]
                self.options.append(ops)
                self.print_(f"Options for detective {detective.id}: {self.options[i]}")

            optionsleft = False
            for i, det in enumerate(self.game.detectives):
                if self.options[i] != [(None, None)]:
                    optionsleft = True

            if not optionsleft:
                for i, det in enumerate(self.game.detectives):
                    det.defeated = True
                return None, None

            if self.trn in disperseTurns:
                self.disperse()
            elif self.trn in closeinTurns:
                self.closein()
            elif self.trn in encircleTurns:
                self.encircle(decisiondepth=2)
            elif self.trn in randomTurns:
                self.random()

        self.print_(f"getting decision for id::{self.id}\nFuture:{self.futureNodes, self.futureTransports}")
        try:
            decision = (self.futureNodes[self.id][0], self.futureTransports[self.id][0])
        except Exception as e:
            # print(vars(self))
            # print(vars(ExampleAIImplementationDetective))
            # raise e
            print("Unexpected empty list, assuming detective is out of moves")
            return None, None
        del self.futureTransports[self.id][0]
        del self.futureNodes[self.id][0]
        if decision == (None, None):
            self.living[self.id] = 0

        self.print_(f"Going to play {decision[1]} from {self.position} to {decision[0]}")

        # input("Press Enter to continue...")
        return decision[0], decision[1]

    def disperse(self):

        def getMetroDistances():
            """
            Compose list of distance of a player to all metros that are within a distance of 3 turns
            Returns: list of tuples with (metropos, shortest path length to metro)
            """
            possibleMetros = [] # list containing lists of tuples, every list corresponds to one detective
            for detective in self.game.detectives:
                metrodist = []
                for metro in const.METRO_STATIONS:
                    pl = nx.shortest_path_length(self.game.board.graph, metro, detective.position)
                    if pl <= 3 and pl > 0:
                        metrodist.append((metro, pl))
                possibleMetros.append(metrodist)
            self.print_(f"Metro's for the detectives: {possibleMetros}")
            return possibleMetros

        def assignMetro():
            """
            Assign a specific metro station to every detective, based on distance. Distance 2 is preferred, followed by distance 1, then 3.
            Returns list of metrostations, the position inside defines to which detective that metro is assigned.
            """
            possibleMetros = getMetroDistances()
            targetMetro = []
            for possibilities in possibleMetros:
                sizeTwo = [metroTup[0] for metroTup in possibilities if metroTup[1] == 2 and metroTup[0] not in targetMetro]
                if sizeTwo:
                    targetMetro.append(random.choice(sizeTwo))
                else:
                    sizeOne = [metroTup[0] for metroTup in possibilities if metroTup[1] == 1 and metroTup[0] not in targetMetro]
                    if sizeOne:
                        targetMetro.append(random.choice(sizeOne))
                    else:
                        sizeThree = [metroTup[0] for metroTup in possibilities if metroTup[1] == 3 and metroTup[0] not in targetMetro]
                        if sizeThree:
                            targetMetro.append(random.choice(sizeThree))
                        else:
                            # Assign random unassigned metro
                            remainingMetros = [metro for metro in const.METRO_STATIONS if metro not in targetMetro]
                            targetMetro.append(random.choice(remainingMetros))
            # print(f"Target metros for detectives: {targetMetro}")
            return targetMetro

        # print("---Disperse algo---")

        targetMetro = assignMetro()

        assert (len(targetMetro) == len(self.game.detectives)), "MetroList and detectiveList not of same length!"

        for i, det in enumerate(self.game.detectives):
            path = nx.shortest_path(self.game.board.graph, det.position, targetMetro[i])
            assert (len(path) != 0), "Path lenght is 0!"
            self.print_(f"Shortest path calculation for detective {i}, path length : {len(path)}")
            transp = []
            taken = []
            for turnToEval in range(1,min([3,len(path)])):
                for j in range(0,i):
                        taken.append(self.futureNodes[j][turnToEval-1])
                        if len(self.futureNodes[j]) > turnToEval:
                            taken.append(self.futureNodes[j][turnToEval]) # prevent next detective from blocking earlier detective by being on needed position before moving

                if path[turnToEval] in taken:
                    self.print_(f"Collision detected for detective {i}, problematic node: {path[turnToEval]}.")
                    #collision: find node with shortest path
                    neighbours = [node[0] for node in self.game.board.getOptions(det, customStartPosition=path[turnToEval-1]) if node[0] not in taken]
                    lengths = [nx.shortest_path_length(self.game.board.graph, pos, targetMetro[i]) for pos in neighbours]
                    index, _ = min(enumerate(lengths), key=itemgetter(1))
                    newpath = nx.shortest_path(self.game.board.graph, neighbours[index], targetMetro[i])
                    path = path[0:turnToEval] + newpath
            transportPos = [transport[1] for transport in self.options[i] if transport[0] == path[1]]
            transp.append(transportPos[0])

            # detective one turn away from a metro station
            if len(path) == 2:
                neighbours = self.getFutureOptions(det, 1, path[1])
                transportToUse = "taxi"
                # print(backforth)
                # print(f"viable taxi positions: {[viable for viable in backforth if viable[1] is transportToUse]} ")
                path.append(random.choice([viable for viable in neighbours if viable[1] is transportToUse])[0])
                transp.append(transportToUse)
                path.append(path[0])
                transp.append(transportToUse)

            if len(path) >3 :
                # only look 2 moves ahead and remove other decisions
                path = path[0:3]

            #add missing transport
            if len(transp) < len(path)-1:
                i0 = len(transp)
                for j in range (i0,len(path)-1):
                    neighbours = self.getFutureOptions(det, j, path[j])
                    postrans = [transport[1] for transport in neighbours if transport[0] == path[j+1]]
                    transp.append(postrans[0])

            if len(transp) > 2:
                transp = transp[:2]

            # print(f"Future moves for detective {i}:  {path}")
            self.futureNodes[i] = path[1:]
            self.futureTransports[i] = transp
            # print(f"Shortened: {self.futureNodes[i]}")
        return 0

    def closein(self):
        self.emptyFutureLists()
        for i in range(len(self.game.detectives)):
            #split list of tuples (postion, transport) into two separate lists [positions] and [transport]
            neighboursPos, neighboursTrans = map(list,zip(*self.getAvailableOptions(i)))

            lengths = [nx.shortest_path_length(self.game.board.graph, pos, self.game.misterx.lastKnownPosition) for pos in neighboursPos]
            if lengths:
                index, _ = min(enumerate(lengths), key=itemgetter(1))
                self.futureNodes[i].append(neighboursPos[index])
                self.futureTransports[i].append(neighboursTrans[index])
            # else:
            #     self.futureNodes[i].append([None])
            #     self.futureTransports[i].append([None])

    def encircle(self, decisiondepth = 1):
        def narrowXPosition():
            """
            Narrow all mrX positions down to a subset of most likely positions
            Returns: list of tuples (position, probability)
            """
            _, dictX = self.game.board.possibleMisterXPositions(returnProbabilities=True)
            possibleX = list(dictX.items())
            # print(f"Possible positions for mr X: {possibleX}")
            averageProb = sum(map(lambda x: x[1], possibleX)) / len(possibleX)
            probableX = [pos for pos in possibleX if pos[1] >= averageProb]
            # print(f"Average probability: {averageProb}\nNarrowed down positions for mr X: {probableX}")
            self.print_(f"Original amount of X positions: {len(possibleX)} now narrowed down to {len(probableX)}")
            return probableX

        def calcEntropy(tupleList):
            """
            Calculate the entropy of a given list of tuples (..., probability)
            Returns: a double
            """
            _, probabilities = map(list, zip(*tupleList))
            entropy = 0.0
            for p in probabilities:
                entropy -= p * log2(p)
            # condpr(f"Entropy: {entropy}")
            return entropy

        def multipleExpand(newDetectivePositions, depth = decisiondepth):
            """
            Expands a given list of mrXpositions with it's neighbours
            Returns: list of tuples (position, probability)
            """
            mrx = self.game.misterx
            # Look up the most recent reveal and slice the history accordingly
            sliceStart = max([i for i in const.MRX_OPEN_TURNS if i <= self.trn])
            moves = [hist[1] for hist in mrx.history[sliceStart:]]
            # print(f"Moves: {moves}")
            for _ in range(0, depth):
                moves.append('black')
            prohibited = self.game.board.getprohibited(sliceStart - len(mrx.doubleMoves))  # Account for double moves disrupting the indices
            prohibited.append(newDetectivePositions)
            # print(f"{prohibited}")
            _, dictX = self.game.board.possiblePositions(
                            mrx.lastKnownPosition,
                            moves=moves,
                            occupied=prohibited,
                            refuseCurrent=True,
                            returnProbabilities=True,
                        )
            possibleX = list(dictX.items())

            if possibleX == []:
                _, dictX = self.game.board.possibleMisterXPositions(returnProbabilities=True)
                possibleX = list(dictX.items())
                if not possibleX:
                    print("Da last buggggg")
                    possibleX = [(self.game.misterx.lastKnownPosition, 0.01)]
            return possibleX

        def optionGenerator(prevOptions = [], skip = []):
            """
            Generates a list of options for each detective
            """
            if prevOptions == [] and skip == []:
                fullOptions = []  # create a list containing all lists of options per detecive

                for i, det in enumerate(self.game.detectives):
                    if not det.defeated:
                        ops = self.game.board.getOptions(det)
                        if ops:
                            fullOptions.append(ops)
                        else:
                            skip.append(i)
                return fullOptions, skip

            if prevOptions:
                oldOptions = prevOptions
                if len(oldOptions) != len(self.game.detectives):
                    print("Breakpoint")
                fullOptions = []
                j = 0
                for i, det in enumerate(self.game.detectives):
                    if (not det.defeated) and (i not in skip):
                        tempOptions = []
                        for nde, _ in oldOptions[j]:
                            ops = self.game.board.getOptions(det, customStartPosition = nde)
                            if ops:
                                tempOptions.extend(ops)
                        if tempOptions:
                            fullOptions.append(tempOptions)
                        else:
                            if oldOptions[j]:
                                fullOptions.append(oldOptions[j])
                            else:
                                assert(False), "Error in optionGenerator"
                        j += 1
                return fullOptions, skip
            assert(False), "Error in optionGenerator"

        def solutionCompute(currentDepth, fullOptions, skip = []):
            """
            Assign a score for every possibility and filter based on that score (entropy + transport)
            Returns list of sets (possibility, score), the average entropy and the list to skip
            """
            fullOptions, skip = optionGenerator(fullOptions, skip)
            crossproduct = list(itertools.product(*fullOptions))  # Giant crossproduct of all possible options
            print(f"Added fan-out of {len(crossproduct)} on level {currentDepth}")
            summedEnt = 0
            solution = []
            scoreList = []
            for i, scenario in enumerate(crossproduct):
                detectivePos = [node[0] for node in scenario]
                expanded = multipleExpand(detectivePos, depth=currentDepth)
                # entropy for score
                ent = calcEntropy(expanded)
                # transport for score
                score = ent
                # OPTION: comment the 3 following lines to remove transport scoring
                for tup in scenario:
                    if tup[1] != "taxi":
                        score += 0.25
                summedEnt += ent
                scoreList.append(score)
                solution.append((crossproduct[i],score))
            averageEnt = summedEnt/len(crossproduct)
            averageScore = sum(scoreList)/len(crossproduct)
            self.print_(f"Average entropy: {averageEnt}, Average score: {averageScore}")

            npScores = np.array(scoreList)

            # k is the amount options chosen for next level
            # OPTION: change value of k to narrow/broaden branching
            k = (int) (60/len(self.game.detectives))

            kk = (int) (min(k,len(npScores)-1))
            indexes = np.argpartition(npScores, kk)[:kk]

            scenarios = []
            for ind in indexes:
                scenarios.append(solution[ind])

            print(f"Filter: reduced fanout of level {currentDepth} from {len(solution)} to {len(scenarios)}")

            #Change scenarios to options
            filteredSol = []
            solutions = [scenario[0] for scenario in scenarios]
            if not solutions:
                print(f"Second bug? Printing some info: \nscenarios: {scenarios}\n k: {k}\n scoreList: {scoreList}\n length of crossproduct: {len(crossproduct)}\n fulloptions: {fullOptions}\n Currentdepth: {currentDepth}")
            for i in range(len(solutions[0])):
                allsols = [scenario[i] for scenario in solutions]
                uniquesols = list(set(allsols))
                filteredSol.append(uniquesols)
            return filteredSol, scenarios, skip

        assert(decisiondepth != 0), "Decisiondepth must be greater than zero"

        #sol is a list of tuples (scenario, entropy)
        sol, scenarios, skp = solutionCompute(1,[],[])

        for i in range(1, decisiondepth):
            sol, scenarios, skp = solutionCompute(i+1, sol, skp)

        if len(scenarios) != 1:
            best = min(scenarios, key = lambda t: t[1])[0]
        else:
            print(scenarios)
            best = scenarios[0][0]

        decissions = []
        j=0
        for i, det in enumerate(self.game.detectives):
            if not det.defeated and i not in skp:
                neighboursPos, neighboursTrans = map(list,zip(*self.getAvailableOptions(i)))
                if neighboursPos and neighboursTrans:
                    lengths = [nx.shortest_path_length(self.game.board.graph, pos, best[j][0]) for pos in neighboursPos]
                    if lengths:
                        index, _ = min(enumerate(lengths), key=itemgetter(1))
                        decissions.append((neighboursPos[index],neighboursTrans[index]))
                else:
                    decissions.append((None,None))
                j+=1
            else:
                decissions.append((None,None))
        for i, decission in enumerate(decissions):
            self.futureNodes[i].append(decission[0])
            self.futureTransports[i].append(decission[1])

    def random(self):

        self.emptyFutureLists()
        self.print_("---Broaden algo---")

        for i, det in enumerate(self.game.detectives):
            decision = self.randomMove(det)
            self.futureNodes[i].append(decision[0])
            self.futureTransports[i].append(decision[1])

    def randomMove(self, det):

        options = self.getAvailableOptions(det)
        decision = random.choice(options)

        return decision[0], decision[1]

    def getFutureOptions(self, detective, turnsAhead, startPosition):
        """
        Returns a list of tuples that will be available in the future, taking into accounts the future moves
        Arguments: a detective, amount of turns into the future, startposition
        Returns: list of tuples (position, transport)
        """
        cards = detective.cards
        index = detective.id

        for i in range(min(len(self.futureTransports[index]), turnsAhead)):
            cards[self.futureTransports[i]] = min(0, cards[self.futureTransports[i]]-1)

        taken = []
        for nodeList in self.futureNodes:
            if turnsAhead < len(nodeList)-1 and i < index:
                taken.append(nodeList[turnsAhead+1]) #Those positions are where an earlier detective plans to go, so it may not be occupied.
            if turnsAhead < len(nodeList):
                taken.append(nodeList[turnsAhead]) #Those positions will still be occupied by other detectives

        transportOptions = self.game.board.getSimulatedOptions(cards, startPosition, taken)
        return transportOptions

    def emptyFutureLists(self):
        for i in range(len(self.game.detectives)):
            del self.futureNodes[i][:]
            del self.futureTransports[i][:]

    def getAvailableOptions(self, detOrId):
        """
        Returns a list of tuples that are available after taking the future decissions, made for earlier detectives, into account
        Arguments: a detective, or a detective id
        Returns: list of tuples (position, transport)
        """

        assert(isinstance(detOrId, int) or (detOrId, Detective))
        if isinstance(detOrId, int):
            i = detOrId
        elif isinstance(detOrId, Detective):
            i= detOrId.id

        # Generate list of unavailable positions for our detective
        taken = []
        for j in range(i):
            if self.futureNodes[j]:
                taken.append(self.futureNodes[j][0]) #Those positions will still be occupied by other detectives
            if len(self.futureNodes[j]) > 1:
                taken.append(self.futureNodes[j][1]) #Those positions are where an earlier detective plans to go, so it may not be occupied.

        options = [node for node in self.options[i] if node[0] not in taken]
        # Return None if options is empty
        if not options:
            return [(None, None)]
        return options

##########_TESTS_##########

    def testMetroStartDists(self, dist):
        metrodists = []
        # startPos = possible position
        for startPos in const.START_POSITIONS['detectives']:
            metrodist = []
            for metro in const.METRO_STATIONS:
                if nx.shortest_path_length(self.game.board.graph, metro, startPos) <= dist:
                    metrodist.append([metro, nx.shortest_path_length(self.game.board.graph, metro, startPos)])
            if(len(metrodist) == 0):

                self.print_(f"PROBLEM: No less than {dist} metro for node {startPos}")

            metrodists.append(metrodist)
        self.print_(metrodists)

###########################


import pandas as pd
import jieba
import jieba.analyse
import numpy as np
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import matplotlib
from gensim import corpora, models, similarities
import gensim
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.model_selection import train_test_split
"""
第一步：进行分词，画词云图，并将处理好的数据放回，以备后期建模使用
"""

# 先分词, 去掉停用词
def handel_cut(data, stopwords):
    content = data.content.values.tolist()
    content_S = []
    for line in content:
        current_segment = jieba.lcut(line)
        if len(current_segment) > 1 and current_segment !='\r\n':
            content_S.append(current_segment)
    df_content = pd.DataFrame({'content_S': content_S})
    contents = df_content.content_S.values.tolist()
    stopwords = stopwords.stopword.values.tolist()
    contents_clean = []
    all_words = []
    for line in contents:
        line_clean = []
        for word in line:
            if word in stopwords:
                continue
            line_clean.append(word)
            all_words.append(str(word))
        contents_clean.append(line_clean)
    return contents_clean,all_words

def plot(df_all_words):
    words_count = df_all_words.groupby(by=['all_words'])['all_words'].agg({"count":np.size})
    words_count = words_count.reset_index().sort_values(by=["count"],ascending=False)
    matplotlib.rcParams['figure.figsize'] = (10.0, 5.0)
    wordcloud = WordCloud(font_path="./data/simhei.ttf",background_color="white",max_font_size=80)
    word_frequence = {x[0]:x[1] for x in words_count.head(50).values}
    wordcloud=wordcloud.fit_words(word_frequence)
    plt.imshow(wordcloud)
    plt.axis('off')
    plt.show()

data = pd.read_excel('F:\\content.xlsx')
data = data['content']
data.dropna()
stopwords = pd.read_csv("F:\\stopwords.txt", index_col=False, sep="\t", quoting=3, names=['stopword'], encoding='utf-8')
contents_clean,all_words = handel_cut(data, stopwords)
df_content = pd.DataFrame({'contents_clean':contents_clean})
df_content['content'] = data['content']
df_content.to_excel('F:\\gujia_content.xlsx')
df_all_words=pd.DataFrame({'all_words':all_words})

"""
第二步：用lda和tfidf建模分类，并进行评分选择最优的一个
"""

# 进行数据处理
def data_handel(x):
    words = []
    for line_index in range(len(x)):
        try:
            #x_train[line_index][word_index] = str(x_train[line_index][word_index])
            words.append(' '.join(x[line_index]))
        except:
            print (line_index,x[line_index])
    return words

# 从分好词的数据中拿出一部分，手动分类并保存，用于模型训练
new_data = pd.read_excel('F:\\gujia_content.xlsx')
clean_content = new_data['category', 'contents_clean']
df_train=pd.DataFrame({'contents_clean': new_data['contents_clean'],'label': new_data['category']})
label_mapping = {"质量": 1, "气味": 2, "快递": 3, "服务": 4, "安装":5}
df_train['label'] = df_train['label'].map(label_mapping)
x_train, x_test, y_train, y_test = train_test_split(df_train['contents_clean'].values, df_train['label'].values,random_state=1)
words = data_handel(x=x_train)
test_words = data_handel(x=x_test)
#做映射，相当于词袋
dictionary = corpora.Dictionary(clean_content)
corpus = [dictionary.doc2bow(sentence) for sentence in clean_content]

# 使用lda模型
lda = gensim.models.ldamodel.LdaModel(corpus=corpus, id2word=dictionary, num_topics=10)
# 词向量转换
vec = CountVectorizer(analyzer='word', max_features=50, lowercase=False)
vec.fit(words)

# 调用贝叶斯算法进行模型的分类
classifier = MultinomialNB()
classifier.fit(vec.transform(words), y_train)
score = classifier.score(vec.transform(test_words), y_test)
print(score)

# 使用Tfidf模型训练
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer = TfidfVectorizer(analyzer='word', max_features=50,  lowercase = False)
vectorizer.fit(words)
classifier = MultinomialNB()
classifier.fit(vectorizer.transform(words), y_train)
sorce = classifier.score(vectorizer.transform(test_words), y_test)
print(score)

# 导入要分类的数据
prodict_data = pd.read_excel('F:\\gujia_content.xlsx')
prodict_data = prodict_data['content']
contents_clean1,all_words1 = handel_cut(prodict_data,stopwords)
contents_clean1 = data_handel(x=contents_clean1)
vectorizer = TfidfVectorizer(analyzer='word', max_features=50,  lowercase = False)
vectorizer.fit(contents_clean1)
classifier = MultinomialNB()
class_f = classifier.predict(contents_clean1)
class_f = pd.DataFrame({'class_f':class_f})
prodict_data['class_f'] = class_f
prodict_data.to_excel('F:\\prodict_data.xlsx')


# -*- coding:utf-8 -*-
import xlsxwriter,xlrd

fname = 'test.xlsx'
data = xlrd.open_workbook(fname)
print(data.sheet_names())

table = data.sheet_by_index(0)
nrows = table.nrows
ncols = table.ncols
print(nrows)
print(ncols)
print(table.row_values(0))
print(table.row_values(1))

workbook = xlsxwriter.Workbook('zm6.xlsx')  #创建一个excel文件
worksheet = workbook.add_worksheet('asd')        #创建一个工作表对象
worksheet.set_column(0,ncols,12)            #设定列的宽度为22像素
worksheet.write(1,1,'asdf')
workbook.close()