import sys
import operator
import socket
import re

from collections import defaultdict, deque
from calvin.runtime.north.reliability_calculator_factory import get_reliability_calculator
from calvin.runtime.north.task_scheduler_factory import get_task_scheduler

from calvin.utilities import calvinconfig
from calvin.utilities.calvinlogger import get_logger

_conf = calvinconfig.get()
_log = get_logger(__name__)

DEFAULT_HISTORY_SIZE = 20
DEFAULT_REPLICATION_HISTORY_SIZE = 5
DEFAULT_REPLICATION_TIME = 2.0
DEFAULT_NODE_RELIABILITY = 0.8
LOST_NODE_TIME = 0.5
MAX_PREFERRED_USAGE = 80

class ResourceManager(object):
    def __init__(self, history_size=DEFAULT_HISTORY_SIZE):
        self.history_size = history_size
        self.usages = defaultdict(lambda: deque(maxlen=self.history_size))
        rc = _conf.get('global', 'reliability_calculator')
        self.reliability_calculator = get_reliability_calculator(rc)
        ts = _conf.get('global', 'task_scheduler')
        self.task_scheduler = get_task_scheduler(ts, self)
        _log.info("Starting resource manager using reliability calculator {} and task scheduler {}".format(rc, ts))
        self.node_uris = {}
        self.node_ids = {}
        self.test_sync = 2
        self._lost_nodes = set()
        self._rep_times = {}

    def register_uri(self, node_id, uri):
        _log.debug("Registering uri: {} - {}".format(node_id, uri))
        if isinstance(uri, list):
            uri = uri[0]

        if uri:
            uri = uri.replace("calvinip://", "").replace("http://", "")
            addr = uri.split(":")[0]
            port = int(uri.split(":")[1])

            is_ip = re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", addr)
            if not is_ip:
                addr = socket.gethostbyname(addr)

            uri = "{}:{}".format(addr, port)
            self.node_uris[node_id] = uri
            self.node_ids[uri] = node_id

    def get_id(self, uri):
        uri = uri.replace("calvinip://", "").replace("http://", "")
        addr = uri.split(":")[0]
        port = int(uri.split(":")[1])

        is_ip = re.match(r"^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$", addr)
        if not is_ip:
            addr = socket.gethostbyname(addr)

        return self.node_ids.get("{}:{}".format(addr, port))

    def register(self, node_id, usage, uri):
        _log.debug("Registering resource usage for node {}: {} with uri {}".format(node_id, usage, uri))
        self.register_uri(node_id, uri)

        if usage:
            self.usages[node_id].append(usage['cpu_percent'])

    def lost_node(self, node_id):
        if node_id in self._lost_nodes:
            return
        self._lost_nodes.add(node_id)
        del self.usages[node_id]
        _log.debug("Registering lost node: {}".format(node_id))

    def _average_usage(self, node_id):
        return sum(self.usages[node_id]) / max(len(self.usages[node_id]), 1)

    #Only used for evaluation
    def get_avg_usages(self):
        usages = {}
        for node_id in self.usages.keys():
            uri = self.node_uris.get(node_id)
            if uri:
                usages[uri] = self._average_usage(node_id)
        return usages

    def least_busy(self):
        """Returns the id of the node with the lowest average CPU usage"""
        min_usage, least_busy = sys.maxint, None
        for node_id in self.usages.keys():
            average = self._average_usage(node_id)
            if average < min_usage:
                min_usage = average
                least_busy = node_id

        return least_busy

    def most_busy(self):
        """Returns the id of the node with the highest average CPU usage"""
        min_usage, most_busy = - sys.maxint, None
        for node_id in self.usages.keys():
            average = self._average_usage(node_id)
            if average > min_usage:
                min_usage = average
                most_busy = node_id

        return most_busy

    def get_reliability(self, node_id, replication_times, failure_info):
        _log.info("Getting reliability for {}".format(node_id))
        uri = self.node_uris.get(node_id)
        if uri:
            replication_time = self.replication_time(replication_times)
            if uri in failure_info:
                failure_info = failure_info[uri]
            else:
                failure_info = []
            rel = self.reliability_calculator.calculate_reliability(failure_info, replication_time)
            _log.info("Returning reliability for {}: {}".format(node_id, rel))
            return rel
        else:
            _log.info("Returning default reliability: {}".format(DEFAULT_NODE_RELIABILITY))
            return DEFAULT_NODE_RELIABILITY

    def replication_time(self, replication_times):
        if not replication_times:
            return DEFAULT_REPLICATION_TIME

        key = "".join([str(t) for t in replication_times])
        if key in self._rep_times:
            return self._rep_times[key]

        value = self.reliability_calculator.replication_time(replication_times) + LOST_NODE_TIME
        self._rep_times[key] = value
        return value

    def get_preferred_nodes(self, node_ids):
        preferred = []
        for node_id in node_ids:
            if self._average_usage(node_id) < MAX_PREFERRED_USAGE:
                preferred.append(node_id)
        return preferred

    def _update_deque(self, new_values, old_values):
        for tup in new_values:
            if tup[0] > old_values[-1][0]:
                old_values.append(tup)

    def sort_nodes_reliability(self, node_ids, replication_times, failure_info):
        return self.task_scheduler.sort(node_ids, replication_times, failure_info)

    def current_reliability(self, current_nodes, replication_times, failure_info):
        current_nodes = list(set(current_nodes))
        _log.debug("Calculating reliability for nodes: {}".format(current_nodes))
        failure = []
        for node_id in current_nodes:
            f = 1 - self.get_reliability(node_id, replication_times, failure_info)
            _log.debug("Failure for {}: {}".format(node_id, f))
            failure.append(f)

        p = 1 - reduce(operator.mul, failure, 1)
        _log.info("Reliability for nodes {} is {}".format(current_nodes, p))
        return p

    def sync_info(self, usages=None):
        if usages:
            self._sync_usages(usages)

        usages = {}
        for (node_id, usage_list) in self.usages.iteritems():
            usages[node_id] = [usage for usage in usage_list]

        return usages

    def _sync_usages(self, usages):
        """
        Sync the usages for each node_id stored on another node.
        usages is a dict of lists but stored as a dict with deques
        """
        _log.debug("\n\nSyncing usages {} with usages {}".format(self.usages, usages))
        for (node_id, usage_list) in usages.iteritems():
            if (not node_id in self.usages.keys() or len(usage_list) > self.usages[node_id]) and not node_id in self._lost_nodes:
                usage_deq = deque(maxlen=self.history_size)
                for u in usage_list:
                    usage_deq.append(u)
                self.usages[node_id] = usage_deq


import os
import matplotlib
import matplotlib.pyplot as plt
from matplotlib import dates
import numpy as np
import numpy.ma as ma
from scipy.io import loadmat
import datetime
import pandas as pd
#import seawater as sw
import gsw

# Some infos:
mab_adcp = 1.58
mooring_depth = 83.0
adcpdir = 'up'

#XLIM = [pd.Timestamp('2010-03-01 15:30:00'), pd.Timestamp('2010-03-01 16:30:00')]
XLIM = [pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00')]
#fig_name = 'S1_outlook_wind.png'
fig_name = 'S1_outlook_meteo.png'

#### ---------- Meteo data --------- ####
meteo_dict = loadmat('SMHI_meteo_55570_201002_201003.mat',squeeze_me=True)
wave_dict = loadmat('SMHI_wave_33004_201002_201003.mat',squeeze_me=True)
wind_dir = { k: meteo_dict[k] for k in ['wind_dir']}
wind_mag = { k: meteo_dict[k] for k in ['wind_mag']}
wave_height = { k: wave_dict[k] for k in ['waveheight_max','waveheight_signif']}
wave_dir = { k: wave_dict[k] for k in ['wavedir_maxenergy']}
wave_period= { k: wave_dict[k] for k in ['waveperiod_mean']}
def matlab2datetime(matlab_datenum):
    day = datetime.datetime.fromordinal(int(matlab_datenum))
    dayfrac = datetime.timedelta(days=matlab_datenum%1) - datetime.timedelta(days = 366)
    return day + dayfrac

# convert Matlab time into list of python datetime objects and put in dictionary
wind_dir['date_time'] = [matlab2datetime(tval) for tval in meteo_dict['wind_matlabtime']]
wind_mag['date_time'] = [matlab2datetime(tval) for tval in meteo_dict['wind_matlabtime']]
wave_height['date_time'] = [matlab2datetime(tval) for tval in wave_dict['wave_matlabtime']]
wave_dir['date_time'] = [matlab2datetime(tval) for tval in wave_dict['wave_matlabtime']]
wave_period['date_time'] = [matlab2datetime(tval) for tval in wave_dict['wave_matlabtime']]

# Wind data from R/V Alkor
#Alkor_dict = loadmat('/media/cyrf0006/Fred32/IOW/IOWdata/AL351_DDL_wind.mat',squeeze_me=True, struct_as_record=False)
Alkor_dict = loadmat('./data/AL351_DDL_wind.mat',squeeze_me=True, struct_as_record=False)
Ualkor =  Alkor_dict['DDL'].wind.speed.data
yearday = Alkor_dict['DDL'].time.data+1
pdTimeAlkor = pd.to_datetime('2010') + pd.to_timedelta(yearday - 1, unit='d')
alkor = pd.DataFrame(Ualkor, index=pdTimeAlkor)
alkor = alkor.resample('30Min').mean()
# --------------------------------------#

#### ---------- IOW SBE Mcats --------- ####
sbe_dict = loadmat('/home/cyrf0006/research/IOW/iow_mooring/AL351_TSC2_S1_sbe_all.mat',squeeze_me=True)
Tmat = sbe_dict['Temperature']
Smat = sbe_dict['Salinity']
Dmat = sbe_dict['Density']
zVecSBE = sbe_dict['SBE_depth']
yearday = sbe_dict['SBE_decday']
pdTimeCats = pd.to_datetime('2010') + pd.to_timedelta(yearday - 1, unit='d')
TT = pd.DataFrame(Tmat, index=pdTimeCats, columns=zVecSBE) # now in m/s
S = pd.DataFrame(Smat, index=pdTimeCats, columns=zVecSBE)
D = pd.DataFrame(Dmat, index=pdTimeCats, columns=zVecSBE)

## NOTE: Use CTD cast from MSS to get profile after storm!!!
# --------------------------------------#

#### ---------- CTD cast --------- ####
# CTD casts before storm (pd is powerful!))
lat = 55
lon = 16
ctd_fname = './data/ctd/0001_01_bav_buo.cnv'
data = pd.read_csv(ctd_fname, header=None, delim_whitespace=True)
Zctd = data[:][7]
Tctd = data[:][1]
Sctd = data[:][5]

Ibtm = np.argmax(Zctd)
Pbin = np.arange(0.5, Zctd[Ibtm], 1)
digitized = np.digitize(Zctd[0:Ibtm], Pbin) #<- this is awesome!
Tbin = np.array([Tctd[0:Ibtm][digitized == i].mean() for i in range(0, len(Pbin))])
Sbin = np.array([Sctd[0:Ibtm][digitized == i].mean() for i in range(0, len(Pbin))])
Z = np.abs(gsw.z_from_p(Pbin,lat))
SA = gsw.SA_from_SP_Baltic(Sbin,lon,lat)
CT = gsw.CT_from_t(SA,Tbin,Pbin)
SIG0 = gsw.sigma0(SA,CT)
N2 = gsw.Nsquared(SA,CT,Pbin,lat)
# ------------------------------------#

#### --------- NIOZ Tsensors --------- ####
T_dict = loadmat('./data/Titp_S1_p4std_10minAve.mat',squeeze_me=True)
T = T_dict['Tbin'].T
zVecT = mooring_depth - np.array(T_dict['habVec'])
pdTimeT = pd.date_range('2010-02-28 11:07:00', '2010-03-04 07:37:00', freq='10min')

# Dataframe
T = pd.DataFrame(T, index=pdTimeT, columns=zVecT)
# --------------------------------------#

#### ---------- MSS cast --------- ####
# CTD casts before storm (pd is powerful!))
lat = 55
lon = 16
MSS_S1_1_dict = loadmat('./data/MSS_DATA/S1_1.mat',squeeze_me=True, struct_as_record=False)
Zmss =  MSS_S1_1_dict['CTD'][2].P
Tmss =  MSS_S1_1_dict['CTD'][2].T
Smss =  MSS_S1_1_dict['CTD'][2].S
timemss1 = pd.Timestamp(MSS_S1_1_dict['STA'][2].date)
digitized = np.digitize(Zmss, Pbin) #<- this is awesome!
TTbin = np.array([Tmss[digitized == i].mean() for i in range(0, len(Pbin))])
SSbin = np.array([Smss[digitized == i].mean() for i in range(0, len(Pbin))])
SA_MSS_01 = gsw.SA_from_SP_Baltic(SSbin,lon,lat)
CT_MSS_01 = gsw.CT_from_t(SA_MSS_01,TTbin,Pbin)
SIG0_MSS_01 = gsw.sigma0(SA_MSS_01,CT_MSS_01)
N2_MSS_01 = gsw.Nsquared(SA_MSS_01,CT_MSS_01,Pbin,lat)
import SW_extras as swe
N2_01 = N2_MSS_01[0]
zN2_01 = N2_MSS_01[1]
N2_period_01 = 60.0/swe.cph(N2_01)

MSS_S1_2_dict = loadmat('./data/MSS_DATA/S1_2.mat',squeeze_me=True, struct_as_record=False)
Zmss =  MSS_S1_2_dict['CTD'][0].P
Tmss =  MSS_S1_2_dict['CTD'][0].T
Smss =  MSS_S1_2_dict['CTD'][0].S
timemss2 = pd.Timestamp(MSS_S1_2_dict['STA'][0].date)
digitized = np.digitize(Zmss, Pbin) #<- this is awesome!
TTbin = np.array([Tmss[digitized == i].mean() for i in range(0, len(Pbin))])
SSbin = np.array([Smss[digitized == i].mean() for i in range(0, len(Pbin))])
SA_MSS_02 = gsw.SA_from_SP_Baltic(SSbin,lon,lat)
CT_MSS_02 = gsw.CT_from_t(SA_MSS_02,TTbin,Pbin)
SIG0_MSS_02 = gsw.sigma0(SA_MSS_02,CT_MSS_02)
N2_MSS_02 = gsw.Nsquared(SA_MSS_02,CT_MSS_02,Pbin,lat)
N2_02 = N2_MSS_02[0]
zN2_02 = N2_MSS_02[1]
N2_period_02 = 60.0/swe.cph(N2_02)

MSS_S1_5_dict = loadmat('./data/MSS_DATA/S1_5.mat',squeeze_me=True, struct_as_record=False)
Zmss =  MSS_S1_5_dict['CTD'][-10].P
Tmss =  MSS_S1_5_dict['CTD'][-10].T
Smss =  MSS_S1_5_dict['CTD'][-10].S
timemss5 = pd.Timestamp(MSS_S1_5_dict['STA'][-10].date)
digitized = np.digitize(Zmss, Pbin) #<- this is awesome!
TTbin = np.array([Tmss[digitized == i].mean() for i in range(0, len(Pbin))])
SSbin = np.array([Smss[digitized == i].mean() for i in range(0, len(Pbin))])
SA_MSS_05 = gsw.SA_from_SP_Baltic(SSbin,lon,lat)
CT_MSS_05 = gsw.CT_from_t(SA_MSS_05,TTbin,Pbin)
SIG0_MSS_05 = gsw.sigma0(SA_MSS_05,CT_MSS_05)
N2_MSS_05 = gsw.Nsquared(SA_MSS_05,CT_MSS_05,Pbin,lat)
N2_05 = N2_MSS_05[0]
zN2_05 = N2_MSS_05[1]
N2_period_05 = 60.0/swe.cph(N2_05)
## # ------------------------------------#

#### --------- plots ---------- ####
days = dates.DayLocator()
hours = dates.HourLocator(interval=6)
dfmt = dates.DateFormatter('%d')
storm2 = pd.Timestamp('2010-03-01 12:00:00') # onset langmuir
zoomx = [pd.Timestamp('2010-03-01 11:00:00'), pd.Timestamp('2010-03-02 1:45:00')]
zoomy = [19,78]

fig = plt.figure(2)

# AX1 - Wind mag.
ax1 = plt.subplot2grid((7, 7), (0, 0), rowspan=1, colspan=6)
df = pd.DataFrame(wind_mag)
df = df.set_index('date_time')
df.plot(ax=ax1, grid='on', legend=False)
alkor.plot(ax=ax1, grid='on', legend=False)
#ax1.set_xlim(pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00 '))
ax1.set_xlim(XLIM)
ax1.set_ylim(0,20)
ax1.tick_params(labelbottom='off')
ax1.xaxis.label.set_visible(False)
ax1.xaxis.set_major_locator(days)
ax1.xaxis.set_major_formatter(dfmt)
ax1.xaxis.set_minor_locator(hours)
ax1.set_ylabel(r'$\overline{U} (m/s)$')
ax1.legend(['SMHI','ship'], bbox_to_anchor=(1., 1.), loc=2)
ax1.text(XLIM[0], 15, '  a', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
#plt.plot([storm2, storm2], [0, 20], '--k')
# add patch
import matplotlib.dates as mdates
start = mdates.date2num(zoomx[0])
end = mdates.date2num(zoomx[1])
rect_x = [start, end, end, start, start]
zoomy = [0, 20]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax1.add_patch(Rgon)

# AX2 - Wave heights
ax2 = plt.subplot2grid((7, 7), (1, 0), rowspan=1, colspan=6)
df = pd.DataFrame(wave_height)
df = df.set_index('date_time')
df.plot(ax=ax2, grid='on')
#XTICKS = axes[0].get_xticks()
#ax2.set_xlim(pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00'))
ax2.set_xlim(XLIM)
ax2.set_ylim(0,7.5)
ax2.tick_params(labelbottom='off')
ax2.xaxis.label.set_visible(False)
ax2.set_ylabel('H(m)')
ax2.xaxis.set_major_locator(days)
ax2.xaxis.set_major_formatter(dfmt)
ax2.xaxis.set_minor_locator(hours)
ax2.legend([r'$\rm H_{max}$', r'$\rm H_{s}$'], bbox_to_anchor=(1., 1.), loc=2)
# bbox_to_anchor is origin; loc=1 is inside; bordaxesoad=0 means directly on the axis
ax2.text(XLIM[0], 6, '  b', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
#plt.plot([storm2, storm2], [0, 7.5], '--k')
# add patch
zoomy = [0, 7.5]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax2.add_patch(Rgon)

# AX3 - Wind/wave direction
ax3 = plt.subplot2grid((7, 7), (2, 0), rowspan=1, colspan=6)
df1 = pd.DataFrame(wind_dir)
df2= pd.DataFrame(wave_dir)
df1 = df1.set_index('date_time')
df2 = df2.set_index('date_time')
df1.plot(ax=ax3, grid='on', label="winddir", legend=True)
df2.plot(secondary_y=False, ax=ax3, grid='on', label="wavedir", legend=True)
#ax3.set_xlim(pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00'))
ax3.set_xlim(XLIM)
ax3.set_ylim(85,365)
ax3.set_ylabel(r'$\rm \theta$')
ax3.xaxis.label.set_visible(False)
ax3.tick_params(labelbottom='off')
#ax3.legend([r'$\rm \theta_{wind}$', r'$\rm \theta_{wave}$'], bbox_to_anchor=(-0.18, 0.8), loc=1, borderaxespad=0)
ax3.legend([r'$\rm wind$', r'$\rm wave$'], bbox_to_anchor=(1., 1.), loc=2)
ax3.xaxis.set_major_locator(days)
ax3.xaxis.set_major_formatter(dfmt)
ax3.xaxis.set_minor_locator(hours)
plt.yticks([90, 180, 270, 360], ['E', 'S', 'W', 'N'], rotation='horizontal')
ax3.text(XLIM[0], 340, '  c', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
#plt.plot([storm2, storm2], [85, 365], '--k')
# add patch
zoomy = [0, 360]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax3.add_patch(Rgon)

# AX4 - Wave period
ax4 = plt.subplot2grid((7, 7), (3, 0), rowspan=1, colspan=6)
df = pd.DataFrame(wave_period)
df = df.set_index('date_time')
df.plot(ax=ax4, grid='on', legend=False)
#ax4.set_xlim(pd.Timestamp('2010-02-28 12:00:00'), pd.Timestamp('2010-03-04 06:00:00'))
ax4.set_xlim(XLIM)
ax4.set_ylim(2,7)
ax4.xaxis.label.set_visible(False)
ax4.tick_params(labelbottom='off')
ax4.set_ylabel(r'$\rm \overline{T}_s (s)$')
ax4.xaxis.set_major_locator(days)
ax4.xaxis.set_major_formatter(dfmt)
ax4.xaxis.set_minor_locator(hours)
ax4.text(XLIM[0], 6, '  d', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
#plt.plot([storm2, storm2], [2, 7], '--k')
# add patch
zoomy = [0, 7]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax4.add_patch(Rgon)

# AX5 - Temperature
ax5 = plt.subplot2grid((7, 7), (4, 0), rowspan=3, colspan=6)
levels = np.linspace(0,10, 21)
levels2 = np.linspace(0,10, 11)
c = plt.contourf(T.index, T.columns, T.T, levels, cmap=plt.cm.RdBu_r)
#plt.plot([storm, storm], [53, 83], '--k')
plt.plot([timemss1, timemss1], [20, 83], '--')
plt.plot([timemss2, timemss2], [20, 83], '--')
plt.plot([timemss5, timemss5], [20, 83], '--')
SBE_time = pd.Timestamp('2010-03-02 00:00:00')
plt.plot([SBE_time, SBE_time], [20, 83], 'r--')
#plt.plot([storm2, storm2], [20, 83], '--k')
cax = plt.axes([0.88,0.08,0.02,0.3])
plt.colorbar(c, cax=cax)
ax5.set_xlim(XLIM[0], XLIM[1])
#ax5.set_ylim(1, 83)
ax5.set_ylim(53, 83)
ax5.tick_params(labelbottom='on')
ax5.tick_params(labelleft='on')
ax5.invert_yaxis()
ax5.xaxis.set_major_locator(days)
ax5.xaxis.set_major_formatter(dfmt)
ax5.xaxis.set_minor_locator(hours)
ax5.set_ylabel('Depth (m)')
ax5.text(XLIM[1], 54, r'  T($^{\circ}$C)', horizontalalignment='left', verticalalignment='center', fontsize=14, color='k')
ax5.text(XLIM[0], 55, '  e', horizontalalignment='left', verticalalignment='center', fontsize=15, color='k')
# add patch
zoomy = [53, 83]
rect_y = [zoomy[0], zoomy[0], zoomy[1], zoomy[1], zoomy[0]]
rect = zip(rect_x, rect_y)
Rgon = plt.Polygon(rect,color='gray', alpha=0.3)
ax5.add_patch(Rgon)

#### --------- Save Figure ------------- ####
fig.set_size_inches(w=8, h=10)
fig.set_dpi(300)
fig.tight_layout()
fig.savefig(fig_name)

#plt.show()


"""
This script shows how you can open and extract data from a fits file in python, you need astropy
which is included in the anaconda distribution. The fits files for lyman alpha luminsoity are on
the box.
"""
from astropy.io import fits
from astropy.table import Table

filename = 'Sobral2018_Integrals_SFRDs_individual.fits' # the file you want to open
hdu_list = fits.open(filename) # open the file, returns an array containing raw data and metadata
evt_data = Table(hdu_list[1].data) # The raw data is contained in the first element
rho_Lya = evt_data.field(3)[0:] # get a list of the all the data from fourth field
for i in rho_Lya:
    print(i)


import numpy as np
import cv2
import imutils

scaling_factorx=0.5
scaling_factory=0.5

cap = cv2.VideoCapture(0)
#cap.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
#cap.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
cap.set(cv2.CAP_PROP_FOURCC, cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'))

print(cap.get(3))
print(cap.get(4))

writer = cv2.VideoWriter('filename.avi', cv2.VideoWriter_fourcc(*"MJPG"), 20, (720, 480))

print(cap.get(3))
print(cap.get(4))

while (True):
    # Capture frame-by-frame
    ret, frame = cap.read()  # ret = 1 if the video is captured; frame is the image

    #frame = cv2.resize(frame, None, fx=scaling_factorx, fy=scaling_factory, interpolation=cv2.INTER_AREA)
    #frame = imutils.resize(frame, width=320)
    # Our operations on the frame come here
    #img = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    writer.write(frame)
    # Display the resulting image
    cv2.imshow('Video Capture', frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):  # press q to quit
        status = cv2.imwrite('img.jpg', frame)
        break

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()


import json
import subprocess
import time

import sys

import base64
import click
import os
import random
from collections import namedtuple
from two1.commands.config import pass_config
from two1.lib.bitcoin.block import CompactBlock
from two1.lib.bitcoin.txn import Transaction
from two1.lib.server import rest_client, message_factory, login
from two1.lib.server.analytics import capture_usage
import two1.commands.config as cmd_config
from two1.commands import status
from two1.commands.status import has_bitcoinkit
from two1.lib.bitcoin.hash import Hash
from two1.lib.server.rest_client import ServerRequestError
from two1.lib.util.exceptions import MiningDisabledError
from two1.lib.util.uxstring import UxString
import two1.lib.bitcoin.utils as utils
import two1.commands.config as app_config

@click.command()
@click.option('--dashboard', default=False, is_flag=True,
              help="Dashboard with mining details")
@click.pass_context
def mine(ctx, dashboard):
    """Mine bitcoin at the command line.

\b
Usage
-----
Invoke this with no arguments to start mining.
$ 21 mine

\b
Then view your new balance.
$ 21 status

\b
View aggregated logs to see your mining progress.
$ 21 log

\b
See a mining dashboard for low-level mining details.
$ 21 mine --dashboard
"""
    config = ctx.obj['config']
    _mine(config, dashboard=dashboard)

@capture_usage
def _mine(config, dashboard=False):
    if has_bitcoinkit():
        if not is_minerd_running():
            start_minerd(config, dashboard)
        elif dashboard:
            show_minertop(dashboard)
        # if minerd is running and we have not specified a dashboard flag
        # do a cpu mine
        else:
            start_cpu_mining(config)
    else:
        config.log(UxString.buy_ad, fg="magenta")
        start_cpu_mining(config)

def is_minerd_running():
    rc = True
    import socket
    try:
        s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        s.connect("/tmp/minerd.sock")
        s.close()
    except Exception:
        rc = False
    return rc

def show_minertop(show_dashboard):
    if show_dashboard:
        click.pause(UxString.mining_show_dashboard_prompt)
        subprocess.call("minertop")
    else:
        click.echo(UxString.mining_show_dashboard_context)

def start_minerd(config, show_dashboard=False):
    # Check if it's already up and running by checking pid file.
    minerd_pid_file = "/run/minerd.pid"
    config.log(UxString.mining_chip_start)
    # Read the PID and check if the process is running
    if os.path.isfile(minerd_pid_file):
        pid = None
        with open(minerd_pid_file, "r") as f:
            pid = int(f.read().rstrip())

        if pid is not None:
            if check_pid(pid):
                # Running, so fire up minertop...
                click.echo(UxString.mining_chip_running)
                show_minertop(show_dashboard)
                return
            else:
                # Stale PID file, so delete it.
                subprocess.call(["sudo", "minerd", "--stop"])

    # Not running, let's start it
    # TODO: make sure config exists in /etc
    # TODO: replace with sys-ctrl command
    minerd_cmd = ["sudo", "minerd", "-u", config.username,
                  app_config.TWO1_POOL_URL]
    try:
        o = subprocess.check_output(minerd_cmd, universal_newlines=True)
    except subprocess.CalledProcessError as e:
        config.log("\nError starting minerd: %r" % e)

    # Now call minertop after it's started
    show_minertop(show_dashboard)

def start_cpu_mining(config):
    """Mine at the CPU.
    >>> from two1.commands.config import Config
    >>> config = Config()
    >>> start_cpu_mining(config)
    """

    client = rest_client.TwentyOneRestClient(cmd_config.TWO1_HOST,
                                             config.machine_auth,
                                             config.username)

    enonce1, enonce2_size, reward = set_payout_address(config, client)

    start_time = time.time()
    config.log(UxString.mining_start.format(config.username, reward))

    work = get_work(config, client)

    found_share = mine_work(work, enonce1=enonce1, enonce2_size=enonce2_size)

    paid_satoshis = save_work(client, found_share, config.username)

    end_time = time.time()
    duration = end_time - start_time

    config.log(
        UxString.mining_success.format(config.username, paid_satoshis, duration),
        fg="magenta")

    click.echo(UxString.mining_status)
    status.status_wallet(config, client)

    click.echo(UxString.mining_finish.format(
        click.style("21 status", bold=True), click.style("21 buy", bold=True)))

def set_payout_address(config, client):
    # set a new address from the HD wallet for payouts
    payout_address = config.wallet.current_address
    auth_resp = client.account_payout_address_post(payout_address)

    user_info = json.loads(auth_resp.text)
    enonce1_base64 = user_info["enonce1"]
    enonce1 = base64.decodebytes(enonce1_base64.encode())
    enonce2_size = user_info["enonce2_size"]
    reward = user_info["reward"]

    return enonce1, enonce2_size, reward

# Copied from: http://stackoverflow.com/questions/568271/how-to-check-if-there-exists-a
# -process-with-a-given-pid
def check_pid(pid):
    if pid < 0:
        return False
    if pid == 0:
        # According to "man 2 kill" PID 0 refers to every process
        # in the process group of the calling process.
        # On certain systems 0 is a valid PID but we have no way
        # to know that in a portable fashion.
        raise ValueError('invalid PID 0')
    try:
        os.kill(pid, 0)
    except ProcessLookupError:
        return False
    except PermissionError:
        # There is a process but we don't have permissions
        return True
    except:
        raise

    return True

def get_work(config, client):
    try:
        work_msg = client.get_work()
    except ServerRequestError as e:
        if e.status_code == 404 or e.status_code == 403:
            click.echo(UxString.mining_limit_reached)
            raise MiningDisabledError(UxString.mining_limit_reached)
        else:
            raise e

    msg_factory = message_factory.SwirlMessageFactory()
    msg = base64.decodebytes(work_msg.content)
    work = msg_factory.read_object(msg)
    return work

Share = namedtuple('Share', ['enonce2', 'nonce', 'otime', 'work_id'])
Work = namedtuple('Work', ['work_id', 'enonce2', 'cb'])

def mine_work(work_msg, enonce1, enonce2_size):
    pool_target = utils.bits_to_target(work_msg.bits_pool)
    for enonce2_num in range(0, 2 ** (enonce2_size * 8)):
        enonce2 = enonce2_num.to_bytes(enonce2_size, byteorder="big")

        cb_txn, _ = Transaction.from_bytes(
            work_msg.coinb1 + enonce1 + enonce2 + work_msg.coinb2)
        cb = CompactBlock(work_msg.height,
                          work_msg.version,
                          Hash(work_msg.prev_block_hash),
                          work_msg.ntime,
                          work_msg.nbits,  # lower difficulty work for testing
                          work_msg.merkle_edge,
                          cb_txn)

        row_counter = 0
        for nonce in range(0xffffffff):

            if nonce % 6e3 == 0:
                click.echo(click.style(u'█', fg='green'), nl=False)
                row_counter += 1
            if row_counter > 40:
                row_counter = 0
                click.echo("")

            cb.block_header.nonce = nonce
            h = cb.block_header.hash.to_int('little')
            if h < pool_target:
                share = Share(
                    enonce2=enonce2,
                    nonce=nonce,
                    work_id=work_msg.work_id,
                    otime=int(time.time()))
                # adds a new line at the end of progress bar
                click.echo("")
                return share

        click.echo("Exhausted enonce1 space. Changing enonce2")

def save_work(client, share, username):
    message_id = random.randint(1, 1e5)
    msg_factory = message_factory.SwirlMessageFactory()
    req_msg = msg_factory.create_submit_share_request(message_id=message_id,
                                                      work_id=share.work_id,
                                                      enonce2=share.enonce2,
                                                      otime=share.otime,
                                                      nonce=share.nonce)

    payment_result = client.send_work(data=req_msg)
    payment_details = json.loads(payment_result.text)
    amount = payment_details["amount"]
    return amount


from django.conf.urls import patterns, include, url
from django.views.generic import TemplateView

urlpatterns = patterns(
    'web02.views',
    # url(r'^home/$', TemplateView.as_view(template_name="web02/home.html")),
    url(r'^home/$', 'home'),
    url(r'^product/$', 'product'),
    url(r'^product/(?P<id>[0-9]+)/$', 'product_detail'),
    url(r'^course/$', 'course'),
    url(r'^course/(?P<id>[0-9]+)/$', 'course_detail'),
    url(r'^accounts/', include('web02.urls_auth')),
)


# -*- coding: utf-8 -*-
"""
This file contains the Qudi hardware implementation for the Red Pitaya counter.

Qudi is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

Qudi is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with Qudi. If not, see <http://www.gnu.org/licenses/>.

Copyright (c) the Qudi Developers. See the COPYRIGHT.txt file at the
top-level directory of this distribution and at <https://github.com/Ulm-IQO/qudi/>
"""

import numpy as np

import socket
import random
import time
import visa

from core.module import Base
from interface.slow_counter_interface import SlowCounterInterface
from interface.slow_counter_interface import SlowCounterConstraints
from interface.slow_counter_interface import CountingMode
from interface.odmr_counter_interface import ODMRCounterInterface

class RedPitayaCounter(Base, SlowCounterInterface, ODMRCounterInterface):

    """This is the Interface class to define the controls for the Red Pitaya counter.
    """
    _modclass = 'RedPitayaCounter'
    _modtype = 'hardware'

    def __init__(self, config, **kwargs):
        super().__init__(config=config, **kwargs)

        self.log.info('The following configuration was found.')

        # checking for the right configuration
        for key in config.keys():
            self.log.info('{0}: {1}'.format(key, config[key]))

    def on_activate(self):
        """ Initialisation performed during activation of the module.
        """

        config = self.getConfiguration()

        if 'ip_address' in config.keys():
            self._ip_address = config['ip_address']
        else:
            self.log.error('No paramerter "ip_address" configured in '
                    'Red Pitaya Counter. This parameter is mandatory.')

        if 'clock_frequency' in config.keys():
            self._clock_frequency=config['clock_frequency']
        else:
            self._clock_frequency = 100
            self.log.warning('No parameter "clock_frequency" configured in '
                    'Red Pitaya Counter, taking the default value of {0} Hz '
                    'instead.'.format(self._clock_frequency))

        if 'samples_number' in config.keys():
            self._samples_number = config['samples_number']
        else:
            self._samples_number = 1
            self.log.warning('No parameter "samples_number" configured in '
                    'Red Pitaya Counter, taking the default value of {0} '
                    'instead.'.format(self._samples_number))

        self.source_channels = 2
        self.odmr_length = 100

        self._connect()

        # Initialize Counter
        # NOTE: Do not do that here! This messes up the counts (why?)
        #self._send_command('COUNT:StopOutput')
        #self._send_command('COUNT:time {:f}'.format(1./self._clock_frequency))
        #self.log.info("Red Pitaya Counter: COUNT:time is "+str(1./self._clock_frequency))
        #self._send_command('COUNT:CountOutput 1000000,0')

        self.log.info('RedPitayaCounter initialized and connected to hardware.')
        return

    def _connect(self):
        try:
            self._ip_connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
            self._ip_connection.connect((self._ip_address, 5000))
        except:
            self.log.error('This is RedPitayaCounter: could not connect to RedPitaya on IP address {0}'
                           ''.format(self._ip_address))
            raise

    def _disconnect(self):
        self._ip_connection.shutdown(socket.SHUT_RDWR)
        self._ip_connection.close()

    def on_deactivate(self):
        """ Deinitialisation performed during deactivation of the module.
        """
        self._disconnect()
        return

    def get_constraints(self):
        """ Return a constraints class for the slow counter."""
        constraints = SlowCounterConstraints()
        constraints.min_count_frequency = 5e-5
        constraints.max_count_frequency = 5e5
        constraints.counting_mode = [
            CountingMode.CONTINUOUS,
            CountingMode.GATED,
            CountingMode.FINITE_GATED]

        return constraints

    def get_odmr_channels(self):
        """ Return a list of channel names.

        @return list(str): channels recorded during ODMR measurement
        """
        return ['Counts']

    def set_up_clock(self, clock_frequency=None, clock_channel=None):
        """ Configures the hardware clock of the NiDAQ card to give the timing.

        @param float clock_frequency: if defined, this sets the frequency of the clock
        @param string clock_channel: if defined, this is the physical channel of the clock

        @return int: error code (0:OK, -1:error)
        """

        # NOTE: Do not do that here, it messes up the counts! (why?)
        """
        if clock_frequency is not None and float(clock_frequency) != self._clock_frequency:
            self._clock_frequency = float(clock_frequency)
            self._send_command('COUNT:StopOutput')
            self._send_command('COUNT:time {:f}'.format(1./self._clock_frequency))
            self.log.info("Red Pitaya Counter: COUNT:time is "+str(1./self._clock_frequency))
            self._send_command('COUNT:CountOutput 1000000,0')
        """

        self.log.info("Count frequency set to %fHz (=%fs)" % (clock_frequency, 1./clock_frequency))
        self._send_command('COUNTER:TIME {:f}'.format(1./clock_frequency))
        self._clock_frequency = clock_frequency

        return 0

    def set_up_counter(self,
                       counter_channels=None,
                       sources=None,
                       clock_channel=None,
                       counter_buffer=None):
        """ Configures the actual counter with a given clock.

        @param string counter_channel: if defined, this is the physical channel of the counter
        @param string photon_source: if defined, this is the physical channel where the photons are to count from
        @param string clock_channel: if defined, this specifies the clock for the counter

        @return int: error code (0:OK, -1:error)
        """

        return 0

    def get_counter(self, samples=None):
        """ Returns the current counts per second of the counter.

        @param int samples: if defined, number of samples to read in one go

        @return float: the photon counts per second
        """

        if samples is None:
            samples = int(self._samples_number)
        else:
            samples = int(samples)

        #self._count_time = float(self._query_command("COUNT:time?"))
        count_data = np.empty([self.source_channels+1, samples], dtype=np.uint32)
        for i in range(samples):
            counts = [float(x) for x in self._query_command('COUNTER:COUNTS?').split(',')]

            counts += [sum(counts)]
            if len(counts) < self.source_channels:
                self.log.error('Red Pitaya Counter got counts from {0} channels, but {1} '
                               'were expected configured.'.format(len(counts), self.source_channels))
                return 0

            for j in range(self.source_channels+1):
                count_data[j, i] = counts[j]

        return count_data

    def get_counter_channels(self):
        """ Returns the list of counter channel names.
        @return tuple(str): channel names
        Most methods calling this might just care about the number of channels, though.
        """
        return ['APD{0}'.format(i) for i in range(self.source_channels)]+["Sum"]

    def _check_error_response(self, resp):
        if resp.startswith("ERR:"):
            raise RuntimeError(resp[4:].strip())

    def _send_command(self, cmd):
        self._query_command(cmd)

    def _query_command(self, cmd):
        try:
            if not cmd.endswith("\r\n"):
                cmd += "\r\n"
            cmd = cmd.encode("ascii")

            # Sending command
            totalsent = 0
            while totalsent < len(cmd):
                sent = self._ip_connection.send(cmd[totalsent:])
                if sent == 0:
                    self.log.error("Connection error to redpitaya...Reconnecting")
                    self._connect()
                else:
                    totalsent += sent

            # Receiving response
            chunks = []
            while True:
                chunk = self._ip_connection.recv(2048)
                chunks.append(chunk)
                if chunk == b'':
                    self.log.error("Connection error to redpitaya...Reconnecting")
                    self._connect()
                elif len(chunk) >= 1 and chunk[-1] == 0xa and chunk[-2] == 0xd:
                    break
            resp = b''.join(chunks).decode("ascii")

            self._check_error_response(resp)
            return resp
        except:
            raise

    def close_counter(self):
        """ Closes the counter and cleans up afterwards.

        @return int: error code (0:OK, -1:error)
        """

        return 0

    def close_clock(self,power=0):
        """ Closes the clock and cleans up afterwards.

        @return int: error code (0:OK, -1:error)
        """

        return 0

    #### ODMR counter interface

    def set_up_odmr_clock(self, clock_frequency=None, clock_channel=None):
        """ Configures the hardware clock of the NiDAQ card to give the timing.
        @param float clock_frequency: if defined, this sets the frequency of the
                                      clock
        @param str clock_channel: if defined, this is the physical channel of
                                  the clock
        @return int: error code (0:OK, -1:error)
        """
        return self.set_up_clock(clock_frequency, clock_channel)

    def set_up_odmr(self, counter_channel=None, photon_source=None,
                    clock_channel=None, odmr_trigger_channel=None):
        """ Configures the actual counter with a given clock.
        @param str counter_channel: if defined, this is the physical channel of
                                    the counter
        @param str photon_source: if defined, this is the physical channel where
                                  the photons are to count from
        @param str clock_channel: if defined, this specifies the clock for the
                                  counter
        @param str odmr_trigger_channel: if defined, this specifies the trigger
                                         output for the microwave
        @return int: error code (0:OK, -1:error)
        """
        return 0

    def set_odmr_length(self, length=100):
        """Set up the trigger sequence for the ODMR and the triggered microwave.
        @param int length: length of microwave sweep in pixel
        @return int: error code (0:OK, -1:error)
        """
        self._odmr_length = length
        return 0

    def count_odmr(self, length = 100):
        """ Sweeps the microwave and returns the counts on that sweep.
        @param int length: length of microwave sweep in pixel
        @return float[]: the photon counts per second
        """
        self.set_odmr_length(length)
        counts = self.get_counter(length)
        return counts[0,:]+counts[1,:] # Return the sum of APD1+APD2

    def close_odmr(self):
        """ Close the odmr and clean up afterwards.
        @return int: error code (0:OK, -1:error)
        """
        return 0

    def close_odmr_clock(self):
        """ Close the odmr and clean up afterwards.
        @return int: error code (0:OK, -1:error)
        """
        return 0


import json
import os
from config import UPLOAD_DIR
from worker import celery
import celery.states as states
from bottle import request, route, view, get, post, template, redirect
from tasks import add

"""
All routes for application that managing UI side.
example
1) user can upload and submit 7 main files that we use for update sales tax in Tract.
2) user can update sales tax by hitting single "Start sales tax Task" button
3)
"""

@get('/')
@get('/upload')
def upload():
    return template('upload' , message="")

@post('/upload', method='POST')
def do_upload():
    """Handle file upload form"""
    message = ""
    if request.method == 'POST':
        uploads = request.files.getall('newfile')
        for file in uploads:

            # only allow upload of text files
            if file.filename not in  ['FLATT.txt','LOCALT.txt','STATET.txt','FLAT.txt','LOCAL.txt','STATE.txt', 'TaxEligibility.txt']:
                message = "Error Saving"
                return template('upload' , message=message)

            save_path = os.path.join(UPLOAD_DIR, file.filename)
            if os.path.isfile(save_path):
                os.remove(save_path)
            file.save(save_path)

    # redirect to home page if it all works ok
    return redirect('/upload')

@get('/sales_task')
def sales_task():
    task = celery.send_task('tasks.sales_tax')
    return template("task", id=task.task_id)

@get('/task_eligibility')
def task_eligibility():
    task = celery.send_task('tasks.tax_eligibility')
    return template("task", id=task.task_id)

# @route('/background_job/<task_id>')
# def background_job(task_id=''):
#     task = sales_tax_task.AsyncResult(task_id)
#     ret = {}
#     if task.state == "PENDING":
#         ret['progress'] = 0
#     elif task.state == "PROGRESS":
#         ret['progress'] = "PROGRESS"
#     else:
#         ret['progress'] = task.state
#     return ret

@route('/background_job/<task_id>')
def background_job(task_id= ""):
    task = celery.AsyncResult(task_id)
    ret = {}
    ret['progress'] = task.state
    return ret


import torch
import torch.nn as nn
from DataInit import DataInit
from WordVecs import WordVecs
from Models.SimpleModel import SimpleModel
from Models.AttnModel import AttnModel
from Trainer import Trainer
from Plotter import Plotter
import argparse

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-ep', '--emdpath',
                        help="where the embeddings file is located",
                        default='Data/fasttext.vec')
    parser.add_argument('-dp', '--datapath',
                        help="where the data file is located",
                        default='Data/train.csv')
    parser.add_argument('-model', '--model',
                        help="which model to use",
                        default='simple')
    parser.add_argument('-lr', '--lr',
                        help="learning rate to use",
                        default=0.0001, type=float)
    parser.add_argument('-batch_size', '--batch_size',
                        help="size of batch",
                        default=128, type=int)
    parser.add_argument('-epochs', '--epochs',
                        help="number of epochs",
                        default=250, type=int)
    parser.add_argument('-test_ratio', '--test_ratio',
                        help="test/train ratio",
                        default=0.1, type=float)
    parser.add_argument('-cuda', '--cuda',
                        help="use cuda",
                        default=True, type=bool)
    args = parser.parse_args()

    # Consts
    print("Init Consts")
    data_file_path = args.datapath
    emmbedings_file_path = args.emdpath
    batch_size = args.batch_size
    epochs = args.epochs
    learning_rate = args.lr
    test_ratio = args.test_ratio
    cuda = args.cuda

    print("Start DataInit")
    data_init = DataInit(data_file_path)

    train_X, test_X, train_y, test_y = data_init.get_train_test_data(test_ratio)

    print("Importing embeddings")
    src_vecs = WordVecs(emmbedings_file_path)

    if args.model == 'simple':
        print("Init simple Model")
        is_attn = False
        model = SimpleModel(src_vecs, cuda)
    else:
        print("Init attention Model")
        is_attn = True
        model = AttnModel(src_vecs, batch_size, cuda)

    print("Model parametrs ", model)
    optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
    criterion = nn.BCEWithLogitsLoss(reduce=True, size_average=False)

    print("Init Trainer")
    trainer = Trainer(model, optimizer, learning_rate, criterion, epochs, batch_size, is_attn, cuda)

    print("Start training with parameters:")
    print("batch_size: {} | epochs: {} | learning_rate: {} | train_ratio: {}".format(batch_size, epochs, learning_rate,
                                                                                test_ratio))
    train_losses, test_losses = trainer.train(train_X, train_y, test_X, test_y)

    plotter = Plotter(train_losses, test_losses)
    plotter.plot()

if __name__ == '__main__':
    main()


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.model_selection import RandomizedSearchCV, GroupKFold
import xgboost as xgb
from scipy import stats
import pickle
from sklearn.model_selection import cross_val_score
from pathlib import Path
import matplotlib
import Model_XGBoost.model_utils as mu
mu.set_figure_art()
matplotlib.rc('text', usetex=True)
abs_path = Path(__file__).parent

file_name = 'ems_optimization_2.1_1000_yes.csv'
data_scenarios = pd.read_csv(abs_path.parents[0] / ('Model_optimization/' + file_name))
scenario_list = np.round(np.linspace(10, data_scenarios['Scenario'].max() + 1, 10))

# RETRAIN_ALL_MODELS = False
# if RETRAIN_ALL_MODELS:
#     model_results = {'best_model': list(),
#                      'mean_norm_rmse_fold': list(),
#                      'std_norm_rmse_fold': list()}
#     xgb_models_trained = list()
#     for scenario_number, scenario_threshold in enumerate(scenario_list):
#         print(f'Scenario: {scenario_number + 1} of {(len(scenario_list))} -- Scenario threshold: {scenario_threshold}')
#
#         idx = data_scenarios['Scenario'] <= scenario_threshold
#         data_scenarios_selected = data_scenarios.loc[idx,:]
#
#         columns_drop=['Scenario','v_1', ' storage_Q']
#         columns_predict=[' storage_P']
#         data_train_clean = data_scenarios_selected.drop(columns=columns_drop)
#
#         x_train = data_train_clean.drop(columns=columns_predict)
#         y_train = data_train_clean.loc[:, columns_predict]
#
#         param_dist = {'n_estimators': stats.randint(50, 1000),
#                       'learning_rate': stats.uniform(0.01, 0.1),
#                       'subsample': stats.uniform(0.5, 1),
#                       'max_depth': [3, 4, 5, 6, 7, 8, 9, 10],
#                       'colsample_bytree': stats.uniform(0.5, 0.45),
#                       'min_child_weight': [1, 2, 3]}
#
#         group_kfold_inner = GroupKFold(n_splits=10)
#         groups_train = data_scenarios_selected['Scenario']
#
#         xgb_regressor_model = xgb.XGBRegressor(objective='reg:squarederror')
#         xgb_regressor_search = RandomizedSearchCV(xgb_regressor_model,
#                                                   param_distributions=param_dist,
#                                                   n_iter=50,
#                                                   scoring='neg_mean_squared_error',
#                                                   cv=group_kfold_inner.split(x_train, groups=groups_train),
#                                                   refit=1,
#                                                   n_jobs=4)
#         xgb_regressor_search.fit(x_train, y_train)
#
#         # Calculate the normalized error and standard deviation from the folds
#         cv_results = pd.DataFrame(xgb_regressor_search.cv_results_)
#         cv_results = cv_results.sort_values(by='rank_test_score')
#
#         norm_factor = np.max(y_train.max()) - np.min(y_train.min())
#
#         folds_results = cv_results.filter(regex='split', axis=1)
#         norm_mse_splits = ((np.sqrt(-folds_results) / norm_factor) * 100)  # Values in percentage
#
#         mean_norm_mse = norm_mse_splits.mean(axis=1)[0]  # Mean performance of best model
#         std_norm_mse = norm_mse_splits.std(axis=1, ddof=0)[0]  # Standard deviation of performance of best model
#
#         # Save results
#         model_results['best_model'].append(xgb_regressor_search.best_estimator_)
#
#         model_results['mean_norm_rmse_fold'].append(mean_norm_mse)
#         model_results['std_norm_rmse_fold'].append(std_norm_mse)
#
#         print(f'Normalized RMSE (%): {round(mean_norm_mse, 2)}')
#
#     pickle.dump((scenario_list, model_results), open(abs_path / 'XGBoost_scenario_analysis.dat', 'wb'))
# else:
#     (scenario_list, model_results) = pickle.load(open(abs_path / 'XGBoost_scenario_analysis.dat', 'rb'))
#
# #%%
# fig, ax = plt.subplots(1, 1, figsize=(4, 2))
# ax.errorbar(scenario_list, model_results['mean_norm_rmse_fold'],
#             yerr=model_results['std_norm_rmse_fold'],
#             marker='o',  markersize=0, linewidth=0.3, color='k')
# ax.set_ylabel('NRMSE [\%]', fontsize=7)
# ax.set_xlabel('Number of scenarios', fontsize='small')
# ax.set_xticks(scenario_list)
# ax.tick_params(axis='both', labelsize=7)
# ax.set_xlim((0, 800))
# plt.tight_layout()
#
# #%% GET THE SAME ANALYSIS THAT THE PREVIOUS LOOP, BUT WITH ONLY ONE MODEL PARAMETER.
# # Analyze the parameters found by the best model found by the sweep of scenarios and random hyperparameter search:
# idx_ = np.argsort(model_results['mean_norm_rmse_fold'])
# opt_params = model_results['best_model'][idx_[0]].get_params()

# Load the best hyperparameters from the nested cross validation on 200 scenarios.
opt_params = pickle.load(open(abs_path / 'XGBoost_optimal_params.dat', 'rb'))
scenario_list_final = np.round(np.linspace(10, data_scenarios['Scenario'].max() + 1, 100))

RETRAIN_ALL_MODELS = False
if RETRAIN_ALL_MODELS:
    cross_val_results = {'norm_rmse_scores': list(),
                         'mean_norm_rmse_scores': list(),
                         'std_norm_rmse_scores': list()}

    # Re-train the model with the optimal parameters
    for scenario_number, scenario_threshold in enumerate(scenario_list_final):
        print(f'Scenario: {scenario_number + 1} of {(len(scenario_list_final))} -- Scenario threshold: {scenario_threshold}')

        idx = data_scenarios['Scenario'] <=scenario_threshold
        data_scenarios_selected = data_scenarios.loc[idx, :]

        columns_drop = ['Scenario', 'v_1', ' storage_Q']
        columns_predict = [' storage_P']
        data_train_clean = data_scenarios_selected.drop(columns=columns_drop)
        x_train = data_train_clean.drop(columns=columns_predict)
        y_train = data_train_clean.loc[:, columns_predict]

        group_kfold_inner = GroupKFold(n_splits=10)
        groups_train = data_scenarios_selected['Scenario']

        xgb_regressor_model = xgb.XGBRegressor(**opt_params)
        norm_factor = np.max(y_train.max()) - np.min(y_train.min())

        scores = cross_val_score(xgb_regressor_model,
                                 x_train, y_train,
                                 cv=group_kfold_inner.split(x_train, groups=groups_train),
                                 scoring='neg_mean_squared_error',
                                 n_jobs=4)

        norm_rmse_scores = (np.sqrt(-scores)/norm_factor) * 100
        cross_val_results['norm_rmse_scores'].append(norm_rmse_scores)
        cross_val_results['mean_norm_rmse_scores'].append(np.mean(norm_rmse_scores))
        cross_val_results['std_norm_rmse_scores'].append(np.std(norm_rmse_scores))

    pickle.dump((scenario_list_final, cross_val_results), open(abs_path / 'XGBoost_scenario_model.dat', 'wb'))
else:
    (scenario_list_final, cross_val_results) = pickle.load(open(abs_path / 'XGBoost_scenario_model.dat', 'rb'))

mean_values = np.array(cross_val_results['mean_norm_rmse_scores'])
std_values = np.array(cross_val_results['std_norm_rmse_scores'])

#%%
fig, ax = plt.subplots(1, 1, figsize=(3.5, 1.5))
plt.subplots_adjust(bottom=0.22, top=0.98, right=0.98, left=0.1)
ax.errorbar(np.array(scenario_list_final)[::2], mean_values[::2],
            yerr=std_values[::2],
            marker='o',
            markersize=0,
            linewidth=0.9,
            color='r',
            elinewidth=0.4,
            capsize=2,
            capthick=0.1, ecolor='k')
ax.set_ylabel('NRMSE [\%]', fontsize=7)
ax.set_xlabel('Number of scenarios', fontsize=7)
# ax.set_xticks(np.array(scenario_list_final)[::2])
ax.tick_params(axis='both', labelsize=8)
ax.axvline(x=200, color='b', linewidth=0.8, linestyle='--')
# plt.tight_layout()


import sys
import os

activate_this = '/home/ubuntu/.local/share/virtualenvs/mian-tNCgafuV/bin/activate_this.py'
with open(activate_this) as file_:
    exec(file_.read(), dict(__file__=activate_this))

sys.path.insert(0, '/var/www/html/mian')

import json
import logging.config

def setup_logging(
    default_path='logging.json',
    default_level=logging.INFO,
    env_key='LOG_CFG'
):
    """Setup logging configuration

    """
    path = default_path
    value = os.getenv(env_key, None)
    if value:
        path = value
    if os.path.exists(path):
        with open(path, 'rt') as f:
            config = json.load(f)
        logging.config.dictConfig(config)
    else:
        logging.basicConfig(level=default_level)

print("App Startup")

from mian.main import app as application

application.secret_key = 'Twilight Sparkle'
application.config['SESSION_TYPE'] = 'filesystem'


import json
import time

import requests
from slugify import slugify

_ua = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
headers = {"Content-Type": "application/json", "User-agent": _ua}
api_url = "https://www.ocado.com/webshop/api/v1/products?skus="

def get_api(sku):
    response = requests.get(api_url + sku, headers=headers)
    time.sleep(2)
    if response.status_code == 200:
        return json.loads(response.content.decode("utf-8"))
    else:
        print("status code: ", response.status_code)
        return None

def product_url(sku):
    products_info = get_api(sku)

    try:
        product_name = products_info[0]["name"]
        product_url = slugify(product_name, lowercase=False)
        product_url = f"https://www.ocado.com/webshop/product/{product_url}/{sku}"

        return product_url
    except:
        return None


# coding=utf-8

import json
import numpy as np
import os
import pandas as pd
import re
import torch
import torch.nn as nn
from gensim.models import KeyedVectors
from gensim.models import FastText
from nltk.corpus import stopwords
from torch.utils.data import Dataset
from torch.utils.data import TensorDataset
from tqdm import tqdm

from model.attention.abcnn1 import ABCNN1Attention
from model.attention.abcnn2 import ABCNN2Attention
from model.blocks.abcnn1 import ABCNN1Block
from model.blocks.abcnn2 import ABCNN2Block
from model.blocks.abcnn3 import ABCNN3Block
from model.blocks.bcnn import BCNNBlock
from model.convolution.conv import Convolution
from model.model import Model
from model.layers.layer import CNNLayer
from model.pooling.allap import AllAP
from model.pooling.widthap import WidthAP
from process import setup_dataset

# Use GPU if available, otherwise use CPU
USE_CUDA = torch.cuda.is_available()

def read_config(config_path):
    """ Reads in the configuration file from the given path.

        Args:
            config_path: string
                The path to the configuration file.

        Returns:
            config: dict
                Contains the information needed to initialize the
                datasets and model. See "config.json" for configuration
                details.
    """
    with open(config_path) as json_file:
        config = json.load(json_file)
        return config

def setup_v2(config):
    """ Handles all of the setup needed to run an ABCNN model.

        Args:
            config: dict
                Contains the information needed to initialize the datasets
                and model.

        Returns:
            datasets: dict of TensorDatasets
                Contains the datasets to be used for training/evaluation.
            model: Model
                The instantiated model.
    """
    # Read in the relevant config info
    data_paths = config["data_paths"]
    embeddings_size = config["embeddings"]["size"]
    max_length = config["model"]["max_length"]

    # Setup the datasets
    datasets = {name: pd.read_csv(data_path) for name, data_path in data_paths.items()}
    datasets, texts, word2index = setup_datasets_v2(datasets, embeddings_size, max_length)

    # Create embedding matrix
    word_vectors = setup_word_vectors(config)
    embeddings = setup_embedding_matrix_v2(word_vectors, word2index, embeddings_size)

    # Create the model
    model = setup_model_v2(embeddings, config)
    return datasets, model

def setup(config):
    """ Handles all of the setup needed to run an ABCNN model.

        Args:
            config: dict
                Contains the information needed to initialize the datasets
                and model. See "config.json" for configuration details.

        Returns:
            datasets: dict of TensorDatasets
                 Contains the datasets to be used for training/evaluation.
            model: nn.Module
                The instantiated model.
    """
    # Read in relevant config info
    data_paths = config["data_paths"]
    embeddings_size = config["embeddings"]["size"]
    max_length = config["model"]["max_length"]

    # Setup the datasets
    datasets = {name: pd.read_csv(data_path) for name, data_path in data_paths.items()}
    word_vectors = setup_word_vectors(config)
    for name, dataset in datasets.items():
        dataset, _ = setup_dataset(dataset, word_vectors, embeddings_size, max_length)
        datasets[name] = dataset

    # Setup the model
    model = setup_model(config)
    return datasets, model

def setup_model_v2(embeddings, config):
    """ Sets up the model for training/evaluation. The architecture here extends
        on the architecture introduced in the ABCNN paper by allowing for multiple
        convolutional layers with different window sizes (computed in parallel, not
        in series).

        Args:
            config: dict
                Contains the information needed to setup the model.

        Returns:
            model: ModelV2 module
                The instantiated model.
    """
    print("Creating the ABCNN model...")

    # Get relevant parameters
    embeddings_size = config["embeddings"]["size"]
    max_length = config["model"]["max_length"]
    layer_configs = config["model"]["layers"]
    use_all_layer_outputs = config["model"]["use_all_layer_outputs"]

    # Create the layers
    layers = []
    layer_sizes = [embeddings_size]
    for layer_config in layer_configs:
        layer, layer_size = setup_layer(max_length, layer_config)
        layers.append(layer)
        layer_sizes.append(layer_size)

    # Compute the size of the FC layer
    final_size = 2 * sum(layer_sizes) if use_all_layer_outputs else 2 * layer_sizes[-1]

    # Put it all together
    model = Model(embeddings, layers, use_all_layer_outputs, final_size).float()
    model = model.cuda() if USE_CUDA else model
    model.apply(weights_init)
    return model

def setup_word_vectors(config):
    """ Loads the pre-trained word vectors. The word vector file can be in
        Word2Vec or FastText formats.

        Args:
            config: dict
                Contains the information needed to initialize the embeddings
                model.

        Returns:
            word_vectors: KeyedVectors, FastTextKeyedVectors, or None
                The pretrained word embeddings. If the embeddings path
                is for a pre-trained Word2Vec model, then a KeyedVectors
                instance is returned. If the embeddings path is for a
                pre-trained FastText model, then a FastTextKeyedVectors
                instance is returned. Otherwise, None is returned.
    """
    # Get relevant parameters from config file
    embeddings_path = config["embeddings"]["path"]
    embeddings_format = config["embeddings"]["format"]
    is_binary = config["embeddings"]["is_binary"]

    # Load pre-trained word embeddings
    word_vectors = None
    if embeddings_format == "word2vec":
        if os.path.isfile(embeddings_path):
            print("Loading Word2Vec word vectors from: {}".format(embeddings_path))
            word_vectors = KeyedVectors.load_word2vec_format(embeddings_path, binary=is_binary)
    elif embeddings_format == "fasttext":
        if os.path.isfile(embeddings_path):
            print("Loading FastText word vectors from: {}".format(embeddings_path))
            embeddings_model = FastText.load_fasttext_format(embeddings_path)
            word_vectors = embeddings_model.wv
    else:
        raise Exception("Unsupported type. Must be one of 'word2vec' or 'fasttext'.")
    return word_vectors

def setup_dataset(examples, word_vectors, embeddings_size, max_length):
    """ Convert question pairs and labels into machine-readable datasets that
        can be used for training and evaluation.

        Args:
            examples: pd.DataFrame or list of lists
                Contains the examples. If examples is a pd.Dataframe instance,
                then the examples must be separated into columns named
                "question1", "question2", and "is_duplicate". If examples is a
                list of lists, then each example will be in the format
                [question1, question2, label]. Each question is a string of words,
                and the label is 0 or 1.
            word_vectors: KeyedVectors, FastTextKeyedVectors, or None
                The pre-trained word vectors. If word_vectors is a KeyedVectors
                instance, then only words in its vocabulary can be used.
                If word_vectors is a FastTextKeyedVectors instance, then OOV
                words can potentially be generated on the fly using n-gram word
                vectors. If word_vectors is None, then a random embedding is used.
            embeddings_size: int
                The dimension of the word embeddings
            max_length: int
                The maximum length of questions/sequences.

        Returns:
            dataset: TensorDataset
                Contains the feature maps and labels for all of the question pairs
                in the given set of examples.
            examples: list of tuples
                Contains the examples in the format (question1, question2, label)
                Each question is represented as a list of words/tokens. The
                label is a 0 or a 1 (integer). Padding tokens (resp. features) are
                added to the examples (resp. feature maps) so that each question
                has max_length tokens.
    """
    # Parse the data into the same format before processing
    if isinstance(examples, pd.DataFrame):
        pairs = examples[["question1", "question2"]].values.tolist()
        labels = examples["is_duplicate"].tolist()
    elif isinstance(examples, list):
        pairs = [(x[0], x[1]) for x in examples]
        labels = [int(x[2]) for x in examples]

    # Convert to tensors / clean up text
    features, texts = texts_to_features(pairs, word_vectors, embeddings_size, max_length)
    examples = [(text[0], text[1], label) for text, label in zip(texts, labels)]
    labels = torch.LongTensor(labels)
    dataset = TensorDataset(features, labels)
    return dataset, examples

def setup_datasets_v2(datasets, embeddings_size, max_length):
    """ Converts the examples from the datasets into a machine-readable format
        useful for training.

        To ensure that all words have a word embedding associated to them, we
        should have text from ALL datasets (note: this is NOT peeking at the
        dataset... this is just to prevent the model from crashing/complaining
        when it sees a word that is OOV.) OOV words are assigned random word
        embeddings.

        Args:
            datasets: dict of pd.DataFrame
                The text we would like to convert to indices into the embedding
                matrix.
            word2index: dict
                Contains the mapping from words to their indices in the
                embedding matrix.
            word_vectors: KeyedVectors, FastTextKeyedVectors, or None
                Contains the pre-trained word vectors, if available.
            embeddings_size: int
                The dimension of the word embeddings.
            max_length: int
                The maximum length of questions/sentences.

        Returns:
            index_mappings: dict of torch.LongTensors
                The keys are the names of the datasets. The values are
                LongTensors of shape (num_examples, 2, max_length).
                Each example is represented as a LongTensor of indices
                into the embedding matrix.
            word2index: dict
                Maps each word to an index of the embedding matrix.
            index2word: dict
                Maps each index of the embedding matrix to a word.
    """
    texts = dict()
    word2index = {"<PAD>": 0}
    question_cols = ["question1", "question2"]

    # Process each dataset
    for name, dataset in datasets.items():

        # Process texts
        labels = []
        indexed_examples = []
        parsed_texts = []
        num_examples = len(dataset)
        for index, example in tqdm(dataset.iterrows(), desc=name, total=num_examples):

            # Process each question separately
            index_map = []
            parsed_text = []
            for column in question_cols:

                # Parse and clean the text
                question = example[column]
                words = text_to_word_list(question)
                words = remove_stop_words(words)

                # Convert words to indices
                indexes = []
                for word in words:

                    # Update word-index lookup if necessary
                    if word not in word2index:
                        word2index[word] = len(word2index)

                    # Add the word's index to the list
                    indexes.append(word2index[word])

                # Truncate if necessary
                length = len(indexes) if len(indexes) < max_length else max_length
                indexes = indexes[:length]
                words = words[:length]

                # Pad if necessary
                if length < max_length:
                    num_padding = max_length - length
                    indexes.extend([0] * num_padding)
                    words.extend(["<PAD>"] * num_padding)

                # Store parsed text and index tensors
                index_map.append(indexes)
                parsed_text.append(words)

            # Store processed text and index tensor map and label
            labels.append(example["is_duplicate"])
            indexed_examples.append(index_map)
            parsed_texts.append(parsed_text)

        # Save the processed result
        labels = torch.LongTensor(labels)
        indexed_examples = torch.LongTensor(indexed_examples)
        dataset = TensorDataset(indexed_examples, labels)
        datasets[name] = dataset
        texts[name] = parsed_texts

    return datasets, texts, word2index

def setup_embedding_matrix_v2(word_vectors, word2index, embeddings_size):
    """ Creates the embedding matrix using the given word embeddings and mapping
        from words to indices.

        Args:
            word_vectors: KeyedVectors, FastTextKeyedVectors, or None
                The pre-trained word-vectors, if available.
            word2index: dict
                Maps words to indices in the embedding matrix.

        Returns
            embeddings: nn.Embedding
                The embedding matrix.
    """
    # Sanity check
    embeddings = np.random.uniform(-0.01, 0.01, (len(word2index) + 1, embeddings_size))
    embeddings[0] = 0   # Padding is just all 0s

    # Replace random vectors with pre-trained vectors if available
    if word_vectors is not None:
        for word, index in tqdm(word2index.items(), desc="embedding matrix"):
            try:
                embeddings[index] = word_vectors[word]
            except (RuntimeError, KeyError):
                pass

    # Convert to nn.Embedding
    embeddings = nn.Embedding.from_pretrained(torch.from_numpy(embeddings))
    return embeddings

def texts_to_features(pairs, word_vectors, embeddings_size, max_length):
    """ Converts a list of texts (strings) into their feature maps.

        Args:
            texts: list of tuple of  string
                The text we would like to featurize in the format (question1, question2).
            word_vectors: KeyedVectors, FastTextKeyedVectors, or None
                The pre-trained word vectors. If word_vectors is a KeyedVectors
                instance, then only words in its vocabulary can be used.
                If word_vectors is a FastTextKeyedVectors instance, then OOV
                words can potentially be generated on the fly using n-gram word
                vectors. If word_vectors is None, then a random embedding is used.
            embeddings_size: int
                The dimension of the word embeddings
            max_length: int
                The maximum length of questions/sequences.

        Returns:
            feature_maps: torch.Tensor of shape (batch_size, max_length, embedding_size)
                The feature maps for each text.
            processed_texts: list of lists
                Contains the processed text for each question. Stop words are removed
                and <unk> tokens are added to pad the sequence to max length.
    """
    # Process texts
    feature_maps = []
    processed_texts = []
    for pair in tqdm(pairs):

        # Process each question separately
        feature_map = []
        processed_text = []
        for question in pair:

            # Parse text
            words = text_to_word_list(question)
            words = remove_stop_words(words, word_vectors)

            # Convert words to features
            # If a word has no word vector, use a random word vector
            features = []
            for word in words:
                try:
                    feature = torch.from_numpy(word_vectors[word])
                except (RuntimeError, KeyError):
                    feature = torch.Tensor(embeddings_size).uniform_(-0.01, 0.01)
                feature = feature.view(1, embeddings_size)
                features.append(feature)

            # Truncate if necessary
            length = len(words) if len(words) < max_length else max_length
            features = features[:length]
            words = words[:length]

            # Pad if necessary
            if length < max_length:
                num_padding = max_length - length
                padding = torch.zeros((num_padding, embeddings_size), dtype=torch.float)
                unks = ["<unk>"] * num_padding
                features.append(padding)
                words.extend(unks)

            # Combine features into single feature map
            features = torch.cat(features, dim=0) # shape (max_length, embedding_size)
            feature_map.append(features.clone())
            processed_text.append(words)

        # Stack feature maps for the question pair
        feature_map = torch.stack(feature_map) # shape (2, batch_size, embedding_size)
        feature_maps.append(feature_map)
        processed_texts.append(processed_text)

    # Stack feature maps for all examples
    feature_maps = torch.stack(feature_maps)
    return feature_maps, processed_texts

def text_to_word_list(text):
    """ Preprocess and convert texts to a list of words. This code was taken
        from Elior Cohen's MaLSTM code, which can be found here:

        https://github.com/eliorc/Medium/blob/master/MaLSTM.ipynb

         Args:
             text: string
                 The text to parse.

         Returns:
             text: list of string
                 The parsed text.
     """
    text = str(text)
    text = text.lower()

    # Clean the text
    text = re.sub(r"[^A-Za-z0-9^,!.\/'+-=]", " ", text)
    text = re.sub(r"what's", "what is ", text)
    text = re.sub(r"\'s", " ", text)
    text = re.sub(r"\'ve", " have ", text)
    text = re.sub(r"can't", "cannot ", text)
    text = re.sub(r"n't", " not ", text)
    text = re.sub(r"i'm", "i am ", text)
    text = re.sub(r"\'re", " are ", text)
    text = re.sub(r"\'d", " would ", text)
    text = re.sub(r"\'ll", " will ", text)
    text = re.sub(r",", " ", text)
    text = re.sub(r"\.", " ", text)
    text = re.sub(r"!", " ! ", text)
    text = re.sub(r"\/", " ", text)
    text = re.sub(r"\^", " ^ ", text)
    text = re.sub(r"\+", " + ", text)
    text = re.sub(r"\-", " - ", text)
    text = re.sub(r"\=", " = ", text)
    text = re.sub(r"'", " ", text)
    text = re.sub(r"(\d+)(k)", r"\g<1>000", text)
    text = re.sub(r":", " : ", text)
    text = re.sub(r" e g ", " eg ", text)
    text = re.sub(r" b g ", " bg ", text)
    text = re.sub(r" u s ", " american ", text)
    text = re.sub(r"\0s", "0", text)
    text = re.sub(r" 9 11 ", "911", text)
    text = re.sub(r"e - mail", "email", text)
    text = re.sub(r"j k", "jk", text)
    text = re.sub(r"\s{2,}", " ", text)

    text = text.split()

    return text

def remove_stop_words(words):
    """ Removes all of the stop words.

        Args:
            words: list of string
                The words in the text.

        Returns:
            words: list of string
                The words in the text with stop words removed.
    """
    stops = set(stopwords.words("english"))
    return list(filter(lambda w: w not in stops, words))

def setup_embedding_matrix(word2index, word_vectors, config):
    """ Creates the embedding matrix.

        This code is based on code from Elior Cohen's MaLSTM notebook,
        which can be found here:

        https://github.com/eliorc/Medium/blob/master/MaLSTM.ipynb

        Args:
            word2index: dict
                Mapping from word/token to index in embeddings matrix.
            word_vectors: KeyedVectors or None
                The pretrained word embeddings. If a pre-trained
                model was provided, then this should be a KeyedVectors
                instance. Otherwise, this should be None.
            config: dict
                Contains the information needed to initialize the model.

        Returns:
            embeddings: nn.Embedding of shape (vocab_size, embeddings_size)
                The matrix of word embeddings.
    """
    print("Creating embedding matrix...")

    # Get relevant parameters
    embeddings_size = config["embeddings"]["size"]
    num_words = len(word2index)

    # Initialize the embedding matrix
    # Words without pre-trained embeddings will be assigned random embeddings
    embeddings = np.random.uniform(-0.01, 0.01, (num_words + 1, embeddings_size))
    embeddings[0] = 0  # So that the padding will be ignored

    # Load in the pre-trained word embeddings
    if word_vectors is not None:
        with tqdm(total=num_words) as pbar:
            for word, index in word2index.items():
                try:
                    embeddings[index] = word_vectors[word]
                except KeyError:
                    pass
                pbar.update(1)

    # Convert to an nn.Embedding layer
    embeddings = nn.Embedding.from_pretrained(torch.from_numpy(embeddings))
    return embeddings

def setup_layer(max_length, layer_config):
    """ Creates a single Layer for the CNN model.

        Args:
            max_length: int
                The maximum length of the input sequences.
            layer_config: dict
                Contains the information needed to create the layer.

        Returns:
            layer: Layer module
                The desired Layer module.
    """
    blocks = []
    output_sizes = []
    for block_config in layer_config:
        block, output_size = setup_block(max_length, block_config)
        blocks.append(block)
        output_sizes.append(output_size)
    layer = CNNLayer(blocks)
    layer_size = sum(output_sizes)
    return layer, layer_size

def setup_block(max_length, block_config):
    """ Creates a single block for the CNN model.

        Args:
            max_length: int
                The maximum length for each sequence/question.
            block_config: dict
                Contains the information needed to create the block.

        Returns:
            block: Block module
                The desired Block module.
    """
    input_size = block_config["input_size"]
    output_size = block_config["output_size"]
    width = block_config["width"]
    dropout_rate = block_config["dropout_rate"]
    match_score = block_config["match_score"]
    share_weights = block_config["share_weights"]

    if block_config["type"] == "bcnn":
        conv = Convolution(input_size, output_size, width, 1)
        pool = WidthAP(width)
        block = BCNNBlock(conv, pool, dropout_rate=dropout_rate)

    elif block_config["type"] == "abcnn1":
        attn = ABCNN1Attention(input_size, max_length, share_weights, match_score)
        conv = Convolution(input_size, output_size, width, 2)
        pool = WidthAP(width)
        block = ABCNN1Block(attn, conv, pool, dropout_rate=dropout_rate)

    elif block_config["type"] == "abcnn2":
        conv = Convolution(input_size, output_size, width, 1)
        attn = ABCNN2Attention(max_length, width, match_score)
        block = ABCNN2Block(conv, attn, dropout_rate=dropout_rate)

    elif block_config["type"] == "abcnn3":
        attn1 = ABCNN1Attention(input_size, max_length, share_weights, match_score)
        conv = Convolution(input_size, output_size, width, 2)
        attn2 = ABCNN2Attention(max_length, width, match_score)
        block = ABCNN3Block(attn1, conv, attn2, dropout_rate=dropout_rate)

    else:
        raise Exception("Unsupported type. Must be one of 'bcnn', 'abcnn1', 'abcnn2', 'abcnn3'.")

    return block, output_size

def weights_init(m):
    """ Initializes the weights for the modules in the CNN model. This function
        is applied recursively to all modules in the model via the "apply"
        function.

        Args:
            m: nn.Module
                The module to initialize.

        Returns:
            None
    """
    classname = m.__class__.__name__
    if classname.find("Conv2d") != -1:
        nn.init.xavier_normal_(m.weight)
        nn.init.constant_(m.bias, 0)
    elif classname.find("Linear") != -1:
        nn.init.xavier_normal_(m.weight)
        nn.init.constant_(m.bias, 0)
    elif classname.find("ABCNN1Attention") != -1:
        nn.init.xavier_normal_(m.W1)
        nn.init.xavier_normal_(m.W2)


import numpy as np
import cv2

hls_dict = dict(H=0, L=1, S=2)
rgb_dict = dict(R=0, G=1, B=2)
luv_dict = dict(l=0, u=1, v=2)

def sobel_filter(img, orient="x", sobel_kernel=3):
    """
    Helper function for applying Sobel filter

    Returns:
        sobel: Sobel filtered image using specified kernel size
    """
    gray = cv2.cvtColor(img, cv2.COLOR_RGB2GRAY)
    # Take the derivative in x or y given orient = 'x' or 'y'
    if orient=="x":
        sobel = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=sobel_kernel)
    elif orient=="y":
        sobel = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=sobel_kernel)
    else:
        raise ValueError("orient has to be x or y!!!")
    return sobel

def abs_sobel_thresh(img, orient='x', thresh=(0, 255), sobel_kernel=3):
    """
    Apply Sobel Filter on image gradient

    Returns:
        binary_output: binary image after applying sobel filter in x or y direction
    """
    sobel = sobel_filter(img, orient=orient, sobel_kernel=sobel_kernel)
    # Take the absolute value of the derivative or gradient
    abs_sobel = np.absolute(sobel)
    # Scale to 8-bit (0 - 255) then convert to type = np.uint8
    scaled_sobel = np.uint8(255*abs_sobel/np.max(abs_sobel))
    # Create mask of ones
    binary_output = np.zeros_like(scaled_sobel)
    binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1  # ToDo: use min max instead of index

    return binary_output

def mag_thresh(img, thresh=(0, 255), sobel_kernel=3):
    """
    Apply Filter on gradient magnitude

    Returns:
        binary_output: binary image filtered on gradient magnitude
    """
    sobelx = sobel_filter(img, orient="x", sobel_kernel=sobel_kernel)
    sobely = sobel_filter(img, orient="y", sobel_kernel=sobel_kernel)
    # Calculate the magnitude
    abs_sobelxy = np.sqrt(sobelx**2 + sobely**2)
    # Scale to 8-bit (0 - 255) and convert to type = np.uint8
    scaled_sobel = np.uint8(255*abs_sobelxy/np.max(abs_sobelxy))
    # Create a binary mask where mag thresholds are met
    binary_output = np.zeros_like(scaled_sobel)
    binary_output[(scaled_sobel >= thresh[0]) & (scaled_sobel <= thresh[1])] = 1
    return binary_output

def dir_thresh(img, thresh=(0, np.pi/2), sobel_kernel=3):
    """
    Apply Filter on gradient direction

    Returns:
        binary_output: binary image filtered on gradient direction
    """

    sobelx = sobel_filter(img, orient="x", sobel_kernel=sobel_kernel)
    sobely = sobel_filter(img, orient="y", sobel_kernel=sobel_kernel)
    # Take the absolute value of the x and y gradients
    abs_sobelx = np.abs(sobelx)
    abs_sobely = np.abs(sobely)
    # Use np.arctan2(abs_sobely, abs_sobelx) to calculate the direction of the gradient
    direction = np.arctan2(abs_sobely, abs_sobelx)
    # Create a binary mask where direction thresholds are met
    binary_output = np.zeros_like(sobelx)
    binary_output[(direction >= thresh[0]) & (direction <= thresh[1])] = 1

    return binary_output

def hls_thresh(img, hls_channel="S", thresh=(0, 255)):
    """
    Apply Filter on image in HLS color space

    Returns:
        binary_output: binary image filtered on specified channel in HLS color space
    """
    # Convert to HLS color space
    hls = cv2.cvtColor(img, cv2.COLOR_BGR2HLS)
    # Apply a threshold to channel
    channel_num = hls_dict[hls_channel]
    channel_dim = hls[:,:,channel_num]
    # Return a binary image of threshold result
    binary_output = np.zeros_like(channel_dim)
    binary_output[(channel_dim > thresh[0]) & (channel_dim <= thresh[1])] = 1

    return binary_output

def rgb_thresh(img, rgb_channel="R", thresh=(0, 255)):
    """
    Apply Filter on image in RGB color space

    Returns:
        binary_output: binary image filtered on specified channel in RGB color space
    """
    rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
    channel_num = rgb_dict[rgb_channel]
    channel_dim = rgb[:,:,channel_num]
    binary_output = np.zeros_like(channel_dim)
    binary_output[(channel_dim > thresh[0]) & (channel_dim <= thresh[1])] = 1

    return binary_output


from collections import deque

def groupby(func, seq):
    result = {}

    for x in seq:
        result[func(x)] = [y for y in seq if func(y) == func(x)]

    return result

def zip_with(func, *iterables):
    zip_iterables = zip(*iterables)
    while True:
        yield func(*next(zip_iterables))

def composer(f, g):
    return lambda x: f(g(x))

def iterate(func):
    yield lambda x: x
    composition = func
    while True:
        yield composition
        composition = composer(func, composition)

def cache(func, cache_size):
    values = {}
    cache_order = deque()

    def func_cashed(*args):
        if args in values:
            return values[args]

        result = func(*args)

        if cache_size:
            if len(values) == cache_size:
                values.pop(cache_order.popleft())
            cache_order.append(args)
            values[args] = result

        return result
    return func_cashed


import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d.axes3d import Axes3D
from matplotlib import cm
from career import *
from compute_fp import compute_fixed_point

wp = workerProblem()
v_init = np.ones((wp.N, wp.N))*100
v = compute_fixed_point(bellman, wp, v_init)

fig = plt.figure(figsize=(8,6))
ax = fig.add_subplot(111, projection='3d')
tg, eg = np.meshgrid(wp.theta, wp.epsilon)
ax.plot_surface(tg, eg, v.T, rstride=2, cstride=2, cmap=cm.jet, alpha=0.5,
        linewidth=0.25)
ax.set_zlim(150, 200)
ax.set_xlabel('theta', fontsize=14)
ax.set_ylabel('epsilon', fontsize=14)
fig.show()


import discord
from discord.ext import commands
from settings import API_URL
from utils import get_gql_client
from queries import get_profile
from mutations import set_player

# Ultima edição feita por @Grievon

client = commands.Bot(command_prefix='r/')

@client.event
async def on_ready():
    print("Let's to work!")

@client.command()
async def cadastrar(bot, usr=None):
    """
    Use registrar para adicionar novos jogadores no server!
    """

    if not usr:
        return await bot.send(f'{bot.author} o comando não pode ser vazio!')

    is_pc = True if usr[2] == '!' else False
    if is_pc:
        author = '<@!' + str(bot.author.id) + '>'
    else:
        author = '<@' + str(bot.author.id) + '>'

    if author != usr:
        return await bot.send(f'{bot.author} por favor fornecer o @SeuNickName correto!')
    else:

        if usr[2] != '!':
            p1, p2 = usr[:2], usr[2:]
            usr = f'{p1}!{p2}'

        payload = set_player(usr)
        api_client = get_gql_client(API_URL)
        response = api_client.execute(payload)
        print(response)
        return await bot.send(f'O usuário {usr} foi cadastrado')

@client.command()
async def perfil(bot, usr=None):
    """
    Visualisar perfil do jogador e seu personagem!
    """

    if not usr:
        return await bot.send(f'{bot.author} por favor informe o @NickName')
    else:

        return await bot.send(f'''Nick:{usr}
''')

@client.command()
async def ajuda(bot):
    return await bot.send('''
Olá, meu nome é Regis e estou aqui para te ajudar!
Seu assistente de registro e monitoramento de Players, caso você precise dos meus serviços basta me informar o comando desejado!

```Para se cadastrar no servidor basta dar o comando: r/cadastrar @SeuNickName

Para verificar o perfil de um(a) jogador(a) basta dar o comando: r/perfil @NickName```

Caso precise de uma ajuda que não esteja nesse tópico entre em contato com a equipe de desenvolvedores. Eles poderão lhe dar mais informações!

Epero ter ajudado o/
    ''')

@client.command()
async def teste(bot, usr):

    print(usr)
    return await bot.send(usr)


import numpy as np
import argparse
import cv2

# Parse the command line arguments
ap = argparse.ArgumentParser()
ap.add_argument("-i", "--image", required=True, help="Path to the image")
args = vars(ap.parse_args())

# Load the image, then split it into R, G, B channels
# Remember that OpenCV stored RGB images as NumPy arrays in reverse
# channel order
image = cv2.imread(args["image"])
(B, G, R) = cv2.split(image)

cv2.imshow("Red", R)
cv2.imshow("Green", G)
cv2.imshow("Blue", B)
cv2.waitKey(0)

# Merge the channels back together with the cv2.merge function
# With the following code we can show the actual colour of the channel
zeros = np.zeros(image.shape[:2], dtype="uint8")
cv2.imshow("Red", cv2.merge([zeros, zeros, R]))
cv2.imshow("Green", cv2.merge([zeros, G, zeros]))
cv2.imshow("Blue", cv2.merge([B, zeros, zeros]))
cv2.waitKey(0)


#coding:utf-8
import hashlib as hasher
import datetime

def create_genesis_block():
    '''Constructs a block with index 0 and arbitrary previous hash'''
    return Block(0, datetime.datetime.now(), "Genesis Block" ,"0","0","0","0")
#操作的是区块对象
class Block():
    def __init__(self, index, timestamp, software,company, addr,cite,previous_hash):
        self.index = index
        self.timestamp = timestamp
        self.software = software
        self.company = company
        self.addr = addr
        self.cite = cite #引用字段
        self.previous_hash = previous_hash
        self.hash = self.hash_block()

    def  hash_block(self):
        sha = hasher.sha256()
        sha.update(str(self.index).encode("utf-8")
            + str(self.timestamp).encode("utf-8")
            + str(self.software).encode("utf-8")
            + str(self.company).encode("utf-8")
            + str(self.addr).encode("utf-8")
            + str(self.cite).encode("utf-8")
            + str(self.previous_hash).encode("utf-8")
        )
        return sha.hexdigest()

# Create the blockchain and add genesis block.
blockchain = [create_genesis_block()]
#print(blockchain[0].cite)
previous_block = blockchain[0]
#print(previous_block)

# How many blocks to add after the genesis block?
#num_of_blocks_to_add = 10

def next_block(last_block):
    this_index = last_block.index + 1
    this_timestamp = datetime.datetime.now()
    #this_data = "Hey! I'm block " + str(this_index)
    this_software = input("请输入第 #{} 个区块的软件名 : ".format(this_index))
    this_company = input("请输入第 #{} 个区块的公司名 : ".format(this_index))
    this_addr = input("请输入第 #{} 个区块的软件地址 : ".format(this_index))
    this_cite = input("请输入第 #{} 个区块的引用记录 : ".format(this_index))
    return Block(this_index, this_timestamp, this_software, this_company,this_addr,this_cite,last_block.hash)

# Add blocks to the chain.
#for i in range(num_of_blocks_to_add):
while 1:
    block_to_add = next_block(previous_block)
    blockchain.append(block_to_add)
    previous_block = block_to_add
    # Tell everyone about it!
    print("区块 #{} 成功上链！".format(block_to_add.index))
    print("软件名 :{}".format(block_to_add.software))
    print("公司名 :{}".format(block_to_add.company))
    print("软件地址 :{}".format(block_to_add.addr))
    print("引用记录: {}".format(block_to_add.cite))
    print("哈希值: {}".format(block_to_add.hash))
    print("时间戳: {}\n".format(block_to_add.timestamp))
    #存入数据库
# print(blockchain[0].previous_hash)


"""Async HTTP RPC Provider."""
from typing import Any, Dict, Optional

import httpx

from ...exceptions import SolanaRpcException, handle_async_exceptions
from ..types import RPCMethod, RPCResponse
from .async_base import AsyncBaseProvider
from .core import DEFAULT_TIMEOUT, _HTTPProviderCore

class AsyncHTTPProvider(AsyncBaseProvider, _HTTPProviderCore):
    """Async HTTP provider to interact with the http rpc endpoint."""

    def __init__(
        self,
        endpoint: Optional[str] = None,
        extra_headers: Optional[Dict[str, str]] = None,
        timeout: float = DEFAULT_TIMEOUT,
    ):
        """Init AsyncHTTPProvider."""
        super().__init__(endpoint, extra_headers)
        self.session = httpx.AsyncClient(timeout=timeout)

    def __str__(self) -> str:
        """String definition for HTTPProvider."""
        return f"Async HTTP RPC connection {self.endpoint_uri}"

    @handle_async_exceptions(SolanaRpcException, Exception)
    async def make_request(self, method: RPCMethod, *params: Any) -> RPCResponse:
        """Make an async HTTP request to an http rpc endpoint."""
        request_kwargs = self._before_request(method=method, params=params, is_async=True)
        raw_response = await self.session.post(**request_kwargs)
        return self._after_request(raw_response=raw_response, method=method)

    async def is_connected(self) -> bool:
        """Health check."""
        try:
            response = await self.session.get(self.health_uri)
            response.raise_for_status()
        except (IOError, httpx.HTTPError) as err:
            self.logger.error("Health check failed with error: %s", str(err))
            return False

        return response.status_code == httpx.codes.OK

    async def __aenter__(self) -> "AsyncHTTPProvider":
        """Use as a context manager."""
        await self.session.__aenter__()
        return self

    async def __aexit__(self, _exc_type, _exc, _tb):
        """Exits the context manager."""
        await self.close()

    async def close(self) -> None:
        """Close session."""
        await self.session.aclose()