from dash import callback_context
from .utils import sync_index, get_nth_chunk

class RowOptionsSelected:
    """
    A convenience wrapper for the 3/4 values from a row of the user drop-downs.
    In each case the first three values are the indices of the
    * field name (i.e. `.item` of the UserOption)
    * transformation (i.e. `.set_transform` of the UserOption)
    * aggregation (i.e. `.set_aggregation` of the UserOption)
    For the secondary fields, a final option of `Name` (i.e. `.perform_lkp` of UserOption)
    is available too.

    Constructors:
    * RowOptionsSelected(3) -- create empty object with 3 values
    * RowOptionsSelected(0,2,1 [,True]) -- create object with 3[4] specified valuaes.

    This class allows named values for easier-to-read code.
    """
    def __init__(self, *args):
        self.n = len(args)
        if self.n == 1:
            assert args[0] in (3,4), "Expecting 3 or 4 arguments for RowOptionsSelected"
            self.n = args[0]
            self.item_id = None
            self.trans_id = None
            self.agg_id = None
            self.perform_lkp = None
            return

        assert self.n in (3,4), "Expecting 3 or 4 arguments for RowOptionsSelected"
        self.item_id = args[0]
        self.trans_id = args[1]
        self.agg_id = args[2]
        if len(args) == 4:
            if isinstance(args[3], bool):
                self.perform_lkp = args[3]
            elif isinstance(args[3], list):
                self.perform_lkp = len(args[3]) > 0
            else:
                raise RuntimeError(f"Unexpected args[4] type: {type(args[3])}")

    @property
    def has_name_flag(self):
        return self.n == 4

    def to_list(self):
        if not self.has_name_flag:
            return [self.item_id, self.trans_id, self.agg_id]
        else:
            return [self.item_id, self.trans_id, self.agg_id, self.perform_lkp]

def app_state_to_opts(args, primary_fields, secondary_fields):
    assert (len(args) - 3) % 4 == 0, "Expecting 3+4n args (n>0)"
    num_secondary = (len(args) - 3) // 4
    use_opts = []
    debug = []
    chunksizes = [3, *([4] * num_secondary)]
    for i in range(num_secondary + 1):
        # Get i'th row of indices selected from the UI
        c_args = get_nth_chunk(i, args, chunksizes)
        debug.append(", ".join([str(x) for x in c_args]))
        selected = RowOptionsSelected(*c_args)
        # Get the field that the user has selected (dropdown column 1)
        if i == 0:
            opt = primary_fields[selected.item_id].copy()
            opt.perform_lkp = False     # no lookups available for primary field.
        elif selected.item_id is None:
            continue   # user has [x] the current item and hit submit without selecting.
        elif selected.item_id > 0:
            opt = secondary_fields[selected.item_id - 1].copy()
            opt.is_secondary = True
        else:
            continue

        # Get the transformation/aggregation selected in dropdown columns 2/3.
        # -------------------------------------------------------------------
        # try/except: might be out of range if dropdowns have changed, and hence None.
        try:
            t = opt.transformations[selected.trans_id]
            opt.set_transform(t)
        except (IndexError, TypeError):
            opt.set_transform(None)
        try:
            a = opt.aggregations[selected.agg_id]
            opt.set_aggregation(a)
        except (IndexError, TypeError):
            opt.set_aggregation(None)

        # Get the 'name' flag in column 4 (n.b. not available in row 1.)
        if i > 0:
            opt.perform_lkp = selected.perform_lkp

        use_opts.append(opt)

    debug = "\n".join(debug)
    return use_opts, debug

def find_in_item_names(query, user_opts, error_about='list'):
    try:
        return [x.item.lower() for x in user_opts].index(query)
    except ValueError as e:
        raise ValueError(f'Standard query: Cannot find in {error_about}: {query}')

def get_trigger(default=None):
    # what called the function?
    ctx = callback_context
    if not ctx.triggered:
        trigger_id = default
    else:
        trigger_id = ctx.triggered[0]['prop_id'].split('.')[0]
    return trigger_id

# ########################### RE STANDARD QUERIES ##################################

def standard_query_to_panel_indices(query, primary_opts, secondary_opts,
                                    secondary_appends_none=True, as_obj=False):
    # primary
    primary_row = RowOptionsSelected(3)
    ix = 0
    primary_row.item_id = find_in_item_names(query[ix][0].lower(), primary_opts,
                                             error_about="primary options")
    chosen_field = primary_opts[primary_row.item_id]
    primary_row.trans_id = chosen_field.transformations.index(query[ix][1])
    primary_row.agg_id = chosen_field.aggregations.index(query[ix][2])
    out = primary_row.to_list()
    out_obj = [primary_row]

    # secondary
    num_secondary = len(query) - 1
    for i in range(num_secondary):
        ix = i + 1
        secondary_row = RowOptionsSelected(4)
        secondary_row.item_id = find_in_item_names(query[ix][0].lower(),
                                                   secondary_opts,
                                                   error_about="secondary options")
        chosen_field = secondary_opts[secondary_row.item_id]
        if secondary_appends_none:
            # If UI appends an additional <None> field to possible fieldnames.
            # Note that we must therefore increment *AFTER* we've selected from opts above
            secondary_row.item_id += 1
        secondary_row.trans_id = chosen_field.transformations.index(query[ix][1])
        secondary_row.agg_id = chosen_field.aggregations.index(query[ix][2])
        secondary_row.perform_lkp = query[ix][3]
        out.extend(secondary_row.to_list())
        out_obj.append(secondary_row)

    return out_obj if as_obj else out

def get_query_from_index(i, queries):
    q_txt = list(queries.keys())[i]
    return queries[q_txt]

def standard_query_to_opts(query, primary_opts, secondary_opts):
    indices = standard_query_to_panel_indices(query, primary_opts, secondary_opts)
    return app_state_to_opts(indices, primary_opts, secondary_opts)


"""
An example implementation of the abstract Node class for use in MCTS
If you run this file then you can play against the computer.
A tic-tac-toe board is represented as a tuple of 9 values, each either None,
True, or False, respectively meaning 'empty', 'X', and 'O'.
The board is indexed by row:
0 1 2
3 4 5
6 7 8
For example, this game board
O - X
O X -
X - -
corrresponds to this tuple:
(False, None, True, False, True, None, True, None, None)
"""
import copy
from collections import namedtuple
from random import choice, getrandbits
from MCTS import MCTS, Node
import numpy as np
import math
from scipy.stats import norm
#from alphazero.NNet import NNetWrapper
#from MCTSaz import MCTSaz
from game.TicTacToeGame import TicTacToeGame
import pickle
from NormalSearchAlgorithm import NormalSearchAlgorithm
_TTTB = namedtuple("TicTacToeBoard", "tup turn winner terminal")

states=dict()
states2=dict()
states_cache_bvoi=dict()
previous_tree = None
alphazero_agent = None
global weak_heuristic_dict
weak_heuristic_dict = None

def load_tictactoe_alphazero():
    alphazero_agent = NNetWrapper()
    alphazero_agent.load_checkpoint('./pretrained_models','best-25eps-25sim-10epch.pth.tar')

def sample(to_simulate,  num_sims = 25):
    sum = 0
    for i in range(num_sims):
        sum += mcts.simulate(to_simulate, invert_reward=False)

    meanvalue = sum / num_sims
    return meanvalue

# Inheriting from a namedtuple is convenient because it makes the class
# immutable and predefines __init__, __repr__, __hash__, __eq__, and others
class TicTacToeBoard(Node):

    def __init__(self, max, tup, turn, winner, terminal, meanvalue, depth):
        standard_derv=(10-depth)*0.3/10
        self.is_max=max
        self.tup=tup
        self.marked=0
        self.turn=turn
        self.winner=winner
        self.terminal=terminal
        self.meanvalue=meanvalue
        self.depth=depth
        self.hash = getrandbits(128)
        self.probability_density = lambda x : norm.pdf(x, loc=meanvalue, scale=standard_derv, size=None)
        self.buckets=[]
        num_of_buckets = 13
        j = 0
        for i in np.linspace(0,1,num_of_buckets + 2):
            self.buckets.append((norm.ppf(i,loc=meanvalue, scale=standard_derv), j/num_of_buckets))
            j = j + 1
        self.buckets=self.buckets[1:-1]
        def dist(x):
            if self.buckets[0]>x:
                return self.buckets[0]
            for i in range(1, len(self.buckets)):

                if x<self.buckets[i]:
                    d1=x-self.buckets[i-1]
                    d2=self.buckets[i]-x
                    if d1>d2:
                        return self.buckets[i-1]
                    else:
                        return self.buckets[i]
            return self.buckets[-1]

        #self.distribution=lambda x : dist(norm.rvs(loc=meanvalue,scale=standard_derv, size=None))

    def find_children(board):
        if board.terminal:  # If the game is finished then no moves can be made
            return set()
        # Otherwise, you can make a move in each of the empty spots
        return {
            board.make_move(i) for i, value in enumerate(board.tup) if value == 0
        }

    def find_children_bvoi(board, distribution_mode="sample"):
        if board.terminal:  # If the game is finished then no moves can be made
            return set()
        # Otherwise, you can make a move in each of the empty spots
        return {
            board.make_move_bvoi(i, distribution_mode = distribution_mode) for i, value in enumerate(board.tup) if value == 0
        }

    def find_random_child(board):
        if board.terminal:
            return None  # If the game is finished then no moves can be made
        empty_spots = [i for i, value in enumerate(board.tup) if value == 0]
        return board.make_move(choice(empty_spots))

    def find_random_child_bvoi(board, distribution_mode = "sample"):
        if board.terminal:
            return None  # If the game is finished then no moves can be made
        empty_spots = [i for i, value in enumerate(board.tup) if value == 0]
        return board.make_move_bvoi(choice(empty_spots), distribution_mode = distribution_mode)

    def reward(board):
        if not board.terminal:
            raise RuntimeError(f"reward called on nonterminal board {board}")
        if board.winner is board.turn:
            # It's your turn and you've already won. Should be impossible.
            print(board.tup)
            print(board.winner)
            print(board.turn)
            print(board.is_max)
            print(board.terminal)
            raise RuntimeError(f"reward called on unreachable board {board}")
        if board.winner is None:
            return 0.5  # Board is a tie

        if board.turn is (not board.winner):
            return 0  # Your opponent has just won. Bad.

        # The winner is neither True, False, nor None
        raise RuntimeError(f"board has unknown winner type {board.winner}")

    def is_terminal(board):
        return board.terminal

    def make_move(board, index):
        tup = board.tup[:index] + ((1,) if board.turn else (-1,)) + board.tup[index + 1 :]
        state=states.get(tup)
        if state is not None:
            return state
        turn = not board.turn
        winner = _find_winner(tup)
        is_terminal = (winner is not None) or not any(v == 0 for v in tup)
        ret = TicTacToeBoard(not board.is_max, tup, turn, winner, is_terminal, 0, board.depth + 1)
        states[tup] = ret
        return ret

    def make_move_bvoi(board, index, distribution_mode = "weak_heuristic"):
        tup = board.tup[:index] + ((1,) if board.turn else (-1,)) + board.tup[index + 1 :]
        state=states_cache_bvoi.get(tup)
        if state is not None:
            return state
        turn = not board.turn
        winner = _find_winner(tup)
        is_terminal = (winner is not None) or not any(v == 0 for v in tup)
        to_simulate = TicTacToeBoard(not board.is_max, tup, turn, winner, is_terminal, 0 , board.depth + 1)
        mcts = MCTS(to_simulate)

        if distribution_mode == "sample":
            meanvalue = sample(to_simulate)
        elif distribution_mode == "weak heuristic":
            global weak_heuristic_dict
            if weak_heuristic_dict.get(to_simulate.tup) is not None:
                meanvalue = weak_heuristic_dict[to_simulate.tup][0]
            else:
                flipped_to_simulate = flip_board(to_simulate)
                if weak_heuristic_dict.get(flipped_to_simulate.tup) is not None:
                    meanvalue = 1 - weak_heuristic_dict[flipped_to_simulate.tup][0]
                else:
                    meanvalue = sample(to_simulate)
        elif distribution_mode == 'none':
            meanvalue = 0
        ret = TicTacToeBoard(not board.is_max, tup, turn, winner, is_terminal, meanvalue, board.depth + 1)
        if distribution_mode != 'none':
            states_cache_bvoi[tup] = ret
        return ret

    def make_move_bvoi_NN(board, index):
        tup = board.tup[:index] + ((1,) if board.turn else (-1,)) + board.tup[index + 1 :]
        state=states_cache_bvoi.get(tup)
        if state is not None:
            return state
        turn = not board.turn
        winner = _find_winner(tup)
        is_terminal = (winner is not None) or not any(v == 0 for v in tup)
        if not board.is_max:
            tup2=tup
        else:
            tup2=[-1*i for i in tup]

        X = np.asarray(tup2).astype('float32')
        X = np.reshape(X, (3, 3,))

        meanvalue = alphazero_agent.predict(X)[1][0]# 0 is pi and 1 is v
        if not not board.is_max:
            meanvalue = -meanvalue
        ret = TicTacToeBoard(not board.is_max, tup, turn, winner, is_terminal, meanvalue , board.depth + 1)
        states_cache_bvoi[tup] = ret
        return ret

    def to_pretty_string(board):
        to_char = lambda v: ("X" if v == 1 else ("O" if v == -1 else " "))
        rows = [
            [to_char(board.tup[3 * row + col]) for col in range(3)] for row in range(3)
        ]
        return (
            "\n  1 2 3\n"
            + "\n".join(str(i + 1) + " " + " ".join(row) for i, row in enumerate(rows))
            + "\n"
        )

def flip_board(board):
    tup2 = []
    for i in range(9):
        tup2.append(-board.tup[i])

    board = TicTacToeBoard(not board.is_max, tuple(tup2), not board.turn, board.winner, board.terminal, 0, board.depth)
    return board

def play_game(mode="uct", mode2 ="uct"):
    board = new_tic_tac_toe_board()
    tree = MCTS(board, mode=mode, distribution_mode="weak heuristic")
    tree2 = MCTS(flip_board(board), mode=mode2)
    game=TicTacToeGame()
    #rival=MCTSaz(game,alphazero_agent)
    #prob, v = alphazero_agent.predict(np.asarray(board.tup).astype('float32').reshape((3, 3)))

    print(board.to_pretty_string())
    while True:
        for i in range(70):
            tree.do_rollout(board)
        board = tree.choose(board)
        print(board.to_pretty_string())
        if board.terminal:
            break

#        board = flip_board(board)

        print(board.tup)
        for i in range(150):
            tree2.do_rollout(board)
        board = tree2.choose(board)

        if board.terminal:
  #          board = flip_board(board)
  #          board.is_max = False
  #          board.winner = False
  #          board.turn = True
            break

 #       board = flip_board(board)
        print(board.to_pretty_string())
    return board

def _winning_combos():
    for start in range(0, 9, 3):  # three in a row
        yield (start, start + 1, start + 2)
    for start in range(3):  # three in a column
        yield (start, start + 3, start + 6)
    yield (0, 4, 8)  # down-right diagonal
    yield (2, 4, 6)  # down-left diagonal

def _find_winner(tup):
    "Returns None if no winner, True if X wins, False if O wins"
    for i1, i2, i3 in _winning_combos():
        v1, v2, v3 = tup[i1], tup[i2], tup[i3]
        if -1 == v1 == v2 == v3:
            return False
        if 1 == v1 == v2 == v3:
            return True
    return None

def new_tic_tac_toe_board():
    return TicTacToeBoard(True, (0,) * 9, True, None, False,0, 0)

def simulate(node, dict):
    path = []
    invert_reward = False
    while True:
        path.append((node))
        if node.is_terminal():
            reward = node.reward()
            reward = 1 - reward if invert_reward else reward
            break
        node = node.find_random_child()
                #print(node.is_terminal())
                #print(node.get_legal_moves(1 if node.is_max else -1))
        invert_reward = not invert_reward
    path2 = reversed(path)
    for node in path2:
        n = node.tup
        if dict.get(n) is None:
            dict[n] = (reward, 1)
        else:
            dict[n] = ((dict[n][0]*dict[n][1] + reward)/(dict[n][1] + 1), dict[n][1] + 1)

def simulate_until_no_tomorrow(load = False):

    if load:
        f = open("weak_heuristic", "rb")
        result_dict = pickle.load(f)
        print(result_dict[new_tic_tac_toe_board().tup])
        f.close()
    else:
        result_dict = {}

    init = new_tic_tac_toe_board()
    for _ in range(50000000):
        simulate(init, result_dict)
    f = open("weak_heuristic", "wb")
    pickle.dump(result_dict, f)
    f.close()
if __name__ == "__main__":

    #alphazero_agent = NNetWrapper()
    #alphazero_agent.load_checkpoint('./pretrained_models/alternative', 'best-25eps-25sim-10epch.pth.tar')

    import time

    mcts = MCTS(new_tic_tac_toe_board())
    print(mcts.compute_max_probability([[(-1, 0.5), (3, 0.7), (10, 1.0)],
                                  [(0, 0.2), (2, 0.4), (8, 1.0)],
                                  [(-2, 0.3), (6, 0.6), (9, 1.0)]]))

    f = open("weak_heuristic", "rb")
    weak_heuristic_dict = pickle.load(f)
    f.close()
    #exit(0)
    #start = time.time()
    #simulate_until_no_tomorrow(load = True)
    #print("Finished")
    #print(time.time() - start)
    #exit(0)

    start = time.time()
    sum = 100
    for i in range(0,25):
        fail=0
        board=play_game( mode= "FT Greedy", mode2="uct")
        if board.reward() != 0.5:
            if not board.is_max:
                print("Kawabanga")
            else:
                print("Pikapika")
            print(board.to_pretty_string())
            fail = 1
        sum=sum-fail

    print("We got ", sum)
    print("tie out of 5")
    end = time.time()
    print("Time:", start-end)


#!/usr/bin/env python3
from ..lib import config
from ..lib.store import Store
import os.path
import re

CONSTANTS = {
    -128: "INT8_MIN",
    128: "INT8_MAX",
    255: "UINT8_MAX",
    -32768: "INT16_MIN",
    32767: "INT16_MAX",
    65535: "UINT16_MAX",
    -2147483648: "INT32_MIN",
    2147483647: "INT32_MAX",
    4294967295: "UINT32_MAX",
    -9223372036854775808: "INT64_MIN",
    9223372036854775807: "INT64_MAX",
}

def parse_constant_type(k):
    left, _ = k.split("_")
    return left

def humanize_num(n):
    return CONSTANTS.get(n, str(n))

def humanize_ival(p):
    if p[0] == p[1]:
        return humanize_num(p[0])
    else:
        return "[%s,%s]" % (humanize_num(p[0]), humanize_num(p[1]))

def humanize_range(r):
    if len(r) == 1:
        return "== " + humanize_ival(r[0])
    if len(r) == 2:
        ((lo1, hi1), (lo2, hi2)) = r
        if hi1 + 2 == lo2 and lo1 in CONSTANTS and hi2 in CONSTANTS:
            if parse_constant_type(CONSTANTS[lo1]) == parse_constant_type(CONSTANTS[hi2]):
              return "!= " + str(hi1 + 1)

    return "in {" + ", ".join(humanize_ival(p) for p in r) + "}"

def print_line(enc_filename):
    try:
        orig_fname, fname, lineno = enc_filename.split(":")
        lineno = int(lineno)
    except ValueError:
        return
    if not os.path.exists(orig_fname):
        return
    expr = re.compile('# ([0-9]+) "([^"]+)".*')
    with open(orig_fname) as fp:
        counter = 1
        curr_line = 1
        curr_fname = os.path.basename(orig_fname)
        for line in fp:
            res = expr.match(line)
            if res is not None:
                curr_fname = res.group(2)
                curr_line = int(res.group(1))
                counter += 1
                continue

            if curr_fname == fname and lineno == curr_line:
                return f"{orig_fname}:{counter}: {line}"

            curr_line += 1
            counter += 1
    return

class BugReport:
    def __init__(self, score, code, key, ctx, references=None):
        self.key = key
        self.ctx = ctx
        self.score = score
        self.code = code
        self.references = references

    def get_references(self, size):
        if len(self.references) == 1 or size == 1:
            refs = "{" + self.references.pop() + "}"
        else:
            i = 0
            refs = "{"
            for x in self.references:
                if i >= size:
                    break
                elif i == len(self.references) - 1 or i == size - 1:
                    refs += x
                else:
                    refs = refs + x + ", "
                i += 1
            refs += "}"
        return refs

    def __repr__(self):
        if self.references is None:
            return "BugReport(score=%.02f, code=%s, key=%s, ctx=%s)" % (
                self.score, self.code, self.key, self.ctx
            )
        else:
            return "BugReport(score=%.02f, code=%s, key=%s, ctx=%s, reference=%s)" % (
                self.score, self.code, self.key, self.ctx,
                self.get_references(self.ctx.config.reference)
            )

    def __str__(self):
        refs = self.get_references(self.ctx.config.reference) if self.references is not None else ""
        ctx = humanize_range(self.ctx)
        line = print_line(self.code)
        line = "\n" + line if line is not None else ""
        return f"{self.score:.2%} {self.code} '{self.key}' {ctx} {refs}{line}"

class Context:
    def __init__(self, config):
        self.total_uses = Store(level=1)
        self.ctx_uses = Store(level=2)
        self.config = config

    def add(self, key, value, code):
        if value is not None:
            self.ctx_uses[key][value].add(code)
        self.total_uses[key].add(code)

    def merge(self, other):
        self.total_uses.merge(other.total_uses)
        self.ctx_uses.merge(other.ctx_uses)

    def get_bugs(self):
        added = set()
        bugs = []
        for key, value in self.ctx_uses.items():
            total = self.total_uses[key]
            for ctx, codes in value.items():
                score = len(codes) / len(total)
                if score >= self.config.threshold and score != 1:
                    diff = total - codes
                    for bug in diff:
                        br = BugReport(score, bug, key, ctx)
                        added.add(bug)
                        bugs.append(br)
        return bugs

class Checker:
    def __init__(self, config):
        self.config = config

    def _initialize_process(self):
        # optional
        pass

    def _finalize_process(self):
        raise NotImplementedError

    def _process_path(self, path):
        raise NotImplementedError

    def process(self, tree):
        self._initialize_process()
        for path in tree:
            self._process_path(path)
        return self._finalize_process()


from keras.datasets import imdb
from keras import preprocessing

max_features = 10000    # Maximum feature to
maxlen = 20             # Will limit the review to max these words

(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features) # This loads integer data
print("x_train shape: ", x_train.shape)
print("sample train data:", x_train[0])
print("Sample train label", y_train[0]) # 0 -> Negative Review, 1 -> Positive Review

# Restricting maximum length to maxlen for each review
x_train = preprocessing.sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = preprocessing.sequence.pad_sequences(x_test, maxlen=maxlen)
print("After limiting x_train shape: ", x_train.shape)

# Build the Embedding Layer and Classifier Layer
from keras.models import Sequential
from keras import layers

model = Sequential()

# The Embedding layer is best understood as a dictionary that maps integer indices
# (which stand for specific words) to dense vectors. It takes integers as input, it looks up
# these integers in an internal dictionary, and it returns the associated vectors. It’s effec-
# tively a dictionary lookup
max_tokens = max_features
embedding_dim = 8

# Add the Embedding layer
# THe output of Embedding layer is 3D tensor of shape (num_samples, max_word_length, vector_dim)
model.add(layers.Embedding(max_tokens, embedding_dim, input_length=maxlen))

model.add(layers.Flatten()) # Here we are loosing the sequence

# Add classifier layer
model.add(layers.Dense(1, activation='sigmoid'))

model.compile(optimizer='rmsprop', loss= 'binary_crossentropy', metrics=['acc'])

model.fit(x_train, y_train, epochs= 5, batch_size= 32, validation_split= 0.2)


import sys
sys.path.insert(0, '../../')
import numpy as np
from jax.experimental import optimizers
import matplotlib.pyplot as plt
import time
from sde_gp import SDEGP
import approximate_inference as approx_inf
import priors
import likelihoods
from utils import softplus_list, plot
from sklearn.preprocessing import StandardScaler
import tikzplotlib

plot_intermediate = False
save_tikz = False

print('loading data ...')
D = np.loadtxt('mcycle.csv', delimiter=',')
X = D[:, 1:2]
Y = D[:, 2:]
N = X.shape[0]

# Standardize
X_scaler = StandardScaler().fit(X)
y_scaler = StandardScaler().fit(Y)
Xall = X_scaler.transform(X)
Yall = y_scaler.transform(Y)
xshift = np.min(X) - np.min(Xall)
xrange = (np.max(X) - np.min(X)) / (np.max(Xall) - np.min(Xall))

# Set test data
XT = np.linspace(np.min(Xall), np.max(Xall), num=200)

var_f1 = 3.  # GP variance
len_f1 = 1.  # GP lengthscale
var_f2 = 3.  # GP variance
len_f2 = 1.  # GP lengthscale

prior1 = priors.Matern32(variance=var_f1, lengthscale=len_f1)
prior2 = priors.Matern32(variance=var_f2, lengthscale=len_f2)
prior = priors.Independent([prior1, prior2])
lik = likelihoods.HeteroscedasticNoise()

# inf_method = approx_inf.ExpectationPropagation(power=0.9, intmethod='UT', damping=0.1)
inf_method = approx_inf.ExpectationPropagation(power=0.1, intmethod='GH', damping=0.5)
# inf_method = approx_inf.VariationalInference(intmethod='GH', damping=0.5)
# inf_method = approx_inf.VariationalInference(intmethod='UT', damping=0.5)
# inf_method = approx_inf.ExtendedEP(power=0, damping=0.5)
# inf_method = approx_inf.ExtendedKalmanSmoother(damping=0.5)
# inf_method = approx_inf.GaussHermiteKalmanSmoother(damping=0.5)
# inf_method = approx_inf.StatisticallyLinearisedEP(intmethod='UT', damping=0.5)
# inf_method = approx_inf.UnscentedKalmanSmoother(damping=0.5)

model = SDEGP(prior=prior, likelihood=lik, t=Xall, y=Yall, t_test=XT, approx_inf=inf_method)

opt_init, opt_update, get_params = optimizers.adam(step_size=5e-2)
# parameters should be a 2-element list [param_prior, param_likelihood]
opt_state = opt_init([model.prior.hyp, model.likelihood.hyp])

def gradient_step(i, state, mod):
    params = get_params(state)
    mod.prior.hyp = params[0]
    mod.likelihood.hyp = params[1]

    # grad(Filter) + Smoother:
    neg_log_marg_lik, gradients = mod.run()
    # neg_log_marg_lik, gradients = mod.run_two_stage()  # <-- less elegant but reduces compile time

    prior_params = softplus_list(params[0])
    print('iter %2d: var_f1=%1.2f len_f1=%1.2f var_f2=%1.2f len_f2=%1.2f, nlml=%2.2f' %
          (i, prior_params[0][0], prior_params[0][1], prior_params[1][0], prior_params[1][1], neg_log_marg_lik))

    if plot_intermediate:
        plot(mod, i)

    return opt_update(i, gradients, state)

print('optimising the hyperparameters ...')
t0 = time.time()
for j in range(250):
    opt_state = gradient_step(j, opt_state, model)
t1 = time.time()
print('optimisation time: %2.2f secs' % (t1-t0))

# calculate posterior predictive distribution via filtering and smoothing at train & test locations:
print('calculating the posterior predictive distribution ...')
t0 = time.time()
posterior_mean, posterior_var, _, nlpd = model.predict(compute_nlpd=False)
t1 = time.time()
print('prediction time: %2.2f secs' % (t1-t0))

# x_pred = xrange * (model.t_all[:, 0] + xshift)
x_pred = X_scaler.inverse_transform(model.t_all[:, 0])
# X_rescale = xrange * (X + xshift)
link = model.likelihood.link_fn
lb = posterior_mean[:, 0, 0] - np.sqrt(posterior_var[:, 0, 0] + link(posterior_mean[:, 1, 0]) ** 2) * 1.96
ub = posterior_mean[:, 0, 0] + np.sqrt(posterior_var[:, 0, 0] + link(posterior_mean[:, 1, 0]) ** 2) * 1.96
post_mean = y_scaler.inverse_transform(posterior_mean[:, 0, 0])
lb = y_scaler.inverse_transform(lb)
ub = y_scaler.inverse_transform(ub)

print('plotting ...')
fig, ax = plt.subplots()
plt.plot(X, Y, 'k.', markersize=0.8)
plt.plot(x_pred, post_mean, 'k', linewidth=0.8)
plt.fill_between(x_pred, lb, ub, color='k', alpha=0.1, edgecolor=(0, 0, 0, 0.12))
plt.xlim(x_pred[0], x_pred[-1])
plt.xlabel('Time (milliseconds)')
plt.ylabel('Accelerometer reading')
ax.tick_params(axis="y", direction="in")
ax.tick_params(axis="x", direction="in")
ax.tick_params(top=True, right=True)
if save_tikz:
    tikzplotlib.save('/Users/wilkinw1/postdoc/inprogress/linearised-inference/paper/fig/mcycle.tex',
                     axis_width='\\figurewidth',
                     axis_height='\\figureheight',
                     tex_relative_path_to_data='./fig/')

lb_1 = posterior_mean[:, 0, 0] - np.sqrt(posterior_var[:, 0, 0]) * 1.96
ub_1 = posterior_mean[:, 0, 0] + np.sqrt(posterior_var[:, 0, 0]) * 1.96
lb_1 = y_scaler.inverse_transform(lb_1)
ub_1 = y_scaler.inverse_transform(ub_1)

fig2, ax2 = plt.subplots()
plt.plot(x_pred, post_mean, 'k', linewidth=0.8, label='posterior mean')
plt.fill_between(x_pred, lb_1, ub_1, color='k', alpha=0.1, edgecolor=(0, 0, 0, 0.12), label='95\% confidence')
plt.plot(X, Y, 'k.', markersize=0.8, label='observations')
plt.xlim(x_pred[0], x_pred[-1])
plt.title('EP: $f^{(1)}$ (mean component)')
# plt.xlabel('Time (milliseconds)')
# plt.ylabel('Accelerometer reading')
# plt.legend(['observations', 'posterior mean', '95% confidence'])
plt.legend(loc=4)
ax2.tick_params(axis="y", direction="in")
ax2.tick_params(axis="x", direction="in")
ax2.tick_params(top=True, right=True)
if save_tikz:
    tikzplotlib.save('/Users/wilkinw1/postdoc/inprogress/linearised-inference/paper/fig/mcycle_f1.tex',
                     axis_width='\\figurewidth',
                     axis_height='\\figureheight',
                     tex_relative_path_to_data='./fig/')

mean_f2 = link(posterior_mean[:, 1, 0])
mean_f2 = y_scaler.scale_ * mean_f2
lb_2 = posterior_mean[:, 1, 0] - np.sqrt(posterior_var[:, 1, 1]) * 1.96
ub_2 = posterior_mean[:, 1, 0] + np.sqrt(posterior_var[:, 1, 1]) * 1.96
lb_2 = y_scaler.scale_ * link(lb_2)
ub_2 = y_scaler.scale_ * link(ub_2)

fig3, ax3 = plt.subplots()
# plt.plot(X, Y, 'k.', markersize=0.8)
plt.plot(x_pred, mean_f2, 'k', linewidth=0.8)
plt.fill_between(x_pred, lb_2, ub_2, color='k', alpha=0.1, edgecolor=(0, 0, 0, 0.12))
plt.xlim(x_pred[0], x_pred[-1])
plt.title('EP: $\phi(f^{(2)})$ (noise standard deviation component)')
# plt.xlabel('Time (milliseconds)')
# plt.ylabel('Accelerometer reading')
ax3.tick_params(axis="y", direction="in")
ax3.tick_params(axis="x", direction="in")
ax3.tick_params(top=True, right=True)
if save_tikz:
    tikzplotlib.save('/Users/wilkinw1/postdoc/inprogress/linearised-inference/paper/fig/mcycle_f2.tex',
                     axis_width='\\figurewidth',
                     axis_height='\\figureheight',
                     tex_relative_path_to_data='./fig/')

fig4, ax4 = plt.subplots()
plt.plot(X, Y, 'k.', markersize=0.8)
plt.plot(x_pred, post_mean, 'k', linewidth=0.8)
plt.fill_between(x_pred, lb, ub, color='k', alpha=0.1, edgecolor=(0, 0, 0, 0.12))
plt.fill_between(x_pred, lb_1, ub_1, color='white', alpha=0.85, edgecolor=(0, 0, 0, 0.12))
plt.xlim(x_pred[0], x_pred[-1])
plt.title('EP: full posterior')
plt.xlabel('Time (milliseconds)')
# plt.ylabel('Accelerometer reading')
ax4.tick_params(axis="y", direction="in")
ax4.tick_params(axis="x", direction="in")
ax4.tick_params(top=True, right=True)
if save_tikz:
    tikzplotlib.save('/Users/wilkinw1/postdoc/inprogress/linearised-inference/paper/fig/mcycle_combined.tex',
                     axis_width='\\figurewidth',
                     axis_height='\\figureheight',
                     tex_relative_path_to_data='./fig/')

plt.show()


import torch.nn.functional as F
from smodels.model_resnet import getmodel_byname
from torchvision import transforms, utils
from Params import Params
import os
from STrain import Trainer
from SDataLoad import DataLoad
from torch.utils.data import DataLoader
import pandas as pd
import numpy as np
import torch
import sys
import torch.nn as nn
from datasets import MXFaceDataset

def check_resume(save_dir):
    saved_models = os.listdir(save_dir)
    try:
        i = saved_models.index("train_results.pkl")
        del saved_models[i]
    except:
        pass

    if len(saved_models) > 0:
        saved_model_ints = [int(x.split("_")[2].split(".")[0]) for x in saved_models]
        saved_model_ints.sort(reverse=True)
        resume_checkpoint = save_dir + "/model_epoch_" + str(saved_model_ints[0]) + ".pth"
        return True, resume_checkpoint
    else:
        return False, None

def main():

    torch.manual_seed(1984)
    Param = Params[1]

    checkpoint = dict()

    # -- Defining Transforms
    INPUT_SIZE = [112, 112] # support: [112, 112] and [224, 224]
    RGB_MEAN = [0.5, 0.5, 0.5] # for normalize inputs to [-1, 1]
    RGB_STD = [128./255., 128./255., 128./255.]
    train_transforms = transforms.Compose([ # refer to https://pytorch.org/docs/stable/torchvision/transforms.html for more build-in online data augmentation
        transforms.RandomHorizontalFlip(),
        transforms.ToTensor(),
        transforms.Normalize(mean = RGB_MEAN,
                             std = RGB_STD),
    ])

    val_transforms = transforms.Compose([
            transforms.CenterCrop(INPUT_SIZE),
            transforms.ToTensor(),
            transforms.Normalize(mean = RGB_MEAN,
                             std = RGB_STD),
    ])

    # -- recfile dataloader
    data_loader_rec_san = MXFaceDataset(Param["data_root"], train_transforms)
    loader = DataLoader(data_loader_rec_san, batch_size=Param["batch_size"], shuffle=True, num_workers=32, pin_memory=True)

    checkpoint["train_trans"] = train_transforms
    checkpoint["val_trans"] = val_transforms
    checkpoint['num_classes'] = data_loader_rec_san.num_classes

    # --- Defining Model --- #
    model = getmodel_byname(Param['model_name'])
    checkpoint["model_name"] = Param['model_name']

    # -- Loading another old model as initial weights
    #old_checkpoint2 = torch.load("/vol/vssp/cvpnobackup/facer2vm/people/safwan/saved_models/EP8s/9-EP8/model_epoch_1.pth")
    #model.load_state_dict(old_checkpoint2["weight"])

    checkpoint["Param"] = Param

    # -- checking if resuming is required
    save_dir = os.path.join(Param['save_dir'], Param['exp_id'])
    resume_flag = False
    if Param['save_dir'] is not None:
        if os.path.isdir(save_dir):
            resume_flag, resume_checkpoint_path = check_resume(save_dir)

    if resume_flag:
        resume_checkpoint = torch.load(resume_checkpoint_path)
        print("Resuming from checkpoint : " + resume_checkpoint_path)
        #model = nn.DataParallel(model)
        #model.load_state_dict(resume_checkpoint["weight"])
        checkpoint = resume_checkpoint

    # ------- Creating and calling the trainer object -- #
    trainer = Trainer(model, Param, F.cross_entropy, checkpoint=checkpoint, save_dir=save_dir, save_freq=2, resume_flag=resume_flag)
    trainer.loop(Param['epochs'], loader)

    # -- For visualizing a batch
    # for data in train_loader:
    #     imshow(data[0])

if __name__ == '__main__': # if running as a script not being loaded as a module

    main()


def quick_sort(arreglo):
    if len(arreglo) <= 1:
        return arreglo
    else:
        pivote = arreglo[0]
        menores = [x for x in arreglo[1:] if x <= pivote]
        mayores = [x for x in arreglo[1:] if x > pivote]
        return quick_sort(menores) + [pivote] + quick_sort(mayores)

lista = [int(x) for x in input().split()]
print(quick_sort(lista))


from pandas import DataFrame
from pandas import concat
from sklearn.preprocessing import MinMaxScaler
from keras.models import Sequential, model_from_json
from keras import layers as N
from keras import losses
from time import sleep
import pymongo

client = pymongo.MongoClient('localhost', 27017)
db = client.bmexdata
trade_collection = db.candles

# convert series to supervised learning
def series_to_supervised(data, n_in=1, n_out=1, dropnan=True):
    n_vars = 1 if type(data) is list else data.shape[1]
    df = DataFrame(data)
    cols, names = list(), list()
    # input sequence (t-n, ... t-1)
    for i in range(n_in, 0, -1):
        cols.append(df.shift(i))
        names += [('var%d(t-%d)' % (j + 1, i)) for j in range(n_vars)]
    # forecast sequence (t, t+1, ... t+n)
    for i in range(0, n_out):
        cols.append(df.shift(-i))
        if i == 0:
            names += [('var%d(t)' % (j + 1)) for j in range(n_vars)]
        else:
            names += [('var%d(t+%d)' % (j + 1, i)) for j in range(n_vars)]
    # put it all together
    agg = concat(cols, axis=1)
    agg.columns = names
    # drop rows with NaN values
    if dropnan:
        agg.dropna(inplace=True)
    return agg

n = 420
price_scale_ratio = 10000
volume_scale_ratio = 1000000
hist = DataFrame(list(trade_collection.find().skip(trade_collection.count() - n)))
hist = hist.set_index('_id')
hist = hist.drop(['ot', 'ct'], axis=1)
hist['o'] = hist['o'] / price_scale_ratio
hist['h'] = hist['h'] / price_scale_ratio
hist['l'] = hist['l'] / price_scale_ratio
hist['c'] = hist['c'] / price_scale_ratio
hist['bv'] = hist['bv'] / volume_scale_ratio
hist['sv'] = hist['sv'] / volume_scale_ratio

# load dataset
dataset = hist
# print(dataset)
# sleep(10)
values = dataset.values
# print(values)
# # integer encode direction
# encoder = LabelEncoder()
# values[:, 4] = encoder.fit_transform(values[:, 4])

# ensure all data is float
values = values.astype('float32')

# # normalize features
# scaler = MinMaxScaler(feature_range=(0, 1))
# scaled = scaler.fit_transform(values)
# # print(scaled)

# specify the number of lag hours
input_sample = 120
output_shift = 60
n_features = 6
# frame as supervised learning
reframed = series_to_supervised(values, input_sample, output_shift)
print(reframed)
reframed.loc[reframed['var2(t-1)'] < reframed['var2(t+%s)' % (output_shift - 1)], 'direction'] = 1
reframed.loc[reframed['var2(t-1)'] == reframed['var2(t+%s)' % (output_shift - 1)], 'direction'] = 0
reframed.loc[reframed['var2(t-1)'] > reframed['var2(t+%s)' % (output_shift - 1)], 'direction'] = -1
print(reframed)
sleep(5)
# split into train and test sets
values = reframed.values
n_test = 0
n_train = reframed.shape[0] - n_test
train = values[:n_train, :]
test = values[n_train:, :]

# split into input and outputs
target_col = -1
n_obs = input_sample * n_features
train_X, train_y = train[:, :n_obs], train[:, target_col]
test_X, test_y = test[:, :n_obs], test[:, target_col]

# reshape input to be 3D [samples, timesteps, features]
train_X = train_X.reshape((train_X.shape[0], input_sample, n_features))
test_X = test_X.reshape((test_X.shape[0], input_sample, n_features))
print('Train', train.shape)
print(train)
print('Train X', train_X.shape)
print(train_X)
print('Train y', train_y.shape)
print(train_y)
print('Test', test.shape)
print(test)
print('Test X', test_X.shape)
print(test_X)
print('Test y', test_y.shape)
print(test_y)
sleep(10)

# # design network
# neurons = 500
# model = Sequential()
# model.add(N.LSTM(neurons, input_shape=(train_X.shape[1], train_X.shape[2]), return_sequences=True))
# model.add(N.LSTM(5, return_sequences=True))
# model.add(N.LSTM(neurons, return_sequences=True))
# model.add(N.LSTM(5))
# model.add(N.Dense(1))
# model.compile(loss=losses.logcosh, optimizer='adam')

# load json and create model
json_file = open('ai_model.json', 'r')
loaded_model_json = json_file.read()
json_file.close()
model = model_from_json(loaded_model_json)
# load weights into new model
model.load_weights("ai_model.h5")
print("Loaded model from disk")
model.compile(loss=losses.logcosh, optimizer='adam')

# fit network
# history = model.fit(train_X, train_y, epochs=20, batch_size=30, validation_data=(test_X, test_y), verbose=1,
history = model.fit(train_X, train_y, epochs=50, batch_size=30, verbose=1, validation_data=(test_X, test_y),
                    shuffle=True)

# evaluate the model
# scores = model.evaluate(train_X, train_y, verbose=0)
# print("%s: %.2f%%" % (model.metrics_names[1], scores[1]*100))

# serialize model to JSON
model_json = model.to_json()
with open("ai_model.json", "w") as json_file:
    json_file.write(model_json)
# serialize weights to HDF5
model.save_weights("ai_model.h5")
print("Saved model to disk")


import A2
import numpy as np
import math
import copy
import matplotlib.pyplot as plt

def create_random_pol(deg=6):
	p = A2.Polynom(deg)
	for i in range(deg+1):
		p[i] = np.random.uniform(-3, 3)
	return p

def find_smallest_error(n=1000):
	f_pairs = A2.create_interval_pairs(A2.f, (-10, 10), 0.02)
	min_err = math.inf
	min_p = None
	for i in range(1000):
		p = create_random_pol()
		p_pairs = A2.create_interval_pairs(p, (-10, 10), 0.02)
		err = A2.mean_absolute_error(f_pairs, p_pairs)
		if err < min_err:
			min_err = err
			min_p = copy.deepcopy(p)

	return (min_p, min_err)

if __name__ == "__main__":
	min_p, err = find_smallest_error()
	print("Koeffizienten: a0={}, a1={}, a2={}, a3={}, a4={}, a5={}, a6={}\nFehlerwert:{}".format(min_p[0],min_p[1], min_p[2], min_p[3], min_p[4], min_p[5], min_p[6], err))

	min_p_pairs = A2.create_interval_pairs(min_p, (-10, 10), 0.02)
	p_x_values = [x for (x, y) in min_p_pairs]
	p_y_values = [y for (x, y) in min_p_pairs]

	plt.plot(p_x_values, p_y_values, "r-", label = "g(x)")
	plt.legend()
	plt.show()


import random
import secrets
import socket
import sys
import time
from pathlib import Path

import sympy

import sys
import util

from sympy import primitive_root
from sympy.ntheory import isprime
signedCert = 0
util.S = "The Server"
util.sState = 0
plaintextFile = 'serverFile.txt'
def register_client(m, client_map):
    # m = (      I,         0 - s,    1-v, 2-A, 3-b,4-Ks,5-B)
    client_map[m[0]] = [int(m[1]), int(m[2]), -1, -1, -1, -1]

def ack_client(m,client_map):
    # m = (I, A)
    m[1] = util.decipher_rsa(int(m[1]),(util.N, util.rsa_d))
    s = client_map[m[0]][0]
    v = client_map[m[0]][1]
    client_map[m[0]][2] = m[1]
    b = util.getCryptoRand()
    client_map[m[0]][3] = b  # set b for later
    B = util.k*v + pow(util.g,b,util.N)
    client_map[m[0]][5] = B  # set B for later
    return util.numbersToByteArr([s, B])

def auth_successful(m1, clientMap):
    for key in clientMap:
        m = clientMap[key]
        return util.numbersToByteArr([util.H([m[2], int(m1[0]), m[4]])])
    return None

def check_challenge(m1, clientMap):
    # m1 - client's response, assuming just 1 client and called in right order
    for key in clientMap:
        m = clientMap[key]
        A = m[2]
        v = m[1]
        b = m[3]
        B = m[5]
        u = util.H([A, B]) % util.N
        Kserv = pow(A*(pow(v, u, util.N)), b, util.N)
        m[4] = Kserv
        m0 = util.H([A, B, Kserv])
        return int(m1[0])==m0
    return None

def getPlainText(client_msg, clientMap):
    data = client_msg[0]
    iv = data[:16]
    ctxt = data[16:]
    Kserv = -1
    for key in clientMap:
        m = clientMap[key]
        Kserv = m[4]
    return util.decryptFile(iv, ctxt, Kserv)

def start(client_map):
    # Create a TCP/IP socket
    global signedCert
    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

    # Bind the socket to the port
    server_address = (util.server_address, util.server_listen_port)
    print('starting up on {} port {}'.format(*server_address))
    sock.bind(server_address)

    # Listen for incoming connections
    while True:
        sock.listen(1)
        while True:
            # Wait for a connection
            print('waiting for a connection')
            connection, client_address = sock.accept()
            client_msg = bytearray()

            print('connection from', client_address)

            # Receive the data in small chunks and retransmit it
            while True:
                data = connection.recv(1024)
                print('server: received {!r}'.format(data))
                client_msg = util.bytesToStringArr(data) if util.sState!=4 else [data]
                if(len(client_msg)==3 and util.sState == 1):
                    print('server: registering client')
                    register_client(client_msg, client_map)
                    connection.sendall(util.numbersToByteArr(["registered"]))
                    util.sState = 2
                elif(len(client_msg)==2 and util.sState == 3):
                    print('server: acknowledging client')
                    connection.sendall(ack_client(client_msg, client_map))
                elif (len(client_msg) == 1 and client_msg[0]==util.getN and util.sState == 0):
                    print('server: sending initial parameters')
                    connection.sendall(util.numbersToByteArr([util.q, util.N, util.g, util.k, util.nL]))
                    util.sState = 1
                elif (len(client_msg) == 1 and util.sState == 2):
                    print('server: sending signed certificate')
                    #server receives len(I)||Ibytes
                    connection.sendall(signedCert)
                    util.sState = 3
                elif (len(client_msg) == 1 and client_msg[0]!='' and util.sState==3):
                    print('server: checking challenge')
                    if(check_challenge(client_msg,client_map)):
                        print('server: passed challenge, sending confirmation')
                        connection.sendall(auth_successful(client_msg, client_map))
                        util.sState = 4
                    else:
                        print('server: failed challenge')
                elif (len(client_msg) == 1 and util.sState == 4):
                    print('server: decrypting message')
                    pt = getPlainText(client_msg, client_map)[2:-1]
                    pt = pt.replace("\\\\", "\\").replace("\\n", "").replace("\\r", "\r").replace("\\\\'","\'").replace("\\\\","\\")
                    f = open(plaintextFile, 'w')
                    print('server: writing to file')
                    f.write(pt)
                    f.close()
                    util.sState = 0
                    connection.sendall(util.numbersToByteArr(["OK"]))
                else:
                    break
                print('server: done')
                #print('no data from', client_address)

def init():
    #print("Setting up parameters, this usually takes under 30 seconds.")
    #while not (util.isprime(util.N)):
    #        util.q = sympy.randprime(util.lower, util.upper)
    #        util.q = debugQ # TO-DO remove this line
    #        util.N = 2*util.q + 1

    #print(util.q)
    #util.g = primitive_root(util.N)
    #util.k = util.H([util.N, util.g])
    #util.nL = util.getBitLength(util.N)
    #util.getSafePrime()
    while True:
        try:
            getCert()
            break
        except ConnectionRefusedError:
            print("Server: Waiting for TTP Cert, will try again.")
            time.sleep(1)

    print("done")

sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
def getCert():
    global signedCert
    server_address = (util.TTP_address, util.TTP_listen_port )
    print('connecting to {} port {}'.format(*server_address))
    sock.connect(server_address)
    TTP_N_TTP_SIG=send(send([bytes(util.signRequest.encode('utf-8'))], f0), f1)
    TTP_SIG = TTP_N_TTP_SIG[128:]
    ba = bytearray(str(util.S).encode('utf-8'))
    l = len(ba)
    l = l.to_bytes(4, byteorder='big')
    signedCert = l+bytes(util.S.encode('utf-8'))+util.rsa_e.to_bytes(128, byteorder='big')+TTP_SIG

def send(pa, fa):
    return util.usend(pa, fa, sock, "server", util.server_listen_port, "")

def f0(pa): # send cert to be signed
    ba = bytearray(str(util.S).encode('utf-8'))
    l = len(ba)
    l = l.to_bytes(4, byteorder='big')
    if("".join(pa[0].decode("utf-8"))=="OK"):return [l+bytes(util.S.encode('utf-8'))+util.rsa_e.to_bytes(128, byteorder='big')]
    else:
        print("Something went wrong with TTP")
        exit(1)

def f1(pa):return pa[0]

def getServerInfo():
    global plaintextFile
    if __debug__:
        print('Debug OFF')
        util.S = str(input("Please enter server name: "))
        plaintextFile = str(input("Please enter file name: "))
    else:
        print('Debug ON')
        plaintextFile = 'serverFile.txt'
        util.S = "The Server"

def driver():
    getServerInfo()
    util.RSA_keyGeneration(5)
    init()
    client_map = {}
    start(client_map)

driver()


# Time Conversion
# Convert AM/PM time to 24hr
# Created for Hackerrank
# 2016-04-18

import sys

time = input().strip()

if (time[-2] == 'a' or time[-2] == 'A'):
	time = time[0:-2]
	if time[0:2] == '12':
		timesplit = time.split(':')
		timesplit[0] = '00'
		time = ':'.join(timesplit)
else:
	if time[0:2] == '12':
		time = time[0:-2]
	else:
		time = time[0:-2]
		timesplit = time.split(':')
		timesplit[0] = str(int(timesplit[0]) + 12)
		time = ':'.join(timesplit)

print(time)


# uncompyle6 version 3.7.4
# Python bytecode 3.6 (3379)
# Decompiled from: Python 3.6.9 (default, Apr 18 2020, 01:56:04)
# [GCC 8.4.0]
# Embedded file name: /Users/sanhehu/Documents/GitHub/troposphere_mate-project/troposphere_mate/dynamodb.py
# Compiled at: 2020-02-12 18:15:55
# Size of source mod 2**32: 8579 bytes
"""
This code is auto generated from troposphere_mate.code_generator.__init__.py scripts.
"""
import sys
if sys.version_info.major >= 3:
    if sys.version_info.minor >= 5:
        from typing import Union, List, Any
import troposphere.dynamodb
from troposphere.dynamodb import AttributeDefinition as _AttributeDefinition, GlobalSecondaryIndex as _GlobalSecondaryIndex, KeySchema as _KeySchema, LocalSecondaryIndex as _LocalSecondaryIndex, PointInTimeRecoverySpecification as _PointInTimeRecoverySpecification, Projection as _Projection, ProvisionedThroughput as _ProvisionedThroughput, SSESpecification as _SSESpecification, StreamSpecification as _StreamSpecification, Tags as _Tags, TimeToLiveSpecification as _TimeToLiveSpecification
from troposphere import Template, AWSHelperFn
from troposphere_mate.core.mate import preprocess_init_kwargs, Mixin
from troposphere_mate.core.sentiel import REQUIRED, NOTHING

class AttributeDefinition(troposphere.dynamodb.AttributeDefinition, Mixin):

    def __init__(self, title=None, AttributeName=REQUIRED, AttributeType=REQUIRED, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         AttributeName=AttributeName,
         AttributeType=AttributeType, **kwargs)
        (super(AttributeDefinition, self).__init__)(**processed_kwargs)

class KeySchema(troposphere.dynamodb.KeySchema, Mixin):

    def __init__(self, title=None, AttributeName=REQUIRED, KeyType=REQUIRED, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         AttributeName=AttributeName,
         KeyType=KeyType, **kwargs)
        (super(KeySchema, self).__init__)(**processed_kwargs)

class ProvisionedThroughput(troposphere.dynamodb.ProvisionedThroughput, Mixin):

    def __init__(self, title=None, ReadCapacityUnits=REQUIRED, WriteCapacityUnits=REQUIRED, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         ReadCapacityUnits=ReadCapacityUnits,
         WriteCapacityUnits=WriteCapacityUnits, **kwargs)
        (super(ProvisionedThroughput, self).__init__)(**processed_kwargs)

class Projection(troposphere.dynamodb.Projection, Mixin):

    def __init__(self, title=None, NonKeyAttributes=NOTHING, ProjectionType=NOTHING, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         NonKeyAttributes=NonKeyAttributes,
         ProjectionType=ProjectionType, **kwargs)
        (super(Projection, self).__init__)(**processed_kwargs)

class SSESpecification(troposphere.dynamodb.SSESpecification, Mixin):

    def __init__(self, title=None, SSEEnabled=REQUIRED, KMSMasterKeyId=NOTHING, SSEType=NOTHING, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         SSEEnabled=SSEEnabled,
         KMSMasterKeyId=KMSMasterKeyId,
         SSEType=SSEType, **kwargs)
        (super(SSESpecification, self).__init__)(**processed_kwargs)

class GlobalSecondaryIndex(troposphere.dynamodb.GlobalSecondaryIndex, Mixin):

    def __init__(self, title=None, IndexName=REQUIRED, KeySchema=REQUIRED, Projection=REQUIRED, ProvisionedThroughput=NOTHING, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         IndexName=IndexName,
         KeySchema=KeySchema,
         Projection=Projection,
         ProvisionedThroughput=ProvisionedThroughput, **kwargs)
        (super(GlobalSecondaryIndex, self).__init__)(**processed_kwargs)

class LocalSecondaryIndex(troposphere.dynamodb.LocalSecondaryIndex, Mixin):

    def __init__(self, title=None, IndexName=REQUIRED, KeySchema=REQUIRED, Projection=REQUIRED, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         IndexName=IndexName,
         KeySchema=KeySchema,
         Projection=Projection, **kwargs)
        (super(LocalSecondaryIndex, self).__init__)(**processed_kwargs)

class PointInTimeRecoverySpecification(troposphere.dynamodb.PointInTimeRecoverySpecification, Mixin):

    def __init__(self, title=None, PointInTimeRecoveryEnabled=NOTHING, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         PointInTimeRecoveryEnabled=PointInTimeRecoveryEnabled, **kwargs)
        (super(PointInTimeRecoverySpecification, self).__init__)(**processed_kwargs)

class StreamSpecification(troposphere.dynamodb.StreamSpecification, Mixin):

    def __init__(self, title=None, StreamViewType=REQUIRED, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         StreamViewType=StreamViewType, **kwargs)
        (super(StreamSpecification, self).__init__)(**processed_kwargs)

class TimeToLiveSpecification(troposphere.dynamodb.TimeToLiveSpecification, Mixin):

    def __init__(self, title=None, AttributeName=REQUIRED, Enabled=REQUIRED, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         AttributeName=AttributeName,
         Enabled=Enabled, **kwargs)
        (super(TimeToLiveSpecification, self).__init__)(**processed_kwargs)

class Table(troposphere.dynamodb.Table, Mixin):

    def __init__(self, title, template=None, validation=True, AttributeDefinitions=REQUIRED, KeySchema=REQUIRED, BillingMode=NOTHING, GlobalSecondaryIndexes=NOTHING, LocalSecondaryIndexes=NOTHING, PointInTimeRecoverySpecification=NOTHING, ProvisionedThroughput=NOTHING, SSESpecification=NOTHING, StreamSpecification=NOTHING, TableName=NOTHING, Tags=NOTHING, TimeToLiveSpecification=NOTHING, **kwargs):
        processed_kwargs = preprocess_init_kwargs(title=title,
         template=template,
         validation=validation,
         AttributeDefinitions=AttributeDefinitions,
         KeySchema=KeySchema,
         BillingMode=BillingMode,
         GlobalSecondaryIndexes=GlobalSecondaryIndexes,
         LocalSecondaryIndexes=LocalSecondaryIndexes,
         PointInTimeRecoverySpecification=PointInTimeRecoverySpecification,
         ProvisionedThroughput=ProvisionedThroughput,
         SSESpecification=SSESpecification,
         StreamSpecification=StreamSpecification,
         TableName=TableName,
         Tags=Tags,
         TimeToLiveSpecification=TimeToLiveSpecification, **kwargs)
        (super(Table, self).__init__)(**processed_kwargs)


#encoding: utf-8

from OpenOrange import *
from Report import Report
from Currency import Currency
from Customer import Customer

class ItemsByCustomerStatistics(Report):

    def defaults(self):
        Report.defaults(self)
        record = self.getRecord()
        record.CompFromDate = record.FromDate
        record.CompToDate = record.ToDate
        record.SortBy = 0
        record.IncludeInvoices=True
        record.IncludeDebitNotes=True
        record.IncludeCreditNotes=True

    def run(self):
        record = self.getRecord()
        self.record = record

        query = Query()
        query.sql  = "SELECT i.{Currency}, i.{BaseRate}, i.{CurrencyRate}, i.{TransDate}, \n "
        query.sql += "c.{Code} {CustCode}, c.{Name} {CustName}, c.{GroupCode}, "
        query.sql += "it.{Code} {ArtCode}, it.{Name} {ArtName}, "
        query.sql += "iir.{Price}, iir.{Qty} {ArtQty}, iir.{RowNet} "
        query.sql += "FROM [Invoice] i \n"
        query.sql += "INNER JOIN [InvoiceItemRow] iir on iir.{masterId} = i.{internalId} \n"
        query.sql += "INNER JOIN [Customer] c on c.{Code} = i.{CustCode} \n"
        query.sql += "INNER JOIN [Item] it on it.{Code} = iir.{ArtCode} \n"
        query.sql += "WHERE?AND (i.{Invalid} = 0 OR i.{Invalid} IS NULL) \n"
        query.sql += "WHERE?AND i.{Status} = i|1| \n"
        if record.CustCode:
            query.sql += "WHERE?AND c.{Code} = s|%s| " % record.CustCode
        if record.CustomerGroup:
            query.sql += "WHERE?AND c.{GroupCode} = s|%s| " % record.CustomerGroup
        if record.Salesman:
            if record.UseSalesman:
                query.sql += " WHERE?AND c.{Salesman} = s|%s| " % record.Salesman
            else:
                query.sql += " WHERE?AND i.{Salesman} = s|%s| " % record.Salesman
        if record.CustCountry:
            query.sql += " WHERE?AND c.{Country}  = s|%s|" % record.CustCountry
        query.sql += " WHERE?AND ((i.{TransDate} BETWEEN d|%s| and d|%s|) \n" %(record.FromDate, record.ToDate)
        query.sql += " OR (i.{TransDate} BETWEEN d|%s| and d|%s|)) \n" %(record.CompFromDate, record.CompToDate)
        query.sql += " ORDER BY c.{Code}"
        if query.open() and query.count():
            self.Data = {}
            for rec in query:
                if not rec.CustCode in self.Data.keys():
                    self.Data[rec.CustCode] = {}
                self.placeRecord(rec)

            self.printReportTitle("Sales Statistic By Item/Customer")
            self.startTable()
            self.startHeaderRow()
            self.addValue("", ColSpan = 2)
            self.addValue("Origin Period ", ColSpan = 2)
            self.addValue("Compared Period ", ColSpan = 2)
            self.addValue("", ColSpan = 2)
            self.endHeaderRow()
            self.startHeaderRow()
            self.addValue("Customer Code")
            self.addValue("Customer Name")
            self.addValue(" %s -  %s" %(record.FromDate.strftime("%d/%m/%Y"), record.ToDate.strftime("%d/%m/%Y")), ColSpan = 2)
            self.addValue(" %s -  %s" %(record.CompFromDate.strftime("%d/%m/%Y"), record.CompToDate.strftime("%d/%m/%Y")), ColSpan = 2)
            self.addValue("Difference between Periods", ColSpan = 2)
            self.endHeaderRow()
            self.startHeaderRow()
            self.addValue("Item Code")
            self.addValue("Item Name")
            self.addValue("Qty")
            self.addValue("Amount")
            self.addValue("Qty")
            self.addValue("Amount")
            self.addValue("Qty")
            self.addValue("Amount")
            self.endHeaderRow()
            for rec in self.Data.keys():
                cust = Customer.bring(rec)
                self.startRow(BGColor = "gray")
                self.addValue(rec)
                self.addValue(cust.Name, ColSpan = 7)
                self.endRow()
                totalOriginQty, totalOriginAmount, totalCompQty, totalCompAmount = 0,0,0,0
                for item in self.Data[rec].keys():
                    self.startRow()
                    self.addValue(item)
                    self.addValue(self.Data[rec][item]["ArtName"])
                    self.addValue(self.Data[rec][item]["OriginPeriod"]["Qty"], Align="Right")
                    totalOriginQty += self.Data[rec][item]["OriginPeriod"]["Qty"]
                    self.addValue(self.Data[rec][item]["OriginPeriod"]["Amount"], Align="Right")
                    totalOriginAmount += self.Data[rec][item]["OriginPeriod"]["Amount"]
                    self.addValue(self.Data[rec][item]["CompPeriod"]["Qty"], Align="Right")
                    totalCompQty += self.Data[rec][item]["CompPeriod"]["Qty"]
                    self.addValue(self.Data[rec][item]["CompPeriod"]["Amount"], Align="Right")
                    totalCompAmount += self.Data[rec][item]["CompPeriod"]["Amount"]
                    self.addValue(self.Data[rec][item]["OriginPeriod"]["Qty"] - self.Data[rec][item]["CompPeriod"]["Qty"], Align="Right")
                    self.addValue(self.Data[rec][item]["OriginPeriod"]["Amount"] - self.Data[rec][item]["CompPeriod"]["Amount"], Align="Right")
                    self.endRow()
                self.startRow(BGColor = "#F0FAFA")
                self.addValue("Total", ColSpan = 2)
                self.addValue(totalOriginQty, Align="Right")
                self.addValue(totalOriginAmount, Align="Right")
                self.addValue(totalCompQty, Align="Right")
                self.addValue(totalCompAmount, Align="Right")
                self.addValue(totalOriginQty-totalCompQty, Align="Right")
                self.addValue(totalOriginAmount-totalCompAmount, Align="Right")
            self.endTable()

    def placeRecord(self, register):
        if register.TransDate >= self.record.FromDate   and register.TransDate <= self.record.ToDate:
            if not register.ArtCode in self.Data[register.CustCode].keys():
                self.Data[register.CustCode][register.ArtCode] = {"ArtName":register.ArtName,"OriginPeriod":{"Qty":0,"Amount":0},"CompPeriod":{"Qty":0,"Amount":0}}
            originqty  = self.Data[register.CustCode][register.ArtCode]["OriginPeriod"]["Qty"] + register.ArtQty
            cy = Currency.bring(register.Currency)
            originprice = self.Data[register.CustCode][register.ArtCode]["OriginPeriod"]["Amount"] + cy.convertToBase(register.RowNet,register.CurrencyRate,register.BaseRate,0)
            self.Data[register.CustCode][register.ArtCode]["OriginPeriod"]["Qty"] = originqty
            self.Data[register.CustCode][register.ArtCode]["OriginPeriod"]["Amount"] = originprice
        if register.TransDate >= self.record.CompFromDate   and register.TransDate <= self.record.CompToDate:
            if not register.ArtCode in self.Data[register.CustCode].keys():
                self.Data[register.CustCode][register.ArtCode] = {"ArtName":register.ArtName,"OriginPeriod":{"Qty":0,"Amount":0},"CompPeriod":{"Qty":0,"Amount":0}}
            compqty  = self.Data[register.CustCode][register.ArtCode]["CompPeriod"]["Qty"] + register.ArtQty
            cy = Currency.bring(register.Currency)
            compprice = self.Data[register.CustCode][register.ArtCode]["CompPeriod"]["Amount"] + cy.convertToBase(register.RowNet,register.CurrencyRate,register.BaseRate,0)
            self.Data[register.CustCode][register.ArtCode]["CompPeriod"]["Qty"] = compqty
            self.Data[register.CustCode][register.ArtCode]["CompPeriod"]["Amount"] = compprice


# -*- coding: utf-8 -*-
"""
Created on Sat Mar 30 01:17:49 2019

@author: isaac
"""
import pandas as pd
import numpy as np
import os

import Floyd.scripts.commons.utils as utils

def create_phon_df(X,Y):
    labels_df = pd.DataFrame(columns=["ix", "phon", "start", "stop", "feats"])
    for ix, y in enumerate(Y):
        feats = []
        start = 0
        x = X[ix] #para cada frase
        for i in range(len(y)-1): #para cada frame (menos el útlimo), mientras sea el mismo fonema extendemos feats, cuando cambie lo guardamos
            if y[i]!=y[i+1]:
                feats.append(x[i,:])
                stop = i
                labels_df = labels_df.append({"ix":ix, "phon": y[i], "start": start, "stop": stop, "feats": np.array(feats)}, ignore_index = True)
                start = i+1
                feats = []
            else:
                feats.append(x[i,:])
    return labels_df

def create_ngram(df, n):
    X = []
    y = []
    for d in df.groupby("ix"):
        for i in range(len(d[1])-n):
            try:
                X.append(np.concatenate(d[1].iloc[i:i+n]["feats"].values))
                y.append(d[1].iloc[i:i+n]["phon"].values)
            except ValueError:
                print("Problema en ix: " + str(d[1].iloc[i]["ix"]) + ", i: " + str(i))
    return X,y

def create_ngram_v2(df, n):
    X = []
    y = []
    for phrase in df.groupby("ix"):
        d = phrase[1].sort_values("start")
        for i in range(len(d)-n):
            try:
                feats = np.concatenate(d.iloc[i:i+n]["feats"].values)
                labels = d.iloc[i:i+n].apply(lambda row: np.repeat(row["phon"], row["stop"] - row["start"] + 1), axis = 1).values
                labels = np.concatenate(labels)

                assert len(labels) == len(feats)

                X.append(feats)
                y.append(labels)
            except ValueError:
                print("Problema en ix: " + str(d.iloc[i]["ix"]) + ", i: " + str(i))
    return X,y

def makedir(path):
    if not os.path.isdir(path):
        os.makedirs(path)

def normalize_data(X_train, X_test, folder_name):
    if os.path.isfile(folder_name + "mean.pkl"):
        mean = utils.open_pickle(folder_name + "mean.pkl")
        std = utils.open_pickle(folder_name + "std.pkl")
    else:
        train_matrix = []
        for x in X_train:
            train_matrix.extend(x)
        train_matrix = np.array(train_matrix)

        mean = np.array(train_matrix).mean(0)
        std = np.array(train_matrix).std(0)

        utils.save_pickle(folder_name + "mean.pkl", mean)
        utils.save_pickle(folder_name + "std.pkl", std)

    X_train = [(x-mean)/std for x in X_train]
    X_test = [(x-mean)/std for x in X_test]

    return np.array(X_train), np.array(X_test)

def create_dfs(data_folder, checkpointfolder):
    if os.path.isfile(checkpointfolder + "train_df.pkl"):
        assert os.path.isfile(checkpointfolder + "test_df.pkl")
        print("dfs already exist")

        return
    else:
        print("Creating dfs")
        #Data extraction and preparation
        X_train_path = data_folder + "X_train.npy"
        X_train = np.array(utils.load_npy(X_train_path))

        X_test_path = data_folder + "X_test.npy"
        X_test = np.array(utils.load_npy(X_test_path))

        y_train_path = data_folder + "y_train.npy"
        y_train = utils.load_npy(y_train_path)
        y_train = np.array([utils.collapse_num_labels(y) for y in y_train])

        y_test_path = data_folder + "y_test.npy"
        y_test = utils.load_npy(y_test_path)
        y_test = np.array([utils.collapse_num_labels(y) for y in y_test])

        #SAVE MEMORY??
        X_train, X_test = normalize_data(X_train, X_test, data_folder)

        train_df = create_phon_df(X_train, y_train)
        del(X_train, y_train)
        train_df.to_pickle(checkpointfolder + "train_df.pkl")

        test_df = create_phon_df(X_test, y_test)
        del(X_test, y_test)
        test_df.to_pickle(checkpointfolder + "test_df.pkl")

if __name__ == "__main__":
    ngram = 7

    rootdir = "./LSTM_ngram/"
    checkpointfolder = rootdir + "checkpoints/"
    checkpointfolder_specific = checkpointfolder + str(ngram) + "gram_v2/"
    makedir(checkpointfolder_specific)

    data_folder = "./PreprocessedData/RBMs_longtrain/hann/v2/400_600/"

    create_dfs(data_folder, checkpointfolder)

    if os.path.isfile(checkpointfolder + "train_df.pkl"):
        assert os.path.isfile(checkpointfolder + "test_df.pkl")

        print("Create .npys")
        train_df = pd.read_pickle(checkpointfolder + "train_df.pkl")
        X_train, y_train = create_ngram_v2(train_df, ngram)
        del(train_df)

        print("save train")
        utils.save_npy(y_train[:len(y_train)//5], checkpointfolder_specific + "y_train_0.npy")
        utils.save_npy(y_train[len(y_train)//5:2*len(y_train)//5], checkpointfolder_specific + "y_train_1.npy")
        utils.save_npy(y_train[2*len(y_train)//5:3*len(y_train)//5], checkpointfolder_specific + "y_train_2.npy")
        utils.save_npy(y_train[3*len(y_train)//5:4*len(y_train)//5], checkpointfolder_specific + "y_train_3.npy")
        utils.save_npy(y_train[4*len(y_train)//5:], checkpointfolder_specific + "y_train_4.npy")

        del(y_train)
        utils.save_npy(X_train[:len(X_train)//5], checkpointfolder_specific + "X_train_0.npy")
        utils.save_npy(X_train[len(X_train)//5:2*len(X_train)//5], checkpointfolder_specific + "X_train_1.npy")
        utils.save_npy(X_train[2*len(X_train)//5:3*len(X_train)//5], checkpointfolder_specific + "X_train_2.npy")
        utils.save_npy(X_train[3*len(X_train)//5:4*len(X_train)//5], checkpointfolder_specific + "X_train_3.npy")
        utils.save_npy(X_train[4*len(X_train)//5:], checkpointfolder_specific + "X_train_4.npy")
        del(X_train)

        print("Create .npys")
        test_df = pd.read_pickle(checkpointfolder + "test_df.pkl")
        X_test, y_test = create_ngram_v2(test_df, ngram)
        del(test_df)

        print("save test")
        utils.save_npy(y_test[:len(y_test)//5], checkpointfolder_specific + "y_test_0.npy")
        utils.save_npy(y_test[len(y_test)//5:2*len(y_test)//5], checkpointfolder_specific + "y_test_1.npy")
        utils.save_npy(y_test[2*len(y_test)//5:3*len(y_test)//5], checkpointfolder_specific + "y_test_2.npy")
        utils.save_npy(y_test[3*len(y_test)//5:4*len(y_test)//5], checkpointfolder_specific + "y_test_3.npy")
        utils.save_npy(y_test[4*len(y_test)//5:], checkpointfolder_specific + "y_test_4.npy")
        del(y_test)

        utils.save_npy(X_test[:len(X_test)//5], checkpointfolder_specific + "X_test_0.npy")
        utils.save_npy(X_test[len(X_test)//5:2*len(X_test)//5], checkpointfolder_specific + "X_test_1.npy")
        utils.save_npy(X_test[2*len(X_test)//5:3*len(X_test)//5], checkpointfolder_specific + "X_test_2.npy")
        utils.save_npy(X_test[3*len(X_test)//5:4*len(X_test)//5], checkpointfolder_specific + "X_test_3.npy")
        utils.save_npy(X_test[4*len(X_test)//5:], checkpointfolder_specific + "X_test_4.npy")
        del(X_test)


from flask import Flask, url_for, redirect, request, render_template, send_from_directory
from flask.ext.sqlalchemy import SQLAlchemy
from model import *
from werkzeug import secure_filename

import os

ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif', 'mp4'])

if not os.path.exists('data/media'):
    os.makedirs('data/media')

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = 'data/media'
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///data/test.db'
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False

with app.app_context():
    db.init_app(app)
    db.drop_all()
    db.create_all()

    # sample data
    sample_courses = [
        Course('chemistry', 'This is a thing that has to do with chemistry', 'chemistry.png'),
        Course('physics', 'This is a thing that has to do with physics', 'physics.png'),
        Course('programming', 'This is a thing that has to do with programming', 'programming.png'),
    ]
    for course in sample_courses:
        db.session.add(course)
    db.session.commit()

@app.route("/")
def home():
    return redirect('/courses')

@app.route('/courses')
def list_courses():
    courses = Course.query.all()
    return render_template('list_courses.html', courses=courses)

@app.route('/courses/<course_id>/media/add', methods=['GET', 'POST'])
def add_media(course_id):
    if request.method == 'POST':
        file = request.files['file']
        if not file or not is_allowed_file(file.filename):
            redirect('/courses/<course_id>/media/add', course_id=course_id)

        upload_media(course_id, file)
        return redirect(url_for('get_course', course_id=course_id))

    course = Course.query.filter_by(ID=course_id).first()
    return render_template('add_media.html', course=course)

def get_courses_dir(course_id):
    result = os.path.join('courses', course_id)
    try:
        os.makedirs(result)
    except OSError:
        pass
    return result

@app.route('/courses/<course_id>')
def get_course(course_id):
    course = Course.query.filter_by(ID=course_id).first()
    media = Media.query.filter_by(course_id=course_id)
    return render_template('course.html', course=course, media=media)

@app.route('/courses/<course_id>/media/<name>')
def get_media(course_id, name):
    media = Media.query.filter_by(course_id=course_id, name=name).first()
    return send_from_directory(app.config['UPLOAD_FOLDER'], media.location)

def upload_media(course_id, file):
    filename = secure_filename(file.filename)
    new_media = Media(course_id, filename, -1, None, get_file_extension(filename))
    new_media.location = os.path.join(get_courses_dir(course_id), filename)
    file.save(os.path.join(app.config['UPLOAD_FOLDER'], new_media.location))

    db.session.add(new_media)
    db.session.commit()

def get_file_extension(filename):
    return filename.rsplit('.', 1)[1]

def is_allowed_file(filename):
    return '.' in filename and \
        filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

if __name__ == "__main__":
    app.run(host="0.0.0.0", debug=True)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations
import django.db.models.deletion
from django.conf import settings

class Migration(migrations.Migration):

    dependencies = [
        ('geo_location', '0001_initial'),
        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
    ]

    operations = [
        migrations.CreateModel(
            name='DocumentType',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(null=True, blank=True)),
                ('name', models.CharField(max_length=60, verbose_name='name', db_index=True)),
                ('abbr', models.CharField(max_length=4, null=True, verbose_name='abbr', blank=True)),
                ('is_active', models.BooleanField(default=True, verbose_name='is_active')),
                ('description', models.CharField(max_length=255, null=True, verbose_name='description', blank=True)),
                ('created_by', models.ForeignKey(related_name='document_type_created_by', on_delete=django.db.models.deletion.PROTECT, verbose_name='created_by', blank=True, to=settings.AUTH_USER_MODEL, null=True)),
                ('updated_by', models.ForeignKey(related_name='document_type_updated_by', on_delete=django.db.models.deletion.PROTECT, verbose_name='updated_by', blank=True, to=settings.AUTH_USER_MODEL, null=True)),
            ],
            options={
                'verbose_name': 'document_type',
                'verbose_name_plural': 'document_types',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Email',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(null=True, blank=True)),
                ('recipients', models.TextField(verbose_name='recipients')),
                ('subject', models.CharField(max_length=255, verbose_name='subject', db_index=True)),
                ('content', models.TextField(null=True, verbose_name='content', blank=True)),
                ('description', models.TextField(max_length=255, null=True, verbose_name='description', blank=True)),
                ('status', models.CharField(default=b'pending_sent', max_length=20, verbose_name='status', choices=[(b'sent', 'email_sent'), (b'pending_sent', 'email_pending_sent'), (b'failed_sent', 'email_failed_sent')])),
                ('is_active', models.BooleanField(default=True, verbose_name='is_active')),
                ('sent_date', models.DateTimeField(null=True, verbose_name='sent_date', blank=True)),
                ('num_recipients', models.IntegerField(default=0, verbose_name='num_recipients')),
                ('num_success_deliveries', models.IntegerField(default=0, verbose_name='num_recipients')),
                ('email_service_response', models.CharField(max_length=500, null=True, verbose_name='email_service_response', blank=True)),
                ('created_by', models.ForeignKey(related_name='email_created_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True)),
                ('updated_by', models.ForeignKey(related_name='email_updated_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True)),
            ],
            options={
                'verbose_name': 'email',
                'verbose_name_plural': 'emails',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='EmailTemplate',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(null=True, blank=True)),
                ('name', models.CharField(max_length=255, verbose_name='name', db_index=True)),
                ('content', models.TextField(null=True, verbose_name='content', blank=True)),
                ('description', models.CharField(max_length=400, null=True, verbose_name='description', blank=True)),
                ('is_active', models.BooleanField(default=True, verbose_name='is_active')),
                ('created_by', models.ForeignKey(related_name='emailtemplate_created_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True)),
                ('updated_by', models.ForeignKey(related_name='emailtemplate_updated_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True)),
            ],
            options={
                'verbose_name': 'email_template',
                'verbose_name_plural': 'email_templates',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Organization',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(null=True, blank=True)),
                ('name', models.CharField(max_length=60, verbose_name='name', db_index=True)),
                ('domain_name', models.CharField(unique=True, max_length=60, verbose_name='domain_name')),
                ('description', models.CharField(max_length=255, null=True, verbose_name='description', blank=True)),
                ('logo', models.ImageField(upload_to=b'uploads/org_structure_organization_logo/', null=True, verbose_name='logo', blank=True)),
                ('is_active', models.BooleanField(default=True, verbose_name='is_active')),
                ('created_by', models.ForeignKey(related_name='org_created_by', on_delete=django.db.models.deletion.PROTECT, verbose_name='created_by', blank=True, to=settings.AUTH_USER_MODEL, null=True)),
                ('location', models.ForeignKey(verbose_name='location', to='geo_location.Location')),
                ('updated_by', models.ForeignKey(related_name='org_updated_by', on_delete=django.db.models.deletion.PROTECT, verbose_name='updated_by', blank=True, to=settings.AUTH_USER_MODEL, null=True)),
            ],
            options={
                'verbose_name': 'organization',
                'verbose_name_plural': 'organizations',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='PopulationGroup',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(null=True, blank=True)),
                ('name', models.CharField(max_length=255, verbose_name='name', db_index=True)),
                ('description', models.CharField(max_length=255, null=True, verbose_name='description', blank=True)),
                ('is_active', models.BooleanField(default=True, verbose_name='is_active')),
                ('created_by', models.ForeignKey(related_name='benpop_created_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True)),
                ('parent', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, verbose_name='parent', blank=True, to='org_structure.PopulationGroup', null=True)),
                ('updated_by', models.ForeignKey(related_name='benpop_updated_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True)),
            ],
            options={
                'verbose_name': 'population_group',
                'verbose_name_plural': 'population_groups',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='ProfileEvent',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(null=True, blank=True)),
                ('due_date', models.DateField(null=True, verbose_name='due_date', blank=True)),
                ('name', models.CharField(max_length=255, verbose_name='name', db_index=True)),
                ('description', models.CharField(max_length=1000, null=True, verbose_name='description', blank=True)),
                ('is_active', models.BooleanField(default=True, verbose_name='is_active')),
                ('created_by', models.ForeignKey(related_name='profileevent_created_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True)),
            ],
            options={
                'verbose_name': 'profile_event',
                'verbose_name_plural': 'profile_events',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='ProfileEventType',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(null=True, blank=True)),
                ('name', models.CharField(max_length=255, verbose_name='name', db_index=True)),
                ('description', models.CharField(max_length=255, null=True, verbose_name='description', blank=True)),
                ('is_active', models.BooleanField(default=True, verbose_name='is_active')),
                ('created_by', models.ForeignKey(related_name='profileeventtype_created_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True)),
                ('updated_by', models.ForeignKey(related_name='profileeventtype_updated_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True)),
            ],
            options={
                'verbose_name': 'profile_event_type',
                'verbose_name_plural': 'profile_event_types',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='Telephone',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(null=True, blank=True)),
                ('number', models.IntegerField(verbose_name='number')),
                ('telephone_type', models.CharField(max_length=30, verbose_name='telephone_type', choices=[(b'cel', 'cellphone'), (b'tel', 'telephone'), (b'fax', 'fax')])),
                ('description', models.CharField(max_length=255, verbose_name='description')),
                ('is_active', models.BooleanField(default=True, verbose_name='is_active')),
                ('created_by', models.ForeignKey(related_name='_telephone_created_by', on_delete=django.db.models.deletion.PROTECT, verbose_name='created_by', blank=True, to=settings.AUTH_USER_MODEL, null=True)),
                ('updated_by', models.ForeignKey(related_name='_telephone_updated_by', on_delete=django.db.models.deletion.PROTECT, verbose_name='updated_by', blank=True, to=settings.AUTH_USER_MODEL, null=True)),
            ],
            options={
                'verbose_name': 'telephone',
                'verbose_name_plural': 'telephones',
            },
            bases=(models.Model,),
        ),
        migrations.CreateModel(
            name='UserProfile',
            fields=[
                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
                ('created_at', models.DateTimeField(auto_now_add=True)),
                ('updated_at', models.DateTimeField(null=True, blank=True)),
                ('profile_type', models.CharField(default=b'P', max_length=5, verbose_name='profile_type', choices=[(b'E', 'enterprise'), (b'P', 'person')])),
                ('name', models.CharField(max_length=90, null=True, verbose_name='name', blank=True)),
                ('first_name', models.CharField(default=b'', max_length=90, null=True, verbose_name='first_name', blank=True)),
                ('second_name', models.CharField(default=b'', max_length=90, null=True, verbose_name='second_name', blank=True)),
                ('last_name', models.CharField(default=b'', max_length=90, null=True, verbose_name='last_name', blank=True)),
                ('second_last_name', models.CharField(default=b'', max_length=90, null=True, verbose_name='second_last_name', blank=True)),
                ('document_id', models.CharField(unique=True, max_length=100, verbose_name='document_id')),
                ('doc_exp_location', models.CharField(max_length=255, null=True, verbose_name='doc_exp_location', blank=True)),
                ('gender', models.CharField(blank=True, max_length=1, null=True, verbose_name='gender', choices=[(b'M', 'male'), (b'F', 'female')])),
                ('birth_date', models.DateField(null=True, verbose_name='birth_date', blank=True)),
                ('age', models.IntegerField(default=0, verbose_name='age', blank=True)),
                ('is_employee', models.BooleanField(default=False, verbose_name='is_employee')),
                ('is_supplier', models.BooleanField(default=False, verbose_name='is_supplier')),
                ('is_generic', models.BooleanField(default=False, verbose_name='is_generic')),
                ('code', models.CharField(max_length=20, null=True, verbose_name='userprofile_code', blank=True)),
                ('cellphone', models.IntegerField(null=True, verbose_name='cellphone', blank=True)),
                ('telephone', models.CharField(max_length=60, null=True, verbose_name='telephone', blank=True)),
                ('address', models.CharField(max_length=120, null=True, verbose_name='address', blank=True)),
                ('email', models.EmailField(db_index=True, max_length=75, null=True, verbose_name='email', blank=True)),
                ('is_active', models.BooleanField(default=True, verbose_name='is_active')),
                ('avatar_image', models.ImageField(upload_to=b'uploads/user_profile_userprofile_avatar_image/', null=True, verbose_name='avatar_image', blank=True)),
                ('address_neighborhood', models.CharField(default=b'', max_length=90, null=True, verbose_name='address_neighborhood', blank=True)),
                ('prof_ref_document_id', models.CharField(default=b'', max_length=90, null=True, verbose_name='prof_ref_document_id', blank=True)),
                ('birth_date_year', models.IntegerField(default=0, null=True, verbose_name='birth_date_year', blank=True)),
                ('profession_occupation', models.CharField(max_length=60, null=True, verbose_name='profession_occupation', blank=True)),
                ('academic_level', models.CharField(blank=True, max_length=60, null=True, verbose_name='academic_level', choices=[(b'primary', 'primary'), (b'high_school', 'high_school'), (b'technical', 'technical'), (b'technologist', 'technologist'), (b'professional', 'professional'), (b'magister', 'magister'), (b'doctor', 'doctor')])),
                ('employment_situation', models.CharField(blank=True, max_length=60, null=True, verbose_name='employment_situation', choices=[(b'unemployed', 'unemployed'), (b'employed', 'employed'), (b'independent', 'independent')])),
                ('profile_reference_kinship', models.CharField(default=b'friend', choices=[(b'friend', 'friend'), (b'relative', 'relative')], max_length=60, blank=True, null=True, verbose_name='profile_reference_kinship')),
                ('is_leader', models.BooleanField(default=False, verbose_name='is_leader')),
                ('imported', models.BooleanField(default=False, verbose_name='imported', editable=False)),
                ('web_info', models.TextField(null=True, blank=True)),
                ('receive_emails', models.BooleanField(default=True, verbose_name='receive_emails')),
                ('address_location', models.ForeignKey(related_name='user_profile_address_location', on_delete=django.db.models.deletion.PROTECT, verbose_name='address_location', blank=True, to='geo_location.Location', null=True)),
                ('created_by', models.ForeignKey(related_name='user_profile_created_by', on_delete=django.db.models.deletion.PROTECT, verbose_name='created_by', blank=True, to=settings.AUTH_USER_MODEL, null=True)),
                ('document_type', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, default=1, verbose_name='document_type', to='org_structure.DocumentType')),
                ('org_domain', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, verbose_name=b'org_domain', blank=True, to='org_structure.Organization', null=True)),
                ('population_groups', models.ManyToManyField(to='org_structure.PopulationGroup', null=True, verbose_name='population_groups', blank=True)),
                ('profile_reference', models.ForeignKey(related_name='children', on_delete=django.db.models.deletion.PROTECT, verbose_name='profile_reference', blank=True, to='org_structure.UserProfile', null=True)),
                ('updated_by', models.ForeignKey(related_name='user_profile_updated_by', on_delete=django.db.models.deletion.PROTECT, verbose_name='updated_by', blank=True, to=settings.AUTH_USER_MODEL, null=True)),
                ('user', models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL)),
            ],
            options={
                'verbose_name': 'user_profile',
                'verbose_name_plural': 'user_profiles',
            },
            bases=(models.Model,),
        ),
        migrations.AddField(
            model_name='telephone',
            name='user_profile',
            field=models.ForeignKey(related_name='telephones', on_delete=django.db.models.deletion.PROTECT, verbose_name='user_profile', to='org_structure.UserProfile'),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='profileevent',
            name='type',
            field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, verbose_name='profile_event_type', to='org_structure.ProfileEventType'),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='profileevent',
            name='updated_by',
            field=models.ForeignKey(related_name='profileevent_updated_by', on_delete=django.db.models.deletion.PROTECT, blank=True, to=settings.AUTH_USER_MODEL, null=True),
            preserve_default=True,
        ),
        migrations.AddField(
            model_name='profileevent',
            name='user_profile',
            field=models.ForeignKey(related_name='profile_events', verbose_name='user_profile', to='org_structure.UserProfile'),
            preserve_default=True,
        ),
    ]


# setting python working environment ...

# import tensorflow as tf
# from tf import keras

import keras
from keras.models import Input, Model
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, concatenate, SpatialDropout1D

def create_model(max_len, n_words, n_tags, pos_tags , max_len_chars, n_chars):

	word_input = Input(shape=(max_len,))
	word_emb = Embedding(input_dim=n_words + 2, output_dim=20,input_length=max_len, mask_zero = True)(word_input)

	print(word_emb)
	char_input = Input(shape =(max_len, max_len_chars))
	char_emb = TimeDistributed(Embedding(input_dim=n_chars + 2, output_dim = 10,input_length= max_len_chars, mask_zero= True))(char_input)
	print(char_emb)
	char_enc = TimeDistributed(LSTM(units = 20, return_sequences=False,recurrent_dropout = 0.6 ))(char_emb)
	print(char_enc)
	x = keras.layers.concatenate([word_emb, char_enc])
	x = SpatialDropout1D(0.3)(x)
	print(x)
	main_lstm = Bidirectional(LSTM(units = 50, return_sequences = True,
							 recurrent_dropout = 0.6))(x)
	print(main_lstm)
	out = TimeDistributed(Dense(n_tags + 1, activation="softmax"))(main_lstm)
	print(out)
	# print(concat_in)
	# print("\nword_input:",word_emb)
	# # print("\npos_input:",pos_input)

	# model = Dropout(0.1)(word_emb)
	# model = Bidirectional(LSTM(50, return_sequences = True, recurrent_dropout= 0.1))(model)
	# # Dense Layer with 50 neurons
	# model = TimeDistributed(Dense(50, activation = "relu"))(model)
	# crf = CRF(n_tags)
	# # output layer is crf
	# out = crf(model)

	# model creation for compilation
	# model = Model(inputs = [word_input,pos_input] , outputs =out)
	# model = Model([word_in, char_in], out)

	model = Model(inputs = [word_input, char_input] , outputs =out)
	# model.compile(optimizer="rmsprop", loss=crf.loss_function, metrics=[crf.accuracy])
	print(model)
	model.compile(optimizer="adam" , loss="sparse_categorical_crossentropy", metrics = ["acc"])
	return model

# utils
def plot_history(history):

	import pandas as pd
	hist = pd.DataFrame(history.history)
	plt.figure(figsize=(12,12))
	plt.plot(hist["acc"])
	plt.plot(hist["val_acc"])
	plt.legend()
	plt.show()

def draw_histogram(sentences):

	plt.style.use("ggplot")
	plt.hist([len(s) for s in sentences ], bins = 50)
	plt.show()


# coding: utf-8

# # Q4_PART ONE
# MOVIE AWARDS ANALYSIS

# Use ‘movies_awards’ data set.
# • You are supposed to extract data from the awards column in this dataset and split it into several
# columns. An example is given below.
# • The awards has details of wins, nominations in general and also wins and nominations in certain categories(e.g. Oscar, BAFTA etc.)
# • You are supposed to create a win and nominated column (these 2 columns contain total number of wins and nominations) and other columns that extract the number of wins and nominations for each category of award.
# • If a movie has 2 Oscar nominations and 4 Oscar won, the columns Oscar_Awards_Won should have value 4 and Oscar_Awards_Nominated should have value 2. You should also have a total won and nominated column which aggregates all the awards (won or nominated).
# • Create two separate columns for each award category (won and nominated).
# • Write your output to a csv file.

# In[25]:

import pandas as pd
import re

# In[26]:

path_local='Data/movies_awards.csv'
data_raw = pd.read_csv(path_local)

# ### Process raw data: only keep Awards column and drop nan from Awards

# In[27]:

df1=data_raw.loc[:,['Awards']]
df1=df1[df1.Awards.notnull()]

# ### functions for filtering win and nomination and awards

# In[78]:

def get_win(x):
    rex='(\d+) win'
    rex1='(\d+) won'
    l=re.findall(rex,x)
    l1=re.findall(rex1,x)
    a=0
    if l1!=[]:
        if l==[]:
            a+=int(l1[0])
        else:
            a=a + int(l1[0]) + int(l[0])
    else:
        if l==[]:
            a+=0
        else:
            a += int(l[0])
    return a

# In[89]:

def get_nom(x):
    rex='(\d+) nom'
    l=re.findall(rex,x)
    a=0
    if l!=[]:
        a += int(l[0])
    else:
        a+=0
    return a

# In[122]:

def get_prime_nom(x):
    rex='Nominated for (\d+) Prime'
    l=re.findall(rex,x)
    a=0
    if l!=[]:
        a += int(l[0])
    else:
        a+=0
    return a

# In[107]:

def get_oscar_nom(x):
    rex='Nominated for (\d+) Oscar'
    l=re.findall(rex,x)
    a=0
    if l!=[]:
        a += int(l[0])
    else:
        a+=0
    return a

# In[112]:

def get_golden_nom(x):
    rex='Nominated for (\d+) Golden'
    l=re.findall(rex,x)
    a=0
    if l!=[]:
        a += int(l[0])
    else:
        a+=0
    return a

# In[124]:

def get_BAFTA_nom(x):
    rex='Nominated for (\d+) BAFTA'
    l=re.findall(rex,x)
    a=0
    if l!=[]:
        a += int(l[0])
    else:
        a+=0
    return a

# In[125]:

def get_prime_win(x):
    rex='Won (\d+) Prime'
    l=re.findall(rex,x)
    a=0
    if l!=[]:
        a += int(l[0])
    else:
        a+=0
    return a

# In[126]:

def get_oscar_win(x):
    rex='Won (\d+) Oscars'
    l=re.findall(rex,x)
    a=0
    if l!=[]:
        a += int(l[0])
    else:
        a+=0
    return a

# In[127]:

def get_golden_win(x):
    rex='Won (\d+) Golden'
    l=re.findall(rex,x)
    a=0
    if l!=[]:
        a += int(l[0])
    else:
        a+=0
    return a

# In[117]:

def get_BAFTA_win(x):
    rex='Won (\d+) BAFTA'
    l=re.findall(rex,x)
    a=0
    if l!=[]:
        a += int(l[0])
    else:
        a+=0
    return a

# ### Create new columns for each kind of award

# In[128]:

df1['Awards_won']=df1['Awards'].apply(lambda x: get_win(x))
df1['Awards_nominated']=df1['Awards'].apply(lambda x: get_nom(x))
df1['Prime_Awards_nominated']=df1['Awards'].apply(lambda x: get_prime_nom(x))
df1['Oscar_Awards_nominated']=df1['Awards'].apply(lambda x: get_oscar_nom(x))
df1['Golden_Globe_Awards_nominated']=df1['Awards'].apply(lambda x: get_golden_nom(x))
df1['BAFTA_Awards_nominated']=df1['Awards'].apply(lambda x: get_BAFTA_nom(x))
df1['Prime_Awards_won']=df1['Awards'].apply(lambda x: get_prime_win(x))
df1['Oscar_Awards_won']=df1['Awards'].apply(lambda x: get_oscar_win(x))
df1['Golden_Globe_Awards_won']=df1['Awards'].apply(lambda x: get_golden_win(x))
df1['BAFTA_Awards_won']=df1['Awards'].apply(lambda x: get_BAFTA_win(x))

# ### A little sample

# In[129]:

df1.head()

# ### Write output to a csv file

# In[130]:

df1.to_csv('Q4_p1.csv')

# In[ ]: