"""
Class for encoding variable-length peptides to fixed-size numerical matrices
"""
from __future__ import (
    print_function,
    division,
    absolute_import,
)

import math
from six import string_types
from functools import partial

import numpy
import pandas

from amino_acid import *

class EncodingError(ValueError):
    """
    Exception raised when peptides cannot be encoded
    """
    def __init__(self, message, supported_peptide_lengths):
        self.supported_peptide_lengths = supported_peptide_lengths
        ValueError.__init__(
            self,
            message + " Supported lengths: %s - %s." % supported_peptide_lengths)

class EncodableSequences(object):
    """
    Class for encoding variable-length peptides to fixed-size numerical matrices

    This class caches various encodings of a list of sequences.

    In practice this is used only for peptides. To encode MHC allele sequences,
    see AlleleEncoding.
    """
    unknown_character = "X"

    @classmethod
    def create(klass, sequences):
        """
        Factory that returns an EncodableSequences given a list of
        strings. As a convenience, you can also pass it an EncodableSequences
        instance, in which case the object is returned unchanged.
        """
        if isinstance(sequences, klass):
            return sequences
        return klass(sequences)

    def __init__(self, sequences):
        if not all(isinstance(obj, string_types) for obj in sequences):
            raise ValueError("Sequence of strings is required")
        self.sequences = numpy.array(sequences)
        lengths = pandas.Series(self.sequences, dtype=numpy.object_).str.len()

        self.min_length = lengths.min()
        self.max_length = lengths.max()

        self.encoding_cache = {}
        self.fixed_sequence_length = None
        if len(self.sequences) > 0 and all(
                len(s) == len(self.sequences[0]) for s in self.sequences):
            self.fixed_sequence_length = len(self.sequences[0])

    def __len__(self):
        return len(self.sequences)

    def variable_length_to_fixed_length_categorical(
            self,
            alignment_method="processing",
            left_edge=4,
            right_edge=4,
            max_length=15):
        """
        Encode variable-length sequences to a fixed-size index-encoded (integer)
        matrix.

        See `sequences_to_fixed_length_index_encoded_array` for details.

        Parameters
        ----------
        alignment_method : string
            One of "pad_middle" or "left_pad_right_pad"
        left_edge : int, size of fixed-position left side
            Only relevant for pad_middle alignment method
        right_edge : int, size of the fixed-position right side
            Only relevant for pad_middle alignment method
        max_length : maximum supported peptide length

        Returns
        -------
        numpy.array of integers with shape (num sequences, encoded length)

        For pad_middle, the encoded length is max_length. For left_pad_right_pad,
        it's 3 * max_length.
        """

        cache_key = (
            "fixed_length_categorical",
            alignment_method,
            left_edge,
            right_edge,
            max_length)

        if cache_key not in self.encoding_cache:
            fixed_length_sequences = (
                self.sequences_to_fixed_length_index_encoded_array(
                    self.sequences,
                    alignment_method=alignment_method,
                    left_edge=left_edge,
                    right_edge=right_edge,
                    max_length=max_length))
            self.encoding_cache[cache_key] = fixed_length_sequences
        return self.encoding_cache[cache_key]

    def variable_length_to_fixed_length_vector_encoding(
            self,
            vector_encoding_name,
            alignment_method="processing",
            left_edge=4,
            right_edge=4,
            max_length=15,
            trim=False,
            pssm=False,
            allow_unsupported_amino_acids=True):
        """
        Encode variable-length sequences to a fixed-size matrix. Amino acids
        are encoded as specified by the vector_encoding_name argument.

        See `sequences_to_fixed_length_index_encoded_array` for details.

        See also: variable_length_to_fixed_length_categorical.

        Parameters
        ----------
        vector_encoding_name : string
            How to represent amino acids.
            One of "BLOSUM62", "one-hot", etc. Full list of supported vector
            encodings is given by available_vector_encodings().
        alignment_method : string
            One of "pad_middle" or "left_pad_right_pad"
        left_edge : int
            Size of fixed-position left side.
            Only relevant for pad_middle alignment method
        right_edge : int
            Size of the fixed-position right side.
            Only relevant for pad_middle alignment method
        max_length : int
            Maximum supported peptide length
        trim : bool
            If True, longer sequences will be trimmed to fit the maximum
            supported length. Not supported for all alignment methods.
        allow_unsupported_amino_acids : bool
            If True, non-canonical amino acids will be replaced with the X
            character before encoding.

        Returns
        -------
        numpy.array with shape (num sequences, encoded length, m)

        where
            - m is the vector encoding length (usually 21).
            - encoded length is max_length if alignment_method is pad_middle;
              3 * max_length if it's left_pad_right_pad.
        """
        cache_key = (
            "fixed_length_vector_encoding",
            vector_encoding_name,
            alignment_method,
            left_edge,
            right_edge,
            max_length,
            trim,
            pssm,
            allow_unsupported_amino_acids)
        if cache_key not in self.encoding_cache:
            fixed_length_sequences = (
                self.sequences_to_fixed_length_index_encoded_array(
                    self.sequences,
                    alignment_method=alignment_method,
                    left_edge=left_edge,
                    right_edge=right_edge,
                    max_length=max_length,
                    trim=trim,
                    allow_unsupported_amino_acids=allow_unsupported_amino_acids))
#            result = fixed_vectors_encoding(fixed_length_sequences,
#                                            ENCODING_DATA_FRAMES[vector_encoding_name],
#                                            pssm=pssm)
            assert fixed_length_sequences.shape[0] == len(self.sequences)
            self.encoding_cache[cache_key] = fixed_length_sequences #result
        return self.encoding_cache[cache_key]

    @classmethod
    def sequences_to_fixed_length_index_encoded_array(
            klass,
            sequences,
            alignment_method="processing",
            left_edge=4,
            right_edge=4,
            max_length=15,
            trim=False,
            allow_unsupported_amino_acids=False):
        """
        Encode variable-length sequences to a fixed-size index-encoded (integer)
        matrix.

        Parameters
        ----------
        sequences : list of string
        alignment_method : string
            One of "pad_middle", "left_pad_right_pad", "processing"
        left_edge : int
            Size of fixed-position left side.
            Only relevant for pad_middle alignment method
        right_edge : int
            Size of the fixed-position right side.
            Only relevant for pad_middle alignment method
        max_length : int
            maximum supported peptide length
        trim : bool
            If True, longer sequences will be trimmed to fit the maximum
            supported length. Not supported for all alignment methods.
        allow_unsupported_amino_acids : bool
            If True, non-canonical amino acids will be replaced with the X
            character before encoding.

        Returns
        -------
        numpy.array of integers with shape (num sequences, encoded length)

        For pad_middle, the encoded length is max_length. For left_pad_right_pad,
        it's 2 * max_length. For left_pad_centered_right_pad, it's
        3 * max_length.
        """
        if allow_unsupported_amino_acids:  # if unrecog AA provided (eg J) replace with X
            fill_value = AMINO_ACID_INDEX['X']

            def get_amino_acid_index(a):
                return AMINO_ACID_INDEX.get(a, fill_value)
        else:
            get_amino_acid_index = AMINO_ACID_INDEX.__getitem__

        result = None
        if alignment_method == 'processing':
            #  create blank array where rows are the supplied sequences and
            #  column length is equal to the desired length
            result = numpy.full(
                fill_value=AMINO_ACID_INDEX['X'],
                shape=(len(sequences), max_length),
                dtype="int32")

            df = pandas.DataFrame({"peptide": sequences}, dtype=numpy.object_)
            df["length"] = df.peptide.str.len()

            for (length, sub_df) in df.groupby("length"):
                diff = max_length - length
                #  identify peptides which are shorter than the max length to pad center
                if diff > 0:
                    split = length // 2
                    if length % 2 != 0:  # if length of peptide is odd
                        split += 1  # make larger end the n terminal end
                    #  create pad that puts 'X' at the middle of the max len peptide,
                    #   where pad is equal to the diff btwn max len and actual len
                    xpad = 'X' * diff
                    sub_df["peptide"] = sub_df["peptide"].str[:split] + xpad + \
                                        sub_df["peptide"].str[split:]  # put pad in mid

                #  Identify peptides that are longer than max length to trim center
                elif diff < 0:
                    split1 = split2 = max_length // 2   # calc length that should be kept
                    if max_length % 2 != 0:             # from each end of peptide
                        split1 += 1  # if max_length is odd, keep 1 more from n term side

                    sub_df["peptide"] = sub_df["peptide"].str[:split1] +\
                                        sub_df["peptide"].str[-split2:]  # trim peptides

                #  Convert sequence string to array of corresponding amino acid idx vals
                fixed_length_sequences = numpy.stack(
                            sub_df.peptide.map(
                                lambda s: numpy.array([
                                    get_amino_acid_index(char) for char in s
                                ])).values)
                #  map array to result at correct indicies
                result[sub_df.index, :] = fixed_length_sequences

        else:
            raise NotImplementedError(
                "Unsupported alignment method: %s" % alignment_method)

        return result


import player.clues
import util
import numpy as np
import disp
# TODO remove logging as it is kinda 'cheating'
import logging

class ClueKeying:
    def __init__(self, n_players, n_handcards):
        r"""Initialize a player.

        Initialize all internal fields and prepare player object for gameplay.

        Parameters
        ----------
        n_players : int
            Total number of players.
        n_handcards : int
            Number of cards in each players hand (before last round).

        """
        self.n_players = n_players
        self.curr_player_idx = None
        self.n_fuses = util.MAX_FUSES
        self.n_clues = util.MAX_CLUES
        self.n_undealt = util.N_CARDS
        self.hands = np.tile(util.CARD_ZEROS[np.newaxis, np.newaxis, ...], (self.n_players, n_handcards, 1, 1))
        self.clues_hands = np.tile(util.CARD_ONES[np.newaxis, np.newaxis, ...],
                                   (self.n_players, n_handcards, 1, 1))
        self.info_hands = np.tile(util.CARD_ZEROS[np.newaxis, np.newaxis, ...],
                                  (self.n_players, n_handcards, 1, 1))
        self.odds_hands = np.tile(util.CARD_ZEROS[np.newaxis, np.newaxis, ...],
                                  (self.n_players, n_handcards, 1, 1))
        self.n_cards = np.zeros((self.n_players,), np.int)
        self.table = np.copy(util.CARD_ZEROS)
        self.discards = np.copy(util.CARD_ZEROS)
        self.last_cluegiver = -1
        self.last_clue = -np.ones((4, 1))
        return

    def start_game(self, first_player_idx):
        r"""Get player ready for the game to start.

        Once provided `first_player_idx`, initialize all internal data in preparation for first turn.

        Parameters
        ----------
        first_player_idx : int
            Index of the first player, clockwise relative to self.

        """
        self.curr_player_idx = first_player_idx
        return

    def play_turn(self):
        r"""Play a turn by telling game object your action.

        Player chooses what action to take and on what cards or what clue to give to whom. Main logic for a player
        ruleset.

        Returns
        ----------
        action : tuple
            Action tuple, where first element is string description {'play', 'discard', 'clue'} and second element is
            action metadata. If 'play' or 'discard', second element is card index. If 'clue', second element is a tuple
            itself of (clue_type, clue_hint, card_idxs).

        """

        # determine action to take
        valid_action = False

        #if person to your right gave last clue or no clues left
        if (self.last_cluegiver == self.n_players - 2 or not self.n_clues):
            # follow last clue instruction
            possible_clues = self.get_possible_clues(self.last_clue[0], -1)
            cluable_player_idxs = self.exclude_idxs(np.array([self.last_clue[0], -1]))
            action = decode_clue(possible_clues, self.last_clue, cluable_player_idxs)

        else:
            top_playable = np.array([])

            # for cluing person to your left
            possible_clues = self.get_possible_clues(-1, 0)

            if self.n_clues > 1:
                # get person to your left to play a card
                ranked_playability_0 = self.rank_playability(0)
                actual_playable_0 = np.any(get_playable_cards(self.table)*self.hands[0], (1, 2))
                top_playable_0 = np.where(actual_playable_0[ranked_playability_0])[0]

                if top_playable_0.size:
                    rank_to_play_0 = np.where(ranked_playability_0 == top_playable_0[0])[0]
                    clue_info = ('play', rank_to_play_0[0])
                    logging.info('Trying to get player to my left to play {} most playable card, card {}'.format(rank_to_play_0[0], ranked_playability_0[rank_to_play_0[0]]))
                    clue = encode_clue(possible_clues, clue_info, self.exclude_idxs(np.array([0, -1])))
                    valid_action = True

            # if no playable cards or only one clue left
            if self.n_clues == 1 or not valid_action:

                # TODO tell to play if next person's communal top discardable card (after clue given? could be impossible, will have to check) is discardable

                ranked_discardability_0 = self.rank_discardability(0, np.array([0, -1]))
                actual_discardable_0 = np.any(get_discardable_cards(self.table)*self.hands[0], (1, 2))
                top_discardable_0 = np.where(actual_discardable_0[ranked_discardability_0])[0]
                if top_discardable_0.size:
                    rank_to_discard_0 = np.where(ranked_discardability_0 == top_discardable_0[0])[0]
                    clue_info = ('discard', rank_to_discard_0[0])
                    logging.info('Trying to get player to my left to discard {} most discardable card, card {}'.format(rank_to_discard_0[0], ranked_discardability_0[rank_to_discard_0[0]]))
                    clue = encode_clue(possible_clues, clue_info, self.exclude_idxs(np.array([0, -1])))
                    valid_action = True

            if valid_action:
                if not clue[1]:
                    # number
                    clue_idxs = np.where(np.sum(self.hands[clue[0], ...], axis=2)[:, clue[2]])[0]
                else:
                    # color
                    clue_idxs = np.where(np.sum(self.hands[clue[0], ...], axis=1)[:, clue[2]])[0]

                action = ('clue', (clue[0], 'color' if clue[1] else 'number', clue[2], clue_idxs))
            else:
                # this only happens if no discardable cards in their hand
                action = ('discard', 0)

        return action

    def invalid_card_played(self, player_idx, card_idx, card, n_fuses):
        r"""Learn that an unplayable card was played.

        Even when this player's card was played, the game lets all players know of an invalid play.

        Parameters
        ----------
        player_idx : int
            Relative index of the player who played the card.
        card_idx : int
            Index of the card played in the players hand.
        card : int
            Card index of the played card.
        n_fuses : int
            Number of fuses remaining after card was played.

        """
        self.discards += util.card2info(card)
        self.n_fuses = n_fuses
        self.reduce_hand(player_idx, card_idx)
        return

    def valid_card_played(self, player_idx, card_idx, card, n_clues):
        r"""Learn that a playable card was played.

        Even when this player's card was played, the game lets all players know of a valid play.

        Parameters
        ----------
        player_idx : int
            Relative index of the player who played the card.
        card_idx : int
            Index of the card played in the players hand.
        card : int
            Card index of the played card.
        n_clues : int
            Number of clues remaining after card was played.

        """
        self.table += util.card2info(card)
        self.n_clues = n_clues
        self.reduce_hand(player_idx, card_idx)
        return

    def card_discarded(self, player_idx, card_idx, card, n_clues):
        r"""Learn that a card was discarded.

        Even when this player's card was discarded, the game lets all players know of a discard.

        Parameters
        ----------
        player_idx : int
            Relative index of the player who discarded the card.
        card_idx : int
            Index of the card discarded in the players hand.
        card : int
            Card index of the discarded card.
        n_clues : int
            Number of clues remaining after card was discarded.

        """
        self.discards += util.card2info(card)
        self.n_clues = n_clues
        self.reduce_hand(player_idx, card_idx)
        return

    def clue_given(self, player_idx, to_player_idx, card_idxs, clue_type, clue_idx):
        r"""Learn that a clue was given.

        Even when this player is giving the clue, the game lets all players know of a clue.

        Parameters
        ----------
        player_idx : int
            Relative index of the player who gave the clue.
        to_player_idx : int
            Relative index of the player who received the clue.
        card_idxs : array of int
            Hand indexes of the cards being informed about.
        clue_type : {'number', 'color'}
            Type of clue given.
        clue_idx : int
            Index for given clue type.

        """
        self.n_clues -= 1
        self.last_cluegiver = player_idx
        self.last_clue[0] = player_idx
        self.last_clue[1] = to_player_idx
        self.last_clue[2] = [1 if clue_type is 'color' else 0]
        self.last_clue[3] = clue_idx

        if clue_type is 'number':
            for card_idx in range(self.n_cards[to_player_idx]):
                if np.isin(card_idx, card_idxs):
                    self.clues_hands[to_player_idx, card_idx, ...] *= get_number_clue_mask(clue_idx)
                else:
                    self.clues_hands[to_player_idx, card_idx, ...] *= get_number_nonclue_mask(clue_idx)
        elif clue_type is 'color':
            for card_idx in range(self.n_cards[to_player_idx]):
                if np.isin(card_idx, card_idxs):
                    self.clues_hands[to_player_idx, card_idx, ...] *= get_color_clue_mask(clue_idx)
                else:
                    self.clues_hands[to_player_idx, card_idx, ...] *= get_color_nonclue_mask(clue_idx)

        return

    def card_dealt(self, player_idx, card, n_cards):
        r"""Learn that a card was dealt.

        Even when this player is dealt a card, the game lets all players know of a deal. Card is always dealt to the
        right-most position, with all other cards shifting left.

        Parameters
        ----------
        player_idx : int
            Relative index of the player who was dealt a card.
        card : int or empty
            Card index of the dealt card, or empty if this player got the new card.
        n_cards : int
            Number of cards in receiving player's hand after card was dealt.

        """
        self.n_cards[player_idx] += 1
        assert (self.n_cards[player_idx] == n_cards)
        if player_idx == -1 or player_idx == self.n_players:
            self.hands[player_idx][self.n_cards[player_idx] - 1, ...] = util.CARD_NOINFO
        else:
            card_info = util.card2info(card)
            self.hands[player_idx][self.n_cards[player_idx] - 1, ...] = card_info
            self.hands[-1][:self.n_cards[player_idx], ...] -= card_info

        return

    def reduce_hand(self, player_idx, card_idx):
        keep_inds = np.where(np.arange(self.n_cards[player_idx]) != card_idx)[0]
        self.n_cards[player_idx] -= 1
        self.hands[player_idx][:self.n_cards[player_idx], ...] = self.hands[player_idx][keep_inds, ...]
        self.hands[player_idx][self.n_cards[player_idx], ...] = util.CARD_ZEROS
        self.clues_hands[player_idx][:self.n_cards[player_idx], ...] = self.clues_hands[player_idx][keep_inds, ...]
        self.clues_hands[player_idx][self.n_cards[player_idx], ...] = util.CARD_ONES
        return

    def rank_discardability(self, , visible_hands):

        # get indices of visible hands
        hand_idxs = np.delete(np.arange(0, self.n_players), np.mod(communal_idxs, self.n_players))
        # all visible cards to communal players
        visible_cards = self.table + self.discards + np.sum(self.hands[hand_idxs], (0, 1))
        discardable_cards = get_discardable_cards(visible_cards)
        hand_odds = norm_hand(util.CARD_NOINFO*self.clues_hands[player_idx, ...])
        odds_discardable = np.sum(discardable_cards[np.newaxis, ...] * hand_odds, (2, 1))

        return np.argsort(odds_discardable)

    def rank_playability(self, player_idx):

        playable_cards = get_playable_cards(self.table)
        hand_odds = norm_hand(util.CARD_NOINFO*self.clues_hands[player_idx, ...])
        odds_playable = np.sum(playable_cards[np.newaxis, ...] * hand_odds, (2, 1))

        return np.argsort(odds_playable)

    def get_possible_clues(self, clue_from, clue_to):
        communal_idxs = np.array([clue_from, clue_to])
        # get indices of visible hands, ordered left to right from clue giver
        hand_idxs = np.delete(np.arange(0, self.n_players), np.mod(communal_idxs, self.n_players))
        hand_idxs = (np.mod(np.sort(np.mod(hand_idxs - clue_from, self.n_players)) + clue_from, self.n_players)).astype(np.uint)
        hands = self.hands[hand_idxs]
        possible_color_clues = np.any(hands, (1, 2))
        possible_number_clues = np.any(hands, (1, 3))
        # disp.hand2string_basic(util.hand2cards(hands[0, :]))
        possible_clues = np.where(np.concatenate(
            (np.reshape(possible_number_clues, (possible_number_clues.shape[0], 1, possible_number_clues.shape[1])),
             np.reshape(possible_color_clues,
                        (possible_color_clues.shape[0], 1, possible_color_clues.shape[1]))), 1))

        # possible clues are in form (relative player_idx, number (0) or color (1), color/number idx)
        return possible_clues

    def exclude_idxs(self, idxs):
        return np.delete(np.arange(0, self.n_players), np.mod(idxs, self.n_players))

    def order_hands(from_idx, to_idx, n_players):
        hand_idxs = np.arange(n_players)
        remaining_idxs = np.delete(hand_idxs, np.mod(hand_idxs, n_players))
        ordered_idxs = (np.mod(np.sort(np.mod(remaining_idxs - from_idx, n_players)) + from_idx, n_players)).astype(
            np.uint)
        return ordered_idxs

    def exclude_order_hands(self, exclude_idxs):
        hand_idxs = exclude_order_idxs(exclude_idxs, self.n_players)
        return self.hands[hand_idxs]

def card_probs_from_clues(card_clues, visible_cards):
    n_cards = card_clues.shape[0]
    unknown_card = util.CARD_NOINFO - np.sum(visible_cards, 0)
    card_probs = np.tile(unknown_card[np.newaxis, ...], (n_cards, 1, 1))
    # I think this relies on the fact that clues apply to all held cards, whether positive or negative
    # so this only works if card_info is from valid clues, not inferred or otherwise learned info
    for ii in range(n_cards):
        card_probs[ii] = norm_card(unknown_card*card_clues[ii])
        unknown_card -= card_probs[ii]
    return card_probs

def rank_discardability(hand_info, visible_cards):

    # get indices of visible hands
    hand_idxs = np.delete(np.arange(0, self.n_players), np.mod(communal_idxs, self.n_players))
    # all visible cards to communal players
    visible_cards = self.table + self.discards + np.sum(self.hands[hand_idxs], (0, 1))
    discardable_cards = get_discardable_cards(visible_cards)
    hand_odds = norm_hand(util.CARD_NOINFO*self.clues_hands[player_idx, ...])
    odds_discardable = np.sum(discardable_cards[np.newaxis, ...] * hand_odds, (2, 1))

    return np.argsort(odds_discardable)

def exclude_order_idxs(exclude_idxs, n_idxs):
    r"""Order a set of indices with exclusions.

    First exclude index is used as the "base" for ordering purposes. Results is indices [0 N) excluding exclude_idxs,
    and ordered ascending from base index.

    Parameters
    ----------
    exclude_idxs : np.array(int)
        Indices to exclude. First element is the base index from which the remaining indices will be sorted mod ascending.
    n_idxs : int
        Number of indices.

    """
    base_idx = exclude_idxs[0]
    hand_idxs = np.arange(n_idxs)
    remaining_idxs = np.delete(hand_idxs, np.mod(exclude_idxs, n_idxs)) # exclude indices
    # order mod ascending from base
    ordered_idxs = (np.mod(np.sort(np.mod(remaining_idxs - base_idx, n_idxs)) + base_idx, n_idxs)).astype(np.uint)
    return ordered_idxs

def cards_possible(table, discards, hands):
    return util.CARD_NOINFO - table - discards - np.sum(hands, axis=(0, 1))

def norm_hand(hand):
    norm = np.sum(hand, (1, 2))[:, np.newaxis, np.newaxis]
    norm[np.where(norm == 0)[0]] = 1
    hand = hand / norm
    return hand

def norm_card(card):
    norm = np.sum(card)
    norm = 1 if norm==0 else norm
    card = card / norm
    return card

def get_playable_cards(table):
    playable_cards = np.concatenate((np.copy(util.CARD_ZEROS), np.zeros((1, table.shape[1]))), 0)
    playable_cards[np.sum(table, axis=0), np.arange(util.N_COLORS)] = 1
    return playable_cards[:-1, :]

def get_discardable_cards(visible_cards):
    discardable_cards = (np.copy(util.CARD_NOINFO) - visible_cards) != 1
    return discardable_cards

def get_useless_cards(table, discards):
    useless_cards = np.copy(util.CARD_ONES)
    max_color_scores = get_max_color_scores(discards)
    for color_idx in range(util.N_COLORS):
        useless_cards[np.sum(table[:, color_idx]):max_color_scores[color_idx], color_idx] = 0
    useless_cards[0, :] = 0
    return useless_cards

def get_max_color_scores(discards):
    # TODO figure out when not enough turns to play remaining cards
    cards_left = np.copy(util.CARD_NOINFO) - discards
    max_color_scores = util.MAX_NUMBER * np.ones((util.N_COLORS,), np.int8)
    for color_idx in range(util.N_COLORS):
        zero_inds = np.where(cards_left[:, color_idx] == 0)[0]
        if zero_inds.size:
            max_color_scores[color_idx] = np.min(zero_inds)
    return max_color_scores

def get_color_clue_mask(color_idx):
    mask = np.copy(util.CARD_ZEROS)
    mask[:, color_idx] = 1
    return mask

def get_number_clue_mask(number_idx):
    mask = np.copy(util.CARD_ZEROS)
    mask[number_idx, :] = 1
    return mask

def get_color_nonclue_mask(color_idx):
    return 1 - get_color_clue_mask(color_idx)

def get_number_nonclue_mask(number_idx):
    return 1 - get_number_clue_mask(number_idx)

def encode_clue(possible_clues, clue_info, clueable_player_idxs):

    logging.info(clue_info)
    if clue_info[0] == 'play':
        poss_idx = 2*clue_info[1]
    else:
        poss_idx = 2*clue_info[1] + 1

    logging.info('Using poss_idx of {}'.format(poss_idx))

    player_idx = clueable_player_idxs[possible_clues[0][poss_idx]]
    clue_type = possible_clues[1][poss_idx]
    clue_idx = possible_clues[2][poss_idx]
    clue = np.array([player_idx, clue_type, clue_idx])

    return clue

def decode_clue(possible_clues, clue_info, clueable_player_idxs):

    rel_clue_idx = np.where(clueable_player_idxs == clue_info[1])[0][0]
    poss_player_match = np.where(possible_clues[0] == rel_clue_idx)[0]
    poss_type_match = np.where(possible_clues[1] == clue_info[2])[0]
    poss_clue_match = np.where(possible_clues[2] == clue_info[3])[0]
    poss_idx = np.intersect1d(poss_player_match, np.intersect1d(poss_type_match, poss_clue_match))[0]

    logging.info('Received poss_idx of {}'.format(poss_idx))

    action_type = 'play' if not np.mod(poss_idx, 2) else 'discard'
    logging.info(action_type)
    card_idx = np.floor(poss_idx/2).astype(np.int)

    action = (action_type, card_idx)

    return action

def odds_sans_players(hands, clues, table, discards, visible_hands):
    return


#!/usr/bin/env python3
""" Parent class for color Adjustments for faceswap.py converter """

import logging
import numpy as np

from plugins.convert._config import Config

logger = logging.getLogger(__name__)  # pylint: disable=invalid-name

def get_config(plugin_name):
    """ Return the config for the requested model """
    return Config(plugin_name).config_dict

class Adjustment():
    """ Parent class for adjustments """
    def __init__(self, config=None):
        logger.debug("Initializing %s: (config: %s)", self.__class__.__name__, config)
        self.config = self.set_config(config)
        logger.debug("config: %s", self.config)
        logger.debug("Initialized %s", self.__class__.__name__)

    def set_config(self, config):
        """ Set the config to either global config or passed in config """
        section = ".".join(self.__module__.split(".")[-2:])
        if config is None:
            retval = get_config(section)
        else:
            config.section = section
            retval = config.config_dict
            config.section = None
        logger.debug("Config: %s", retval)
        return retval

    def process(self, old_face, new_face, raw_mask):
        """ Override for specific color adjustment process """
        raise NotImplementedError

    def run(self, old_face, new_face, raw_mask):
        """ Perform selected adjustment on face """
        logger.trace("Performing color adjustment")
        # Remove Mask for processing
        reinsert_mask = False
        if new_face.shape[2] == 4:
            reinsert_mask = True
            final_mask = new_face[:, :, -1]
            new_face = new_face[:, :, :3]
        new_face = self.process(old_face, new_face, raw_mask)
        new_face = np.clip(new_face, 0.0, 1.0)
        if reinsert_mask and new_face.shape[2] != 4:
            # Reinsert Mask
            new_face = np.concatenate((new_face, np.expand_dims(final_mask, axis=-1)), -1)
        logger.trace("Performed color adjustment")
        return new_face


import numpy as np
import cv2
import tensorflow as tf
from keras.preprocessing import image
from keras_vggface.vggface import VGGFace
from keras_vggface import utils
from keras.applications.resnet50 import preprocess_input

model = tf.keras.models.load_model("new resnet 50.model")

cap = cv2.VideoCapture(0)
classes = ["anger", "hap", "neut", "sad", "surp"]
IMG_SIZE = 224

while(True):
    ret, frame = cap.read()
    face_cascade = cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

    gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
    cropgray = gray[0:1080, 280:1000]
    resized = cv2.resize(cropgray, (IMG_SIZE, IMG_SIZE))
    faces = face_cascade.detectMultiScale(gray, 1.3, 5)
    for (x, y, w, h) in faces:
        crop = gray[y-50:y + h+50, x:x + w]
        crop = preprocess_input(crop)
        cv2.imwrite("capimg.jpg", crop)

    img = image.load_img('capimg.jpg', target_size=(224, 224))
    x = image.img_to_array(img)
    x = np.expand_dims(x, axis=0)
    # x = utils.preprocess_input(x, version=1)  # or version=2
    preds = model.predict(x)
    index = preds.argmax(axis=-1)
    print('Predicted:', classes[index[0]])

    faces = face_cascade.detectMultiScale(frame, 1.3, 5)
    font = cv2.FONT_HERSHEY_SIMPLEX
    org = (50, 50)
    fontScale = 1
    color = (255, 0, 0)
    thickness = 2
    for (x, y, w, h) in faces:
        cv2.rectangle(frame, (x, y), (x+w+10, y+h+10), (0, 255, 0), 2)
        cv2.putText(frame, classes[index[0]], (x, y-10), font,
                    fontScale, color, thickness, cv2.LINE_AA)
    disimg = cv2.putText(frame, classes[index[0]], org, font,
                         fontScale, color, thickness, cv2.LINE_AA)

    cv2.imshow('frame', disimg)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()


import cv2
import argparse
import numpy as np

ap = argparse.ArgumentParser()
ap.add_argument('-i','--image',required=True,help='path to input image')
ap.add_argument('-c','--config',required=True,help='path to yolo config file')
ap.add_argument('-w','--weights',required=True,help='path to yolo pre-trained weights')
ap.add_argument('-cl','--classes',required=True,help='path to text file containing class names')
args = ap.parse_args()

def get_output_layers(net):

    layer_names = net.getLayerNames()

    output_layers = [layer_names[i[0]-1] for i in net.getUnconnectedOutLayers()]

    return output_layers

def draw_prediction(img,class_id,confidence,x,y,x_plus_w,y_plus_h):

    label = str(classes[class_id])

    color = COLORS[class_id]

    cv2.rectangle(img,(x,y),(x_plus_w,y_plus_h),color,2)

    cv2.putText(img,label,(x-10,y-10),cv2.FONT_HERSHEY_SIMPLEX,0.5,color,2)

image = cv2.imread(args.image)

Width = image.shape[1]
Height = image.shape[0]
scale = 0.00392

classes = None
with open(args.classes,'r') as f:
    classes = [line.strip() for line in f.readlines()]

COLORS = np.random.uniform(0,255,size=(len(classes),3))

net = cv2.dnn.readNet(args.weights,args.config)

blob = cv2.dnn.blobFromImage(image,scale,(416,416),(0,0,0),True,crop=False)

net.setInput(blob)

outs = net.forward(get_output_layers(net))

class_ids = []
confidences = []
boxes = []
conf_threshold = 0.5
nms_threshold = 0.4

for out in outs:
    for detection in out:
        scores=detection[5:]
        class_id = np.argmax(scores)
        confidence = scores[class_id]
        if confidence > 0.5:
            center_x=int(detection[0]*Width)
            center_y=int(detection[1]*Height)
            w=int(detection[2]*Width)
            h=int(detection[3]*Height)
            x=center_x-w/2
            y=center_y-h/2
            class_ids.append(class_id)
            confidences.append(float(confidence))
            boxes.append([x,y,w,h])

indices = cv2.dnn.NMSBoxes(boxes,confidences,conf_threshold,nms_threshold)

for i in indices:
    i = i[0]
    box = boxes[i]
    x = box[0]
    y = box[1]
    w = box[2]
    h = box[3]
    draw_prediction(image,class_ids[i],confidences[i],round(x),round(y),round(x+w),round(y+h))

cv2.imshow("object detection",image)
cv2.waitKey()

cv2.imwrite("object-detection.jpg",image)
cv2.destroyAllWindows()


from keras.models import load_model
import cv2
import numpy as np

#Loading the model that has the highest accuracy and lowest loss
trainedmodel = load_model('model-050.model')

#The haar cascade classifier algorithm is being used to locate the face
faceclassifier=cv2.CascadeClassifier('haarcascade_frontalface_default.xml')

#Opens the default web camera for the device
webcamera=cv2.VideoCapture(0)

#Dictionaries that contain the labels and the colours for the bounding boxes
labels={0:'NO MASK',1:'MASK'}
colours={0:(0,0,255),1:(0,255,0)}

#keeps looping as long as the web camera is open
while(True):
    #reads in the frame from the webcam
    ret,frame=webcamera.read()
    #converts the image to gray scale to normalise the data
    grey=cv2.cvtColor(frame,cv2.COLOR_BGR2GRAY)
    faces=faceclassifier.detectMultiScale(grey,1.3,5)

    #for each frame in faces do this:
    for (x,y,w,h) in faces:
        #normalise and reshape the face
        face=grey[y:y+w,x:x+w]
        resized=cv2.resize(face,(100,100))
        normalised=resized/255.0
        reshaped=np.reshape(normalised,(1,100,100,1))
        #predict if the face is wearing a mask or not
        result=trainedmodel.predict(reshaped)

        #save the labels that were found in prediction
        label=np.argmax(result,axis=1)[0]

        #Draw the bounding box around the face and the result i.e 'MASK' and green box or 'NO MASK' and red box
        cv2.rectangle(frame,(x,y),(x+w,y+h),colours[label],2)
        cv2.rectangle(frame,(x,y-40),(x+w,y),colours[label],-1)
        cv2.putText(frame, labels[label], (x, y-10),cv2.FONT_HERSHEY_SIMPLEX,0.8,(255,255,255),2)

    #display window containing the image
    cv2.imshow('Face mask detector by Kyle Hennessy, Jordan Marah, Aaron Finlay',frame)
    #wait for a key press
    key=cv2.waitKey(1)

    #if the key press is "escape" then we break out of the loop
    if(key==27):
        break

#Destroy the window after the loop has finished and release the webcam
cv2.destroyAllWindows()
webcamera.release()


import time
import os
import argparse
import torch
import numpy as np
import glob
from pathlib import Path
from tqdm import tqdm
from ppg_extractor import load_model
import librosa
import soundfile as sf
from utils.load_yaml import HpsYaml

from encoder.audio import preprocess_wav
from encoder import inference as speacker_encoder
from vocoder.hifigan import inference as vocoder
from ppg2mel import MelDecoderMOLv2
from utils.f0_utils import compute_f0, f02lf0, compute_mean_std, get_converted_lf0uv

def _build_ppg2mel_model(model_config, model_file, device):
    ppg2mel_model = MelDecoderMOLv2(
        **model_config["model"]
    ).to(device)
    ckpt = torch.load(model_file, map_location=device)
    ppg2mel_model.load_state_dict(ckpt["model"])
    ppg2mel_model.eval()
    return ppg2mel_model

@torch.no_grad()
def convert(args):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    output_dir = args.output_dir
    os.makedirs(output_dir, exist_ok=True)

    step = os.path.basename(args.ppg2mel_model_file)[:-4].split("_")[-1]

    # Build models
    print("Load PPG-model, PPG2Mel-model, Vocoder-model...")
    ppg_model = load_model(
        Path('./ppg_extractor/saved_models/24epoch.pt'),
        device,
    )
    ppg2mel_model = _build_ppg2mel_model(HpsYaml(args.ppg2mel_model_train_config), args.ppg2mel_model_file, device)
    # vocoder.load_model('./vocoder/saved_models/pretrained/g_hifigan.pt', "./vocoder/hifigan/config_16k_.json")
    vocoder.load_model('./vocoder/saved_models/24k/g_02830000.pt')
    # Data related
    ref_wav_path = args.ref_wav_path
    ref_wav = preprocess_wav(ref_wav_path)
    ref_fid = os.path.basename(ref_wav_path)[:-4]

    # TODO: specify encoder
    speacker_encoder.load_model(Path("encoder/saved_models/pretrained_bak_5805000.pt"))
    ref_spk_dvec = speacker_encoder.embed_utterance(ref_wav)
    ref_spk_dvec = torch.from_numpy(ref_spk_dvec).unsqueeze(0).to(device)
    ref_lf0_mean, ref_lf0_std = compute_mean_std(f02lf0(compute_f0(ref_wav)))

    source_file_list = sorted(glob.glob(f"{args.wav_dir}/*.wav"))
    print(f"Number of source utterances: {len(source_file_list)}.")

    total_rtf = 0.0
    cnt = 0
    for src_wav_path in tqdm(source_file_list):
        # Load the audio to a numpy array:
        src_wav, _ = librosa.load(src_wav_path, sr=16000)
        src_wav_tensor = torch.from_numpy(src_wav).unsqueeze(0).float().to(device)
        src_wav_lengths = torch.LongTensor([len(src_wav)]).to(device)
        ppg = ppg_model(src_wav_tensor, src_wav_lengths)

        lf0_uv = get_converted_lf0uv(src_wav, ref_lf0_mean, ref_lf0_std, convert=True)
        min_len = min(ppg.shape[1], len(lf0_uv))

        ppg = ppg[:, :min_len]
        lf0_uv = lf0_uv[:min_len]

        start = time.time()
        _, mel_pred, att_ws = ppg2mel_model.inference(
            ppg,
            logf0_uv=torch.from_numpy(lf0_uv).unsqueeze(0).float().to(device),
            spembs=ref_spk_dvec,
        )
        src_fid = os.path.basename(src_wav_path)[:-4]
        wav_fname = f"{output_dir}/vc_{src_fid}_ref_{ref_fid}_step{step}.wav"
        mel_len = mel_pred.shape[0]
        rtf = (time.time() - start) / (0.01 * mel_len)
        total_rtf += rtf
        cnt += 1
        # continue
        mel_pred= mel_pred.transpose(0, 1)
        y, output_sample_rate = vocoder.infer_waveform(mel_pred.cpu())
        sf.write(wav_fname, y.squeeze(), output_sample_rate, "PCM_16")

    print("RTF:")
    print(total_rtf / cnt)

def get_parser():
    parser = argparse.ArgumentParser(description="Conversion from wave input")
    parser.add_argument(
        "--wav_dir",
        type=str,
        default=None,
        required=True,
        help="Source wave directory.",
    )
    parser.add_argument(
        "--ref_wav_path",
        type=str,
        required=True,
        help="Reference wave file path.",
    )
    parser.add_argument(
        "--ppg2mel_model_train_config", "-c",
        type=str,
        default=None,
        required=True,
        help="Training config file (yaml file)",
    )
    parser.add_argument(
        "--ppg2mel_model_file", "-m",
        type=str,
        default=None,
        required=True,
        help="ppg2mel model checkpoint file path"
    )
    parser.add_argument(
        "--output_dir", "-o",
        type=str,
        default="vc_gens_vctk_oneshot",
        help="Output folder to save the converted wave."
    )

    return parser

def main():
    parser = get_parser()
    args = parser.parse_args()
    convert(args)

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
#
#    Project: Peak finder in a single 2D diffraction frame
#             https://github.com/silx-kit/pyFAI
#
#
#    Copyright (C) 2014-2019 European Synchrotron Radiation Facility, Grenoble, France
#
#    Principal author:       Jérôme Kieffer (Jerome.Kieffer@ESRF.eu)
#
#
#  Permission is hereby granted, free of charge, to any person obtaining a copy
#  of this software and associated documentation files (the "Software"), to deal
#  in the Software without restriction, including without limitation the rights
#  to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
#  copies of the Software, and to permit persons to whom the Software is
#  furnished to do so, subject to the following conditions:
#  .
#  The above copyright notice and this permission notice shall be included in
#  all copies or substantial portions of the Software.
#  .
#  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
#  IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
#  FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
#  AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
#  LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
#  OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
#  THE SOFTWARE.

__authors__ = ["Jérôme Kieffer"]
__license__ = "MIT"
__date__ = "06/12/2019"
__copyright__ = "2014-2019, ESRF, Grenoble"
__contact__ = "jerome.kieffer@esrf.fr"

import logging
from collections import OrderedDict
import numpy
from ..utils import EPS32
from .azim_csr import OCL_CSR_Integrator, BufferDescription, EventDescription, mf, calc_checksum, pyopencl

logger = logging.getLogger(__name__)

class OCL_PeakFinder(OCL_CSR_Integrator):
    BLOCK_SIZE = 512  # unlike in OCL_CSR_Integrator, here we need large blocks
    buffers = [BufferDescription("output", 1, numpy.float32, mf.WRITE_ONLY),
               BufferDescription("output4", 4, numpy.float32, mf.READ_WRITE),
               BufferDescription("image_raw", 1, numpy.float32, mf.READ_ONLY),
               BufferDescription("image", 1, numpy.float32, mf.READ_WRITE),
               BufferDescription("variance", 1, numpy.float32, mf.READ_WRITE),
               BufferDescription("dark", 1, numpy.float32, mf.READ_WRITE),
               BufferDescription("dark_variance", 1, numpy.float32, mf.READ_ONLY),
               BufferDescription("flat", 1, numpy.float32, mf.READ_ONLY),
               BufferDescription("polarization", 1, numpy.float32, mf.READ_ONLY),
               BufferDescription("solidangle", 1, numpy.float32, mf.READ_ONLY),
               BufferDescription("absorption", 1, numpy.float32, mf.READ_ONLY),
               BufferDescription("mask", 1, numpy.int8, mf.READ_ONLY),
               BufferDescription("peak_position", 1, numpy.int32, mf.READ_WRITE),
               BufferDescription("radius2d", 1, numpy.float32, mf.READ_ONLY),
               ]
    kernel_files = ["pyfai:openCL/kahan.cl",
                    "pyfai:openCL/preprocess.cl",
                    "pyfai:openCL/memset.cl",
                    "pyfai:openCL/ocl_azim_CSR.cl",
                    "pyfai:openCL/peak_finder.cl"
                    ]

    def __init__(self, lut, image_size, checksum=None,
                 empty=None, unit=None, bin_centers=None,
                 radius=None, mask=None,
                 ctx=None, devicetype="all", platformid=None, deviceid=None,
                 block_size=None, profile=False):
        """
        :param lut: 3-tuple of arrays
            data: coefficient of the matrix in a 1D vector of float32 - size of nnz
            indices: Column index position for the data (same size as data)
            indptr: row pointer indicates the start of a given row. len nbin+1
        :param image_size: Expected image size: image.size
        :param checksum: pre-calculated checksum of the LUT to prevent re-calculating it :)
        :param empty: value to be assigned to bins without contribution from any pixel
        :param unit: Storage for the unit related to the LUT
        :param bin_centers: the radial position of the bin_center
        :param radius: radial position of every-pixel center.
        :param ctx: actual working context, left to None for automatic
                    initialization from device type or platformid/deviceid
        :param devicetype: type of device, can be "CPU", "GPU", "ACC" or "ALL"
        :param platformid: integer with the platform_identifier, as given by clinfo
        :param deviceid: Integer with the device identifier, as given by clinfo
        :param block_size: preferred workgroup size, may vary depending on the outpcome of the compilation
        :param profile: switch on profiling to be able to profile at the kernel level,
                        store profiling elements (makes code slightly slower)
        """
        self.radius = radius
        if mask is not None:
            self.mask = numpy.ascontiguousarray(mask, "int8")
        else:
            self.mask = None
        assert image_size == radius.size
        nbin = lut[2].size-1
        self.buffers += [BufferDescription("radius1d", nbin, numpy.float32, mf.READ_ONLY),
                         BufferDescription("counter", 1, numpy.float32, mf.WRITE_ONLY),
                         ]
        OCL_CSR_Integrator.__init__(self, lut, image_size, checksum,
                 empty, unit, bin_centers,
                 ctx, devicetype, platformid, deviceid,
                 block_size, profile)

        if self.mask is not None:
            self.send_buffer(self.mask, "mask")
            self.cl_kernel_args["corrections4"]["do_mask"] = numpy.int8(1)
        if self.bin_centers is not None:
            self.send_buffer(self.bin_centers, "radius1d")
        if self.radius is not None:
            self.send_buffer(self.radius, "radius2d")

    def set_kernel_arguments(self):
        OCL_CSR_Integrator.set_kernel_arguments(self)
        self.cl_kernel_args["find_peaks"] = OrderedDict((("output4", self.cl_mem["output4"]),
                                                          ("radius2d", self.cl_mem["radius2d"]),
                                                          ("radius1d", self.cl_mem["radius1d"]),
                                                          ("averint", self.cl_mem["averint"]),
                                                          ("stderr", self.cl_mem["stderr"]),
                                                          ("radius_min", numpy.float32(0.0)),
                                                          ("radius_max", numpy.float32(numpy.finfo(numpy.float32).max)),
                                                          ("cutoff", numpy.float32(5.0)),
                                                          ("noise", numpy.float32(1.0)),
                                                          ("counter", self.cl_mem["counter"]),
                                                          ("peak_position", self.cl_mem["peak_position"])))

    def peak_finder(self, data, dark=None, dummy=None, delta_dummy=None,
                   variance=None, dark_variance=None,
                   flat=None, solidangle=None, polarization=None, absorption=None,
                   dark_checksum=None, flat_checksum=None, solidangle_checksum=None,
                   polarization_checksum=None, absorption_checksum=None, dark_variance_checksum=None,
                   safe=True, error_model=None,
                   normalization_factor=1.0,
                   cutoff=4.0, cycle=5, noise=1,
                   radial_range=None,
                   out_avgint=None, out_stderr=None, out_merged=None):
        """
        Perform a sigma-clipping iterative filter within each along each row.
        see the doc of scipy.stats.sigmaclip for more descriptions.

        If the error model is "azimuthal": the variance is the variance within a bin,
        which is refined at each iteration, can be costly !

        Else, the error is propagated according to:

        .. math::

            signal = (raw - dark)
            variance = variance + dark_variance
            normalization  = normalization_factor*(flat * solidangle * polarization * absortoption)
            count = number of pixel contributing

        Integration is performed using the CSR representation of the look-up table on all
        arrays: signal, variance, normalization and count

        :param dark: array of same shape as data for pre-processing
        :param dummy: value for invalid data
        :param delta_dummy: precesion for dummy assessement
        :param variance: array of same shape as data for pre-processing
        :param dark_variance: array of same shape as data for pre-processing
        :param flat: array of same shape as data for pre-processing
        :param solidangle: array of same shape as data for pre-processing
        :param polarization: array of same shape as data for pre-processing
        :param dark_checksum: CRC32 checksum of the given array
        :param flat_checksum: CRC32 checksum of the given array
        :param solidangle_checksum: CRC32 checksum of the given array
        :param polarization_checksum: CRC32 checksum of the given array
        :param safe: if True (default) compares arrays on GPU according to their checksum, unless, use the buffer location is used
        :param preprocess_only: return the dark subtracted; flat field & solidangle & polarization corrected image, else
        :param normalization_factor: divide raw signal by this value
        :param cutoff: discard all points with |value - avg| > cutoff * sigma. 3-4 is quite common
        :param cycle: perform at maximum this number of cycles. 5 is common.
        :param out_avgint: destination array or pyopencl array for sum of all data
        :param out_stderr: destination array or pyopencl array for sum of the number of pixels
        :param out_merged: destination array or pyopencl array for averaged data (float8!)
        :return: out_avgint, out_stderr, out_merged
        """
        events = []
        if isinstance(error_model, str):
            error_model = error_model.lower()
        else:
            error_model = ""
        with self.sem:
            self.send_buffer(data, "image")
            wg = self.workgroup_size["memset_ng"]
            wdim_bins = (self.bins + wg[0] - 1) & ~(wg[0] - 1),
            memset = self.kernels.memset_out(self.queue, wdim_bins, wg, *list(self.cl_kernel_args["memset_ng"].values()))
            events.append(EventDescription("memset_ng", memset))
            kw_corr = self.cl_kernel_args["corrections4"]
            kw_int = self.cl_kernel_args["csr_sigma_clip4"]
            kw_proj = self.cl_kernel_args["find_peaks"]

            if dummy is not None:
                do_dummy = numpy.int8(1)
                dummy = numpy.float32(dummy)
                if delta_dummy is None:
                    delta_dummy = numpy.float32(0.0)
                else:
                    delta_dummy = numpy.float32(abs(delta_dummy))
            else:
                do_dummy = numpy.int8(0)
                dummy = numpy.float32(self.empty)
                delta_dummy = numpy.float32(0.0)

            kw_corr["do_dummy"] = do_dummy
            kw_corr["dummy"] = dummy
            kw_corr["delta_dummy"] = delta_dummy
            kw_corr["normalization_factor"] = numpy.float32(normalization_factor)

            if variance is not None:
                self.send_buffer(variance, "variance")
            if dark_variance is not None:
                if not dark_variance_checksum:
                    dark_variance_checksum = calc_checksum(dark_variance, safe)
                if dark_variance_checksum != self.on_device["dark_variance"]:
                    self.send_buffer(dark_variance, "dark_variance", dark_variance_checksum)
            else:
                do_dark = numpy.int8(0)
            kw_corr["do_dark"] = do_dark

            if dark is not None:
                do_dark = numpy.int8(1)
                # TODO: what is do_checksum=False and image not on device ...
                if not dark_checksum:
                    dark_checksum = calc_checksum(dark, safe)
                if dark_checksum != self.on_device["dark"]:
                    self.send_buffer(dark, "dark", dark_checksum)
            else:
                do_dark = numpy.int8(0)
            kw_corr["do_dark"] = do_dark

            if flat is not None:
                do_flat = numpy.int8(1)
                if not flat_checksum:
                    flat_checksum = calc_checksum(flat, safe)
                if self.on_device["flat"] != flat_checksum:
                    self.send_buffer(flat, "flat", flat_checksum)
            else:
                do_flat = numpy.int8(0)
            kw_corr["do_flat"] = do_flat

            if solidangle is not None:
                do_solidangle = numpy.int8(1)
                if not solidangle_checksum:
                    solidangle_checksum = calc_checksum(solidangle, safe)
                if solidangle_checksum != self.on_device["solidangle"]:
                    self.send_buffer(solidangle, "solidangle", solidangle_checksum)
            else:
                do_solidangle = numpy.int8(0)
            kw_corr["do_solidangle"] = do_solidangle

            if polarization is not None:
                do_polarization = numpy.int8(1)
                if not polarization_checksum:
                    polarization_checksum = calc_checksum(polarization, safe)
                if polarization_checksum != self.on_device["polarization"]:
                    self.send_buffer(polarization, "polarization", polarization_checksum)
            else:
                do_polarization = numpy.int8(0)
            kw_corr["do_polarization"] = do_polarization

            if absorption is not None:
                do_absorption = numpy.int8(1)
                if not absorption_checksum:
                    absorption_checksum = calc_checksum(absorption, safe)
                if absorption_checksum != self.on_device["absorption"]:
                    self.send_buffer(absorption, "absorption", absorption_checksum)
            else:
                do_absorption = numpy.int8(0)
            kw_corr["do_absorption"] = do_absorption

            if error_model.startswith("poisson"):
                kw_corr["poissonian"] = numpy.int8(1)
            else:
                kw_corr["poissonian"] = numpy.int8(0)
            wg = self.workgroup_size["corrections4"]
            ev = self.kernels.corrections4(self.queue, self.wdim_data, wg, *list(kw_corr.values()))
            events.append(EventDescription("corrections", ev))

            wg = self.workgroup_size["csr_sigma_clip4"][0]
            kw_proj["cutoff"] = kw_int["cutoff"] = numpy.float32(cutoff)
            kw_int["cycle"] = numpy.int32(cycle)
            if error_model.startswith("azim"):
                kw_int["azimuthal"] = numpy.int32(1)
            else:
                kw_int["azimuthal"] = numpy.int32(0)

            wdim_bins = (self.bins * wg),
            if wg == 1:
                raise RuntimeError("csr_sigma_clip4 is not yet available in single threaded OpenCL !")
                integrate = self.kernels.csr_integrate4_single(self.queue, wdim_bins, (wg,), *kw_int.values())
                events.append(EventDescription("integrate4_single", integrate))
            else:
                integrate = self.kernels.csr_sigma_clip4(self.queue, wdim_bins, (wg,), *kw_int.values())
                events.append(EventDescription("csr_sigma_clip4", integrate))

            # now perform the calc_from_1d on the device and count the number of pixels
            memset1 = self.program.memset_int(self.queue, self.wdim_data, self.workgroup_size["corrections"], self.cl_mem["peak_position"], numpy.int32(0), numpy.int32(self.size))
            memset2 = self.program.memset_int(self.queue, (1,), (1,), self.cl_mem["counter"], numpy.int32(0), numpy.int32(1))
            events+=[EventDescription("memset peak_position", memset1), EventDescription("memset counter", memset2)]

            if radial_range is not None:
                kw_proj["radius_min"] = numpy.float32(min(radial_range))
                kw_proj["radius_max"] = numpy.float32(max(radial_range) * EPS32)
            else:
                kw_proj["radius_min"] = numpy.float32(0.0)
                kw_proj["radius_max"] = numpy.float32(numpy.finfo(numpy.float32).max)

            kw_proj["noise"] = numpy.float32(noise)

            peak_search = self.program.find_peaks(self.queue, self.wdim_data, self.workgroup_size["corrections4"], *list(kw_proj.values()))
            events.append(EventDescription("peak_search", peak_search))
            # call the find_peaks kernel

            # Return the number of peaks
            cnt = numpy.empty(1, dtype=numpy.int32)
            ev = pyopencl.enqueue_copy(self.queue, cnt, self.cl_mem["counter"])
            events.append(EventDescription("copy D->H counter", ev))
            ev.wait()
            high = numpy.empty(cnt, dtype=numpy.int32)
            ev = pyopencl.enqueue_copy(self.queue, high, self.cl_mem["peak_position"])
            events.append(EventDescription("copy D->H high_position", ev))
#             if out_merged is None:
#                 merged = numpy.empty((self.bins, 8), dtype=numpy.float32)
#             else:
#                 merged = out_merged.data
#             if out_avgint is None:
#                 avgint = numpy.empty(self.bins, dtype=numpy.float32)
#             else:
#                 avgint = out_avgint.data
#             if out_stderr is None:
#                 stderr = numpy.empty(self.bins, dtype=numpy.float32)
#             else:
#                 stderr = out_stderr.data
#
#             ev = pyopencl.enqueue_copy(self.queue, avgint, self.cl_mem["averint"])
#             events.append(EventDescription("copy D->H avgint", ev))
#
#             ev = pyopencl.enqueue_copy(self.queue, stderr, self.cl_mem["stderr"])
#             events.append(EventDescription("copy D->H stderr", ev))
#             ev = pyopencl.enqueue_copy(self.queue, merged, self.cl_mem["merged8"])
#             events.append(EventDescription("copy D->H merged8", ev))
        if self.profile:
            self.events += events
        #res = Integrate1dtpl(self.bin_centers, avgint, stderr, merged[:, 0], merged[:, 2], merged[:, 4], merged[:, 6])
        #"position intensity error signal variance normalization count"
        return high

    # Name of the default "process" method
    __call__ = peak_finder


# execute mask_rcnn
# save txt file
# output : " frame_id, -1, x1, y1, w, h, scores, -1, -1, -1 "<- deep sort want this format

import torch
import numpy as np
import os
from PIL import Image

from maskrcnn_benchmark.config import cfg
from demo.predictor import COCODemo
import cv2

def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

GPU_NUM = 0
device = torch.device(f'cuda:{GPU_NUM}' if torch.cuda.is_available() else 'cpu')
torch.cuda.set_device(device)

test_split = 'test.txt'
test_vids = []
with open(test_split, 'r') as f:
    for line in f:
        vid = line.split(' ')[0]
        if 'positive' in vid:
            vid_name = vid.replace('positive/', '')
            vid_name = vid_name.replace('.npz', '')
            test_vids.append(vid_name)

#choose the pretrained model.
config_file = "configs/caffe2/e2e_mask_rcnn_R_50_FPN_1x_caffe2.yaml"
# update the config options with the config file
cfg.merge_from_file(config_file)
#Load the model
coco_demo = COCODemo(cfg, min_image_size=800, confidence_threshold=0.7,)

vid_root = '/HDD/accident_anticipation/Data/CCD/Crash'

test_data = {}

for i in range(len(test_vids)):
    vid_name = test_vids[i]
    vid_path = os.path.join(vid_root, vid_name+'.mp4')
    cap = cv2.VideoCapture(vid_path)
    cnt = 0
    frames = []
    while(cap.isOpened()):
        _, img = cap.read()
        img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
        img_pil = Image.fromarray(img_rgb)
        frames.append(img_pil)
        cnt+=1
        if cnt > 49:
            break
    cap.release()

    test_data[vid_name] = frames

save_root = '/HDD/accident_anticipation/Data/CCD/BBox/test'
vid_list = list(test_data.keys())
for vid_name in vid_list[92:]:
    frames = test_data[vid_name]
    data = None

    for i in range(len(frames)):
        image = frames[i]
        image = np.array(image)[:,:,[2,1,0]]

        bbox_predictions = coco_demo.compute_prediction(image)
        top_predictions = coco_demo.select_top_predictions(bbox_predictions)

        boxes = top_predictions.bbox
        labels = top_predictions.get_field("labels").tolist()
        scores = top_predictions.get_field("scores").tolist()

        boxes_arr = np.array(boxes) #[x1,y1,x2,y2]

        #start frame is 0
        frame_id = i

        for k in range(len(scores)):
            label_id = labels[k]
            # Only person(1), bicycle(2), car(3), motorcycle(4), bus(6), truck(8)
            if label_id == 1 or label_id==2 or label_id==3 or label_id==4 or label_id==6 or label_id==8:
                if data is not None: #if i==0 and k==0:
                    data_temp = np.array([frame_id, -1, boxes_arr[k][0], boxes_arr[k][1], boxes_arr[k][2] - boxes[k][0],
                                          boxes_arr[k][3] - boxes_arr[k][1], scores[k], -1, -1, -1])
                    data = np.vstack([data, data_temp])

                else:
                    data = np.array([frame_id, -1, boxes_arr[k][0], boxes_arr[k][1], boxes_arr[k][2] - boxes[k][0],
                                     boxes_arr[k][3] - boxes_arr[k][1], scores[k], -1, -1, -1])

    try:
        save_folder = os.path.join(save_root, vid_name)
        os.makedirs(save_folder, exist_ok = True)
        save_path = os.path.join(save_folder, vid_name+'.txt')
        np.savetxt(save_path, data, fmt='%.3f', delimiter=',')
    except:
        print(vid_name, 'passed')
        pass


#!/usr/bin/python3
'''
Abstract:
    This is a program for convert the SED data of SWIRE ELAIS N2 survey (2005) to AI model readable format.
Usage:
    tbl_to_sed_ELAIS_N2_2005.py [ELAIS N2 2005 table]
Output:
    1. The SED of sources in JHK, IRAC, and MIPS
    2. The coordinate of sources.
    3. The fake source type
    4. The fake Av
    5. The Quality label
Editor:
    Jacob975

##################################
#   Python3                      #
#   This code is made in python3 #
##################################

20190719
####################################
update log
20190719 version alpha 1:
    1. The code works.
'''
import time
import numpy as np
from sys import argv
import sys
import convert_lib
np.set_printoptions(threshold=sys.maxsize)
#--------------------------------------------
# Main code
if __name__ == "__main__":
    VERBOSE = 0
    # Measure time
    start_time = time.time()
    #-----------------------------------
    # Load argv
    if len(argv) != 2:
        print ("The number of arguments is wrong.")
        print ("tbl_to_sed_ELAIS_N2_2005.py [ELAIS_N2 2005 table]")
        exit()
    inp_table_name = argv[1]
    #-----------------------------------
    # Load data from input table
    inp_table = np.loadtxt(inp_table_name, dtype = str)
    inp_table[inp_table == '-99.00'] = 'null'
    inp_table[inp_table == '-99.0'] = 'null'
    #Jmag   = np.array(np.transpose([inp_table[:,4] , inp_table[:, 5]]), dtype = float)
    #Hmag   = np.array(np.transpose([inp_table[:,6] , inp_table[:, 7]]), dtype = float)
    #Kmag   = np.array(np.transpose([inp_table[:,8] , inp_table[:, 9]]), dtype = float)
    IR1flux = np.array(np.transpose([inp_table[:, 2], inp_table[:, 3]]))
    IR2flux = np.array(np.transpose([inp_table[:, 4], inp_table[:, 5]]))
    IR3flux = np.array(np.transpose([inp_table[:, 6], inp_table[:, 7]]))
    IR4flux = np.array(np.transpose([inp_table[:, 8], inp_table[:, 9]]))
    MP1flux = np.array(np.transpose([inp_table[:,10], inp_table[:,11]]))
    print ('Loading, done.')
    flux_sed = np.array(np.transpose([
                                        #Jflux[:,0],
                                        #Hflux[:,0],
                                        #Kflux[:,0],
                                        IR1flux[:,0],
                                        IR2flux[:,0],
                                        IR3flux[:,0],
                                        IR4flux[:,0],
                                        MP1flux[:,0],
                                        #Jflux[:,1],
                                        #Hflux[:,1],
                                        #Kflux[:,1],
                                        IR1flux[:,1],
                                        IR2flux[:,1],
                                        IR3flux[:,1],
                                        IR4flux[:,1],
                                        MP1flux[:,1],
                                        ]))
    flux_sed[flux_sed == 'null'] = '0.0'
    flux_sed = np.array(flux_sed, dtype = float)/1000
    print ('Arrange, done.')
    # coordinate, Av, source type, and Q label.
    coord = np.transpose(np.array([inp_table[:,0], inp_table[:,1]]))
    fake_Av = np.ones((len(inp_table), 2))
    source_type = np.zeros((len(inp_table), 3))
    source_type[:,0] = 1
    fake_Q = np.chararray((len(inp_table), 5))
    fake_Q[:] = 'A'
    fake_Q = np.array(fake_Q, dtype= str)
    #-----------------------------------
    # Save the data
    np.savetxt('ELAIS_N2_sed.txt', flux_sed)
    np.savetxt('ELAIS_N2_coord.txt', coord, fmt = '%s')
    np.savetxt('ELAIS_N2_c2d2007_Sp.txt', source_type, header = '# fake source type')
    np.savetxt('ELAIS_N2_Av.txt', fake_Av, header = '# fake Av')
    np.savetxt('ELAIS_N2_Q.txt', fake_Q, fmt = '%s', header = '# fake quality label')
    #-----------------------------------
    # Measure time
    elapsed_time = time.time() - start_time
    print("Exiting Main Program, spending ", elapsed_time, "seconds.")


# backend/main.py

import uvicorn
from fastapi import FastAPI

import config
import numpy as np
import tensorflow as tf
from bert import bert_tokenization
import tensorflow_hub as hub
from pydantic import BaseModel

# IMPORTANT STATE VARIABLES
module_url = 'https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2'
bert_layer = hub.KerasLayer(module_url, trainable=True)

vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = bert_tokenization.FullTokenizer(vocab_file, do_lower_case)

# UTILITY FUNCTIONS
saved_model = tf.keras.models.load_model(
    f'{config.MODEL_PATH}/{config.MODEL_NAME}')

def bert_encode(texts, tokenizer, max_len=512):
    all_tokens = []
    all_masks = []
    all_segments = []

    for text in texts:
        text = tokenizer.tokenize(text)

        text = text[:max_len-2]
        input_sequence = ["[CLS]"] + text + ["[SEP]"]
        pad_len = max_len - len(input_sequence)

        tokens = tokenizer.convert_tokens_to_ids(
            input_sequence) + [0] * pad_len
        pad_masks = [1] * len(input_sequence) + [0] * pad_len
        segment_ids = [0] * max_len

        all_tokens.append(tokens)
        all_masks.append(pad_masks)
        all_segments.append(segment_ids)

    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)

# FASTAPI

class Item(BaseModel):
    input_text: str

app = FastAPI()

@app.get("/")
def helloWorld():
    return {"message": "Welcome, I am CyberPolice! I will help you determine if a text is sexually harassing."}

@app.post("/predict")
async def predict(item: Item):
    """predicts if sentence is sexually harassing

    Parameters
    ----------
    item : Item
        BaseModel derived class object containing input_text(str).

    Returns
    -------
    dict
        Example dict
        {
            "confidence": 0.014952600002288818,
            "CyberPolice": "CYBER POLICE: says this sentence is not Sexually Harassing."
        }
    """
    inp = bert_encode([item.input_text], tokenizer, max_len=config.MAX_LEN)
    pred = saved_model.predict(inp)
    msg = 'CYBER POLICE: says this sentence is not Sexually Harassing.'
    if(pred[0][0] > 0.5):
        msg = 'CYBER POLICE: ALERT! This sentence is Sexually Harassing. Call 155260 or visit https://cybercrime.gov.in  to report this crime.'
    print(pred, pred[0][0], type(pred), type(pred[0][0]))
    return {"confidence": pred.item(), "CyberPolice": msg}

if __name__ == "__main__":
    uvicorn.run("main:app", host="0.0.0.0", port=8000)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import numpy as np

from rasa_core.actions.action import ACTION_LISTEN_NAME

class Customer:
    def __init__(self, indecisiveness=0.01, informativeness=1.1):
        self.cuisines = ['french', 'indian', 'chinese', 'sushi', 'british']
        self.amount_of_people = ['2', '3', '5', '6']
        self.city = ['london', 'paris', 'berlin', 'san francisco']
        self.price = ['moderate', 'expensive', 'cheap']
        self.happy = False
        self.complete = False
        self.indecisiveness = indecisiveness
        self.informativeness = informativeness
        self.preferences = {
            'cuisine': np.random.choice(self.cuisines),
            'people': np.random.choice(self.amount_of_people),
            'location': np.random.choice(self.city),
            'price': np.random.choice(self.price)
        }
        init_size = np.random.poisson(informativeness - 1) + 1
        initial_information = np.random.choice(list(self.preferences.keys()),
                                               size=init_size, replace=False)
        entity_strings = ','.join(['{}={}'.format(key, self.preferences[key])
                                   for key in initial_information])

        self.opening_inform = '_inform[{}]'.format(entity_strings)

        self.response_dict = {
            'action_ask_location': '.location',
            'action_ask_numpeople': '.people',
            'action_ask_price': '.price',
            'action_ask_cuisine': '.cuisine',
            'action_ack_findalternatives': '_deny',
            'action_ack_makereservation': 'happy_test',
            'action_ask_moreupdates': '_deny',
            'action_ask_helpmore': 'reshuf',
            'action_on_it': '_deny',
            'action_search_restaurants': '_deny',
            'action_store_slot': '_deny',
            'action_suggest': 'compare',
            'action_greet': self.opening_inform,
            'action_ask_howcanhelp': self.opening_inform,
            'action_goodbye': 'reset'
        }

    def _slot_values(self, tracker):
        values = {}
        for k, s in tracker.slots.items():
            values[k] = s.value
        return values

    def respond_to_action(self, tracker):
        if tracker is None:
            return '_greet'
        else:
            previous_actions = tracker.latest_action_ids
            if (len(previous_actions) < 2 or
                    previous_actions[-1] != ACTION_LISTEN_NAME or
                    previous_actions[-2] not in self.response_dict):
                # if the robot is not listening
                return 0
            output = self.response_dict[previous_actions[-2]]
            if self.complete is True:
                return '_goodbye'
            elif output.startswith('_'):
                return output
            elif output.startswith('.'):
                entity_string = '{}={}'.format(output[1:],
                                               self.preferences[output[1:]])
                return '_inform[{}]'.format(entity_string)
            elif output == 'coinflip':
                response = int(np.random.binomial(1, 0.5))
                return ['_affirm', '_deny'][response]
            elif output == 'reshuf':
                self.rethink()
            elif output == 'compare':
                if self._slot_values(tracker) == self.preferences:
                    self.happy = True
                    return '_affirm'
                else:
                    set_guess = set([k
                                     for k, s in tracker.slots.items()
                                     if s.value])
                    set_true = set(self.preferences.items())
                    fixes = set_true - set_guess
                    print(fixes)
                    entity_string = ','.join(['{}={}'.format(
                            fix[0], fix[1]) for fix in fixes])
                    return '_inform[{}]'.format(entity_string)
            elif output == 'happy_test':
                if self.happy is True:
                    self.complete = True
                    return '_affirm'
                else:
                    return '_deny'
            elif output == 'reset':
                self.__init__(indecisiveness=self.indecisiveness,
                              informativeness=self.informativeness)
                tracker._reset()
                return 'reset'
        self.rethink()

    def rethink(self):
        if np.random.rand() < self.indecisiveness:
            new_preferences = {
                'cuisine': np.random.choice(self.cuisines),
                'people': np.random.choice(self.amount_of_people),
                'location': np.random.choice(self.city),
                'price': np.random.choice(self.price)
            }
            random_key = np.random.choice(new_preferences)
            self.preferences[random_key] = new_preferences[random_key]
            return '_inform[{}:{}]'.format(
                    random_key, new_preferences[random_key])
        else:
            return '_deny'


# coding: utf-8

# # 生成电视剧剧本
#
# 在这个项目中，你将使用 RNN 创作你自己的[《辛普森一家》](https://zh.wikipedia.org/wiki/%E8%BE%9B%E6%99%AE%E6%A3%AE%E4%B8%80%E5%AE%B6)电视剧剧本。你将会用到《辛普森一家》第 27 季中部分剧本的[数据集](https://www.kaggle.com/wcukierski/the-simpsons-by-the-data)。你创建的神经网络将为一个在 [Moe 酒馆](https://simpsonswiki.com/wiki/Moe's_Tavern)中的场景生成一集新的剧本。
# ## 获取数据
# 我们早已为你提供了数据。你将使用原始数据集的子集，它只包括 Moe 酒馆中的场景。数据中并不包括酒馆的其他版本，比如 “Moe 的山洞”、“燃烧的 Moe 酒馆”、“Moe 叔叔的家庭大餐”等等。

# In[1]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
import helper

data_dir = './data/simpsons/moes_tavern_lines.txt'
text = helper.load_data(data_dir)
# Ignore notice, since we don't use it for analysing the data
text = text[81:]

# ## 探索数据
# 使用 `view_sentence_range` 来查看数据的不同部分。

# In[2]:

view_sentence_range = (0, 10)

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
import numpy as np

print('Dataset Stats')
print('Roughly the number of unique words: {}'.format(len({word: None for word in text.split()})))
scenes = text.split('\n\n')
print('Number of scenes: {}'.format(len(scenes)))
sentence_count_scene = [scene.count('\n') for scene in scenes]
print('Average number of sentences in each scene: {}'.format(np.average(sentence_count_scene)))

sentences = [sentence for scene in scenes for sentence in scene.split('\n')]
print('Number of lines: {}'.format(len(sentences)))
word_count_sentence = [len(sentence.split()) for sentence in sentences]
print('Average number of words in each line: {}'.format(np.average(word_count_sentence)))

print()
print('The sentences {} to {}:'.format(*view_sentence_range))
print('\n'.join(text.split('\n')[view_sentence_range[0]:view_sentence_range[1]]))

# ## 实现预处理函数
# 对数据集进行的第一个操作是预处理。请实现下面两个预处理函数：
#
# - 查询表
# - 标记符号的字符串
#
# ### 查询表
# 要创建词嵌入，你首先要将词语转换为 id。请在这个函数中创建两个字典：
#
# - 将词语转换为 id 的字典，我们称它为 `vocab_to_int`
# - 将 id 转换为词语的字典，我们称它为 `int_to_vocab`
#
# 请在下面的元组中返回这些字典
#  `(vocab_to_int, int_to_vocab)`

# In[3]:

import numpy as np
import problem_unittests as tests

def create_lookup_tables(text):
    """
    Create lookup tables for vocabulary
    :param text: The text of tv scripts split into words
    :return: A tuple of dicts (vocab_to_int, int_to_vocab)
    """
    # TODO: Implement Function
    vocab_to_int = {}
    int_vocab = {}
    for w in text:
        if w not in vocab_to_int:
            vocab_to_int[w] = len(vocab_to_int)
            int_vocab[len(vocab_to_int)-1] = w
    return vocab_to_int,int_vocab

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_create_lookup_tables(create_lookup_tables)

# ### 标记符号的字符串
# 我们会使用空格当作分隔符，来将剧本分割为词语数组。然而，句号和感叹号等符号使得神经网络难以分辨“再见”和“再见！”之间的区别。
#
# 实现函数 `token_lookup` 来返回一个字典，这个字典用于将 “!” 等符号标记为 “||Exclamation_Mark||” 形式。为下列符号创建一个字典，其中符号为标志，值为标记。
#
# - period ( . )
# - comma ( , )
# - quotation mark ( " )
# - semicolon ( ; )
# - exclamation mark ( ! )
# - question mark ( ? )
# - left parenthesis ( ( )
# - right parenthesis ( ) )
# - dash ( -- )
# - return ( \n )
#
# 这个字典将用于标记符号并在其周围添加分隔符（空格）。这能将符号视作单独词汇分割开来，并使神经网络更轻松地预测下一个词汇。请确保你并没有使用容易与词汇混淆的标记。与其使用 “dash” 这样的标记，试试使用“||dash||”。

# In[4]:

def token_lookup():
    """
    Generate a dict to turn punctuation into a token.
    :return: Tokenize dictionary where the key is the punctuation and the value is the token
    """
    # TODO: Implement Function
    token_dict = {}
    token_dict['.'] = '||period||'
    token_dict[','] = '||comma||'
    token_dict['"'] = '||quotation_mark||'
    token_dict[';'] = '||semicolon||'
    token_dict['!'] = '||exclamation_mark||'
    token_dict['?'] = '||question_mark||'
    token_dict['('] = '||left_parenthesis||'
    token_dict[')'] = '||right_parenthesis||'
    token_dict['--'] = '||dash||'
    token_dict['\n'] = '||return||'
    return token_dict

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_tokenize(token_lookup)

# ## 预处理并保存所有数据
# 运行以下代码将预处理所有数据，并将它们保存至文件。

# In[5]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
# Preprocess Training, Validation, and Testing Data
helper.preprocess_and_save_data(data_dir, token_lookup, create_lookup_tables)

# # 检查点
# 这是你遇到的第一个检点。如果你想要回到这个 notebook，或需要重新打开 notebook，你都可以从这里开始。预处理的数据都已经保存完毕。

# In[6]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
import helper
import numpy as np
import problem_unittests as tests

int_text, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()

# ## 创建神经网络
# 你将通过实现下面的函数，来创造用于构建 RNN 的必要元素：
#
# - get_inputs
# - get_init\_cell
# - get_embed
# - build_rnn
# - build_nn
# - get_batches
#
# ### 检查 TensorFlow 版本并访问 GPU

# In[7]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
from distutils.version import LooseVersion
import warnings
import tensorflow as tf

# Check TensorFlow Version
assert LooseVersion(tf.__version__) >= LooseVersion('1.0'), 'Please use TensorFlow version 1.0 or newer'
print('TensorFlow Version: {}'.format(tf.__version__))

# Check for a GPU
if not tf.test.gpu_device_name():
    warnings.warn('No GPU found. Please use a GPU to train your neural network.')
else:
    print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))

# ### 输入
#
# 实现函数 `get_inputs()` 来为神经网络创建 TF 占位符。它将创建下列占位符：
#
# - 使用 [TF 占位符](https://www.tensorflow.org/api_docs/python/tf/placeholder) `name` 参量输入 "input" 文本占位符。
# - Targets 占位符
# - Learning Rate 占位符
#
# 返回下列元组中的占位符 `(Input, Targets, LearningRate)`

# In[8]:

def get_inputs():
    """
    Create TF Placeholders for input, targets, and learning rate.
    :return: Tuple (input, targets, learning rate)
    """
    # TODO: Implement Function
    return tf.placeholder(tf.int32,shape=(None, None), name='input'), tf.placeholder(tf.int32,shape=(None, None), name='targets'), tf.placeholder(tf.float32,shape=(None), name='LearningRate')

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_get_inputs(get_inputs)

# ### 创建 RNN Cell 并初始化
#
# 在 [`MultiRNNCell`](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/MultiRNNCell) 中堆叠一个或多个 [`BasicLSTMCells`](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/BasicLSTMCell)
#
# - 使用 `rnn_size` 设定 RNN 大小。
# - 使用 MultiRNNCell 的 [`zero_state()`](https://www.tensorflow.org/api_docs/python/tf/contrib/rnn/MultiRNNCell#zero_state) 函数初始化 Cell 状态
# - 使用 [`tf.identity()`](https://www.tensorflow.org/api_docs/python/tf/identity) 为初始状态应用名称 "initial_state"
#
#
# 返回 cell 和下列元组中的初始状态 `(Cell, InitialState)`

# In[9]:

def get_init_cell(batch_size, rnn_size):
    """
    Create an RNN Cell and initialize it.
    :param batch_size: Size of batches
    :param rnn_size: Size of RNNs
    :return: Tuple (cell, initialize state)
    """
    # TODO: Implement Function
    cells = tf.contrib.rnn.BasicLSTMCell(rnn_size)
    stacked_rnn_cell = tf.contrib.rnn.MultiRNNCell([cells])
    State = stacked_rnn_cell.zero_state(batch_size=batch_size,dtype=tf.float32)
    InitialState = tf.identity(State,name='initial_state')

    return stacked_rnn_cell, InitialState

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_get_init_cell(get_init_cell)

# ### 词嵌入
# 使用 TensorFlow 将嵌入运用到 `input_data` 中。
# 返回嵌入序列。

# In[10]:

def get_embed(input_data, vocab_size, embed_dim):
    """
    Create embedding for <input_data>.
    :param input_data: TF placeholder for text input.
    :param vocab_size: Number of words in vocabulary.
    :param embed_dim: Number of embedding dimensions
    :return: Embedded input.
    """
    # TODO: Implement Function
    word_embed = tf.get_variable("word_embeddings",[vocab_size, embed_dim],dtype=tf.float32)
    embed = tf.nn.embedding_lookup(word_embed,ids=input_data)
    return embed

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_get_embed(get_embed)

# ### 创建 RNN
# 你已经在 `get_init_cell()` 函数中创建了 RNN Cell。是时候使用这个 Cell 来创建 RNN了。
#
# - 使用 [`tf.nn.dynamic_rnn()`](https://www.tensorflow.org/api_docs/python/tf/nn/dynamic_rnn) 创建 RNN
# - 使用 [`tf.identity()`](https://www.tensorflow.org/api_docs/python/tf/identity) 将名称 "final_state" 应用到最终状态中
#
#
# 返回下列元组中的输出和最终状态`(Outputs, FinalState)`

# In[11]:

def build_rnn(cell, inputs):
    """
    Create a RNN using a RNN Cell
    :param cell: RNN Cell
    :param inputs: Input text data
    :return: Tuple (Outputs, Final State)
    """
    # TODO: Implement Function
    outputs, state = tf.nn.dynamic_rnn(cell=cell,
                                   inputs=inputs,
                                   dtype=tf.float32)
    final_state = tf.identity(state,name='final_state')
    return outputs, final_state

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_build_rnn(build_rnn)

# ### 构建神经网络
# 应用你在上面实现的函数，来：
#
# - 使用你的 `get_embed(input_data, vocab_size, embed_dim)` 函数将嵌入应用到 `input_data` 中
# - 使用 `cell` 和你的 `build_rnn(cell, inputs)` 函数来创建 RNN
# - 应用一个完全联通线性激活和 `vocab_size` 的分层作为输出数量。
#
# 返回下列元组中的 logit 和最终状态 `Logits, FinalState`

# In[12]:

def build_nn(cell, rnn_size, input_data, vocab_size, embed_dim):
    """
    Build part of the neural network
    :param cell: RNN cell
    :param rnn_size: Size of rnns
    :param input_data: Input data
    :param vocab_size: Vocabulary size
    :param embed_dim: Number of embedding dimensions
    :return: Tuple (Logits, FinalState)
    """
    # TODO: Implement Function
    embed = get_embed(input_data, vocab_size, embed_dim)
    rnn_out,finalstate = build_rnn(cell, embed)
    logits = tf.layers.dense(rnn_out,vocab_size)
    return logits, finalstate

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_build_nn(build_nn)

# ### 批次
#
# 实现 `get_batches` 来使用 `int_text` 创建输入与目标批次。这些批次应为 Numpy 数组，并具有形状 `(number of batches, 2, batch size, sequence length)`。每个批次包含两个元素：
#
# - 第一个元素为**输入**的单独批次，并具有形状 `[batch size, sequence length]`
# - 第二个元素为**目标**的单独批次，并具有形状 `[batch size, sequence length]`
#
# 如果你无法在最后一个批次中填入足够数据，请放弃这个批次。
#
# 例如 `get_batches([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15], 2, 3)` 将返回下面这个 Numpy 数组：
# ```
# [
#   # First Batch
#   [
#     # Batch of Input
#     [[ 1  2  3], [ 7  8  9]],
#     # Batch of targets
#     [[ 2  3  4], [ 8  9 10]]
#   ],
#
#   # Second Batch
#   [
#     # Batch of Input
#     [[ 4  5  6], [10 11 12]],
#     # Batch of targets
#     [[ 5  6  7], [11 12 13]]
#   ]
# ]
# ```

# In[13]:

def get_batches(int_text, batch_size, seq_length):
    """
    Return batches of input and target
    :param int_text: Text with the words replaced by their ids
    :param batch_size: The size of batch
    :param seq_length: The length of sequence
    :return: Batches as a Numpy array
    """
    # TODO: Implement Function

    n_batches = int(len(int_text) / (batch_size * seq_length))
    xdata = np.array(int_text[: n_batches * batch_size * seq_length])
    ydata = np.array(int_text[1: n_batches * batch_size * seq_length + 1])
    x_batches = np.split(xdata.reshape(batch_size, -1), n_batches, 1)
    y_batches = np.split(ydata.reshape(batch_size, -1), n_batches, 1)
    return np.array(list(zip(x_batches, y_batches)))

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_get_batches(get_batches)

# ## 神经网络训练
# ### 超参数
# 调整下列参数:
#
# - 将 `num_epochs` 设置为训练次数。
# - 将 `batch_size` 设置为程序组大小。
# - 将 `rnn_size` 设置为 RNN 大小。
# - 将 `embed_dim` 设置为嵌入大小。
# - 将 `seq_length` 设置为序列长度。
# - 将 `learning_rate` 设置为学习率。
# - 将 `show_every_n_batches` 设置为神经网络应输出的程序组数量。

# In[14]:

# Number of Epochs
num_epochs = 300
# Batch Size
batch_size = 128
# RNN Size
rnn_size = 256
# Embedding Dimension Size
embed_dim = 300
# Sequence Length
seq_length = 100
# Learning Rate
learning_rate = .01
# Show stats for every n number of batches
show_every_n_batches = 10

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
save_dir = './save'

# ### 创建图表
# 使用你实现的神经网络创建图表。

# In[15]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
from tensorflow.contrib import seq2seq

train_graph = tf.Graph()
with train_graph.as_default():
    vocab_size = len(int_to_vocab)
    input_text, targets, lr = get_inputs()
    input_data_shape = tf.shape(input_text)
    cell, initial_state = get_init_cell(input_data_shape[0], rnn_size)
    logits, final_state = build_nn(cell, rnn_size, input_text, vocab_size, embed_dim)

    # Probabilities for generating words
    probs = tf.nn.softmax(logits, name='probs')

    # Loss function
    cost = seq2seq.sequence_loss(
        logits,
        targets,
        tf.ones([input_data_shape[0], input_data_shape[1]]))

    # Optimizer
    optimizer = tf.train.AdamOptimizer(lr)

    # Gradient Clipping
    gradients = optimizer.compute_gradients(cost)
    capped_gradients = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gradients if grad is not None]
    train_op = optimizer.apply_gradients(capped_gradients)

# ## 训练
# 在预处理数据中训练神经网络。如果你遇到困难，请查看这个[表格](https://discussions.udacity.com/)，看看是否有人遇到了和你一样的问题。

# In[16]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
batches = get_batches(int_text, batch_size, seq_length)

with tf.Session(graph=train_graph) as sess:
    sess.run(tf.global_variables_initializer())

    for epoch_i in range(num_epochs):
        state = sess.run(initial_state, {input_text: batches[0][0]})

        for batch_i, (x, y) in enumerate(batches):
            feed = {
                input_text: x,
                targets: y,
                initial_state: state,
                lr: learning_rate}
            train_loss, state, _ = sess.run([cost, final_state, train_op], feed)

            # Show every <show_every_n_batches> batches
            if (epoch_i * len(batches) + batch_i) % show_every_n_batches == 0:
                print('Epoch {:>3} Batch {:>4}/{}   train_loss = {:.3f}'.format(
                    epoch_i,
                    batch_i,
                    len(batches),
                    train_loss))

    # Save Model
    saver = tf.train.Saver()
    saver.save(sess, save_dir)
    print('Model Trained and Saved')

# ## 储存参数
# 储存 `seq_length` 和 `save_dir` 来生成新的电视剧剧本。

# In[17]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
# Save parameters for checkpoint
helper.save_params((seq_length, save_dir))

# # 检查点

# In[18]:

"""
DON'T MODIFY ANYTHING IN THIS CELL
"""
import tensorflow as tf
import numpy as np
import helper
import problem_unittests as tests

_, vocab_to_int, int_to_vocab, token_dict = helper.load_preprocess()
seq_length, load_dir = helper.load_params()

# ## 实现生成函数
# ### 获取 Tensors
# 使用 [`get_tensor_by_name()`](https://www.tensorflow.org/api_docs/python/tf/Graph#get_tensor_by_name)函数从 `loaded_graph` 中获取 tensor。  使用下面的名称获取 tensor：
#
# - "input:0"
# - "initial_state:0"
# - "final_state:0"
# - "probs:0"
#
# 返回下列元组中的 tensor `(InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor)`

# In[19]:

def get_tensors(loaded_graph):
    """
    Get input, initial state, final state, and probabilities tensor from <loaded_graph>
    :param loaded_graph: TensorFlow graph loaded from file
    :return: Tuple (InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor)
    """
    # TODO: Implement Function
    InputTensor = loaded_graph.get_tensor_by_name(name="input:0")
    InitialStateTensor = loaded_graph.get_tensor_by_name(name="initial_state:0")
    FinalStateTensor = loaded_graph.get_tensor_by_name(name="final_state:0")
    ProbsTensor = loaded_graph.get_tensor_by_name(name="probs:0")
    return InputTensor, InitialStateTensor, FinalStateTensor, ProbsTensor

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_get_tensors(get_tensors)

# ### 选择词汇
# 实现 `pick_word()` 函数来使用 `probabilities` 选择下一个词汇。

# In[20]:

def pick_word(probabilities, int_to_vocab):
    """
    Pick the next word in the generated text
    :param probabilities: Probabilites of the next word
    :param int_to_vocab: Dictionary of word ids as the keys and words as the values
    :return: String of the predicted word
    """
    # TODO: Implement Function
    return np.random.choice(list(int_to_vocab.values()), 1, p=probabilities)[0]

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
tests.test_pick_word(pick_word)

# ## 生成电视剧剧本
# 这将为你生成一个电视剧剧本。通过设置 `gen_length` 来调整你想生成的剧本长度。

# In[21]:

gen_length = 200
# homer_simpson, moe_szyslak, or Barney_Gumble
prime_word = 'moe_szyslak'

"""
DON'T MODIFY ANYTHING IN THIS CELL THAT IS BELOW THIS LINE
"""
loaded_graph = tf.Graph()
with tf.Session(graph=loaded_graph) as sess:
    # Load saved model
    loader = tf.train.import_meta_graph(load_dir + '.meta')
    loader.restore(sess, load_dir)

    # Get Tensors from loaded model
    input_text, initial_state, final_state, probs = get_tensors(loaded_graph)

    # Sentences generation setup
    gen_sentences = [prime_word + ':']
    prev_state = sess.run(initial_state, {input_text: np.array([[1]])})

    # Generate sentences
    for n in range(gen_length):
        # Dynamic Input
        dyn_input = [[vocab_to_int[word] for word in gen_sentences[-seq_length:]]]
        dyn_seq_length = len(dyn_input[0])

        # Get Prediction
        probabilities, prev_state = sess.run(
            [probs, final_state],
            {input_text: dyn_input, initial_state: prev_state})

        pred_word = pick_word(probabilities[dyn_seq_length-1], int_to_vocab)

        gen_sentences.append(pred_word)

    # Remove tokens
    tv_script = ' '.join(gen_sentences)
    for key, token in token_dict.items():
        ending = ' ' if key in ['\n', '(', '"'] else ''
        tv_script = tv_script.replace(' ' + token.lower(), key)
    tv_script = tv_script.replace('\n ', '\n')
    tv_script = tv_script.replace('( ', '(')

    print(tv_script)

# # 这个电视剧剧本是无意义的
# 如果这个电视剧剧本毫无意义，那也没有关系。我们的训练文本不到一兆字节。为了获得更好的结果，你需要使用更小的词汇范围或是更多数据。幸运的是，我们的确拥有更多数据！在本项目开始之初我们也曾提过，这是[另一个数据集](https://www.kaggle.com/wcukierski/the-simpsons-by-the-data)的子集。我们并没有让你基于所有数据进行训练，因为这将耗费大量时间。然而，你可以随意使用这些数据训练你的神经网络。当然，是在完成本项目之后。
# # 提交项目
# 在提交项目时，请确保你在保存 notebook 前运行了所有的单元格代码。请将 notebook 文件保存为 "dlnd_tv_script_generation.ipynb"，并将它作为 HTML 文件保存在 "File" -> "Download as" 中。请将 "helper.py" 和 "problem_unittests.py" 文件一并提交。