import re
import re
import warnings

import numpy as np
import pandas as pd
from pandas.core.common import SettingWithCopyWarning
from scipy.sparse import csr_matrix
from scipy.stats import hmean, fisher_exact, rankdata, norm
from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.linear_model import ElasticNet
from sklearn.linear_model import LogisticRegression
from sklearn.linear_model import RidgeClassifierCV, LassoCV

from scattertext.CSRMatrixTools import delete_columns
from scattertext.Common import DEFAULT_BETA, DEFAULT_SCALER_ALGO, DEFAULT_BACKGROUND_SCALER_ALGO, \
	DEFAULT_BACKGROUND_BETA
from scattertext.FeatureOuput import FeatureLister
from scattertext.frequencyreaders.DefaultBackgroundFrequencies import DefaultBackgroundFrequencies
from scattertext.indexstore import IndexStore, IndexStoreFromList
from scattertext.termranking import AbsoluteFrequencyRanker
from scattertext.termscoring.CornerScore import CornerScore

warnings.simplefilter(action="ignore", category=SettingWithCopyWarning)

from scattertext.termscoring.ScaledFScore import InvalidScalerException, ScaledFScore

SPACY_ENTITY_TAGS = ['person', 'norp', 'facility', 'org', 'gpe',
                     'loc', 'product', 'event', 'work_of_art', 'language',
                     'type', 'date', 'time', 'percent', 'money', 'quantity',
                     'ordinal', 'cardinal']

MY_ENGLISH_STOP_WORDS = set(ENGLISH_STOP_WORDS) | {'hasn', 'won', 'don', 'haven', 'shouldn', 'isn', 'couldn', 'wouldn',
                                                   'aren', 'didn', 'wasn', 'dosen', 'weren', 'doesn'}

class CannotCreateATermDocMatrixWithASignleCategoryException(Exception):
	pass

class TermDocMatrix(object):
	'''
	!!! to do: refactor score functions into classes
	'''

	def __init__(self,
	             X, mX, y,
	             term_idx_store,
	             category_idx_store,
	             metadata_idx_store,
	             unigram_frequency_path=None):
		'''

		Parameters
		----------
		X : csr_matrix
			term document matrix
		mX : csr_matrix
			metadata-document matrix
		y : np.array
			category index array
		term_idx_store : IndexStore
			Term indices
		category_idx_store : IndexStore
			Catgory indices
		metadata_idx : IndexStore
		  Document metadata indices
		unigram_frequency_path : str or None
			Path to term frequency file.
		'''
		if all(y == y[0]):
			raise CannotCreateATermDocMatrixWithASignleCategoryException(
				'Documents must be labeled with more than one category. All documents were labeled '
				'with category: "' + str(category_idx_store.getval(y[0])) + '"')
		self._X, self._mX, self._y, self._term_idx_store, self._category_idx_store = \
			X, mX, y, term_idx_store, category_idx_store
		self._metadata_idx_store = metadata_idx_store
		self._unigram_frequency_path = unigram_frequency_path
		self._background_corpus = None

	def get_default_stoplist(self):
		return MY_ENGLISH_STOP_WORDS

	def compact(self, compactor):
		'''
		Compact term document matrix.

		Parameters
		----------
		compactor : object
			Object that takes a Term Doc Matrix as its first argument, and has a compact function which returns a
			Term Doc Matrix like argument

		Returns
		-------
		TermDocMatrix
		New Term Doc Matrix
		'''
		return compactor.compact(self)

	def get_num_terms(self):
		'''
		Returns
		-------
		The number of terms registered in the term doc matrix
		'''
		return len(self._term_idx_store)

	def get_categories(self):
		'''

		Returns
		-------
		list
		Category names
		'''
		return self._category_idx_store.values()

	def get_num_docs(self):
		'''

		Returns
		-------
		int, number of documents
		'''
		return len(self._y)

	def get_total_unigram_count(self):
		return self._get_unigram_term_freq_df().sum()

	def _get_unigram_term_freq_df(self):
		return self._get_corpus_unigram_freq(
			self.get_term_freq_df().sum(axis=1)
		)

	def old_get_term_freq_df(self):
		d = {'term': self._term_idx_store._i2val}
		for i, category in self._category_idx_store.items():
			d[category + ' freq'] = self._X[self._y == i].sum(axis=0).A1
		return pd.DataFrame(d).set_index('term')

	def get_term_freq_df(self, label_append=' freq'):
		'''
		Parameters
		-------
		label_append : str

		Returns
		-------
		pd.DataFrame indexed on terms, with columns giving frequencies for each
		'''

		'''
		row = self._row_category_ids()
		newX = csr_matrix((self._X.data, (row, self._X.indices)))
		return self._term_freq_df_from_matrix(newX)
		'''
		mat = self.get_term_freq_mat()
		return pd.DataFrame(mat,
		                    index=pd.Series(self.get_terms(), name='term'),
		                    columns=[c + label_append for c in self.get_categories()])

	def get_term_freq_mat(self):
		'''
		Returns
		-------
		np.array with columns as categories and rows as terms
		'''
		freq_mat = np.zeros(shape=(len(self.get_terms()), len(self.get_categories())), dtype=int)
		for cat_i in range(len(self._category_idx_store._i2val)):
			freq_mat[:, cat_i] = self._X[self._y == cat_i, :].sum(axis=0)
		return freq_mat

	def get_term_count_mat(self):
		'''
		Returns
		-------
		np.array with columns as categories and rows as terms
		'''
		freq_mat = np.zeros(shape=(len(self.get_terms()), len(self.get_categories())), dtype=int)
		for cat_i in range(len(self._category_idx_store._i2val)):
			X = (self._X[self._y == cat_i, :] > 0).astype(int)
			freq_mat[:, cat_i] = X.sum(axis=0)
		return freq_mat

	def get_term_doc_count_df(self, label_append=' freq'):
		'''

		Returns
		-------
		pd.DataFrame indexed on terms, with columns the number of documents each term appeared in
		'''
		# row = self._row_category_ids()
		# newX = csr_matrix(((self._X.data > 0).astype(int), (row, self._X.indices)))
		# return self._term_freq_df_from_matrix(newX)
		mat = self.get_term_count_mat()
		return pd.DataFrame(mat,
		                    index=self.get_terms(),
		                    columns=[c + label_append for c in self.get_categories()])

	def _term_freq_df_from_matrix(self, catX):
		return self._get_freq_df_using_idx_store(catX, self._term_idx_store)

	def _get_freq_df_using_idx_store(self, catX, idx_store):
		d = {'term': idx_store._i2val}
		for idx, cat in self._category_idx_store.items():
			try:
				d[cat + ' freq'] = catX[idx, :].A[0]
			except IndexError:
				self._fix_problem_when_final_category_index_has_no_terms(cat, catX, d)
		return pd.DataFrame(d).set_index('term')

	def _fix_problem_when_final_category_index_has_no_terms(self, cat, catX, d):
		d[cat + ' freq'] = np.zeros(catX.shape[1])

	def get_metadata_freq_df(self):
		'''
		Returns
		-------
		pd.DataFrame indexed on metadata, with columns giving frequencies for each
		'''
		row = self._row_category_ids_for_meta()
		newX = csr_matrix((self._mX.data, (row, self._mX.indices)))
		return self._metadata_freq_df_from_matrix(newX)

	def _row_category_ids(self):
		row = self._X.tocoo().row
		for i, cat in enumerate(self._y):
			row[row == i] = cat
		return row

	def _row_category_ids_for_meta(self):
		row = self._mX.tocoo().row
		for i, cat in enumerate(self._y):
			row[row == i] = cat
		return row

	def _metadata_freq_df_from_matrix(self, catX):
		return self._get_freq_df_using_idx_store(catX, self._metadata_idx_store)

	def get_terms(self):
		'''
		Returns
		-------
		np.array of unique terms
		'''
		return self._term_idx_store._i2val

	def get_category_names_by_row(self):
		'''
		Returns
		-------
		np.array of the category name for each row
		'''
		return np.array(self.get_categories())[self._y]

	def get_unigram_corpus(self):
		'''
		Returns
		-------
		A new TermDocumentMatrix consisting of only unigrams in the current TermDocumentMatrix.
		'''
		terms_to_ignore = self._get_non_unigrams()
		return self.remove_terms(terms_to_ignore)

	def _get_non_unigrams(self):
		return [term for term
		        in self._term_idx_store._i2val
		        if ' ' in term or "'" in term]

	def remove_infrequent_words(self, minimum_term_count, term_ranker=AbsoluteFrequencyRanker):
		'''
		Returns
		-------
		A new TermDocumentMatrix consisting of only terms which occur at least minimum_term_count.
		'''
		tdf = term_ranker(self).get_ranks().sum(axis=1)
		return self.remove_terms(list(tdf[tdf <= minimum_term_count].index))

	def remove_entity_tags(self):
		'''
		Returns
		-------
		A new TermDocumentMatrix consisting of only terms in the current TermDocumentMatrix
		 that aren't spaCy entity tags.

		Note: Used if entity types are censored using FeatsFromSpacyDoc(tag_types_to_censor=...).
		'''
		terms_to_remove = [term for term in self._term_idx_store._i2val
		                   if any([word in SPACY_ENTITY_TAGS for word in term.split()])]
		return self.remove_terms(terms_to_remove)

	def get_stoplisted_unigram_corpus(self, stoplist=None):
		'''
		Parameters
		-------
		stoplist : list, optional

		Returns
		-------
		A new TermDocumentMatrix consisting of only unigrams in the current TermDocumentMatrix.
		'''
		if stoplist is None:
			stoplist = self.get_default_stoplist()
		else:
			stoplist = [w.lower() for w in stoplist]
		return self._remove_terms_from_list(stoplist)

	def get_stoplisted_unigram_corpus_and_custom(self,
	                                             custom_stoplist):
		'''
		Parameters
		-------
		stoplist : list of lower-cased words, optional

		Returns
		-------
		A new TermDocumentMatrix consisting of only unigrams in the current TermDocumentMatrix.
		'''
		if type(custom_stoplist) == str:
			custom_stoplist = [custom_stoplist]
		return self._remove_terms_from_list(set(self.get_default_stoplist())
		                                    | set(w.lower() for w in custom_stoplist))

	def _remove_terms_from_list(self, stoplist):
		terms_to_ignore = [term for term
		                   in self._term_idx_store._i2val
		                   if ' ' in term or "'" in term
		                   or term in stoplist]
		return self.remove_terms(terms_to_ignore)

	def term_doc_lists(self):
		'''
		Returns
		-------
		dict
		'''
		doc_ids = self._X.transpose().tolil().rows
		terms = self._term_idx_store.values()
		return dict(zip(terms, doc_ids))

	def get_term_doc_count_df(self):
		'''

		Returns
		-------
		pd.DataFrame indexed on terms, with columns giving the
		 number of docs containing each term
		'''
		row = self._row_category_ids()
		catX = self._change_document_type_in_matrix(self._X, row)
		return self._term_freq_df_from_matrix(catX)

	def _change_document_type_in_matrix(self, X, new_doc_ids):
		new_data = self._make_all_positive_data_ones(X.data)
		newX = csr_matrix((new_data, (new_doc_ids, X.indices)))
		return newX

	def _make_all_positive_data_ones(self, newX):
		# type: (sparse_matrix) -> sparse_matrix
		return (newX > 0).astype(np.int32)

	def remove_terms(self, terms, ignore_absences=False):
		'''Non destructive term removal.

		Parameters
		----------
		terms : list
			list of terms to remove
		ignore_absences : bool, False by default
			if term does not appear, don't raise an error, just move on.

		Returns
		-------
		TermDocMatrix, new object with terms removed.
		'''
		idx_to_delete_list = self._build_term_index_list(ignore_absences, terms)
		return self.remove_terms_by_indices(idx_to_delete_list)

	def _build_term_index_list(self, ignore_absences, terms):
		idx_to_delete_list = []
		for term in terms:
			if term not in self._term_idx_store:
				if not ignore_absences:
					raise KeyError('Term %s not found' % (term))
				continue
			idx_to_delete_list.append(self._term_idx_store.getidx(term))
		return idx_to_delete_list

	def keep_only_these_categories(self, categories, ignore_absences=False):
		'''
		Non destructive category removal.

		Parameters
		----------
		categories : list
			list of categories to keep
		ignore_absences : bool, False by default
			if categories does not appear, don't raise an error, just move on.

		Returns
		-------
		TermDocMatrix, new object with categories removed.
		'''
		if not ignore_absences:
			assert set(self.get_categories()) & set(categories) == set(categories)
		categories_to_remove = [c for c in self.get_categories() if c not in categories]
		return self.remove_categories(categories_to_remove)

	def remove_categories(self, categories, ignore_absences=False):
		'''
		Non destructive category removal.

		Parameters
		----------
		categories : list
			list of categories to remove
		ignore_absences : bool, False by default
			if categories does not appear, don't raise an error, just move on.

		Returns
		-------
		TermDocMatrix, new object with categories removed.
		'''
		idx_to_delete_list = []
		existing_categories = set(self.get_categories())
		for category in categories:
			if category not in existing_categories:
				if not ignore_absences:
					raise KeyError('Category %s not found' % (category))
				continue
			idx_to_delete_list.append(self._category_idx_store.getidx(category))
		new_category_idx_store = self._category_idx_store.batch_delete_idx(idx_to_delete_list)

		columns_to_delete = np.nonzero(np.isin(self._y, idx_to_delete_list))
		new_X = delete_columns(self._X.T, columns_to_delete).T
		new_mX = delete_columns(self._mX.T, columns_to_delete).T
		intermediate_y = self._y[~np.isin(self._y, idx_to_delete_list)]
		old_y_to_new_y = [self._category_idx_store.getidx(x)
		                  for x in new_category_idx_store._i2val]
		new_y = np.array([old_y_to_new_y.index(i) if i in old_y_to_new_y else None
		                  for i in range(intermediate_y.max() + 1)])[intermediate_y]

		new_metadata_idx_store = self._metadata_idx_store

		if self.metadata_in_use():
			meta_idx_to_delete = np.nonzero(new_mX.sum(axis=0).A1 == 0)[0]
			new_metadata_idx_store = self._metadata_idx_store.batch_delete_idx(meta_idx_to_delete)

		term_idx_to_delete = np.nonzero(new_X.sum(axis=0).A1 == 0)[0]
		new_term_idx_store = self._term_idx_store.batch_delete_idx(term_idx_to_delete)
		new_X = delete_columns(new_X, term_idx_to_delete)

		term_doc_mat_to_ret = self._make_new_term_doc_matrix(new_X,
		                                                     new_mX,
		                                                     new_y,
		                                                     new_term_idx_store,
		                                                     new_category_idx_store,
		                                                     new_metadata_idx_store,
		                                                     ~np.isin(self._y, idx_to_delete_list))
		return term_doc_mat_to_ret

	def metadata_in_use(self):
		'''
		Returns True if metadata values are in term doc matrix.

		Returns
		-------
		bool
		'''
		return len(self._metadata_idx_store) > 0

	def remove_terms_by_indices(self, idx_to_delete_list):
		'''
		Parameters
		----------
		idx_to_delete_list, list

		Returns
		-------
		TermDocMatrix
		'''
		new_X, new_term_idx_store = self._get_X_after_delete_terms(idx_to_delete_list)
		return self._make_new_term_doc_matrix(new_X,
		                                      self._mX,
		                                      self._y,
		                                      new_term_idx_store,
		                                      self._category_idx_store,
		                                      self._metadata_idx_store,
		                                      self._y == self._y)

	def _get_X_after_delete_terms(self, idx_to_delete_list):
		new_term_idx_store = self._term_idx_store.batch_delete_idx(idx_to_delete_list)
		new_X = delete_columns(self._X, idx_to_delete_list)
		return new_X, new_term_idx_store

	def remove_terms_used_in_less_than_num_docs(self, threshold):
		'''
		Parameters
		----------
		threshold: int
			Minimum number of documents term should appear in to be kept

		Returns
		-------
		TermDocMatrix, new object with terms removed.
		'''
		term_counts = self._X.astype(bool).astype(int).sum(axis=0).A[0]
		terms_to_remove = np.where(term_counts < threshold)[0]
		return self.remove_terms_by_indices(terms_to_remove)

	def _make_new_term_doc_matrix(self,
	                              new_X,
	                              new_mX,
	                              new_y,
	                              new_term_idx_store,
	                              new_category_idx_store,
	                              new_metadata_idx_store,
	                              new_y_mask):
		return TermDocMatrix(X=new_X,
		                     mX=new_mX,
		                     y=new_y,
		                     term_idx_store=new_term_idx_store,
		                     category_idx_store=new_category_idx_store,
		                     metadata_idx_store=new_metadata_idx_store,
		                     unigram_frequency_path=self._unigram_frequency_path)

	def get_posterior_mean_ratio_scores(self, category):
		''' Computes posterior mean score.
		Parameters
		----------
		category : str
			category name to score

		Returns
		-------
			np.array
		'''
		return self._get_posterior_mean_ratio_from_category(category)

	def get_corner_scores(self, category):
		''' Computes corner score, which is inversely correlated
		to the Rudder score to the nearest upper-left or lower-right corner.
		Parameters
		----------
		category : str
			category name to score

		Returns
		-------
			np.array
		'''
		return CornerScore.get_scores(
			*self._get_catetgory_and_non_category_word_counts(category)
		)

	def get_rudder_scores(self, category):
		''' Computes Rudder score.
		Parameters
		----------
		category : str
			category name to score

		Returns
		-------
			np.array
		'''
		category_percentiles = self._get_term_percentiles_in_category(category)
		not_category_percentiles = self._get_term_percentiles_not_in_category(category)
		rudder_scores = self._get_rudder_scores_for_percentile_pair(category_percentiles,
		                                                            not_category_percentiles)
		return rudder_scores

	def _get_posterior_mean_ratio_from_category(self, category):
		cat_word_counts, not_cat_word_counts = self._get_catetgory_and_non_category_word_counts(category)
		return self._get_posterior_mean_ratio_from_counts(cat_word_counts, not_cat_word_counts)

	def _get_posterior_mean_ratio_from_counts(self, cat_word_counts, not_cat_word_counts):
		cat_posterior_mean = self._get_posterior_mean_from_counts(cat_word_counts, not_cat_word_counts)
		not_cat_posterior_mean = self._get_posterior_mean_from_counts(not_cat_word_counts, cat_word_counts)
		return np.log(cat_posterior_mean / not_cat_posterior_mean) / np.log(2)

	def _get_posterior_mean_from_counts(self, cat_word_counts, not_cat_word_counts):
		a = cat_word_counts
		b = cat_word_counts.sum() - cat_word_counts
		beta = ((cat_word_counts.sum() + not_cat_word_counts.sum())
		        / (cat_word_counts + not_cat_word_counts) - 1)
		posterior_mean = (1. + a) / (1. + a + b + beta)
		return posterior_mean

	def get_logistic_regression_coefs_l2(self, category,
	                                     clf=RidgeClassifierCV()):
		''' Computes l2-penalized logistic regression score.
		Parameters
		----------
		category : str
			category name to score

		category : str
			category name to score
		Returns
		-------
			(coefficient array, accuracy, majority class baseline accuracy)
		'''
		from sklearn.cross_validation import cross_val_predict
		y = self._get_mask_from_category(category)
		X = TfidfTransformer().fit_transform(self._X)
		clf.fit(X, y)
		y_hat = cross_val_predict(clf, X, y)
		acc, baseline = self._get_accuracy_and_baseline_accuracy(y, y_hat)
		return clf.coef_[0], acc, baseline

	def _get_accuracy_and_baseline_accuracy(self, y, y_hat):
		acc = sum(y_hat == y) * 1. / len(y)
		baseline = max([sum(y), len(y) - sum(y)]) * 1. / len(y)
		return acc, baseline

	def get_logistic_regression_coefs_l1(self, category,
	                                     clf=LassoCV(alphas=[0.1, 0.001],
	                                                 max_iter=10000,
	                                                 n_jobs=-1)):
		''' Computes l1-penalized logistic regression score.
		Parameters
		----------
		category : str
			category name to score

		Returns
		-------
			(coefficient array, accuracy, majority class baseline accuracy)
		'''
		from sklearn.cross_validation import cross_val_predict
		y = self._get_mask_from_category(category)
		y_continuous = self._get_continuous_version_boolean_y(y)
		# X = TfidfTransformer().fit_transform(self._X)
		X = self._X

		clf.fit(X, y_continuous)
		y_hat = (cross_val_predict(clf, X, y_continuous) > 0)
		acc, baseline = self._get_accuracy_and_baseline_accuracy(y, y_hat)
		clf.fit(X, y_continuous)
		return clf.coef_, acc, baseline

	def get_regression_coefs(self, category, clf=ElasticNet()):
		''' Computes regression score of tdfidf transformed features
		Parameters
		----------
		category : str
			category name to score
		clf : sklearn regressor

		Returns
		-------
		coefficient array
		'''
		self._fit_tfidf_model(category, clf)
		return clf.coef_

	def get_logreg_coefs(self, category, clf=LogisticRegression()):
		''' Computes regression score of tdfidf transformed features
		Parameters
		----------
		category : str
			category name to score
		clf : sklearn regressor

		Returns
		-------
		coefficient array
		'''
		self._fit_tfidf_model(category, clf)
		return clf.coef_[0]

	def _fit_tfidf_model(self, category, clf):
		y = self._get_mask_from_category(category)
		y_continuous = self._get_continuous_version_boolean_y(y)
		X = TfidfTransformer().fit_transform(self._X)
		clf.fit(X, y_continuous)

	def _get_continuous_version_boolean_y(self, y_bool):
		return 1000 * (y_bool * 2. - 1)

	def get_doc_lengths(self):
		'''
		Returns a list of document lengths in words

		Returns
		-------
		np.array
		'''
		idx_to_delete_list = self._build_term_index_list(True, self._get_non_unigrams())
		unigram_X, _ = self._get_X_after_delete_terms(idx_to_delete_list)
		return unigram_X.sum(axis=1).A1

	def get_scaled_f_scores(self,
	                        category,
	                        scaler_algo=DEFAULT_SCALER_ALGO,
	                        beta=DEFAULT_BETA):
		''' Computes scaled-fscores
		Parameters
		----------
		category : str
			category name to score
		scaler_algo : str
		  Function that scales an array to a range \in [0 and 1]. Use 'percentile', 'normcdf'. Default.
		beta : float
			Beta in (1+B^2) * (Scale(P(w|c)) * Scale(P(c|w)))/(B^2*Scale(P(w|c)) + Scale(P(c|w))). Default.
		Returns
		-------
			np.array of harmonic means of scaled P(word|category) and scaled P(category|word)
		'''

		assert beta > 0
		cat_word_counts, not_cat_word_counts = self._get_catetgory_and_non_category_word_counts(category)
		scores = self._get_scaled_f_score_from_counts(cat_word_counts, not_cat_word_counts, scaler_algo, beta)
		return np.array(scores)

	def _get_scaled_f_score_from_counts(self, cat_word_counts, not_cat_word_counts, scaler_algo,
	                                    beta=DEFAULT_BETA):
		'''
		scaler = self._get_scaler_function(scaler_algo)
		p_word_given_category = cat_word_counts.astype(np.float64) / cat_word_counts.sum()
		p_category_given_word = cat_word_counts.astype(np.float64) / (cat_word_counts + not_cat_word_counts)
		scores \
			= self._computer_harmoic_mean_of_probabilities_over_non_zero_in_category_count_terms(
			cat_word_counts, p_category_given_word, p_word_given_category, scaler
		)
		'''
		return ScaledFScore.get_scores(cat_word_counts, not_cat_word_counts, scaler_algo, beta=beta)

	def _computer_harmoic_mean_of_probabilities_over_non_zero_in_category_count_terms(self,
	                                                                               cat_word_counts,
	                                                                               p_category_given_word,
	                                                                               p_word_given_category,
	                                                                               scaler):
		df = pd.DataFrame({
			'cat_word_counts': cat_word_counts,
			'p_word_given_category': p_word_given_category,
			'p_category_given_word': p_category_given_word
		})
		df_with_count = df[df['cat_word_counts'] > 0]
		df_with_count['scale p_word_given_category'] = scaler(df_with_count['p_word_given_category'])
		df_with_count['scale p_category_given_word'] = scaler(df_with_count['p_category_given_word'])
		df['scale p_word_given_category'] = 0
		df.loc[df_with_count.index, 'scale p_word_given_category'] = df_with_count['scale p_word_given_category']
		df['scale p_category_given_word'] = 0
		df.loc[df_with_count.index, 'scale p_category_given_word'] \
			= df_with_count['scale p_category_given_word']
		score = hmean([df_with_count['scale p_category_given_word'],
		               df_with_count['scale p_word_given_category']])
		df['score'] = 0
		df.loc[df_with_count.index, 'score'] = score
		return df['score']

	def _get_scaler_function(self, scaler_algo):
		scaler = None
		if scaler_algo == 'percentile':
			scaler = lambda x: rankdata(x).astype(np.float64) / len(x)
		elif scaler_algo == 'normcdf':
			# scaler = lambda x: ECDF(x[cat_word_counts != 0])(x)
			scaler = lambda x: norm.cdf(x, x.mean(), x.std())
		elif scaler_algo == 'none':
			scaler = lambda x: x
		else:
			raise InvalidScalerException("Invalid scaler alogrithm.  Must be either percentile or normcdf.")
		return scaler

	def get_fisher_scores(self, category):
		cat_word_counts, not_cat_word_counts = self._get_catetgory_and_non_category_word_counts(category)
		return self._get_fisher_scores_from_counts(cat_word_counts, not_cat_word_counts)

	def get_fisher_scores_vs_background(self):
		'''
		Returns
		-------
			pd.DataFrame of fisher scores vs background
		'''
		df = self.get_term_and_background_counts()
		odds_ratio, p_values = self._get_fisher_scores_from_counts(
			df['corpus'], df['background'])
		df['Odds ratio'] = odds_ratio
		df['Bonferroni-corrected p-values'] = p_values * len(df)
		df.sort_values(by=['Bonferroni-corrected p-values', 'Odds ratio'],
		               ascending=[True, False])
		return df

	def get_posterior_mean_ratio_scores_vs_background(self):
		'''
		Returns
		-------
			pd.DataFrame of posterior mean  scores vs background
		'''
		df = self.get_term_and_background_counts()
		df['Log Posterior Mean Ratio'] = self._get_posterior_mean_ratio_from_counts(df['corpus'],
		                                                                            df['background'])
		return df.sort_values('Log Posterior Mean Ratio', ascending=False)

	def _get_catetgory_and_non_category_word_counts(self, category):
		self._validate_category(category)
		cat_word_counts = self._X[self._get_mask_from_category(category)].sum(axis=0).A1
		not_cat_word_counts = self._X[self._y != self._category_idx_store.getidx(category)].sum(axis=0).A1
		return cat_word_counts, not_cat_word_counts

	def _validate_category(self, category):
		if category not in self.get_categories():
			raise Exception("Invalid category: %s, valid: %s" % (category, self.get_categories()))

	def _get_fisher_scores_from_counts(self, cat_word_counts, not_cat_word_counts):
		cat_not_word_counts = cat_word_counts.sum() - cat_word_counts
		not_cat_not_word_counts = not_cat_word_counts.sum() - not_cat_word_counts

		def do_fisher_exact(x):
			return fisher_exact([[x[0], x[1]], [x[2], x[3]]], alternative='greater')

		odds_ratio, p_values = np.apply_along_axis(
			do_fisher_exact,
			0,
			np.array([cat_word_counts, cat_not_word_counts, not_cat_word_counts, not_cat_not_word_counts]))
		return odds_ratio, p_values

	def get_rudder_scores_vs_background(self):
		'''
		Returns
		-------
    pd.DataFrame of rudder scores vs background
		'''
		df = self.get_term_and_background_counts()
		corpus_percentiles = self._get_percentiles_from_freqs(df['corpus'])
		background_percentiles = self._get_percentiles_from_freqs(df['background'])
		df['Rudder'] = (self._get_rudder_scores_for_percentile_pair(corpus_percentiles,
		                                                            background_percentiles))
		df = df.sort_values(by='Rudder', ascending=True)
		return df

	def _rescale_labels_to_neg_one_pos_one(self, category):
		return (self._get_mask_from_category(category)) * 2 - 1

	def set_background_corpus(self, background):
		'''
		Parameters
		----------
		background

		'''
		if issubclass(type(background), TermDocMatrix):
			self._background_corpus = pd.DataFrame(background
			                                       .get_term_freq_df()
			                                       .sum(axis=1),
			                                       columns=['background']).reset_index()
			self._background_corpus.columns = ['word', 'background']
		elif (type(background) == pd.DataFrame
		      and set(background.columns) == set(['word', 'background'])):
			self._background_corpus = background
		else:
			raise Exception('The argument named background must be a subclass of TermDocMatrix or a ' \
			                + 'DataFrame with columns "word" and "background", where "word" ' \
			                + 'is the term text, and "background" is its frequency.')

	def get_background_corpus(self):
		if self._background_corpus is not None:
			return self._background_corpus
		return DefaultBackgroundFrequencies.get_background_frequency_df(self._unigram_frequency_path)

	def get_term_and_background_counts(self):
		'''
		Returns
		-------
		A pd.DataFrame consisting of unigram term counts of words occurring
		 in the TermDocumentMatrix and their corresponding background corpus
		 counts.  The dataframe has two columns, corpus and background.

		>>> corpus.get_unigram_corpus.get_term_and_background_counts()
		                  corpus  background
		obama              702.0    565739.0
		romney             570.0    695398.0
		barack             248.0    227861.0
		...
		'''
		background_df = self._get_background_unigram_frequencies()
		term_freq_df = self.get_term_freq_df()
		corpus_freq_df = pd.DataFrame({'corpus': term_freq_df.sum(axis=1)})
		corpus_unigram_freq = self._get_corpus_unigram_freq(corpus_freq_df)
		df = corpus_unigram_freq.join(background_df, how='outer').fillna(0)
		del df.index.name
		return df

	def _get_corpus_unigram_freq(self, corpus_freq_df):
		unigram_validator = re.compile('^[A-Za-z]+$')
		corpus_unigram_freq = corpus_freq_df.ix[[term for term
		                                         in corpus_freq_df.index
		                                         if unigram_validator.match(term) is not None]]
		return corpus_unigram_freq

	def _get_background_unigram_frequencies(self):
		if self.get_background_corpus() is not None:
			return self.get_background_corpus()
		return DefaultBackgroundFrequencies.get_background_frequency_df(self._unigram_frequency_path)

	def list_extra_features(self):
		'''
		Returns
		-------
		List of dicts.  One dict for each document, keys are metadata, values are counts
		'''
		return FeatureLister(self._mX,
		                     self._metadata_idx_store,
		                     self.get_num_docs()).output()

	def get_scaled_f_scores_vs_background(self,
	                                      scaler_algo=DEFAULT_BACKGROUND_SCALER_ALGO,
	                                      beta=DEFAULT_BACKGROUND_BETA):
		'''
		Parameters
		----------
		scaler_algo : str
			see get_scaled_f_scores, default 'none'
		beta : float
		  default 1.
		Returns
		-------
		pd.DataFrame of scaled_f_score scores compared to background corpus
		'''
		df = self.get_term_and_background_counts()
		df['Scaled f-score'] = ScaledFScore.get_scores_for_category(
			df['corpus'], df['background'], scaler_algo, beta
		)
		return df.sort_values(by='Scaled f-score', ascending=False)

	def _get_rudder_scores_for_percentile_pair(self, category_percentiles, not_category_percentiles):
		return np.linalg.norm(np.array([1, 0])
		                      - np.array(list(zip(category_percentiles, not_category_percentiles))),
		                      axis=1)

	def _get_term_percentiles_in_category(self, category):
		mask = self._get_mask_from_category(category)
		return self._get_frequency_percentiles(mask)

	def _get_mask_from_category(self, category):
		return self._y == self._category_idx_store.getidx(category)

	def _get_term_percentiles_not_in_category(self, category):
		mask = self._y != self._category_idx_store.getidx(category)
		return self._get_frequency_percentiles(mask)

	def _get_frequency_percentiles(self, mask):
		freqs = self._X[mask].sum(axis=0).A1
		percentiles = self._get_percentiles_from_freqs(freqs)
		return percentiles

	def _get_percentiles_from_freqs(self, freqs):
		return rankdata(freqs) / len(freqs)

	def get_term_category_frequencies(self, scatterchartdata):
		'''
		Applies the ranker in scatterchartdata to term-category frequencies.

		Parameters
		----------
		scatterchartdata : ScatterChartData

		Returns
		-------
		pd.DataFrame
		'''
		term_ranker = scatterchartdata.term_ranker(self)
		if scatterchartdata.use_non_text_features:
			term_ranker.use_non_text_features()
		return term_ranker.get_ranks()

	def apply_ranker(self, term_ranker):
		'''
		Parameters
		----------
		term_ranker : TermRanker

		Returns
		-------
		pd.Dataframe
		'''
		return term_ranker(self).get_ranks()

	def get_category_ids(self):
		'''
		Returns array of category ids

		Returns
		-------
		np.array
		'''
		return self._y

	def get_term_doc_mat(self):
		'''
		Returns sparse matrix representation of term-doc-matrix

		Returns
		-------
		scipy.sparse.csr_matrix
		'''
		return self._X

	def get_category_index_store(self):
		'''
		Returns IndexStore object mapping categories to ids

		Returns
		-------
		IndexStore
		'''
		return self._category_idx_store

	def recategorize(self, new_categories):
		'''
		Parameters
		----------
		new_categories : array like
		String names of new categories. Length should be equal to number of documents

		Returns
		-------
		TermDocMatrix
		'''
		assert len(new_categories) == self.get_num_docs()

		new_category_idx_store = IndexStoreFromList.build(set(new_categories))
		new_y = np.array(new_category_idx_store.getidxstrictbatch(new_categories))

		new_tdm = self._make_new_term_doc_matrix(self._X,
		                                         self._mX,
		                                         new_y,
		                                         self._term_idx_store,
		                                         new_category_idx_store,
		                                         self._metadata_idx_store,
		                                         new_y == new_y)
		return new_tdm


#!/usr/bin/env python
# coding: utf-8

# # Word Embeddings: Ungraded Practice Notebook
#
# In this ungraded notebook, you'll try out all the individual techniques that you learned about in the lecture. Practicing on small examples will prepare you for the graded assignment, where you will combine the techniques in more advanced ways to create word embeddings from a real-life corpus.
#
# This notebook is made of two main parts: data preparation, and the continuous bag-of-words (CBOW) model.
#
# To get started, import and initialize all the libraries you will need.

# In[1]:

# In[2]:

import re
import nltk
from nltk.tokenize import word_tokenize
import emoji
import numpy as np

from utils2 import get_dict

nltk.download('punkt')  # download pre-trained Punkt tokenizer for English

# # Data preparation

# In the data preparation phase, starting with a corpus of text, you will:
#
# - Clean and tokenize the corpus.
#
# - Extract the pairs of context words and center word that will make up the training data set for the CBOW model. The context words are the features that will be fed into the model, and the center words are the target values that the model will learn to predict.
#
# - Create simple vector representations of the context words (features) and center words (targets) that can be used by the neural network of the CBOW model.

# ## Cleaning and tokenization
#
# To demonstrate the cleaning and tokenization process, consider a corpus that contains emojis and various punctuation signs.

# In[3]:

corpus = 'Who ❤ "word embeddings" in 2020? I do!!!'

# First, replace all interrupting punctuation signs — such as commas and exclamation marks — with periods.

# In[4]:

print(f'Corpus:  {corpus}')
data = re.sub(r'[,!?;-]+', '.', corpus)
print(f'After cleaning punctuation:  {data}')

# Next, use NLTK's tokenization engine to split the corpus into individual tokens.

# In[5]:

print(f'Initial string:  {data}')
data = nltk.word_tokenize(data)
print(f'After tokenization:  {data}')

# Finally, as you saw in the lecture, get rid of numbers and punctuation other than periods, and convert all the remaining tokens to lowercase.

# In[6]:

print(f'Initial list of tokens:  {data}')
data = [ ch.lower() for ch in data
         if ch.isalpha()
         or ch == '.'
         or emoji.get_emoji_regexp().search(ch)
       ]
print(f'After cleaning:  {data}')

# Note that the heart emoji is considered as a token just like any normal word.
#
# Now let's streamline the cleaning and tokenization process by wrapping the previous steps in a function.

# In[7]:

def tokenize(corpus):
    data = re.sub(r'[,!?;-]+', '.', corpus)
    data = nltk.word_tokenize(data)  # tokenize string to words
    data = [ ch.lower() for ch in data
             if ch.isalpha()
             or ch == '.'
             or emoji.get_emoji_regexp().search(ch)
           ]
    return data

# Apply this function to the corpus that you'll be working on in the rest of this notebook: "I am happy because I am learning"

# In[8]:

corpus = 'I am happy because I am learning'
print(f'Corpus:  {corpus}')
words = tokenize(corpus)
print(f'Words (tokens):  {words}')

# **Now try it out yourself with your own sentence.**

# In[9]:

tokenize("Now it's your turn: try with your own sentence!")

# ## Sliding window of words

# Now that you have transformed the corpus into a list of clean tokens, you can slide a window of words across this list. For each window you can extract a center word and the context words.
#
# The `get_windows` function in the next cell was introduced in the lecture.

# In[10]:

def get_windows(words, C):
    i = C
    while i < len(words) - C:
        center_word = words[i]
        context_words = words[(i - C):i] + words[(i+1):(i+C+1)]
        yield context_words, center_word
        i += 1

# The first argument of this function is a list of words (or tokens). The second argument, `C`, is the context half-size. Recall that for a given center word, the context words are made of `C` words to the left and `C` words to the right of the center word.
#
# Here is how you can use this function to extract context words and center words from a list of tokens. These context and center words will make up the training set that you will use to train the CBOW model.

# In[11]:

for x, y in get_windows(
            ['i', 'am', 'happy', 'because', 'i', 'am', 'learning'],
            2
        ):
    print(f'{x}\t{y}')

# The first example of the training set is made of:
#
# - the context words "i", "am", "because", "i",
#
# - and the center word to be predicted: "happy".
#
# **Now try it out yourself. In the next cell, you can change both the sentence and the context half-size.**

# In[12]:

for x, y in get_windows(tokenize("Now it's your turn: try with your own sentence!"), 1):
    print(f'{x}\t{y}')

# ## Transforming words into vectors for the training set

# To finish preparing the training set, you need to transform the context words and center words into vectors.
#
# ### Mapping words to indices and indices to words
#
# The center words will be represented as one-hot vectors, and the vectors that represent context words are also based on one-hot vectors.
#
# To create one-hot word vectors, you can start by mapping each unique word to a unique integer (or index). We have provided a helper function, `get_dict`, that creates a Python dictionary that maps words to integers and back.

# In[13]:

word2Ind, Ind2word = get_dict(words)

# Here's the dictionary that maps words to numeric indices.

# In[14]:

word2Ind

# You can use this dictionary to get the index of a word.

# In[15]:

print("Index of the word 'i':  ",word2Ind['i'])

# And conversely, here's the dictionary that maps indices to words.

# In[16]:

Ind2word

# In[17]:

print("Word which has index 2:  ",Ind2word[2] )

# Finally, get the length of either of these dictionaries to get the size of the vocabulary of your corpus, in other words the number of different words making up the corpus.

# In[18]:

V = len(word2Ind)
print("Size of vocabulary: ", V)

# ### Getting one-hot word vectors
#
# Recall from the lecture that you can easily convert an integer, $n$, into a one-hot vector.
#
# Consider the word "happy". First, retrieve its numeric index.

# In[19]:

n = word2Ind['happy']
n

# Now create a vector with the size of the vocabulary, and fill it with zeros.

# In[20]:

center_word_vector = np.zeros(V)
center_word_vector

# You can confirm that the vector has the right size.

# In[21]:

len(center_word_vector) == V

# Next, replace the 0 of the $n$-th element with a 1.

# In[22]:

center_word_vector[n] = 1

# And you have your one-hot word vector.

# In[23]:

center_word_vector

# **You can now group all of these steps in a convenient function, which takes as parameters: a word to be encoded, a dictionary that maps words to indices, and the size of the vocabulary.**

# In[24]:

def word_to_one_hot_vector(word, word2Ind, V):
    # BEGIN your code here
    one_hot_vector = np.zeros(V)
    one_hot_vector[word2Ind[word]] = 1
    # END your code here
    return one_hot_vector

# Check that it works as intended.

# In[25]:

word_to_one_hot_vector('happy', word2Ind, V)

# **What is the word vector for "learning"?**

# In[26]:

# BEGIN your code here
word_to_one_hot_vector('learning', word2Ind, V)
# END your code here

# Expected output:
#
#     array([0., 0., 0., 0., 1.])

# ### Getting context word vectors

# To create the vectors that represent context words, you will calculate the average of the one-hot vectors representing the individual words.
#
# Let's start with a list of context words.

# In[27]:

context_words = ['i', 'am', 'because', 'i']

# Using Python's list comprehension construct and the `word_to_one_hot_vector` function that you created in the previous section, you can create a list of one-hot vectors representing each of the context words.

# In[28]:

context_words_vectors = [word_to_one_hot_vector(w, word2Ind, V) for w in context_words]
context_words_vectors

# And you can now simply get the average of these vectors using numpy's `mean` function, to get the vector representation of the context words.

# In[29]:

np.mean(context_words_vectors, axis=0)

# Note the `axis=0` parameter that tells `mean` to calculate the average of the rows (if you had wanted the average of the columns, you would have used `axis=1`).
#
# **Now create the `context_words_to_vector` function that takes in a list of context words, a word-to-index dictionary, and a vocabulary size, and outputs the vector representation of the context words.**

# In[30]:

def context_words_to_vector(context_words, word2Ind, V):
    # BEGIN your code here
    context_words_vectors = [word_to_one_hot_vector(w, word2Ind, V) for w in context_words]
    context_words_vectors = np.mean(context_words_vectors, axis=0)
    # END your code here
    return context_words_vectors

# And check that you obtain the same output as the manual approach above.

# In[31]:

context_words_to_vector(['i', 'am', 'because', 'i'], word2Ind, V)

# **What is the vector representation of the context words "am happy i am"?**

# In[32]:

# BEGIN your code here
context_words_to_vector(['am', 'happy', 'i', 'am'], word2Ind, V)
# END your code here

# Expected output:
#
#     array([0.5 , 0.  , 0.25, 0.25, 0.  ])

#
# ## Building the training set

# You can now combine the functions that you created in the previous sections, to build a training set for the CBOW model, starting from the following tokenized corpus.

# In[33]:

words

# To do this you need to use the sliding window function (`get_windows`) to extract the context words and center words, and you then convert these sets of words into a basic vector representation using `word_to_one_hot_vector` and `context_words_to_vector`.

# In[34]:

for context_words, center_word in get_windows(words, 2):  # reminder: 2 is the context half-size
    print(f'Context words:  {context_words} -> {context_words_to_vector(context_words, word2Ind, V)}')
    print(f'Center word:  {center_word} -> {word_to_one_hot_vector(center_word, word2Ind, V)}')
    print()

# In this practice notebook you'll be performing a single iteration of training using a single example, but in this week's assignment you'll train the CBOW model using several iterations and batches of example.
# Here is how you would use a Python generator function (remember the `yield` keyword from the lecture?) to make it easier to iterate over a set of examples.

# In[35]:

def get_training_example(words, C, word2Ind, V):
    for context_words, center_word in get_windows(words, C):
        yield context_words_to_vector(context_words, word2Ind, V), word_to_one_hot_vector(center_word, word2Ind, V)

# The output of this function can be iterated on to get successive context word vectors and center word vectors, as demonstrated in the next cell.

# In[36]:

for context_words_vector, center_word_vector in get_training_example(words, 2, word2Ind, V):
    print(f'Context words vector:  {context_words_vector}')
    print(f'Center word vector:  {center_word_vector}')
    print()

# Your training set is ready, you can now move on to the CBOW model itself.

# # The continuous bag-of-words model

# The CBOW model is based on a neural network, the architecture of which looks like the figure below, as you'll recall from the lecture.
#
#
 Figure 1

#
# This part of the notebook will walk you through:
#
# - The two activation functions used in the neural network.
#
# - Forward propagation.
#
# - Cross-entropy loss.
#
# - Backpropagation.
#
# - Gradient descent.
#
# - Extracting the word embedding vectors from the weight matrices once the neural network has been trained.

# ## Activation functions

# Let's start by implementing the activation functions, ReLU and softmax.

# ### ReLU

# ReLU is used to calculate the values of the hidden layer, in the following formulas:
#
# \begin{align}
#  \mathbf{z_1} &= \mathbf{W_1}\mathbf{x} + \mathbf{b_1}  \tag{1} \\
#  \mathbf{h} &= \mathrm{ReLU}(\mathbf{z_1})  \tag{2} \\
# \end{align}
#

# Let's fix a value for $\mathbf{z_1}$ as a working example.

# In[37]:

np.random.seed(10)
z_1 = 10*np.random.rand(5, 1)-5
z_1

# To get the ReLU of this vector, you want all the negative values to become zeros.
#
# First create a copy of this vector.

# In[38]:

h = z_1.copy()

# Now determine which of its values are negative.

# In[39]:

h < 0

# You can now simply set all of the values which are negative to 0.

# In[40]:

h[h < 0] = 0

# And that's it: you have the ReLU of $\mathbf{z_1}$!

# In[41]:

h

# **Now implement ReLU as a function.**

# In[42]:

def relu(z):
    # BEGIN your code here
    result = z.copy()
    result[result < 0] = 0
    # END your code here

    return result

# **And check that it's working.**

# In[43]:

z = np.array([[-1.25459881], [ 4.50714306], [ 2.31993942], [ 0.98658484], [-3.4398136 ]])
relu(z)

# Expected output:
#
#     array([[0.        ],
#            [4.50714306],
#            [2.31993942],
#            [0.98658484],
#            [0.        ]])

# ### Softmax

# The second activation function that you need is softmax. This function is used to calculate the values of the output layer of the neural network, using the following formulas:
#
# \begin{align}
#  \mathbf{z_2} &= \mathbf{W_2}\mathbf{h} + \mathbf{b_2}   \tag{3} \\
#  \mathbf{\hat y} &= \mathrm{softmax}(\mathbf{z_2})   \tag{4} \\
# \end{align}
#
# To calculate softmax of a vector $\mathbf{z}$, the $i$-th component of the resulting vector is given by:
#
# $$ \textrm{softmax}(\textbf{z})_i = \frac{e^{z_i} }{\sum\limits_{j=1}^{V} e^{z_j} }  \tag{5} $$
#
# Let's work through an example.

# In[44]:

z = np.array([9, 8, 11, 10, 8.5])
z

# You'll need to calculate the exponentials of each element, both for the numerator and for the denominator.

# In[45]:

e_z = np.exp(z)
e_z

# The denominator is equal to the sum of these exponentials.

# In[46]:

sum_e_z = np.sum(e_z)
sum_e_z

# And the value of the first element of $\textrm{softmax}(\textbf{z})$ is given by:

# In[47]:

e_z[0]/sum_e_z

# This is for one element. You can use numpy's vectorized operations to calculate the values of all the elements of the $\textrm{softmax}(\textbf{z})$ vector in one go.
#
# **Implement the softmax function.**

# In[48]:

def softmax(z):
    # BEGIN your code here
    e_z = np.exp(z)
    sum_e_z = np.sum(e_z)
    return e_z / sum_e_z
    # END your code here

# **Now check that it works.**

# In[49]:

softmax([9, 8, 11, 10, 8.5])

# Expected output:
#
#     array([0.08276948, 0.03044919, 0.61158833, 0.22499077, 0.05020223])

# ## Dimensions: 1-D arrays vs 2-D column vectors
#
# Before moving on to implement forward propagation, backpropagation, and gradient descent, let's have a look at the dimensions of the vectors you've been handling until now.
#
# Create a vector of length $V$ filled with zeros.

# In[50]:

x_array = np.zeros(V)
x_array

# This is a 1-dimensional array, as revealed by the `.shape` property of the array.

# In[51]:

x_array.shape

# To perform matrix multiplication in the next steps, you actually need your column vectors to be represented as a matrix with one column. In numpy, this matrix is represented as a 2-dimensional array.
#
# The easiest way to convert a 1D vector to a 2D column matrix is to set its `.shape` property to the number of rows and one column, as shown in the next cell.

# In[52]:

x_column_vector = x_array.copy()
x_column_vector.shape = (V, 1)  # alternatively ... = (x_array.shape[0], 1)
x_column_vector

# The shape of the resulting "vector" is:

# In[53]:

x_column_vector.shape

# So you now have a 5x1 matrix that you can use to perform standard matrix multiplication.

# ## Forward propagation

# Let's dive into the neural network itself, which is shown below with all the dimensions and formulas you'll need.
#
#
 Figure 2

# Set $N$ equal to 3. Remember that $N$ is a hyperparameter of the CBOW model that represents the size of the word embedding vectors, as well as the size of the hidden layer.

# In[58]:

N = 3

# ### Initialization of the weights and biases

# Before you start training the neural network, you need to initialize the weight matrices and bias vectors with random values.
#
# In the assignment you will implement a function to do this yourself using `numpy.random.rand`. In this notebook, we've pre-populated these matrices and vectors for you.

# In[55]:

W1 = np.array([[ 0.41687358,  0.08854191, -0.23495225,  0.28320538,  0.41800106],
               [ 0.32735501,  0.22795148, -0.23951958,  0.4117634 , -0.23924344],
               [ 0.26637602, -0.23846886, -0.37770863, -0.11399446,  0.34008124]])

W2 = np.array([[-0.22182064, -0.43008631,  0.13310965],
               [ 0.08476603,  0.08123194,  0.1772054 ],
               [ 0.1871551 , -0.06107263, -0.1790735 ],
               [ 0.07055222, -0.02015138,  0.36107434],
               [ 0.33480474, -0.39423389, -0.43959196]])

b1 = np.array([[ 0.09688219],
               [ 0.29239497],
               [-0.27364426]])

b2 = np.array([[ 0.0352008 ],
               [-0.36393384],
               [-0.12775555],
               [-0.34802326],
               [-0.07017815]])

# **Check that the dimensions of these matrices match those shown in the figure above.**

# In[56]:

# BEGIN your code here
print(f'V (vocabulary size): {V}')
print(f'N (embedding size / size of the hidden layer): {N}')
print(f'size of W1: {W1.shape} (NxV)')
print(f'size of b1: {b1.shape} (Nx1)')
print(f'size of W2: {W2.shape} (VxN)')
print(f'size of b2: {b2.shape} (Vx1)')
# END your code here

# ### Training example

# Run the next cells to get the first training example, made of the vector representing the context words "i am because i", and the target which is the one-hot vector representing the center word "happy".
#
# > You don't need to worry about the Python syntax, but there are some explanations below if you want to know what's happening behind the scenes.

# In[59]:

training_examples = get_training_example(words, 2, word2Ind, V)

# > `get_training_examples`, which uses the `yield` keyword, is known as a generator. When run, it builds an iterator, which is a special type of object that... you can iterate on (using a `for` loop for instance), to retrieve the successive values that the function generates.
# >
# > In this case `get_training_examples` `yield`s training examples, and iterating on `training_examples` will return the successive training examples.

# In[60]:

x_array, y_array = next(training_examples)

# > `next` is another special keyword, which gets the next available value from an iterator. Here, you'll get the very first value, which is the first training example. If you run this cell again, you'll get the next value, and so on until the iterator runs out of values to return.
# >
# > In this notebook `next` is used because you will only be performing one iteration of training. In this week's assignment with the full training over several iterations you'll use regular `for` loops with the iterator that supplies the training examples.

# The vector representing the context words, which will be fed into the neural network, is:

# In[61]:

x_array

# The one-hot vector representing the center word to be predicted is:

# In[62]:

y_array

# Now convert these vectors into matrices (or 2D arrays) to be able to perform matrix multiplication on the right types of objects, as explained above.

# In[63]:

x = x_array.copy()
x.shape = (V, 1)
print('x')
print(x)
print()

y = y_array.copy()
y.shape = (V, 1)
print('y')
print(y)

# ### Values of the hidden layer
#
# Now that you have initialized all the variables that you need for forward propagation, you can calculate the values of the hidden layer using the following formulas:
#
# \begin{align}
#  \mathbf{z_1} = \mathbf{W_1}\mathbf{x} + \mathbf{b_1}  \tag{1} \\
#  \mathbf{h} = \mathrm{ReLU}(\mathbf{z_1})  \tag{2} \\
# \end{align}
#
# First, you can calculate the value of $\mathbf{z_1}$.

# In[64]:

z1 = np.dot(W1, x) + b1

# > `np.dot` is numpy's function for matrix multiplication.
#
# As expected you get an $N$ by 1 matrix, or column vector with $N$ elements, where $N$ is equal to the embedding size, which is 3 in this example.

# In[65]:

z1

# You can now take the ReLU of $\mathbf{z_1}$ to get $\mathbf{h}$, the vector with the values of the hidden layer.

# In[66]:

h = relu(z1)
h

# Applying ReLU means that the negative element of $\mathbf{z_1}$ has been replaced with a zero.

# ### Values of the output layer
#
# Here are the formulas you need to calculate the values of the output layer, represented by the vector $\mathbf{\hat y}$:
#
# \begin{align}
#  \mathbf{z_2} &= \mathbf{W_2}\mathbf{h} + \mathbf{b_2}   \tag{3} \\
#  \mathbf{\hat y} &= \mathrm{softmax}(\mathbf{z_2})   \tag{4} \\
# \end{align}
#
# **First, calculate $\mathbf{z_2}$.**

# In[67]:

# BEGIN your code here
z2 = np.dot(W2, h) + b2
# END your code here

z2

# Expected output:
#
#     array([[-0.31973737],
#            [-0.28125477],
#            [-0.09838369],
#            [-0.33512159],
#            [-0.19919612]])

# This is a $V$ by 1 matrix, where $V$ is the size of the vocabulary, which is 5 in this example.

# **Now calculate the value of $\mathbf{\hat y}$.**

# In[68]:

# BEGIN your code here
y_hat = softmax(z2)
# END your code here

y_hat

# Expected output:
#
#     array([[0.18519074],
#            [0.19245626],
#            [0.23107446],
#            [0.18236353],
#            [0.20891502]])

# As you've performed the calculations with random matrices and vectors (apart from the input vector), the output of the neural network is essentially random at this point. The learning process will adjust the weights and biases to match the actual targets better.
#
# **That being said, what word did the neural network predict?**

#
#
#     Solution
#
#
The neural network predicted the word "happy": the largest element of $\mathbf{\hat y}$ is the third one, and the third word of the vocabulary is "happy".

#
Here's how you could implement this in Python:

#
print(Ind2word[np.argmax(y_hat)])

# Well done, you've completed the forward propagation phase!
#
# ## Cross-entropy loss
#
# Now that you have the network's prediction, you can calculate the cross-entropy loss to determine how accurate the prediction was compared to the actual target.
#
# > Remember that you are working on a single training example, not on a batch of examples, which is why you are using *loss* and not *cost*, which is the generalized form of loss.
#
# First let's recall what the prediction was.

# In[69]:

y_hat

# And the actual target value is:

# In[70]:

y

# The formula for cross-entropy loss is:
#
# $$ J=-\sum\limits_{k=1}^{V}y_k\log{\hat{y}_k} \tag{6}$$
#
# **Implement the cross-entropy loss function.**
#
# Here are a some hints if you're stuck.

#
#
#     Hint 1
#
#
To multiply two numpy matrices (such as y and y_hat) element-wise, you can simply use the * operator.

#
#
#     Hint 2
#
#
Once you have a vector equal to the element-wise multiplication of y and y_hat, you can use np.sum to calculate the sum of the elements of this vector.

# In[71]:

def cross_entropy_loss(y_predicted, y_actual):
    # BEGIN your code here
    loss = np.sum(-np.log(y_predicted)*y_actual)
    # END your code here
    return loss

# **Now use this function to calculate the loss with the actual values of $\mathbf{y}$ and $\mathbf{\hat y}$.**

# In[72]:

cross_entropy_loss(y_hat, y)

# Expected output:
#
#     1.4650152923611106

# This value is neither good nor bad, which is expected as the neural network hasn't learned anything yet.
#
# The actual learning will start during the next phase: backpropagation.
#
# ## Backpropagation
#
# The formulas that you will implement for backpropagation are the following.
#
# \begin{align}
#  \frac{\partial J}{\partial \mathbf{W_1}} &= \rm{ReLU}\left ( \mathbf{W_2^\top} (\mathbf{\hat{y}} - \mathbf{y})\right )\mathbf{x}^\top \tag{7}\\
#  \frac{\partial J}{\partial \mathbf{W_2}} &= (\mathbf{\hat{y}} - \mathbf{y})\mathbf{h^\top} \tag{8}\\
#  \frac{\partial J}{\partial \mathbf{b_1}} &= \rm{ReLU}\left ( \mathbf{W_2^\top} (\mathbf{\hat{y}} - \mathbf{y})\right ) \tag{9}\\
#  \frac{\partial J}{\partial \mathbf{b_2}} &= \mathbf{\hat{y}} - \mathbf{y} \tag{10}
# \end{align}
#
# > Note: these formulas are slightly simplified compared to the ones in the lecture as you're working on a single training example, whereas the lecture provided the formulas for a batch of examples. In the assignment you'll be implementing the latter.
#
# Let's start with an easy one.
#
# **Calculate the partial derivative of the loss function with respect to $\mathbf{b_2}$, and store the result in `grad_b2`.**
#
# $$\frac{\partial J}{\partial \mathbf{b_2}} = \mathbf{\hat{y}} - \mathbf{y} \tag{10}$$

# In[73]:

# BEGIN your code here
grad_b2 = y_hat - y
# END your code here

grad_b2

# Expected output:
#
#     array([[ 0.18519074],
#            [ 0.19245626],
#            [-0.76892554],
#            [ 0.18236353],
#            [ 0.20891502]])

# **Next, calculate the partial derivative of the loss function with respect to $\mathbf{W_2}$, and store the result in `grad_W2`.**
#
# $$\frac{\partial J}{\partial \mathbf{W_2}} = (\mathbf{\hat{y}} - \mathbf{y})\mathbf{h^\top} \tag{8}$$
#
# > Hint: use `.T` to get a transposed matrix, e.g. `h.T` returns $\mathbf{h^\top}$.

# In[74]:

# BEGIN your code here
grad_W2 = np.dot(y_hat - y, h.T)
# END your code here

grad_W2

# Expected output:
#
#     array([[ 0.06756476,  0.11798563,  0.        ],
#            [ 0.0702155 ,  0.12261452,  0.        ],
#            [-0.28053384, -0.48988499,  0.        ],
#            [ 0.06653328,  0.1161844 ,  0.        ],
#            [ 0.07622029,  0.13310045,  0.        ]])

# **Now calculate the partial derivative with respect to $\mathbf{b_1}$ and store the result in `grad_b1`.**
#
# $$\frac{\partial J}{\partial \mathbf{b_1}} = \rm{ReLU}\left ( \mathbf{W_2^\top} (\mathbf{\hat{y}} - \mathbf{y})\right ) \tag{9}$$

# In[75]:

# BEGIN your code here
grad_b1 = relu(np.dot(W2.T, y_hat - y))
# END your code here

grad_b1

# Expected output:
#
#     array([[0.        ],
#            [0.        ],
#            [0.17045858]])

# **Finally, calculate the partial derivative of the loss with respect to $\mathbf{W_1}$, and store it in `grad_W1`.**
#
# $$\frac{\partial J}{\partial \mathbf{W_1}} = \rm{ReLU}\left ( \mathbf{W_2^\top} (\mathbf{\hat{y}} - \mathbf{y})\right )\mathbf{x}^\top \tag{7}$$

# In[76]:

# BEGIN your code here
grad_W1 = np.dot(relu(np.dot(W2.T, y_hat - y)), x.T)
# END your code here

grad_W1

# Expected output:
#
#     array([[0.        , 0.        , 0.        , 0.        , 0.        ],
#            [0.        , 0.        , 0.        , 0.        , 0.        ],
#            [0.04261464, 0.04261464, 0.        , 0.08522929, 0.        ]])

# Before moving on to gradient descent, double-check that all the matrices have the expected dimensions.

# In[77]:

# BEGIN your code here
print(f'V (vocabulary size): {V}')
print(f'N (embedding size / size of the hidden layer): {N}')
print(f'size of grad_W1: {grad_W1.shape} (NxV)')
print(f'size of grad_b1: {grad_b1.shape} (Nx1)')
print(f'size of grad_W2: {grad_W2.shape} (VxN)')
print(f'size of grad_b2: {grad_b2.shape} (Vx1)')
# END your code here

# ## Gradient descent
#
# During the gradient descent phase, you will update the weights and biases by subtracting $\alpha$ times the gradient from the original matrices and vectors, using the following formulas.
#
# \begin{align}
#  \mathbf{W_1} &:= \mathbf{W_1} - \alpha \frac{\partial J}{\partial \mathbf{W_1}} \tag{11}\\
#  \mathbf{W_2} &:= \mathbf{W_2} - \alpha \frac{\partial J}{\partial \mathbf{W_2}} \tag{12}\\
#  \mathbf{b_1} &:= \mathbf{b_1} - \alpha \frac{\partial J}{\partial \mathbf{b_1}} \tag{13}\\
#  \mathbf{b_2} &:= \mathbf{b_2} - \alpha \frac{\partial J}{\partial \mathbf{b_2}} \tag{14}\\
# \end{align}
#
# First, let set a value for $\alpha$.

# In[78]:

alpha = 0.03

# The updated weight matrix $\mathbf{W_1}$ will be:

# In[79]:

W1_new = W1 - alpha * grad_W1

# Let's compare the previous and new values of $\mathbf{W_1}$:

# In[80]:

print('old value of W1:')
print(W1)
print()
print('new value of W1:')
print(W1_new)

# The difference is very subtle (hint: take a closer look at the last row), which is why it takes a fair amount of iterations to train the neural network until it reaches optimal weights and biases starting from random values.
#
# **Now calculate the new values of $\mathbf{W_2}$ (to be stored in `W2_new`), $\mathbf{b_1}$ (in `b1_new`), and $\mathbf{b_2}$ (in `b2_new`).**
#
# \begin{align}
#  \mathbf{W_2} &:= \mathbf{W_2} - \alpha \frac{\partial J}{\partial \mathbf{W_2}} \tag{12}\\
#  \mathbf{b_1} &:= \mathbf{b_1} - \alpha \frac{\partial J}{\partial \mathbf{b_1}} \tag{13}\\
#  \mathbf{b_2} &:= \mathbf{b_2} - \alpha \frac{\partial J}{\partial \mathbf{b_2}} \tag{14}\\
# \end{align}

# In[81]:

# BEGIN your code here
W2_new = W2 - alpha * grad_W2
b1_new = b1 - alpha * grad_b1
b2_new = b2 - alpha * grad_b2
# END your code here

print('W2_new')
print(W2_new)
print()
print('b1_new')
print(b1_new)
print()
print('b2_new')
print(b2_new)

# Expected output:
#
#     W2_new
#     [[-0.22384758 -0.43362588  0.13310965]
#      [ 0.08265956  0.0775535   0.1772054 ]
#      [ 0.19557112 -0.04637608 -0.1790735 ]
#      [ 0.06855622 -0.02363691  0.36107434]
#      [ 0.33251813 -0.3982269  -0.43959196]]
#
#     b1_new
#     [[ 0.09688219]
#      [ 0.29239497]
#      [-0.27875802]]
#
#     b2_new
#     [[ 0.02964508]
#      [-0.36970753]
#      [-0.10468778]
#      [-0.35349417]
#      [-0.0764456 ]]

# Congratulations, you have completed one iteration of training using one training example!
#
# You'll need many more iterations to fully train the neural network, and you can optimize the learning process by training on batches of examples, as described in the lecture. You will get to do this during this week's assignment.
#
# ## Extracting word embedding vectors
#
# Once you have finished training the neural network, you have three options to get word embedding vectors for the words of your vocabulary, based on the weight matrices $\mathbf{W_1}$ and/or $\mathbf{W_2}$.
#
# ### Option 1: extract embedding vectors from $\mathbf{W_1}$
#
# The first option is to take the columns of $\mathbf{W_1}$ as the embedding vectors of the words of the vocabulary, using the same order of the words as for the input and output vectors.
#
# > Note: in this practice notebook the values of the word embedding vectors are meaningless after a single iteration with just one training example, but here's how you would proceed after the training process is complete.
#
# For example $\mathbf{W_1}$ is this matrix:

# In[82]:

W1

# The first column, which is a 3-element vector, is the embedding vector of the first word of your vocabulary. The second column is the word embedding vector for the second word, and so on.
#
# The first, second, etc. words are ordered as follows.

# In[83]:

for i in range(V):
    print(Ind2word[i])

# So the word embedding vectors corresponding to each word are:

# In[84]:

# loop through each word of the vocabulary
for word in word2Ind:
    # extract the column corresponding to the index of the word in the vocabulary
    word_embedding_vector = W1[:, word2Ind[word]]

    print(f'{word}: {word_embedding_vector}')

# ### Option 2: extract embedding vectors from $\mathbf{W_2}$

# The second option is to take $\mathbf{W_2}$ transposed, and take its columns as the word embedding vectors just like you did for $\mathbf{W_1}$.

# In[85]:

W2.T

# In[86]:

# loop through each word of the vocabulary
for word in word2Ind:
    # extract the column corresponding to the index of the word in the vocabulary
    word_embedding_vector = W2.T[:, word2Ind[word]]

    print(f'{word}: {word_embedding_vector}')

# ### Option 3: extract embedding vectors from $\mathbf{W_1}$ and $\mathbf{W_2}$

# The third option, which is the one you will use in this week's assignment, uses the average of $\mathbf{W_1}$ and $\mathbf{W_2^\top}$.

# **Calculate the average of $\mathbf{W_1}$ and $\mathbf{W_2^\top}$, and store the result in `W3`.**

# In[87]:

# BEGIN your code here
W3 = (W1+W2.T)/2
# END your code here

W3

# Expected output:
#
#     array([[ 0.09752647,  0.08665397, -0.02389858,  0.1768788 ,  0.3764029 ],
#            [-0.05136565,  0.15459171, -0.15029611,  0.19580601, -0.31673866],
#            [ 0.19974284, -0.03063173, -0.27839106,  0.12353994, -0.04975536]])

# Extracting the word embedding vectors works just like the two previous options, by taking the columns of the matrix you've just created.

# In[88]:

# loop through each word of the vocabulary
for word in word2Ind:
    # extract the column corresponding to the index of the word in the vocabulary
    word_embedding_vector = W3[:, word2Ind[word]]

    print(f'{word}: {word_embedding_vector}')

# You're now ready to take on this week's assignment!

# ### How this practice relates to and differs from the upcoming graded assignment
#
# - In the assignment, for each iteration of training you will use batches of examples instead of a single example. The formulas for forward propagation and backpropagation will be modified accordingly, and you will use cross-entropy cost instead of cross-entropy loss.
# - You will also complete several iterations of training, until you reach an acceptably low cross-entropy cost, at which point you can extract good word embeddings from the weight matrices.
# - After extracting the word embedding vectors, you will use principal component analysis (PCA) to visualize the vectors, which will enable you to perform an intrinsic evaluation of the quality of the vectors, as explained in the lecture.


import os
import re
import pandas as pd
import numpy as np
from markdown import markdown
from pygments import highlight
from pygments.lexers import PythonLexer
from pygments.formatters import HtmlFormatter
from subprocess import check_output
#  ￥之间的为需要填写的，#%开头的为提示。
#################################################
prob = '''
题目要求：
编写一个Python程序，从example.com中抓取h1标签。

'''
###————————————————————————————————————————————————————
code_source = """
from urllib.request import urlopen
from bs4 import BeautifulSoup
html = urlopen('http://www.example.com/')
bsh = ￥BeautifulSoup￥(html.read(), 'html.parser')
print(bsh.h1)
"""
###########################################################
###########################################################
"""
其它备注信息：
1. with ... as ...语句用于方便的处理文件打开、读写、关闭过程中的异常。

原文版本为 highlight.py
简答题：参考文献：https://m.hanspub.org/journal/paper/37823 ALBERT等神经网络模型。
论述题：EASE
"""
######################################################################
#exec(code.replace('￥', ''))
ZZ = 1  # =0，表示不执行程序；==1:执行程序，并显示结果；==2, 返回名为df或result的array或pandas对象时则显示结果。
frmt = '{:1.0f}'  # 数组显示格式
sol = ''  # 显示答案时的解析步骤

## 这部分设置一般不用修改
expect = re.findall(r'￥(.*?)￥', code_source)[0]
hints = re.findall(r'#%(.*?)\n', code_source)
## 清掉答案和帮助文本
code = re.sub(r'￥.*?￥', '_________________________', code_source)
code = re.sub(r'#%.*?\n', '\n', code)
code = re.sub(r'"', "'", code)  # 以单引号代替双引号
code_exec = re.sub(r'￥', '', code_source)

##############################################################
## 把数组和pandas转化为mathjax，用于显示。
def array_disp(ZZ, frmt):
    import array_to_latex as a2l
    from bs4 import BeautifulSoup
    global code_exec
    if ZZ==0:
        return ''
    elif ZZ==1:
        with open("./test001.py", 'w') as fa:
            fa.write(code_exec)
        lines = check_output("python3 ./test001.py", shell=True).decode('utf-8')
        lines = lines.replace('<', '<')
        lines = lines.replace('>', '>')
        lines = '执行上述程序，输出:\n'+lines
        print(lines)
        return markdown(lines.replace('\n', '
'))
    elif ZZ==2:
        exec(code_exec)
        if 'df' in locals():
            df =df
        elif 'result' in locals():
            df = result
        else:
            print('找不到ndarray或pandas的结果变量')
        lx = a2l.to_ltx(df, frmt=frmt,
                            arraytype='bmatrix', print_out=False)
        with open("./latex_txt.text", 'w') as fa:
            fa.write(lx)
        os.system('pandoc latex_txt.text -s --mathjax --metadata title="latexTxt" -o math_mathjax.html')
        # time.sleep(1.0)
        with open('./math_mathjax.html', 'r') as fr:
            lines = fr.readlines()
        soup = BeautifulSoup(''.join(lines), 'html.parser')
        return str(soup.find_all('span')[0])
    else:
        return ''

## 处理答案和提示等部分
def ans_hint(expect, sol, hints):
    ss = """
    <description>将程序中长下划线部分所缺的代码，填入下框中（请使用单引号）：</description>
    <customresponse cfn="check_ans" expect="YYYY">
        <textline size="30"/>
    </customresponse>
    """
    ss = ss.replace('YYYY', expect)
    detail_solution = """
    <solution>

解析：

    </solution>
    """
    ss = ss + detail_solution
    if len(sol) > 0:
        ss = ss.replace('解析：', sol)
    if len(hints) > 0:
        ss = ss + '<demandhint>' + '\n'
        for ht in hints:
            ss = ss + '      <hint>' + ht + '</hint>\n'
        ss = ss + '</demandhint>'
    ss = ss + '\n</problem>'
    return ss

def main(code, prob, ZZ, frmt, expect, sol, hints):
    ## 两个评分函数，一个比例评分，一个正误评分
    # to do: tfidf的中文评阅
    correct_grader = """
<problem>

    """

    code_str = highlight(code, PythonLexer(), HtmlFormatter(noclasses=True))
    prob = markdown(prob.replace('\n', '
'))
    ppp = correct_grader + prob + code_str +\
        array_disp(ZZ, frmt) + ans_hint(expect, sol, hints)
    # print(ppp)
    try:
        import clipboard as _clipboard

        _clipboard.copy(ppp)
    except ImportError:
        print('\nPackage ''clipboard'' 没有安装')
        print('pip install clipboard\nor install via other ',
              'means to use this function')

if __name__ == "__main__":
    main(code, prob, ZZ, frmt, expect, sol, hints)


#Incomplete, broken, not novel, do not use in current state

import numpy as np
import pandas as pd

treeshere = 100
treesdone = 0

f = open("dump.txt")

niceModel = {"base":0.5, "conts":{"x1":[], "x2":[],"x3":[],"x4":[]}}

def pred(df, model):
 df["preds"]=model["base"]
 for col in model["conts"]:
  df["mult"]=1
  for thing in model["conts"][col]
   df["mult"][df[col]>=thing[0]] = thing[1]
  df["preds"]=df["preds"]*df["mult"]
 return df["preds"]

while treesdone>treeshere:
 deadline = f.readline()
 specline = f.readline()
 yesline = f.readline()
 noline = f.readline()

 spec = specline.split("[")[1].split("]")[0]
 specfeat, specval = spec.split("<")
 specval=float(specval)

 if specval in [c for c in niceModel["conts"][specfeat]

 yesshift = np.exp(float(yesline.split("=")[1]))
 noshift = np.exp(float(noline.split("=")[1]))


# -*- coding: utf-8 -*-
# @Author : Prabhu Appalapuri<prabhu.appalapuri@gmail.com>
# @Time : 11.12.19 21:03

import librosa
import soundfile
import os,glob
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.neural_network import MLPClassifier
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
import seaborn as sns
import matplotlib.pyplot as plt
import joblib
import argparse

# DataFlair - Extract features (mfcc, chroma, mel) from a sound file

def extract_feature(file_name, mfcc, chroma, mel):
    with soundfile.SoundFile(file_name) as sound_file:
        X = sound_file.read(dtype="float32")
        sample_rate = sound_file.samplerate
        # print(sample_rate)
        if chroma:
            stft = np.abs(librosa.stft(X))
            # print(librosa.stft(X).shape)
        result = np.array([])
        if mfcc:
            mfccs = np.mean(librosa.feature.mfcc(y=X, sr=sample_rate, n_mfcc=40).T, axis=0)
            result = np.hstack((result, mfccs))
        if chroma:
            chroma = np.mean(librosa.feature.chroma_stft(S=stft, sr=sample_rate).T, axis=0)
            result = np.hstack((result, chroma))
        if mel:
            mel = np.mean(librosa.feature.melspectrogram(X, sr=sample_rate).T, axis=0)
        result = np.hstack((result, mel))
        # print(result.shape)
    return result

#DataFlair - Emotions in the RAVDESS dataset

emotions={
  '01':'neutral',
  '02':'calm',
  '03':'happy',
  '04':'sad',
  '05':'angry',
  '06':'fearful',
  '07':'disgust',
  '08':'surprised'
}

#DataFlair - Emotions to observe
observed_emotions=['calm', 'happy', 'fearful', 'disgust']

# DataFlair - Load the data and extract features for each sound file

def load_data(test_size=0.2):
    x, y = [], []
    for file in glob.glob("data/ravdess-data/Actor_*/*.wav"):
        file_name = os.path.basename(file)
        emotion = emotions[file_name.split("-")[2]]
        if emotion not in observed_emotions:
            continue # continue will jump back to the top of the loop. pass will continue processing.
        feature = extract_feature(file, mfcc=True, chroma=True, mel=True)
        x.append(feature)
        y.append(emotion)
        # train_test_split(np.array(x), y, test_size=test_size, random_state=9)
        # print(np.array(x).shape)
    # return train_test_split(np.array(x), y, test_size=test_size, random_state=9)
    print("Data has been loaded.......")
    return np.array(x), y

def train(test_size=True, save_model= False, plot_fig= True):

    global x_train, y_train, y_test, x_test

    X,y = load_data(test_size=0.2)
    if test_size:
        x_train,x_test,y_train,y_test = train_test_split(X, y, test_size=0.2, random_state=9)
    layer_size= [400, 450, 500, 550, 600, 650, 700, 750, 800, 850, 900]
    scores = []
    models = []
    for size in layer_size:
        clf= MLPClassifier(alpha=0.001, batch_size=256, epsilon=1e-08,
                             hidden_layer_sizes=(size,), learning_rate='adaptive', max_iter=1000, verbose=True)
        if test_size:
            model = clf.fit(x_train,y_train)
            models.append(model)
            ac = accuracy_score(y_test, model.predict(x_test))
            scores.append(ac)
        else:
            score = cross_val_score(clf, X, y, cv=10)
            scores.append(score.mean())
            model = clf.fit(X, y)
            models.append(model)
    if plot_fig:
        sns.lineplot(layer_size, scores)
        plt.title("Model Performance")
        plt.xlabel("Hidden layer size")
        plt.ylabel("Accuracies")
        plt.savefig("plots/model_performance.png")
        plt.close()

    print("All accuracies:", scores)
    best_hlayer_size = layer_size[scores.index(max(scores))]
    print("Best hidden layer neuron size:", best_hlayer_size)
    max_acc = max(scores)
    print("Best accuracy:", max_acc)
    if save_model:
        print("Saving best model.......")
        best_model = models[scores.index(max(scores))]
        joblib.dump(best_model, 'models/best_model.sav')
    with open("output.txt", "w") as f:
        f.write("All accuracies: {} \nBest hidden layer neuron size: {} \nBest accuracy achieved :{}"
                .format(scores, best_hlayer_size, max_acc))

    return scores

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Emotion detection from audio file')
    parser.add_argument("--test_size", action='store_true')
    parser.add_argument("--plot",  action='store_true')
    parser.add_argument("--save_model", action='store_false')
    args = parser.parse_args()
    sc = train(test_size=args.test_size, save_model=False)
    print(sc)