"""
	Construct account, product and review features given a review dataset.
"""
import nltk
import nltk.tokenize.punkt
from nltk.tokenize import word_tokenize

import numpy as np

import codecs
import re
import string
from datetime import datetime

import json
import pickle
import gzip
import sys
sys.path.insert(0, '../')
from iohelper import *

date_time_format_str = '%Y-%m-%d'

def MNR(data):
	"""
		Normalized maximum number of reviews in a day for a user/product
		Args:
			data is a dictionary with key=u_id or p_id and value = tuples of (neighbor id, rating, label, posting time)
		Return:
			dictionary with key = u_id or p_id and value = MNR
	"""
	# maximum number of reviews written in a day for user / product
	feature = {}
	for i, d in data.items():
		# key = posting date; value = number of reviews
		frequency = {}
		for t in d:
			if t[3] not in frequency:
				frequency[t[3]] = 1
			else:
				frequency[t[3]] += 1
		feature[i] = max(frequency.values())

	# normalize it
	normalizer = max(feature.values())

	for k in feature.keys():
		feature[k] /= normalizer
	return feature

def PR_NR(data):
	"""
		Ratio of positive and negative reviews of a user or product
		Args:
			data is a dictionary with key=u_id or p_id and value = tuples of (neighbor id, rating, label, posting time)
		Return:
			dictionary with key = u_id or p_id and value = (PR, NR)
	"""
	feature = {}

	for i, d in data.items():
		positives = [1 for t in d if t[1] > 3]
		negatives = [1 for t in d if t[1] < 3]
		feature[i] = (float(len(positives)) / len(d), float(len(negatives)) / len(d))
	return feature

def avgRD(user_data, product_data):
	"""
		Average rating deviation of each user / product.
		For a user i, avgRD(i) = average(r_ij - avg_j | for all r_ij of the user i)
		For a product j, avgRD(j) = average(r_ij - avg_j | for all r_ij of the user i) = 0!?
		Return:
			average rating deviation on users, as defined in the paper
				Detecting product review spammers using rating behaviors, CIKM, 2010
			The paper does not define avgRD for the products.
	"""
# find the average rating of each product
	p_avg = {}
	for i, d in product_data.items():
		p_avg[i] = np.mean(np.array([t[1] for t in d]))

	# find average rating deviation of each user
	u_avgRD = {}
	for i, d in user_data.items():
		u_avgRD[i] = np.mean(np.array([abs(t[1] - p_avg[t[0]]) for t in d]))

	return u_avgRD

def BST(user_data):
	""" Burstiness of reviews by users. Spammers are often short term
		members of the site: so BST(i) = 0, if L(i) - F(i) > tau else BST(i) = 1 -
		(L(i) - F(i))/tau, where, L(i) - F(i) are number of days between first and
		last review of i, tau = 28 days

		Args:
			user_data is a dictionary with key=u_id value = tuples of (prod_id, rating, label, posting time)
		Return:
			dictionary with key = u_id and value = BST
	"""
	bst = {}
	tau = 28.0	# 28 days
	for i, d in user_data.items():
		post_dates = sorted([datetime.strptime(t[3], date_time_format_str) for t in d])
		delta_days = (post_dates[-1] - post_dates[0]).days
		if delta_days > tau:
			bst[i] = 0.0
		else:
			bst[i] = 1.0 - (delta_days / tau)
	return bst

def ERD(data):
	"""
		Entropy of the rating distribution of each user (product)
	"""
	erd = {}
	for i, d in data.items():
		ratings = [t[1] for t in d]
		h, _ = np.histogram(ratings, bins = np.arange(1,7))
		h = h / h.sum()
		h = h[np.nonzero(h)]
		erd[i] = (- h * np.log2(h)).sum()
	return erd

def ETG(data):
	"""
		Entropy of the gaps between any two consecutive ratings.
	"""
	etg = {}
# [0,1) -> 1
# [1,3) -> 2
# ...
# [17, 33) -> 6
# anything larger than 33 will be discarded.
	bins = [0,1,3,5,9,17,33]
	for i, d in data.items():

		# if there is only one posting time, then entropy = 0
		if len(d) <= 1:
			etg[i] = 0
			continue
		# sort posting dates from the past to the future
		posting_dates = sorted([datetime.strptime(t[3], date_time_format_str) for t in d])
		# find the difference in days between two consecutive dates
		#delta = posting_dates[1:] - posting_dates[:-1]
		delta_days = [(posting_dates[i+1] - posting_dates[i]).days for i in range(len(posting_dates) - 1)]

		# bin to the 6 bins, discarding any differences that are greater than 33 days
		h, _ = np.histogram(delta_days, bins = bins)
		#h, _ = np.histogram(delta_days, bins = bins)[:-1]
		if h.sum() == 0:
			etg[i] = 0
			continue
		h = h / h.sum()
		h = h[np.nonzero(h)]
		etg[i] = (- h * np.log2(h)).sum()
	return etg

def RL(data, text_features, isUser = True):
	"""
		Average review length of each user / product
		Args:
			data: a dictionary with key=u_id or p_id and value = tuples of (neighbor id, rating, label, posting time)
			text_features: dictionary with key=(u_id, p_id) and value = dict of feature_name:feature_value
			isUser: is dealing with user data or product data
		Return:
			dictionary with key = u_id or p_id and value = (RL)
	"""
	rl = {}
	for i, d in data.items():
		if isUser:
			# d[0] is product id
			lengths = [text_features[(i, t[0])]['L'] for t in d]
		else:
			lengths = [text_features[(t[0], i)]['L'] for t in d]
		rl[i] = np.mean(lengths)
	return rl

def RD(product_data):
	"""Calculate the deviation of the review ratings to the product average.

		Args:
			prod_data:
		Return:
			a dictionary with key = (u_id, p_id), value = deviation of the rating of this review to the average rating of the target product
	"""
	rd = {}
	for i, d in product_data.items():
		avg = np.mean(np.array([t[1] for t in d]))
		for t in d:
			rd[(t[0], i)] = abs(t[1] - avg)
	return rd

def EXT(product_data):
	"""
		Whether a rating is extreme or not
		Args:
			product_data is a dictionary with key=p_id and value = tuples of (u_id, rating, label, posting time)
		Return:
			a dictionary with key = (u_id, p_id) and value = 0 (not extreme) / 1 (extreme)
	"""
	ext = {}
	for i, d in product_data.items():
		for t in d:
			if int(t[1]) == 5 or int(t[1]) == 1:
				ext[(t[0], i)] = 1
			else:
				ext[(t[0], i)] = 0
	return ext

def DEV(product_data):
	"""
		Deviation of each rating from the average rating of the target product.
		Need to use "recursive minimal entropy partitioning" to find beta_1
		Args:
			product_data is a dictionary with key=p_id and value = tuples of (neighbor id, rating, label, posting time)
		Return:
			a dictionary with key = (u_id, p_id) and value = (RD_ij, RD_ij / 4 > 0.63 ? 1: 0)
			where RD_ij = |r_ij - average rating of product j|
	"""
	beta_1 = 0.63
	dev = {}
# i is a product id
	for i, d in product_data.items():
		# find the average rating of each product
		p_avg_rating = np.mean(np.array([t[1] for t in d]))
		for t in d:
			u_id = t[0]	# user id
			if (abs(p_avg_rating - t[1]) / 4.0 > 0.63):
				dev[(u_id, i)] = 1	# absolute difference between current rating and product average rating
			else:
				dev[(u_id, i)] = 0	# absolute difference between current rating and product average rating
	return dev

def ETF(product_data):
	"""
		Binary feature: 0 if ETF_prime <= beta_3, 1 otherwise.
		Need to use "recursive minimal entropy partitioning" to find beta_1
		ETF_prime = 1 - (date of last review of user i on product p from the date of the first review of the product / 7 months)
	"""

	beta_3 = 0.69

# for each product j, find the time of the earliest review F(j)
	first_time_product = {}
	for i,d in product_data.items():
		for t in d:
			if i not in first_time_product:
				first_time_product[i] = datetime.strptime(t[3], date_time_format_str)
			elif datetime.strptime(t[3], date_time_format_str) < first_time_product[i]:
				first_time_product[i] = datetime.strptime(t[3], date_time_format_str)

	etf = {}	# key = (u_id, p_id), value = maximum difference between reviews (u_id, p_id) and first review of the product
	for i, d in product_data.items():
		for t in d:
			td = datetime.strptime(t[3], date_time_format_str) - first_time_product[i]
			if (t[0], i) not in etf:
				etf[(t[0], i)] = td
# find the largest td for the review
			elif td > etf[(t[0], i)]:
				etf[(t[0], i)] = td

	for k, v in etf.items():
		if v.days > 7*30:
			etf[k] = 0
		elif v.days / (7 * 30) > beta_3:
			etf[k] = 1
		else:
			etf[k] = 0
	return etf

def ISR(user_data):
	"""
		Check if a user posts only one review
	"""
	isr = {}
	for i, d in user_data.items():
		# go through all review of this user
		for t in d:
			if len(d) == 1:
				isr[(i, t[0])] = 1
	return isr

def low_level_text_features(review_filename, output_filename):
	"""
		Since extracting low level text feature can take some time, we extract and write the features to files for later use.
		Output to file lines of review features, in the same order as the review text file.

		List of features:
			1. word counts in reviews
			2. ratio of all Capital words to all words
			3. ratio of capital letters to all letters
			4. PP1: number of first person names.
			5. RES: ratio of sentences with exclamations.
		Args:
			review_filename: the review text file
			output_filename: the json file that contains the dictionary = {key = (uid, pid): value = dict(key = text feature name, value = feature value)}
	"""
# prepare various lexicons
	PP1_filename = '1PP-words.txt'

	PP1 = set()

	with open(PP1_filename, 'r') as f:
		for line in f:
			PP1.add(line.strip())

	tag = ['CC', 'CD', 'DT', 'EX', 'FW', 'IN', 'TO', 'UH', 'PDT', 'SYM', 'RP']
	noun = ['NN', 'NNS', 'NP', 'NPS']
	adj = ['JJ', 'JJR', 'JJS']
	pronoun = ['PP', 'PP$', 'WP', 'WP$']
	verb = ['VB', 'VBD', 'VBG', 'VBN', 'VBP', 'VBZ']
	adverb = ['RB', 'RBR', 'RBS', 'WRB']

	# = 'SentiWordNet_3.0.0_20100705.txt'
	#swn = SentiWordNetCorpusReader(swn_filename)

	review_text_features = dict()

# use the rt mode to read strings instead of binary
	infile = gzip.open(review_filename, 'rt')
	line_num = 0
	for line in infile:
		line_num += 1
		if line_num % 10000 == 0:
			print (line_num)

		items = re.split("\s+", line, 3)
		u_id = items[0]
		p_id = items[1]
		time = items[2]
		texts = items[3]

		words = nltk.word_tokenize(texts)

		lower_words = texts.lower()

		wordcount = float(len(words))

		allCapitalCount = 0

		PP1Count = 0

# count the number of capital letters
		countCapital = sum(1 for c in line if c.isupper())
		countAllChars = sum(1 for c in line)

		for w in words:
			# count the numebr of words that have all capital letters
			if w.isupper():
				allCapitalCount += 1
			# count the number of 1st person words
			if w in PP1:
				PP1Count += 1

# find ratios of subjective and objective words

		#objCount = 0
		#subCount = 0
		#for w in word_tag:
		#	word = w[0]
		#	pos_tag = w[1]
		#	if not pos_tag in tag:
		#		if pos_tag in noun:
		#			pos_Char = 'n'
		#		if pos_tag in adj:
		#			pos_Char = 'a'
		#		if pos_tag in pronoun:
		#			pos_Char = 'p'
		#		if pos_tag in verb:
		#			pos_Char = 'v'
		#		if pos_tag in adverb:
		#			pos_Char = 'r'
		#		else:
		#			pos_Char = 'none'
		#	try:
		#		if pos_Char == 'none':
		#			s = swn.senti_synsets(word)
		#		else:
		#			s = swn.senti_synsets(word, pos_Char)
		#		scores = list(s)[0]
		#		if score.obj_score > 0.5:
		#			objCount += 1
		#		elif score.neg_score + score.pos_score > 0.5:
		#			subCount += 1
		#	except:
		#		pass
		#
		#if objCount + subCount > 0:
		#	ratioObj = float(objCount) / (objCount + subCount)
		#	ratioSub = float(subCount) / (objCount + subCount)
		#else:
		#	ratioObj = 0.0
		#	ratioSub = 0.0

		sents = nltk.sent_tokenize(texts)
		countExc = 0
		countSent = 0
		for sentence in sents:
		    countSent += 1
		    if '!' in sentence:
		        countExc += 1
		if countSent > 0:
		    ratioExcSent = float(countExc)/countSent
		else:
		    ratioExcSent = 0.0

		# feature vector for a review: number of words, ratio of allCap words, ratio of allCap Chars, number of 1st persons, number of 2nd persons, ratio of sentences end with exclamations
		review_text_features[(u_id, p_id)] = {"L":wordcount,
			"PCW" : allCapitalCount / wordcount,
			"PC" : float(countCapital) / countAllChars,
			"PP1" : PP1Count,
			"RES" : ratioExcSent}

	with open(output_filename, 'wb') as out_f:
		pickle.dump(review_text_features, out_f)

	print ('%d lines processed.' % line_num)
	print ('Text features written to %s' % output_filename)

def add_feature(existing_features, new_features, feature_names):
	"""
		Add feature(s) of a set of nodes of the same type to the existing feature(s).
		Args:
			existing_features: a dictionary {node_id:dict{feature_name:feature_value}}
			new_features: new feature(s) to be added. A dict {node_id: list of feature values}
			feature_names: the name of the new feature. A list of feature names, in the same order of the list of feature values in new_features
	"""

	for k, v in new_features.items():
		# k is the node id and v is the feature value
		if k not in existing_features:
			existing_features[k] = dict()
		# add the new feature to the dict of the node
		for i in range(len(feature_names)):
			if len(feature_names) > 1:
				existing_features[k][feature_names[i]] = v[i]
			else:
				existing_features[k][feature_names[i]] = v

#def construct_all_features(metadata_filename, text_feature_filename, user_feature_filename, prod_feature_filename, review_feature_filename):
def construct_all_features(user_data, prod_data, text_features):
	"""
		Main entry to feature construction.
		Args:
			metadata_filename:
			text_feature_filename:
		Return:
			user, product and review features
	"""

	UserFeatures = {}	# key = user id, value = dict of {feature_name: feature_value}
	ProdFeatures = {}	# key = product id, value = dict of {feature_name: feature_value}

# go through feature functions
	print ('\nadding user and product features......\n')
	uf = MNR(user_data)
	add_feature(UserFeatures, uf, ["MNR"])
	pf = MNR(prod_data)
	add_feature(ProdFeatures, pf, ["MNR"])

	uf = PR_NR(user_data)
	add_feature(UserFeatures, uf, ["PR", "NR"])
	pf = PR_NR(prod_data)
	add_feature(ProdFeatures, pf, ["PR", "NR"])

	uf = avgRD(user_data, prod_data)
	add_feature(UserFeatures, uf, ["avgRD"])

	uf = BST(user_data)
	add_feature(UserFeatures, uf, ["BST"])

	uf = ERD(user_data)
	add_feature(UserFeatures, uf, ["ERD"])
	pf = ERD(prod_data)
	add_feature(ProdFeatures, pf, ["ERD"])

	uf = ETG(user_data)
	add_feature(UserFeatures, uf, ["ETG"])
	pf = ETG(prod_data)
	add_feature(ProdFeatures, pf, ["ETG"])

	uf = RL(user_data, text_features)
	add_feature(UserFeatures, uf, ['RL'])
	pf = RL(prod_data, text_features, isUser = False)
	add_feature(ProdFeatures, pf, ['RL'])

# go through review features
	print ('\nadding review features......\n')
	ReviewFeatures = {}
	rf = RD(prod_data)
	add_feature(ReviewFeatures, rf, ['RD'])

	rf = EXT(prod_data)
	add_feature(ReviewFeatures, rf, ['EXT'])

	rf = DEV(prod_data)
	add_feature(ReviewFeatures, rf, ['DEV'])

	rf = ETF(prod_data)
	add_feature(ReviewFeatures, rf, ['ETF'])

	rf = ISR(prod_data)
	add_feature(ReviewFeatures, rf, ['ISR'])

# add low level text features
	print ('\nadding low level review features......\n')
	for k, v in text_features.items():
		# k is a tuple (u_id, p_id)
		# v is a dict (key=feature name, value = feature value)
		names = [name for name, value in v.items()]
		values = [value for name, value in v.items()]

# recall that the arguments to add_feature
		add_feature(ReviewFeatures, {k:values}, names)

	return UserFeatures, ProdFeatures, ReviewFeatures

def calculateNodePriors(node_features, when_suspicious):
	"""
		Calculate priors of nodes P(y=1|node) using node features.
		Args:
			node_features: a dictionary with key = node_id and value = dict of feature_name:feature_value
			when_suspicious: a dictionary with key = feature name and value = 'H' (the higher the more suspicious) or 'L' (the opposite)
		Return:
			A dictionary with key = node_id and value = S score (see the SpEagle paper for the definition)
	"""
# return value
	priors = {}

	feature_names = set()
	for k, v in node_features.items():
		for fn, fv in v.items():
			# only include those in when_suspicious
			if fn in when_suspicious:
				feature_names.add(fn)

	feature_names = list(feature_names)
	n_features = len(feature_names)

	print ('number of features: %d' % n_features)

	feature_values = {name:[] for name in feature_names}

	# go through the instances
	for k, v in node_features.items():
		# go through the features
		for fn in feature_names:
			if fn not in v:
				feature_values[fn].append(0)		# add 0 by defaulty
			else:
				feature_values[fn].append(v[fn])	# add the non-zero feature values

	for k,v in feature_values.items():
		assert len(v) == len(node_features), 'number of feature values is different from number of instances'

	#def ecdf(x):
	#    x = np.sort(x)
	#	def result(v):
	#			        return np.searchsorted(x, v, side='right') / x.size
	#				    return result

	# compute CDF for each feature
	all_cdfs = {}
	for fn in feature_names:
		print (fn)
		if fn not in feature_values:
			print (fn + ' not in feature values')
			continue
		bins = sorted(list(set(feature_values[fn])))
		#if fn == 'DEV':
		print (bins[0])
		if len(bins) == 1:
			print (fn + ' has only one value.')
# bin1=[a,b), bin2=[b,c), ... last_bin = [y,z]
		h, d = np.histogram(feature_values[fn], bins = bins)
		cdf = np.cumsum(h)
		normalizer= 1.0 / cdf[-1]
		cdf = normalizer * cdf

# for later lookup
		print (len(bins), len(d), len(cdf))
		all_cdfs[fn] = {d[i]:cdf[i] for i in range(1, len(d)-1)}
		all_cdfs[fn][d[0]] = 0	# the least value has cdf 0
		all_cdfs[fn][d[-1]] = 1	# the least value has cdf 0

	#print (list(all_cdfs['BST'].keys()))
# for each node, find the S score for each feature
	for k, v in node_features.items():
		s = 0
		for fn in feature_names:
			x = 0 if fn not in v else v[fn]
			c = all_cdfs[fn][x]
			fx = 1.0 - c if when_suspicious[fn] == '+' else c
			s += fx * fx
		s = 1 - np.sqrt(s / n_features)
		priors[k] = s

	return priors

if __name__ == '__main__':

# path to the folder containing the files
	#prefix = '/home/sihong/data/yelp_reviews/'
	prefix = '/Users/bon/Documents/data mining/datasets/yelp_reviews/nyc/'

# raw data file names
	metadata_filename = prefix + 'metadata.gz'
	review_filename = prefix + 'reviewContent.gz'

# feature file names
	user_feature_filename = prefix + 'UserFeatures.pickle'
	prod_feature_filename = prefix + 'ProdFeatures.pickle'
	review_feature_filename = prefix + 'ReviewFeatures.pickle'

# prior file names
	user_prior_filename = prefix + 'UserPriors.pickle'
	prod_prior_filename = prefix + 'ProdPriors.pickle'
	review_prior_filename = prefix + 'ReviewPriors.pickle'

# output file names
	text_feature_filename = prefix + 'text_features.pickle'

# feature configuration
	feature_suspicious_filename = '../feature_configuration.txt'

# low level text features
	# just need to construct text features once
	#print ('Starting constructing low level text features\n')
	#low_level_text_features(review_filename, text_feature_filename)

	# can just load the constructed low level text features
	#tf = load_text_feature(text_feature_filename)
	#print ('Finished constructing low level text features\n')

# all high level features
	print ('Starting constructing high level user, product and review features\n')
	user_data, prod_data = read_graph_data(metadata_filename)
	text_features = load_text_feature(text_feature_filename)
	UserFeatures, ProdFeatures, ReviewFeatures = construct_all_features(user_data, prod_data, text_features)
	with open(user_feature_filename, 'wb') as f:
		pickle.dump(UserFeatures, f)

	with open(prod_feature_filename, 'wb') as f:
		pickle.dump(ProdFeatures, f)

	with open(review_feature_filename, 'wb') as f:
		pickle.dump(ReviewFeatures, f)
	print ('Finished constructing high level user, product and review features\n')

# Priors
	print ('Start calculating user, product and review priors.\n')

	feature_config = load_feature_config(feature_suspicious_filename)
	#print (feature_config)

	with open(user_feature_filename, 'rb') as f:
		user_features = pickle.load(f)
	user_priors = calculateNodePriors(user_features, feature_config)
	with open(user_prior_filename, 'wb') as f:
		pickle.dump(user_priors, f)

	with open(prod_feature_filename, 'rb') as f:
		prod_features = pickle.load(f)
	prod_priors = calculateNodePriors(prod_features, feature_config)
	with open(prod_prior_filename, 'wb') as f:
		pickle.dump(prod_priors, f)

	with open(review_feature_filename, 'rb') as f:
		review_features = pickle.load(f)
	review_priors = calculateNodePriors(review_features, feature_config)
	with open(review_prior_filename, 'wb') as f:
		pickle.dump(review_priors, f)
	print ('Finished calculating user, product and review priors.\n')


# Licensed to Cloudera, Inc. under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  Cloudera, Inc. licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from django.core.exceptions import ImproperlyConfigured
from django.core.management.base import NoArgsCommand, CommandError, OutputWrapper
from django.core import serializers
from django.db import router, DEFAULT_DB_ALIAS
from django.utils.datastructures import SortedDict
from django.contrib.auth.models import User

from optparse import make_option
import logging
import sys

#class Command(BaseCommand):
#    option_list = BaseCommand.option_list + (
#        make_option('--format', default='json', dest='format',
#            help='Specifies the output serialization format for fixtures.'),
#        make_option('--indent', default=None, dest='indent', type='int',
#            help='Specifies the indent level to use when pretty-printing output'),
#        make_option('--database', action='store', dest='database',
#            default=DEFAULT_DB_ALIAS, help='Nominates a specific database to dump '
#                'fixtures from. Defaults to the "default" database.'),
#        make_option('-e', '--exclude', dest='exclude',action='append', default=[],
#            help='An appname or appname.ModelName to exclude (use multiple --exclude to exclude multiple apps/models).'),
#        make_option('-n', '--natural', action='store_true', dest='use_natural_keys', default=False,
#            help='Yay CHRIS Use natural keys if they are available.'),
#        make_option('-a', '--all', action='store_true', dest='use_base_manager', default=False,
#            help="Use Django's base manager to dump all models stored in the database, including those that would otherwise be filtered or modified by a custom manager."),
#        make_option('--pks', dest='primary_keys', help="Only dump objects with "
#            "given primary keys. Accepts a comma seperated list of keys. "
#            "This option will only work when you specify one model."),
#    )
#    help = ("Output the contents of the database as a fixture of the given "
#            "format (using each model's default manager unless --all is "
#            "specified).")
#    args = '[appname appname.ModelName ...]'

#    def handle(self, *app_labels, **options):
class Command(NoArgsCommand):
  def handle_noargs(self, *app_labels, **options):
        from django.db.models import get_app, get_apps, get_model

        format = options.get('format')
        indent = options.get('indent')
        using = options.get('database')
        excludes = options.get('exclude')
        show_traceback = options.get('traceback')
        use_natural_keys = options.get('use_natural_keys')
        use_base_manager = options.get('use_base_manager')
        pks = options.get('primary_keys')
        user = options.get('user')
        userid = user.id

        stdout = OutputWrapper(options.get('stdout', sys.stdout))

        if pks:
            primary_keys = pks.split(',')
        else:
            primary_keys = []

        excluded_apps = set()
        excluded_models = set()
        if excludes:
            for exclude in excludes:
                if '.' in exclude:
                    app_label, model_name = exclude.split('.', 1)
                    model_obj = get_model(app_label, model_name)
                    if not model_obj:
                        raise CommandError('Unknown model in excludes: %s' % exclude)
                    excluded_models.add(model_obj)
                else:
                    try:
                        app_obj = get_app(exclude)
                        excluded_apps.add(app_obj)
                    except ImproperlyConfigured:
                        raise CommandError('Unknown app in excludes: %s' % exclude)

        if len(app_labels) == 0:
            if primary_keys:
                raise CommandError("You can only use --pks option with one model")
            app_list = SortedDict((app, None) for app in get_apps() if app not in excluded_apps)
        else:
            if len(app_labels) > 1 and primary_keys:
                raise CommandError("You can only use --pks option with one model")
            app_list = SortedDict()
            for label in app_labels:
                try:
                    app_label, model_label = label.split('.')
                    try:
                        app = get_app(app_label)
                    except ImproperlyConfigured:
                        raise CommandError("Unknown application: %s" % app_label)
                    if app in excluded_apps:
                        continue
                    model = get_model(app_label, model_label)
                    if model is None:
                        raise CommandError("Unknown model: %s.%s" % (app_label, model_label))

                    if app in app_list.keys():
                        if app_list[app] and model not in app_list[app]:
                            app_list[app].append(model)
                    else:
                        app_list[app] = [model]
                except ValueError:
                    if primary_keys:
                        raise CommandError("You can only use --pks option with one model")
                    # This is just an app - no model qualifier
                    app_label = label
                    try:
                        app = get_app(app_label)
                    except ImproperlyConfigured:
                        raise CommandError("Unknown application: %s" % app_label)
                    if app in excluded_apps:
                        continue
                    app_list[app] = None

        # Check that the serialization format exists; this is a shortcut to
        # avoid collating all the objects and _then_ failing.
        if format not in serializers.get_public_serializer_formats():
            try:
                serializers.get_serializer(format)
            except serializers.SerializerDoesNotExist:
                pass

            raise CommandError("Unknown serialization format: %s" % format)

        def get_objects():
            # Collate the objects to be serialized.
            for model in sort_dependencies(app_list.items()):
                if model in excluded_models:
                    continue
                if not model._meta.proxy and router.allow_syncdb(using, model):
                    if use_base_manager:
                        objects = model._base_manager
                    else:
                        objects = model._default_manager

                    queryset = objects.using(using).order_by(model._meta.pk.name)
                    if primary_keys:
                        queryset = queryset.filter(pk__in=primary_keys)
                    queryset = queryset.filter(owner_id=userid)
                    for obj in queryset.iterator():
                        yield obj

        try:
            stdout.ending = None
#            self.stdout.ending = None
            serializers.serialize(format, get_objects(), indent=indent,
                    use_natural_keys=use_natural_keys, stream=stdout)
            stdout
#            serializers.serialize(format, get_objects(), indent=indent,
#                    use_natural_keys=use_natural_keys, stream=self.stdout)
        except Exception as e:
            if show_traceback:
                raise
            raise CommandError("Unable to serialize database: %s" % e)

def sort_dependencies(app_list):
    """Sort a list of app,modellist pairs into a single list of models.

    The single list of models is sorted so that any model with a natural key
    is serialized before a normal model, and any model with a natural key
    dependency has it's dependencies serialized first.
    """
    from django.db.models import get_model, get_models
    # Process the list of models, and get the list of dependencies
    model_dependencies = []
    models = set()
    for app, model_list in app_list:
        if model_list is None:
            model_list = get_models(app)

        for model in model_list:
            models.add(model)
            # Add any explicitly defined dependencies
            if hasattr(model, 'natural_key'):
                deps = getattr(model.natural_key, 'dependencies', [])
                if deps:
                    deps = [get_model(*d.split('.')) for d in deps]
            else:
                deps = []

            # Now add a dependency for any FK or M2M relation with
            # a model that defines a natural key
            for field in model._meta.fields:
                if hasattr(field.rel, 'to'):
                    rel_model = field.rel.to
                    if hasattr(rel_model, 'natural_key') and rel_model != model:
                        deps.append(rel_model)
            for field in model._meta.many_to_many:
                rel_model = field.rel.to
                if hasattr(rel_model, 'natural_key') and rel_model != model:
                    deps.append(rel_model)
            model_dependencies.append((model, deps))

    model_dependencies.reverse()
    # Now sort the models to ensure that dependencies are met. This
    # is done by repeatedly iterating over the input list of models.
    # If all the dependencies of a given model are in the final list,
    # that model is promoted to the end of the final list. This process
    # continues until the input list is empty, or we do a full iteration
    # over the input models without promoting a model to the final list.
    # If we do a full iteration without a promotion, that means there are
    # circular dependencies in the list.
    model_list = []
    while model_dependencies:
        skipped = []
        changed = False
        while model_dependencies:
            model, deps = model_dependencies.pop()

            # If all of the models in the dependency list are either already
            # on the final model list, or not on the original serialization list,
            # then we've found another model with all it's dependencies satisfied.
            found = True
            for candidate in ((d not in models or d in model_list) for d in deps):
                if not candidate:
                    found = False
            if found:
                model_list.append(model)
                changed = True
            else:
                skipped.append((model, deps))
        if not changed:
            raise CommandError("Can't resolve dependencies for %s in serialized app list." %
                ', '.join('%s.%s' % (model._meta.app_label, model._meta.object_name)
                for model, deps in sorted(skipped, key=lambda obj: obj[0].__name__))
            )
        model_dependencies = skipped

    return model_list


#!/usr/bin/env python
# coding: utf-8

# In[5]:

import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import matplotlib
import os
import sys
matplotlib.rcParams['pdf.fonttype'] = 42
matplotlib.rcParams['ps.fonttype'] = 42
matplotlib.rcParams['font.family'] = 'Arial'
sns.set_context('poster', font_scale=1.0)
# matplotlib.rcParams['legend.title_fontsize'] = 'xx-small'

# In[6]:

def read_param(path_home, pathtest, num):
    df = pd.read_csv(path_home + '/' + pathtest + '/fitparms-result_{}.csv'.format(num))
#     pB = ufloat(*df[df['param']=='pB'][['fitval', 'fiterror']].values[0])
#     dwB = ufloat(*df[df['param']=='dwB'][['fitval', 'fiterror']].values[0])
#     kexAB = ufloat(*df[df['param']=='kexAB'][['fitval', 'fiterror']].values[0])
#     rchi2 = ufloat(*df[df['param']=='rchi2'][['fitval', 'fiterror']].values[0])
    pB = df[df['param']=='pB']['fitval'].values[0]*100
    dwB = df[df['param']=='dwB']['fitval'].values[0]
    kexAB = df[df['param']=='kexAB']['fitval'].values[0]
    rchi2 = df[df['param']=='rchi2']['fitval'].values[0]

    return [pB, dwB, kexAB, rchi2]

path = os.getcwd()
num = num = int(path.split('_')[-1])
pb_value = []
dw_value = []
kex_value = []

foldername = [name for name in os.listdir(path) if os.path.isdir(os.path.join(path, name))]
kex_foldername = [name for name in foldername if len(name.split('_')) == 3 and name.split('_')[1] == 'kex']
pb_foldername = [name for name in foldername if len(name.split('_')) == 3 and name.split('_')[1] == 'pb']
dw_foldername = [name for name in foldername if len(name.split('_')) == 3 and name.split('_')[1] == 'dw']

# In[3]:

test_opt_value = read_param(path, 'test', num)
for i in pb_foldername:
    pb_value.append(read_param(path, i, num))
pb_value.append(test_opt_value)
pb_value = np.array(pb_value)

for i in kex_foldername:
    kex_value.append(read_param(path, i, num))
kex_value.append(test_opt_value)
kex_value = np.array(kex_value)

for i in dw_foldername:
    dw_value.append(read_param(path, i, num))
dw_value.append(test_opt_value)
dw_value = np.array(dw_value)

# In[4]:

fig, ax = plt.subplots(3, 1, figsize = (8 , 12))

ax[0].scatter(pb_value[:,0], pb_value[:,-1])
ax[1].scatter(dw_value[:,1], dw_value[:,-1])
ax[2].scatter(kex_value[:,2], kex_value[:,-1])

ax[0].scatter(test_opt_value[0], test_opt_value[-1], color = 'r', label = 'best fit')
ax[1].scatter(test_opt_value[1], test_opt_value[-1], color = 'r', label = 'best fit')
ax[2].scatter(test_opt_value[2], test_opt_value[-1], color = 'r', label = 'best fit')
for i in range(3):
    ax[i].set_xlabel(['$p$$_{ES}$ (%)', '$\Delta$$\omega$ (ppm)', '$k$$_{ex}$ (s$^{-1}$)'][i])
    ax[i].set_ylabel('r$\chi$$^2$')

name = sys.argv[1]
ax[0].set_title(name)
plt.tight_layout()
plt.savefig('{}.pdf'.format(name), dpi = 300, transparent = True)

# In[ ]:


import logging

import numpy as np
import tensorflow as tf

from lib.stream.reader.image_reader import ImageReader
from lib.tf_od_api.models.research.object_detection.utils import label_map_util
from lib.tf_od_api.models.research.object_detection.utils import visualization_utils as vis_util
from tqdm import tqdm

def load_inference_graph(path):
    logging.info(f'Loading {path} model...')
    graph = tf.compat.v1.Graph()

    with graph.as_default():
        od_graph_def = tf.compat.v1.GraphDef()
        with tf.io.gfile.GFile(path, 'rb') as fid:
            serialized_graph = fid.read()
            od_graph_def.ParseFromString(serialized_graph)
            tf.import_graph_def(od_graph_def, name='')

    return graph

def get_category_index(label_map_path, classes):
    logging.info(f'Loading labels from {label_map_path}...')
    label_map = label_map_util.load_labelmap(label_map_path)
    categories = label_map_util.convert_label_map_to_categories(
        label_map,
        max_num_classes=len(classes),
        use_display_name=True
    )
    return label_map_util.create_category_index(categories)

class Model:
    def __init__(self, inference_graph_path, label_map_path, classes):
        self.graph = load_inference_graph(inference_graph_path)
        self.__category_index = get_category_index(label_map_path, classes)

    def predict_samples(self, samples, min_score=.70):
        results = []
        reader = ImageReader(paths=[s.image.path for s in samples])
        with self.graph.as_default():
            with tf.compat.v1.Session(graph=self.graph) as session:
                with tqdm(total=len(samples), position=0, leave=True, unit='f') as pbar:
                    for frame in reader:
                        results.append(self.predict_frame(session, frame, min_score))
                        pbar.update(1)
        return results

    def predict_frame(self, session, frame, min_score=.70):
        # Expand dimensions since the model expects images to have shape: [1, None, None, 3]
        image_np_expanded = np.expand_dims(frame.raw, axis=0)

        image_tensor = self.graph.get_tensor_by_name('image_tensor:0')

        # Each box represents a part of the image where a particular object was detected.
        boxes = self.graph.get_tensor_by_name('detection_boxes:0')

        # Each score represent how level of confidence for each of the objects.
        # Score is shown on the result image, together with the class label.
        scores = self.graph.get_tensor_by_name('detection_scores:0')
        classes = self.graph.get_tensor_by_name('detection_classes:0')

        num_detections = self.graph.get_tensor_by_name('num_detections:0')

        # Actual detection.
        (boxes, scores, classes, num_detections) = session.run(
            [boxes, scores, classes, num_detections],
            feed_dict={image_tensor: image_np_expanded})

        # Visualization of the results of a detection.
        vis_util.visualize_boxes_and_labels_on_image_array(
            frame.raw,
            np.squeeze(boxes),
            np.squeeze(classes).astype(np.int32),
            np.squeeze(scores),
            self.__category_index,
            use_normalized_coordinates=True,
            line_thickness=4,
            max_boxes_to_draw=50,
            min_score_thresh=min_score
        )
        return frame


from dataclasses import InitVar, dataclass

@dataclass(unsafe_hash=True)
class Block:
    tag: int
    set: int
    model: InitVar["Model"]

    def __post_init__(self, model: "Model") -> None:
        self.address = (self.tag << (model.log_sets()) | self.set)

    def __repr__(self) -> str:
        return f"Block({self.tag}, {self.set}, #{self.address})"

    @staticmethod
    def from_address(model: "Model", address: int) -> "Block":
        tag = address >> (model.log_sets())
        set = (address) & ((1 << model.log_sets()) - 1)
        return Block(tag, set, model)


import webapp2, json, sys, views
from google.appengine.ext import ndb
sys.path.insert(0,'libs')
import models

class FindMusiciansHandler(views.Template):
  def get(self):
    states = models.musician.Musician.fetch_distinct_states()
    genres = models.videos.Videos.fetch_distinct_genres()
    states_select = sorted([x.musician_state for x in states])
    genre = sorted([x.genre_tag for x in genres])
    genre.insert(0,'All')
    states_select.insert(0,'All')
    selected_genre = self.request.get("g")
    selected_state = self.request.get("s")

    template_values = {'states':states_select, 'cities': ['All'], 'genres':genre,
      'selected_genre':selected_genre, 'selected_state':selected_state}
    self.render('find_musicians.html', template_values)

  def post(self):
    # NOTE: we are posting genre, popularity, distance, keywords and the cursor from a previous request or null if this is the initial one.
    genre = self.request.get('genre')
    state = self.request.get('state')
    city = self.request.get('city')

    #get all musicians
    musicians = models.musician.Musician.query()

    #filter results by state
    if state != 'All': musicians = musicians.filter(models.musician.Musician.musician_state == state)

    #filter by city
    if city != 'All': musicians = musicians.filter(models.musician.Musician.musician_city == city)

    #filter by genre
    if genre != 'All': musicians = musicians.filter(models.musician.Musician.band_genre == genre)

    musicians = musicians.order(models.musician.Musician.band_name).fetch()

    if musicians:
        data =[]
        for x in musicians:
            d = x.to_dict()

            #delete Non JSON variables
            del d['user_key'], d['profile_pic'],d['latest_update'],d['account_created'],d['DOB'],d['address'][0]['geo_code']

            #add user key back in as urlsafe
            d['key']= x.key.urlsafe()

            data.append(d)
        data = {'musicians':data, 'error':''}

    else:
        data = {'musicians':'','error':'No matches found for given parameters'}

    self.response.headers['Content-Type'] = 'application/json'
    self.response.out.write(json.dumps(data))

app = webapp2.WSGIApplication([

    ('/find_musicians*', FindMusiciansHandler)

], debug=True)


import os
import sys
import json
import logging
import pickle

def getPathsToExtractedLinesFiles():
    curDir = os.getcwd()
    directoryWithCharacterFiles = os.path.join(curDir, 'extractedLines')
    return [
        os.path.join(directoryWithCharacterFiles, workPath)
        for workPath in os.listdir(directoryWithCharacterFiles)
        if os.path.isfile(os.path.join(directoryWithCharacterFiles, workPath))
    ]

def getPathsToLinesFiles(pathToCharacterJson):
    """
    Get all the paths for JSON files
    Arguments:
        pathToCharacterJson - a path to where our lines files are stored
    Returns:
        An array of all absolute paths to files in that directory
    """
    curDir = os.getcwd()
    directoryWithCharacterFiles = os.path.join(curDir, pathToCharacterJson)
    return [
        os.path.join(directoryWithCharacterFiles, workPath)
        for workPath in os.listdir(directoryWithCharacterFiles)
        if os.path.isfile(os.path.join(directoryWithCharacterFiles, workPath))
    ]

def getPathsToGenderFile(pathToGenders, allLinesFilePaths):
    """
    Get the path to a specific gender file
    Arguments:
        pathToGenders - a path to where our gender files are stored
        allLinesFilePaths - an array of paths to the lines files containing characters and their lines
    Returns:
        allGenderFilePaths - an array of paths to gender lookup files, that aligns perfectly with the ordering of the json files
    """
    curDir = os.getcwd()
    directoryWithGenderFiles = os.path.join(curDir, pathToGenders)
    # Get the file name of the jsonFilePath
    allGenderFilePaths = []
    for curLinesFilePath in allLinesFilePaths:
        baseOfLinesFile = os.path.basename(curLinesFilePath)
        baseOfGenderFile = baseOfLinesFile.replace(".json", "-genders.json")
        curGenderFilePath = os.path.join(directoryWithGenderFiles, baseOfGenderFile)
        print(curGenderFilePath)
        if not os.path.isfile(curGenderFilePath):
            logging.exception(f"Error: No gender file at expected location: {curGenderFilePath}")
            return
        allGenderFilePaths.append(curGenderFilePath)
    return allGenderFilePaths

def buildVocabToNumMapping(vocab):
    """
    Create lookup tables for vocabulary used
    Parameters:
        vocab - A list of all unique words in our vocabulary
    Returns:
        vocabToNum - A dict of vocab terms to numeric values
    """
    # Index starts at 4 so we reseve 0-3 as a special characters
    index = 4
    vocabToNum = {}
    for word in vocab:
        if word not in vocabToNum:
            vocabToNum[word] = index
            index += 1
    print(f"Length of vocab: {index}")

    return (vocabToNum)

def pathsToLinesAndGenders():
    pathToLines = sys.argv[1] if len(sys.argv) > 1 else ""
    pathToGenders = sys.argv[2] if len(sys.argv) > 2 else ""
    if not pathToLines:
        pathToLines = f'json'
    if not pathToGenders:
        pathToGenders = f'genders'
    return (pathToLines, pathToGenders)

def buildLookupTablesFromFilePaths(allLinesFilePaths):
    vocab = set()
    for path in allLinesFilePaths:
        fileJson = json.load(open(path, 'r'))
        for char in fileJson:
            lines = fileJson[char]
            for line in lines:
                vocab |= set(line.split(" "))
    return buildVocabToNumMapping(vocab)

def buildGenderToNumMapping():
    genderToNum = {}
    genderToNum["M"] = 0
    genderToNum["F"] = 1
    return (genderToNum)

def lineToIntegerVector(line, vocabToNum):
    """
    Transform a string of words into an array of corresponding numbers, based on lookup table
    Parameters:
        line - a string of words, stripped of punctuation and separated by spaces
        vocabToNum - a dictionary that translates words found in the entire corpus to corresponding, unique integers
    Returns:
        numericLine - an array of numbers, where each number corresponds to a unique word in our corpus vocabulary
    """
    numericLine = []
    for word in line.split(" "):
        numericLine.append(vocabToNum[word])
    return numericLine;

def genderToInteger(gender, genderToNum):
    return genderToNum[gender]

def labelEachLineInFile(characterToLines, characterToGender, vocabToNum, genderToNum):
    """
    Get an array of tuples with lines spoken by characters and the gender of the speaker
    Parameters:
        characterToGender - A dict of each character in a work and their gender
        characterToLines - A dict of each character and an array of their lines
    REturns:
        labeledLines - A list of tuples, with each line followed by the corresponding gender
    """
    encodedLinesByFile = []
    encodedGenderByFile = []
    for char in characterToLines:
        print(char)
        gender = characterToGender[char]
        lines = characterToLines[char]
        # Get encodings by line
        encodedLinesByCharacter = []
        for l in lines:
            encodedLine = lineToIntegerVector(l, vocabToNum)
            encodedLinesByCharacter.append(encodedLine)
            if gender == 'F':
                encodedLinesByCharacter.append(encodedLine)
                encodedLinesByCharacter.append(encodedLine)
                encodedLinesByCharacter.append(encodedLine)
        # Add all line encodings to the encodings by file
        encodedGender = genderToInteger(gender, genderToNum)
        encodedGenderByCharacter = [encodedGender] * len(encodedLinesByCharacter)
        encodedGenderByFile += encodedGenderByCharacter
        encodedLinesByFile += encodedLinesByCharacter
        # Add a corresponding number of gender labellings
    return (encodedLinesByFile, encodedGenderByFile)

def labelAllLines(allLinesFilePaths, allGenderFilePaths, vocabToNum, genderToNum):
    allLinesEncoded  = []
    allGendersEncoded = []
    # Get the encodings for each file
    for index in range(0, len(allLinesFilePaths)):
        print(allLinesFilePaths[index])
        curLinesFile = json.load(open(allLinesFilePaths[index], 'r'))
        curGenderFile = json.load(open(allGenderFilePaths[index], 'r'))
        (encodedLinesByFile, encodedGendersByFile) = labelEachLineInFile(curLinesFile, curGenderFile, vocabToNum, genderToNum)
        allLinesEncoded += encodedLinesByFile
        allGendersEncoded += encodedGendersByFile
    return (allLinesEncoded, allGendersEncoded)

def pickleAllData(tupleOfData):
    curDir = os.getcwd()
    outputFilePath = os.path.join(curDir, 'output', 'output.pkl')
    outputFile = open(outputFilePath, 'wb')
    pickle.dump(tupleOfData, outputFile)

def main():
    # Get paths to lines and gender files
    (pathToLines, pathToGenders) = pathsToLinesAndGenders()
    # Get paths to each json files
    allLinesFilePaths = getPathsToLinesFiles(pathToLines)
    allGenderFilePaths = getPathsToGenderFile(pathToGenders, allLinesFilePaths)
    # Build our vocabulary and our lookup tables -- make sure to include files that were extracted and the ones we're building our model with
    (vocabToNum) = buildLookupTablesFromFilePaths(allLinesFilePaths + getPathsToExtractedLinesFiles())
    (genderToNum) = buildGenderToNumMapping()
    (allEncodedLines, allEncodedGender) = labelAllLines(allLinesFilePaths, allGenderFilePaths, vocabToNum, genderToNum)
    print("lens of allLines and allGender labels: ", len(allEncodedLines), "  ", len(allEncodedGender))
    pickleAllData((allEncodedLines, allEncodedGender, vocabToNum, genderToNum));

if __name__ == '__main__':
    main()
    print('run')


# This comparator was designed to work for a packet preamble of exactly [1,1,1,0], it was not tested under any other cases.

import numpy as np
import matplotlib.pyplot as plt
import time

debug = False				#Generate debugging plots

def compare_signal(signal, samples_per_bit, Packet):

	#t = time.time()

	SPB = int(samples_per_bit)
	ratio = 0.3
	max_signal = max(signal)
	min_signal = min(signal)

	#threshold =  max(max(signal), 0.006)*ratio
	threshold = (max_signal * ratio + min_signal * (1-ratio))

	index = -1
	end_result = [0] * int(len(signal)/samples_per_bit)

	packet_size_samples = int(Packet.packet_size * samples_per_bit)
	cooldown_margin = int(packet_size_samples * 1.2)
	#print(cooldown_margin)
	cooldown = -1 * packet_size_samples

	signal_zero_centered = [(x - threshold) for x in signal]
	signal_zero_centered = signal_zero_centered[0:-packet_size_samples]

	transitions = np.where(np.diff(np.signbit(signal_zero_centered)))[0]

	RPrP = []       #Relative preamble harvest positions
	RPaP = []       #Relative packet harvest positions

	y = int(samples_per_bit*0.5)
	for w in range(Packet.preamble_len):
		RPrP.append(round(w*samples_per_bit)+y)			#an in version cannot be used as the sucessive roundings will eventually cause a synchro error, hence not using variable SPB

	for w in range(Packet.packet_size):
		RPaP.append(round(w*samples_per_bit)+y)

	real_transitions = []
	#print(SPB)
	#print(RPaP)

	for x in transitions:
		#print(x-cooldown)
		if x - cooldown > cooldown_margin:
			real_transitions.append(x)
			preamble_match = []
			for i in range(len(RPrP)):
				preamble_match.append(binary_threshold(signal[x + RPrP[i]],threshold))
			if preamble_match == Packet.preamble :
				for i in range(len(RPaP)):
					position = x + RPaP[i]
					position2 = int(position/samples_per_bit)
					end_result[position2] = binary_threshold(signal[position], threshold)
				cooldown = x
				real_transitions.append(x)

	if debug == True:

		plt.plot(signal)

		POC = []  # Points of collection
		for y in real_transitions:
			for u in RPaP:
				POC.append(y+u)

		t_threshold = [threshold] * len(real_transitions)
		t2_signal = [(signal[x]) for x in POC]
		plt.axhline(y= threshold, color='k')
		plt.scatter(POC, t2_signal, color='orange')
		plt.scatter(real_transitions, t_threshold, color='black')

		plt.title('FAST Comparator', fontsize = 20)
		plt.xlabel('Sample index')
		plt.ylabel('Quantization level')

		#pylab.subplot(2,1,2)
		#fft_signal = np.fft(signal)
		#pylab.plot(fft_signal)
		plt.legend(['Signal', 'Threshold', 'Chosen samples', 'Temporal synchronization Points'], loc = 1)

		plt.show()

	#print(end_result)
	return end_result

def binary_threshold(value, threshold):
	if value>threshold:
		return 1
	else:
		return 0


# -*- coding: utf-8 -*-
"""
Created on Wed Nov 21 13:37:15 2018

@author: 沈鴻儒
"""

import numpy as np
import cv2

path = r"D:\desktop_D\ImageProcessing\python_opencv\License plate recognition"

for i in range(1, 4):
    image = cv2.imread(path + "\\far_{}.jpg".format(i))
    shrink = cv2.resize(image, (500, 500), interpolation=cv2.INTER_AREA)
    gray = cv2.cvtColor(shrink, cv2.COLOR_BGR2GRAY)
    blurred = cv2.GaussianBlur(gray, (9, 9), 0)
    binaryIMG = cv2.Canny(blurred, 20, 160)
    (_, cnts, _) = cv2.findContours(binaryIMG.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
    clone = shrink.copy()

#    cnt = cnts[0]
#    M = cv2.moments(cnt)
#    print(M)

#    for c in cnts:
#        # CV2.moments會傳回一系列的moments值，我們只要知道中點X, Y的取得方式是如下進行即可。
#        M = cv2.moments(c)
#        cX = int(M["m10"] / M["m00"])
#        cY = int(M["m01"] / M["m00"])
#        # 在中心點畫上黃色實心圓
#        cv2.circle(clone, (cX, cY), 10, (1, 227, 254), -1)

    cv2.drawContours(clone, cnts, -1, (0, 255, 0), 2)

#    #計算面積
#    area = cv2.contourArea(cnts)
#    #計算周長
#    perimeter = cv2.arcLength(cnts, True)
#    #印出結果
#    print("Contour #%d — area: %.2f, perimeter: %.2f" % (i + 1, area, perimeter))
    #各参数依次是：照片/添加的文字/左上角坐标/字体/字体大小/颜色/字体粗细
#    cv2.putText(clone, '#%d'%(i + 1), (50,150), cv2.FONT_HERSHEY_COMPLEX, 5, (0, 255, 0), 12)

    cv2.imshow("binary image", binaryIMG)
    cv2.imshow("All Contour", clone)
    cv2.waitKey(0)
    cv2.destroyAllWindows()
#    cv2.imwrite(path + "\Contour_{}.jpg".format(i), contour)


import numpy as np
import pickle
import matplotlib
matplotlib.use('Agg')
from matplotlib import pyplot
import pdb
import shutil

######################################################################
#Parameters
######################################################################
threshold = 0.6

benign_ss_path  = "/home/Drive2/aditya/benign_patches_info.npy"
normal_ss_path  = "/home/Drive2/aditya/normal_patches_info.npy"
insitu_ss_path  = "/home/Drive2/aditya/insitu_patches_info.npy"
invasive_ss_path  = "/home/Drive2/aditya/invasive_patches_info.npy"

benign_ss = np.load(benign_ss_path)
insitu_ss = np.load(insitu_ss_path)
invasive_ss = np.load(invasive_ss_path)
normal_ss = np.load(normal_ss_path)

with open('/home/Drive2/aditya/Benign.txt', 'rb') as f:
    benign = pickle.load(f)
with open('/home/Drive2/aditya/InSitu.txt', 'rb') as f:
    insitu = pickle.load(f)
with open('/home/Drive2/aditya/Invasive.txt', 'rb') as f:
    invasive = pickle.load(f)
with open('/home/Drive2/aditya/Normal.txt', 'rb') as f:
    normal = pickle.load(f)

with open('/home/Drive2/aditya/Benign_conf.txt', 'rb') as f:
    benign_confidence = pickle.load(f)
with open('/home/Drive2/aditya/InSitu_conf.txt', 'rb') as f:
    insitu_confidence = pickle.load(f)
with open('/home/Drive2/aditya/Invasive_conf.txt', 'rb') as f:
    invasive_confidence = pickle.load(f)
with open('/home/Drive2/aditya/Normal_conf.txt', 'rb') as f:
    normal_confidence = pickle.load(f)

#benign = np.load('/home/Drive2/aditya/Benign.npy')
#insitu = np.load('/home/Drive2/aditya/InSitu.npy')
#invasive = np.load('/home/Drive2/aditya/Invasive.npy')
#normal = np.load('/home/Drive2/aditya/Normal.npy')
no_samples = 25
######################################################################
#pdb.set_trace()

benign_hist = np.zeros((no_samples,4))
insitu_hist = np.zeros((no_samples,4))
invasive_hist = np.zeros((no_samples,4))
normal_hist = np.zeros((no_samples,4))

confusion_matrix = np.zeros((4,4))

benign_conf = np.zeros((4,1))
insitu_conf = np.zeros((4,1))
invasive_conf = np.zeros((4,1))
normal_conf = np.zeros((4,1))

# 0: Benign, 1:InSitu, 2:Invasive, 3:Normal
for i in range(no_samples):

    benign[i][(benign_confidence[i][:]<threshold)] = -1
    benign_hist[i,0] = (benign[i][:] == 0).sum()
    benign_hist[i,1] = (benign[i][:] == 1).sum()
    benign_hist[i,2] = (benign[i][:] == 2).sum()
    benign_hist[i,3] = (benign[i][:] == 3).sum()
    benign_hist[i,:] = benign_hist[i,:]/len(benign[i][:])
    #print(np.sum(benign_hist[i,:]))
    print((benign[i][:]==-1).sum())
    if np.sum(benign_hist[i,:]) == 0:
        print("Reduce threshold requested by Benign",i)

    benign_class_temp = np.argmax(benign_hist[i,:])

    if benign_hist[i,3] == benign_hist[i,benign_class_temp]:
        benign_class = 3
    if benign_hist[i,0] == benign_hist[i,benign_class_temp]:
        benign_class = 0
    if benign_hist[i,1] == benign_hist[i,benign_class_temp]:
        benign_class = 1
    if benign_hist[i,2] == benign_hist[i,benign_class_temp]:
        benign_class = 2

    benign_conf[benign_class]+=1

    if benign_class == 1 or benign_class == 3 or benign_class == 2:
        for j in range(benign_ss[i]):
            shutil.copyfile("/home/Drive2/aditya/Aug_Data/Validation/Benign/"+str(i)+"_"+str(j)+".png","/home/Drive2/aditya/Wrong_Class/Benign/"+str(i)+"_"+str(j)+"_TC_"+str(benign_class)+"_AC_0_PC_"+str(int(benign[i][j]))+".png")
        print(" Benign " + str(i) + " incorrectly classified as " + str(benign_class))

    insitu[i][(insitu_confidence[i][:]<threshold)] = -1
    insitu_hist[i,0] = (insitu[i][:] == 0).sum()
    insitu_hist[i,1] = (insitu[i][:] == 1).sum()
    insitu_hist[i,2] = (insitu[i][:] == 2).sum()
    insitu_hist[i,3] = (insitu[i][:] == 3).sum()
    insitu_hist[i,:] = insitu_hist[i,:]/len(insitu[i][:])
    #print(np.sum(insitu_hist[i,:]))

    if np.sum(insitu_hist[i,:]) == 0:
        print("Reduce threshold requested by insitu",i)

    insitu_class_temp = np.argmax(insitu_hist[i,:])

    if insitu_hist[i,3] == insitu_hist[i,insitu_class_temp]:
        insitu_class = 3
    if insitu_hist[i,0] == insitu_hist[i,insitu_class_temp]:
        insitu_class = 0
    if insitu_hist[i,1] == insitu_hist[i,insitu_class_temp]:
        insitu_class = 1
    if insitu_hist[i,2] == insitu_hist[i,insitu_class_temp]:
        insitu_class = 2

    insitu_conf[insitu_class]+=1

    if insitu_class == 0 or insitu_class == 3 or insitu_class == 2:
        for j in range(insitu_ss[i]):
            shutil.copyfile("/home/Drive2/aditya/Aug_Data/Validation/InSitu/"+str(i)+"_"+str(j)+".png","/home/Drive2/aditya/Wrong_Class/InSitu/"+str(i)+"_"+str(j)+"_TC_"+str(insitu_class)+"_AC_1_PC_"+str(int(insitu[i][j]))+".png")
        print(" InSitu " + str(i) + " incorrectly classified as " + str(insitu_class))

    invasive[i][(invasive_confidence[i][:]<threshold)] = -1
    invasive_hist[i,0] = (invasive[i][:] == 0).sum()
    invasive_hist[i,1] = (invasive[i][:] == 1).sum()
    invasive_hist[i,2] = (invasive[i][:] == 2).sum()
    invasive_hist[i,3] = (invasive[i][:] == 3).sum()
    invasive_hist[i,:] = invasive_hist[i,:]/len(invasive[i][:])
    #print(np.sum(invasive_hist[i,:]))

    if np.sum(invasive_hist[i,:]) == 0:
        print("Reduce threshold requested by invasive",i)

    #invasive_class = np.argmax(invasive_hist[i,:])
    invasive_class_temp = np.argmax(invasive_hist[i,:])

    if invasive_hist[i,3] == invasive_hist[i,invasive_class_temp]:
        invasive_class = 3
    if invasive_hist[i,0] == invasive_hist[i,invasive_class_temp]:
        invasive_class = 0
    if invasive_hist[i,1] == invasive_hist[i,invasive_class_temp]:
        invasive_class = 1
    if invasive_hist[i,2] == invasive_hist[i,invasive_class_temp]:
        invasive_class = 2

    invasive_conf[invasive_class]+=1

    if invasive_class == 0 or invasive_class == 3 or invasive_class == 1:
        for j in range(invasive_ss[i]):
            shutil.copyfile("/home/Drive2/aditya/Aug_Data/Validation/Invasive/"+str(i)+"_"+str(j)+".png","/home/Drive2/aditya/Wrong_Class/Invasive/"+str(i)+"_"+str(j)+"_TC_"+str(invasive_class)+"_AC_2_PC_"+str(int(invasive[i][j]))+".png")

        print(" Invasive " + str(i) + " incorrectly classified as " +str(invasive_class))

    normal[i][(normal_confidence[i][:]<threshold)] = -1
    normal_hist[i,0] = (normal[i][:] == 0).sum()
    normal_hist[i,1] = (normal[i][:] == 1).sum()
    normal_hist[i,2] = (normal[i][:] == 2).sum()
    normal_hist[i,3] = (normal[i][:] == 3).sum()
    normal_hist[i,:] = normal_hist[i,:]/len(normal[i][:])
    #print(np.sum(normal_hist[i,:]))

    if np.sum(normal_hist[i,:]) == 0:
        print("Reduce threshold requested by normal",i)

    #normal_class = np.argmax(normal_hist[i,:])
    normal_class_temp = np.argmax(normal_hist[i,:])

    if normal_hist[i,3] == normal_hist[i,normal_class_temp]:
        normal_class = 3
    if normal_hist[i,0] == normal_hist[i,normal_class_temp]:
        normal_class = 0
    if normal_hist[i,1] == normal_hist[i,normal_class_temp]:
        normal_class = 1
    if normal_hist[i,2] == normal_hist[i,normal_class_temp]:
        normal_class = 2

    normal_conf[normal_class]+=1

    if normal_class == 0 or normal_class == 2 or normal_class == 1:
        for j in range(normal_ss[i]):
            shutil.copyfile("/home/Drive2/aditya/Aug_Data/Validation/Normal/"+str(i)+"_"+str(j)+".png","/home/Drive2/aditya/Wrong_Class/Normal/"+str(i)+"_"+str(j)+"_TC_"+str(normal_class)+"_AC_3_PC_"+str(int(normal[i][j]))+".png")
        print(" Normal " + str(i) + " incorrectly classified as " +str(normal_class))

print("")
print("Confusion Matrix: 1 : Benign, 2 : InSitu, 3 : Invasive, 4 : Normal")
print("")
#pdb.set_trace()
print("Beni | InSi | Inv | Norm")
confusion_matrix[:,0] = benign_conf[:,0]
confusion_matrix[:,1] = insitu_conf[:,0]
confusion_matrix[:,2] = invasive_conf[:,0]
confusion_matrix[:,3] = normal_conf[:,0]

print(confusion_matrix)
print("")

print("Benign Acc: ", benign_conf[0]/no_samples)
print("Insitu Acc: ", insitu_conf[1]/no_samples)
print("Invasive Acc: ", invasive_conf[2]/no_samples)
print("Normal Acc: ", normal_conf[3]/no_samples)


import pygame
import random
import os
import sys
import sqlite3

pygame.init()

FPS = 50
WIDTH = 320
HEIGHT = 735
difficult = 10

screen = pygame.display.set_mode((WIDTH, HEIGHT))
clock = pygame.time.Clock()
pygame.display.set_caption("Tetris")

title_sound_play = False

con = sqlite3.connect('data/tetris_records.db')
cur = con.cursor()

colors = [
        (219, 31, 31),
        (255, 116, 10),
        (226, 232, 50),
        (31, 162, 19),
        (13, 135, 167),
        (50, 57, 178),
    ]
# Список возможных цветов фигур

def load_image(name, color_key=None):
    fullname = os.path.join('data', name)
    try:
        image = pygame.image.load(fullname).convert()
    except pygame.error as message:
        print('Cannot load image:', name)
        raise SystemExit(message)

    if color_key is not None:
        if color_key == -1:
            color_key = image.get_at((0, 0))
        image.set_colorkey(color_key)
    else:
        image = image.convert_alpha()
    return image

def terminate():
    pygame.quit()
    sys.exit()

def start_screen():
    global title_sound_play

    if title_sound_play is False:
        title_sound.play(loops=-1)
        title_sound_play = True
    intro_text = ["Матвей Хамков"]

    fon = pygame.transform.scale(load_image('Начальный экран.jpeg'), (WIDTH, HEIGHT))
    screen.blit(fon, (0, 0))
    font = pygame.font.Font(None, 20)
    text_coord = 600

    start_game = load_image('button_start_game.jpg', -1)
    start_game = pygame.transform.scale(start_game, (200, 70))
    screen.blit(start_game, (60, 300))
    table_of_records = load_image('button_table_record.jpg', -1)
    table_of_records = pygame.transform.scale(table_of_records, (200, 70))
    screen.blit(table_of_records, (60, 370))
    exit_game = load_image('button_exit_game.jpg', -1)
    exit_game = pygame.transform.scale(exit_game, (200, 70))
    screen.blit(exit_game, (60, 440))

    for line in intro_text:
        string_rendered = font.render(line, 1, pygame.Color('black'))
        intro_rect = string_rendered.get_rect()
        text_coord += 10
        intro_rect.top = text_coord
        intro_rect.x = 100
        text_coord += intro_rect.height
        screen.blit(string_rendered, intro_rect)

    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                terminate()
            elif event.type == pygame.MOUSEBUTTONDOWN:
                if 60 <= event.pos[0] <= 260 and 300 <= event.pos[1] <= 370:
                    level_hard()
                    return
                if 60 <= event.pos[0] <= 260 and 370 <= event.pos[1] <= 440:
                    show_table_of_records()
                    return
                if 60 <= event.pos[0] <= 260 and 440 <= event.pos[1] <= 510:
                    terminate()
                    return
        pygame.display.flip()
        clock.tick(FPS)

def level_hard():
    global difficult
    intro_text = ["Выберите сложность:"]

    fon = pygame.transform.scale(load_image('Начальный экран.jpeg'), (WIDTH, HEIGHT))
    screen.blit(fon, (0, 0))
    font = pygame.font.Font(None, 30)
    text_coord = 250
    for line in intro_text:
        string_rendered = font.render(line, 1, pygame.Color('black'))
        intro_rect = string_rendered.get_rect()
        text_coord += 10
        intro_rect.top = text_coord
        intro_rect.x = 65
        text_coord += intro_rect.height
        screen.blit(string_rendered, intro_rect)

    difficult1 = load_image('button_1.jpg', -1)
    difficult1 = pygame.transform.scale(difficult1, (200, 70))
    screen.blit(difficult1, (60, 300))
    difficult2 = load_image('button_2.jpg', -1)
    difficult2 = pygame.transform.scale(difficult2, (200, 70))
    screen.blit(difficult2, (60, 370))
    difficult3 = load_image('button_3.jpg', -1)
    difficult3 = pygame.transform.scale(difficult3, (200, 70))
    screen.blit(difficult3, (60, 440))
    back = load_image('button_back.jpg', -1)
    back = pygame.transform.scale(back, (200, 70))
    screen.blit(back, (60, 510))

    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                terminate()
            elif event.type == pygame.MOUSEBUTTONDOWN:
                if 60 <= event.pos[0] <= 260 and 300 <= event.pos[1] <= 370:
                    difficult = 2
                    return
                if 60 <= event.pos[0] <= 260 and 370 <= event.pos[1] <= 440:
                    difficult = 3
                    return
                if 60 <= event.pos[0] <= 260 and 440 <= event.pos[1] <= 510:
                    difficult = 4
                    return
                if 60 <= event.pos[0] <= 260 and 510 <= event.pos[1] <= 580:
                    start_screen()
                    return
        pygame.display.flip()
        clock.tick(FPS)

def show_table_of_records():
    fon = pygame.transform.scale(load_image('Начальный экран.jpeg'), (WIDTH, HEIGHT))
    screen.blit(fon, (0, 0))
    font = pygame.font.SysFont('Calibri', 25, True, False)

    back = load_image('button_back.jpg', -1)
    back = pygame.transform.scale(back, (200, 70))
    screen.blit(back, (60, 620))
    table = load_image('table_of_record.png', -1)
    table = pygame.transform.scale(table, (300, 200))
    screen.blit(table, (10, 10))
    screen.blit(table, (10, 210))
    screen.blit(table, (10, 410))

    con = sqlite3.connect('data/tetris_records.db')
    cur = con.cursor()
    records_1 = cur.execute("SELECT * FROM records_1").fetchall()
    records_2 = cur.execute("SELECT * FROM records_2").fetchall()
    records_3 = cur.execute("SELECT * FROM records_3").fetchall()
    records_all = [records_1, records_2, records_3]
    coord_x = 48
    coord_y = 25
    for number, difficult in enumerate(records_all):
        string_rendered = font.render("Сложность " + str(number + 1), True, pygame.Color('black'))
        screen.blit(string_rendered, [coord_x, coord_y])
        coord_y += 40
        for record in difficult:
            for el in record:
                string_rendered = font.render(str(el), True, pygame.Color('black'))
                screen.blit(string_rendered, [coord_x, coord_y])
                coord_x += 60
            coord_y += 50
            coord_x -= 120
        coord_y += 10
    while True:
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                terminate()
            elif event.type == pygame.MOUSEBUTTONDOWN:
                if 60 <= event.pos[0] <= 260 and 620 <= event.pos[1] <= 690:
                    start_screen()
                    return
        pygame.display.flip()
        clock.tick(FPS)

title_sound = pygame.mixer.Sound("data/title_1.mp3")
in_game_sound = pygame.mixer.Sound("data/game.mp3")
fall_sound = pygame.mixer.Sound("data/fall.mp3")
break_sound = pygame.mixer.Sound("data/break.mp3")
game_over_sound = pygame.mixer.Sound("data/game_over.mp3")

start_screen()

class Figure:
    global colors

    figures = [
        [[1, 5, 9, 13], [4, 5, 6, 7]],
        [[4, 5, 9, 10], [2, 6, 5, 9]],
        [[6, 7, 9, 10], [1, 5, 6, 10]],
        [[1, 2, 5, 9], [0, 4, 5, 6], [1, 5, 9, 8], [4, 5, 6, 10]],
        [[1, 2, 6, 10], [5, 6, 7, 9], [2, 6, 10, 11], [3, 5, 6, 7]],
        [[1, 4, 5, 6], [1, 4, 5, 9], [4, 5, 6, 9], [1, 5, 6, 9]],
        [[1, 2, 5, 6]],
    ]
    # Форма фигур и варианты их поворота, на поле 4 на 4. Например фигура Г выглядит так:
    #   0   *   *   3
    #   4   *   6   7
    #   8   *   10  11
    #   12  13  14  15
    # То есть [1, 2, 5, 9]

    def __init__(self, field_x, field_y):
        self.field_x = field_x
        self.field_y = field_y
        self.type = random.randint(0, len(self.figures) - 1)
        self.color = random.choice(colors)
        self.rotation = 0
        # В инициализации выбирается фигура и её цвет (так же можно добавить выбор градуса поворота,
        # но тогда надо уравнять кол-во елементов в списках)

    def image(self):
        return self.figures[self.type][self.rotation]
        # Возвращается елемент списка figures, заданного типа и поворота

    def rotate(self):
        self.rotation = (self.rotation + 1) % len(self.figures[self.type])
        # Изменение значения rotate для поворота фигуры

class Tetris:
    global colors

    running_game = True
    cell_count_height = 0
    cell_count_width = 0
    field_x = 10
    field_y = 10
    cell_size = 30
    figure = None
    # данные для инициализации: состояние игры, кол-во ячеек в столбце, кол-во ячеек в линии,
    # положение поля по x и y, размер клетки (сторона квадрата), состояние падающей сейчас фигуры

    def __init__(self, cell_count_height, cell_count_width, count=0, difficulty=2):
        self.cell_count_height = cell_count_height
        self.cell_count_width = cell_count_width
        self.difficulty = difficulty
        self.field = []
        self.score = 0
        self.running_game = True
        self.count = count
        self.chase_figure = False
        for _ in range(cell_count_height):
            new_line = []
            for _ in range(cell_count_width):
                new_line.append(0)
            self.field.append(new_line)
            # создание матрицы поля

    def new_figure(self):
        if self.count == 0:
            self.figure = Figure(3, 0)
        elif self.count == 1:
            self.figure = Figure(6, 20)
        # добавление новой фигуры

    def intersects(self):
        intersection = False
        for i in range(4):
            for j in range(4):
                if i * 4 + j in self.figure.image():
                    if i + self.figure.field_y > self.cell_count_height - 1 or \
                            j + self.figure.field_x > self.cell_count_width - 1 or \
                            j + self.figure.field_x < 0 or \
                            self.field[i + self.figure.field_y][j + self.figure.field_x] is not 0:
                        intersection = True
        return intersection
        # поиск пересечений падающей фигуры с полем

    def break_lines(self):
        break_was = False
        lines = 0
        places = []

        for i in range(1, self.cell_count_height):
            zeros = 0
            for j in range(self.cell_count_width):
                if self.field[i][j] == 0:
                    zeros += 1
                    break
            if zeros == 0:
                places.append((i, self.field[i].copy()))
                break_was = True
                lines += 1
                for i1 in range(i, 1, -1):
                    for j1 in range(self.cell_count_width):
                        self.field[i1][j1] = self.field[i1 - 1][j1]

        if break_was:
            break_sound.play()
            for place in places:
                for pos_x in range(9):
                    create_particles((self.field_x + self.cell_size * pos_x,
                                      self.field_y + self.cell_size * place[0]),
                                     colors.index(place[1][pos_x]))
                break
        self.score += lines ** 2
        self.chase_figure = True
        # функция для уничтожения линий. Проверяет на наличие 0 в матрице, и если их нет, то убирает линию

    def go_space(self):
        while not self.intersects():
            self.figure.field_y += 1
        self.figure.field_y -= 1
        self.stop_game()
        # функция клавиши "пробел" (фигура моментально падает как можно ниже)

    def go_down(self):
        self.figure.field_y += 1
        if self.intersects():
            self.figure.field_y -= 1
            self.stop_game()
        # функция клавиши "вниз" (фигура ускоренно падает вниз)

    def stop_game(self):
        if self.count == 0:
            # проверка фигуры (статичная или динамичная она)
            for i in range(4):
                for j in range(4):
                    if i * 4 + j in self.figure.image():
                        self.field[i + self.figure.field_y][j + self.figure.field_x] = self.figure.color
        self.break_lines()
        fall_sound.play()
        self.new_figure()
        if self.intersects():
            self.running_game = False
            game_over_sound.play()
            in_game_sound.stop()
        if self.count == 1:
            self.chase_figure = False
        # останавливает игру, вызывается другими функциями при наличии пересечений

    def go_side(self, dx):
        old_x = self.figure.field_x
        self.figure.field_x += dx
        if self.intersects():
            self.figure.field_x = old_x
        # функция клавиш "влево" и "вправо". Перемещает фигуру влево или вправо

    def rotate(self):
        old_rotation = self.figure.rotation
        self.figure.rotate()
        if self.intersects():
            self.figure.rotation = old_rotation
        # функция клавиши "вверх" поворачивает фигуру против часовой стрелки

class Particle(pygame.sprite.Sprite):
    # сгенерируем частицы разного размера
    white = load_image("particle_white.png")
    red = load_image("particle_red.png")
    orange = load_image("particle_orange.png")
    yellow = load_image("particle_yellow.png")
    green = load_image("particle_green.png")
    sky_blue = load_image("particle_sky_blue.png")
    blue = load_image("particle_blue.png")
    particles_colors = [red, orange, yellow, green, sky_blue, blue, white]
    particles = []

    def __init__(self, pos, dx, dy, color):
        super().__init__(all_sprites)
        self.image = self.particles_colors[color]
        self.rect = self.image.get_rect()

        # у каждой частицы своя скорость — это вектор
        self.velocity = [dx, dy]
        # и свои координаты
        self.rect.x, self.rect.y = pos

        # гравитация будет одинаковой (значение константы)
        self.gravity = GRAVITY

        for scale in (5, 10, 20):
            self.particles.append(pygame.transform.scale(self.particles_colors[color], (scale, scale)))

    def update(self):
        # применяем гравитационный эффект:
        # движение с ускорением под действием гравитации
        self.velocity[1] += self.gravity
        # перемещаем частицу
        self.rect.x += self.velocity[0]
        self.rect.y += self.velocity[1]
        # убиваем, если частица ушла за экран
        if not self.rect.colliderect(0, 0, WIDTH, HEIGHT):
            self.kill()

def create_particles(position, color):
    # количество создаваемых частиц
    particle_count = 5
    # возможные скорости
    numbers = range(-5, 6)
    for _ in range(particle_count):
        Particle(position, random.choice(numbers), random.choice(numbers), color)

BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
GRAY = (128, 128, 128)
GRAVITY = 0.5

size = (WIDTH, HEIGHT)
screen = pygame.display.set_mode(size)
all_sprites = pygame.sprite.Group()
pygame.display.set_caption("Tetris")

running = True
clock = pygame.time.Clock()
fps = 25
game = Tetris(20, 10, 0, difficulty=difficult)
title_sound.stop()
title_sound_play = False
in_game_sound.play(loops=-1)

figure_in = Tetris(20, 10, 1, difficulty=difficult)
# создание статичной фигуры в нижнем правом углу

counter = 0
# создание игры. counter - счетчик отвечающий за отсчет время падения

pressing_down = False

while running:
    if game.figure is None:
        game.new_figure()
    if figure_in.figure is None:
        figure_in.new_figure()
    counter += 1
    if counter > 6000:
        counter = 0

    if counter % (fps // game.difficulty // 2) == 0 or pressing_down:
        if game.running_game is True:
            game.go_down()

    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            running = False
        if game.running_game is True:
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_UP:
                    game.rotate()
                if event.key == pygame.K_DOWN:
                    pressing_down = True
                if event.key == pygame.K_LEFT:
                    game.go_side(-1)
                if event.key == pygame.K_RIGHT:
                    game.go_side(1)
                if event.key == pygame.K_SPACE:
                    game.go_space()
        if game.running_game is False:
            if event.type == pygame.KEYDOWN:
                if event.key == pygame.K_ESCAPE:
                    last_score = game.score
                    last_dif = game.difficulty
                    if last_dif == 2:
                        rec_dif_1 = cur.execute("SELECT * FROM records_1").fetchall()
                        for num, el in enumerate(rec_dif_1):
                            if last_score >= el[1]:
                                pre_score = el[1]
                                cur.execute(f'UPDATE records_1 SET score=? WHERE place=?', (last_score, num + 1))
                                num += 1
                                while num < 2:
                                    cur.execute(f'UPDATE records_1 SET score=? WHERE place=?', (pre_score, num + 1))
                                    num += 1
                                break
                    if last_dif == 3:
                        rec_dif_2 = cur.execute("SELECT * FROM records_2").fetchall()
                        for num, el in enumerate(rec_dif_2):
                            if last_score >= el[1]:
                                pre_score = el[1]
                                cur.execute(f'UPDATE records_2 SET score=? WHERE place=?', (last_score, num + 1))
                                num += 1
                                while num < 2:
                                    cur.execute(f'UPDATE records_2 SET score=? WHERE place=?', (pre_score, num + 1))
                                    num += 1
                                break
                    if last_dif == 4:
                        rec_dif_3 = cur.execute("SELECT * FROM records_3").fetchall()
                        for num, el in enumerate(rec_dif_3):
                            if last_score >= el[1]:
                                pre_score = el[1]
                                cur.execute(f'UPDATE records_3 SET score=? WHERE place=?', (last_score, num + 1))
                                num += 1
                                while num < 2:
                                    cur.execute(f'UPDATE records_3 SET score=? WHERE place=?', (pre_score, num + 1))
                                    num += 1
                                break
                    con.commit()

                    title_sound.play(loops=-1)
                    title_sound_play = True
                    level_hard()
                    title_sound.stop()
                    title_sound_play = False
                    in_game_sound.play(loops=-1)
                    game.__init__(20, 10, difficulty=difficult)
                    game.figure = None

        if event.type == pygame.KEYUP:
            if event.key == pygame.K_DOWN:
                pressing_down = False
        # обработка событий

    screen.fill(BLACK)

    for i in range(game.cell_count_height):
        for j in range(game.cell_count_width):
            pygame.draw.rect(screen, GRAY, [game.field_x + game.cell_size * j, game.field_y + game.cell_size * i,
                                            game.cell_size, game.cell_size], 1)
            if game.field[i][j] is not 0:
                pygame.draw.rect(screen, game.field[i][j],
                                 [game.field_x + game.cell_size * j + 1, game.field_y + game.cell_size * i + 1,
                                  game.cell_size - 2, game.cell_size - 1])
                # отрисовка поля и сетки

    if game.figure is not None:
        for i in range(4):
            for j in range(4):
                p = i * 4 + j
                if p in game.figure.image():
                    pygame.draw.rect(screen, game.figure.color,
                                     [game.field_x + game.cell_size * (j + game.figure.field_x) + 1,
                                      game.field_y + game.cell_size * (i + game.figure.field_y) + 1,
                                      game.cell_size - 2, game.cell_size - 2])
                if p in figure_in.figure.image():
                    pygame.draw.rect(screen, figure_in.figure.color,
                                     [figure_in.field_x + figure_in.cell_size * (j + figure_in.figure.field_x) + 1,
                                      figure_in.field_y + figure_in.cell_size * (i + figure_in.figure.field_y) + 1,
                                      figure_in.cell_size - 2, figure_in.cell_size - 2])
        # отрисовка падающей и статичной фигуры

    font = pygame.font.SysFont('Calibri', 25, True, False)
    font1 = pygame.font.SysFont('Calibri', 50, True, False)
    text_score = font.render("Score:", True, WHITE)
    text_score_num = font.render(str(game.score), True, WHITE)
    text_game_over = font1.render("Game Over", True, WHITE)
    text_game_over1 = font1.render("Press ESC", True, WHITE)
    text_figure_1 = font.render("Next", True, WHITE)
    text_figure_2 = font.render("figure:", True, WHITE)

    screen.blit(text_score, [25, 625])
    screen.blit(text_score_num, [50, 650])
    screen.blit(text_figure_1, [120, 625])
    screen.blit(text_figure_2, [120, 650])

    if game.running_game is False:
        screen.blit(text_game_over, [40, 200])
        screen.blit(text_game_over1, [60, 265])

    if game.chase_figure:
        game.chase_figure = False
        game.figure.color = figure_in.figure.color
        game.figure.type = figure_in.figure.type
        figure_in = Tetris(20, 10, 1, difficulty=difficult)
        # замена фигуры падуюший на статичную

    all_sprites.update()
    all_sprites.draw(screen)
    pygame.display.flip()
    clock.tick(fps)

pygame.quit()

файл с кодом для запуска через интерпритатор/Tetris файл кода.py


import functools
import glob
from multiprocessing import Pool
import os
import pandas as pd
import sys
import tempfile
import traceback

from cluster_utils.io import makedirs_safely
from core_maths.scale_split import merge_split
from db_tools.ezfuncs import query
from db_queries import get_location_metadata, get_cause_metadata, get_ids
from get_draws.api import get_draws
import gbd.constants as gbd
from gbd.decomp_step import decomp_step_id_from_decomp_step
from gbd.estimation_years import gbd_round_from_gbd_round_id
from test_support.profile_support import profile

from split_models.exceptions import IllegalSplitCoDArgument
from split_models.job_classes import SplitCoDSwarm
from split_models.validate import (
    validate_decomp_step_input,
    validate_ids
)

if sys.version_info > (3,):
    long = int

REPORTING_CAUSE_SET_ID, COMPUTATION_CAUSE_SET_ID = 3, 2

# Create list of valid cause_ids
VALID_CAUSE_IDS = get_ids(table='cause').cause_id.unique()
VALID_MEIDS = get_ids(table='modelable_entity').modelable_entity_id.unique()

@profile
def _launch_cod_splits(source_cause_id, target_cause_ids, target_meids,
                       prop_meas_id, gbd_round_id, decomp_step, output_dir,
                       project):
    """
    Split the given source_cause_id given target_meid proportions, saved
    to the target_cause_ids in output_dir.

    Arguments:
        source_cause_id (int): cause_id for the draws to be split
        target_cause_ids (intlist): list of cause ids that you want the new
            outputted subcauses to be identified by
        target_meids (intlist): list of proportion models' modelable_entity_ids
            that you want the source_cause_id to be split by, to make the
            target_cause_ids. Target_cause_ids and target_me_ids must be
            specified in the same order
        prop_meas_id (int): The measure_id that identifies the proportion
            in the target_meids to use for the split.
        gbd_round_id (int): the gbd_round_id for models being split.
        decomp_step (str): Specifies which decomposition step the returned
            estimates should be from. If using interpolate for GBD round 6 and
            above, must specify one of 'step1', 'step2', 'step3', 'step4',
            'step5', or 'iterative'.
        output_dir (str): directory where you want final results stored
        project (str): The SGE project to launch split_cod_model subjobs
            to using SplitCodSwarm.

    Returns:
        A list of tuples with each location_id paired with either 0, or an
                error message. This is then parsed in the central function
                draw_ops.split_cod_model into errors or success messages
    """

    # setup years, sex restrictions, most detailed locations, etc.

    if gbd_round_id >= 6:
        cause_set_id = COMPUTATION_CAUSE_SET_ID
    else:
        cause_set_id = REPORTING_CAUSE_SET_ID
    causes = get_cause_metadata(
        cause_set_id=cause_set_id,
        gbd_round_id=gbd_round_id,
        decomp_step=decomp_step
    ).query("cause_id==@source_cause_id")
    sex_ids = []
    if causes['male'].item() != 0:
        sex_ids.append(1)
    if causes['female'].item() != 0:
        sex_ids.append(2)
    if not sex_ids:
        raise ValueError(
            "Source_cause_id {} is restricted for both males and females, "
            "according to cause metadata"
            .format(source_cause_id)
        )

    most_detailed_locs = list(
        get_location_metadata(
            35, gbd_round_id=gbd_round_id, decomp_step=decomp_step
        ).query('most_detailed==1').location_id.unique())
    meid_cause_map = dict(zip(target_meids, target_cause_ids))

    # run interpolating/extrapolating
    intermediate_dir = os.path.join(
        output_dir, 'intermediate_{}'.format(source_cause_id)
    )
    if not os.path.exists(intermediate_dir):
        makedirs_safely(intermediate_dir)

    swarm = SplitCoDSwarm(
        source_id=source_cause_id,
        proportion_ids=target_meids,
        proportion_measure_id=prop_meas_id,
        sex_ids=sex_ids,
        gbd_round_id=gbd_round_id,
        decomp_step=decomp_step,
        intermediate_dir=intermediate_dir,
        outdir=output_dir,
        project=project
    )
    swarm.add_interpolate_tasks()
    exit_code = swarm.run()
    if exit_code != 0:
        raise RuntimeError(
            "Interpolating CoD years failed. Check logs in {}."
            .format(output_dir)
        )

    # run splitting
    for cid in target_cause_ids:
        cid_dir = os.path.join(output_dir, str(cid))
        if not os.path.exists(cid_dir):
            makedirs_safely(cid_dir)
    file_list = glob.glob(os.path.join('{}/*.h5'.format(intermediate_dir)))

    # read in draws for source cause
    source = _get_draws(source_cause_id, gbd_round_id, decomp_step, sex_ids)
    # create a temporary directory to store all the draws from the source cause
    tmpdir = tempfile.TemporaryDirectory(dir=output_dir)
    # save source cause draws to temporary directory
    source.to_hdf(
        os.path.join(tmpdir.name, 'source_cause_draws.h5'),
        key='draws', mode='w', format='table',
        data_columns=['location_id', 'year_id', 'sex_id', 'age_group_id']
    )

    run_splits = functools.partial(_parallel_merge_split,
                                   meid_cause_map,
                                   file_list,
                                   output_dir,
                                   tmpdir)
    pool = Pool(30)
    res = pool.map(run_splits, most_detailed_locs)
    pool.close()
    pool.join()
    # clean up tempdir
    tmpdir.cleanup()
    return res

def _get_draws(source_cause_id, gbd_round_id, decomp_step, sex_ids):
    """Read in all the draws from the source cause id for each sex."""
    cd = []
    for sex in sex_ids:
        version_id = best_versions(source_cause_id, gbd_round_id, sex,
                                   decomp_step)
        cd.append(get_draws(gbd_id_type='cause_id', gbd_id=source_cause_id,
                            source='codem', gbd_round_id=gbd_round_id,
                            decomp_step=decomp_step, version_id=version_id))
    cd = pd.concat(cd)
    return cd

def best_versions(cause_id, gbd_round_id, sex_id, decomp_step):
    decomp_step_id = decomp_step_id_from_decomp_step(decomp_step, gbd_round_id)
    q = """
        SELECT model_version_id
        FROM cod.model_version
        WHERE cause_id={cid} AND
              sex_id = {sex} AND
              gbd_round_id={gbd} AND
              is_best = 1 AND
              model_version_type_id IN (3, 4) AND
              decomp_step_id = {decomp}
        """.format(cid=cause_id, sex=sex_id, gbd=gbd_round_id,
                   decomp=decomp_step_id)
    res = query(q, conn_def='cod')
    mvids = res.model_version_id.tolist()
    bad_mvids_msg = ("Error: Returned more than one model_version_id: "
                     "{mvids} for cause_id: {cid}. Cannot split model."
                     .format(mvids=mvids, cid=cause_id))
    if len(mvids) != 1:
        raise RuntimeError(bad_mvids_msg)
    return mvids[0]

@profile
def _parallel_merge_split(meid_cause_map, interp_files, output_dir, tmpdir,
                          location_id):
    try:
        epi_draw = []
        for f in interp_files:
            epi_draw.append(pd.read_hdf(f, 'draws',
                            where=["location_id=={}".format(location_id)]))
        epi_draws = pd.concat(epi_draw)

        cd = pd.read_hdf(
            os.path.join(tmpdir.name, 'source_cause_draws.h5'), 'draws',
            where=['location_id=={}'.format(location_id)]
        )

        draw_cols = [col for col in cd.columns if 'draw_' in col]
        epi_draws = epi_draws[
            epi_draws['age_group_id'].isin(cd['age_group_id'].unique())]

        # these columns are not needed and cause maths.merge_split to break
        drop_cols = ['measure_id', 'model_version_id', 'metric_id']
        cd.drop(drop_cols, axis=1, inplace=True, errors='ignore')
        epi_draws.drop(drop_cols, axis=1, inplace=True, errors='ignore')

        cout = merge_split(
            cd, epi_draws,
            ['year_id', 'age_group_id', 'sex_id', 'location_id'],
            draw_cols)

        cout = cout.merge(
            cd[['year_id', 'age_group_id', 'sex_id', 'location_id',
                'envelope']],
            how='left')
        cout['cause_id'] = cout['modelable_entity_id']
        cout['cause_id'] = cout['cause_id'].replace(meid_cause_map)
        cout['measure_id'] = 1
        for cid in cout.cause_id.unique():
            cid_dir = '{}/{}'.format(output_dir, int(cid))
            cid_dir = cid_dir.replace("\r", "")
            if not os.path.exists(cid_dir):
                makedirs_safely(cid_dir)
            fn = '{}/death_{}.csv'.format(cid_dir, location_id)
            cout.query('cause_id=={}'.format(cid)).to_csv(fn, index=False)
        return location_id, 0
    except Exception:
        tb_str = traceback.format_exc()
        return location_id, tb_str

@profile
def split_cod_model(source_cause_id,
                    target_cause_ids,
                    target_meids,
                    project,
                    prop_meas_id=18,
                    decomp_step=None,
                    gbd_round_id=gbd.GBD_ROUND_ID,
                    output_dir=None):

    """Returns a dataframe containing only the name of the out_dir where this
    function will save your new draws.

    Arguments:
        source_cause_id (int): the cause_id to be split into target_cause_ids.

        target_cause_ids (intlist): the cause_ids that should be produced as a
            result of the split. These should be provided in the same order as
            the target_meids which define the split proportions.

        target_meids (intlist): the modelable_entity_ids containing the
            proportion models by which to split the source_cause_id. These
            should be provided in the same order as the target_cause_ids.

        project (str): the team-specific SGE cluster project to launch the
            subjobs of the split_cod_model swarm to.

        prop_meas_id (int): The measure_id that identifies the proportion in
            the target_meids to use for the split. Defaults to measure_id 18,
            proportion.

        decomp_step (str): Specifies which decomposition step the returned
            estimates should be from. Default to None. "Allowed values are
            None, iterative', 'step1', 'step2', 'step3', 'step4', and 'step5'
            depending on the value of gbd_round_id."

        gbd_round_id (int): the gbd_round_id for models being split. Defaults
            to current GBD round id.

        output_dir (str): place where you want new draws to be saved.

    Returns:
        Pandas.DataFrame:
        The output directory where either the draws or an errors logfile
        can be found.

    Raises:
        IllegalSplitCoDArgument: If the source_cause_id, any of the
            target_cause_ids or target_meids are invalid, or if the lists
            of target causes and target meids are not one-to-one.
        RuntimeError: If any under-the-hood errors thrown directing the
            user to the log file.
    """
    # Validate all incoming arguments.
    validate_decomp_step_input(decomp_step, gbd_round_id)

    # Validate source cause_id
    validate_ids(source_cause_id, VALID_CAUSE_IDS, 'source_cause_id')
    # Validate target_cause_ids
    validate_ids(target_cause_ids, VALID_CAUSE_IDS, 'target_cause_ids')

    # Validate modelable_entity_ids
    validate_ids(target_meids, VALID_MEIDS, 'proportion_meids')

    if len(target_cause_ids) != len(target_meids):
        raise IllegalSplitCoDArgument(
            "target_cause_ids and target_meids lists must represent a 1-to-1 "
            "mapping of cause_ids to modelable_entity_ids. Received: {t_ids} "
            "target ids and {p_ids} proportion ids."
            .format(t_ids=len(target_cause_ids), p_ids=len(target_meids))
        )

    if not output_dir:
        output_dir = 'FILEPATH'

    if not os.path.exists(output_dir):
        makedirs_safely(output_dir)

    res = _launch_cod_splits(source_cause_id,
                             target_cause_ids,
                             target_meids,
                             prop_meas_id,
                             gbd_round_id,
                             decomp_step,
                             output_dir,
                             project)
    errors = [r for r in res if r[1] != 0]

    if len(errors) == 0:
        return pd.DataFrame({'output_dir': output_dir}, index=[0])
    else:
        logfile = '{}/{}_errors.log'.format(
            output_dir, str(source_cause_id))
        with open(logfile, 'w') as f:
            estr = "\n".join([str(r) for r in errors])
            f.write(estr)

        return pd.DataFrame({'error_dir': logfile}, index=[0])


import httpx, json
import logging, traceback
import secrets
from notify import send_email

logging.basicConfig(filename=secrets.logpath+'traffic.log', level=logging.INFO,
                    format="%(asctime)s - TRAFFIC - %(levelname)s - %(message)s",
                    datefmt="%Y-%m-%d %H:%M:%S")

def commute():
    api = 'https://maps.googleapis.com/maps/api/directions/json?'
    nodes = 'origin='+secrets.point_a + '&' + 'destination='+secrets.point_b
    options = "&departure_time=now&alternatives=true"
    key = '&key='+ secrets.maps_key
    link = api+nodes+options+key

    res = httpx.get(link)
    routes = res.json()['routes']
    tups = []

    # make tuples out of routes, Tuple(duration, steps)
    for r in routes:
        duration = r['legs'][0]['duration_in_traffic']['value']
        steps = []
        for step in r['legs'][0]['steps']:
            steps.append(step['html_instructions'])
        tup = (duration, steps)
        tups.append(tup)

    best_route = min(tups)[1]
    traffic = not all(any(string in step for step in best_route) for string in ['I-195 N', '183B'])

    if traffic:
        subject = "Traffic Alert"
        body = "I-64 is not the fastest route"
        send_email(subject, body)
        logging.info("Morning, trigger email: YES")
    else:
        logging.info("Morning, trigger email: NO")

def main():
    try:
        commute()
    except:
        tb = traceback.format_exc()
        logging.exception("Morning commute() function did not run properly")
        send_email("Notis logs: traffic_morning.py didn't run properly", tb)

if __name__ == "__main__":
    main()