import csv
import matplotlib.pyplot as plt
import numpy as np
import json
from sklearn.linear_model import LogisticRegression
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier

def getdays(date1):
	year1 = int(date1[0] + date1[1] + date1[2] + date1[3])
	month1 = int(date1[4] + date1[5])
	date1 = int(date1[6] + date1[7])

	answer = 365*year1 + 30*month1 + date1
	return answer

def getmonth(date1):
	month1 = int(date1[4] + date1[5])
	return month1

# Reorders all csv files to fixed column ordering and loads to a ndarray
def create_add_year_matches(input_path, output_path):

	with open(input_path, 'r') as infile, open(output_path, 'w') as outfile:
	    # Output dict needs a list for new column ordering
	    fieldnames = ['tourney_id', #0
	                  'tourney_name', #1
	                  'surface', #2
	                  'draw_size', #3
	                  'tourney_level', #4
	                  'tourney_date', #5
	                  'match_num', #6
	                  'winner_id', #7
	                  'winner_seed', #8
	                  'winner_entry', #9
	                  'winner_name', #10
	                  'winner_hand', #11
	                  'winner_ht', #12
	                  'winner_ioc', #13
	                  'winner_age', #14
	                  'winner_rank', #15
	                  'winner_rank_points', #16
	                  'loser_id', #17
	                  'loser_seed', #18
	                  'loser_entry', #19
	                  'loser_name', #20
	                  'loser_hand', #21
	                  'loser_ht', #22
	                  'loser_ioc', #23
	                  'loser_age', #24
	                  'loser_rank', #25
	                  'loser_rank_points', #26
	                  'score', #27
	                  'best_of', #28
	                  'round', #29
	                  'minutes', #30
	                  'w_ace', #31
	                  'w_df', #32
	                  'w_svpt', #33
	                  'w_1stIn', #34
	                  'w_1stWon', #35
	                  'w_2ndWon', #36
	                  'w_SvGms', #37
	                  'w_bpSaved', #38
	                  'w_bpFaced', #39
	                  'l_ace', #40
	                  'l_df', #41
	                  'l_svpt', #42
	                  'l_1stIn', #43
	                  'l_1stWon', #44
	                  'l_2ndWon', #45
	                  'l_SvGms', #46
	                  'l_bpSaved', #47
	                  'l_bpFaced'] #48

	    writer = csv.DictWriter(outfile, fieldnames=fieldnames)
	    # reorder the header first
	    writer.writeheader()
	    for row in csv.DictReader(infile):
	        # writes the reordered rows to the new file
	        writer.writerow(row)

	inputs = np.loadtxt(output_path, dtype=np.dtype(str), delimiter=',', skiprows=1)
	return inputs

with open('atp_matches_1997.csv', 'r') as infile, open('reordered_atp_matches_1997.csv', 'w') as outfile:
    # output dict needs a list for new column ordering
    fieldnames = ['tourney_id', 'tourney_name', 'surface', 'draw_size', 'tourney_level', 'tourney_date', 'match_num', 'winner_id', 'winner_seed', 'winner_entry', 'winner_name', 'winner_hand', 'winner_ht', 'winner_ioc', 'winner_age', 'winner_rank', 'winner_rank_points' ,'loser_id', 'loser_seed'	,'loser_entry', 'loser_name', 'loser_hand', 'loser_ht' , 'loser_ioc', 'loser_age', 'loser_rank', 'loser_rank_points', 'score', 'best_of' ,'round', 'minutes', 'w_ace', 'w_df', 'w_svpt', 'w_1stIn', 'w_1stWon', 'w_2ndWon', 'w_SvGms', 'w_bpSaved', 'w_bpFaced', 'l_ace', 'l_df', 'l_svpt', 'l_1stIn', 'l_1stWon', 'l_2ndWon', 'l_SvGms', 'l_bpSaved', 'l_bpFaced']

    writer = csv.DictWriter(outfile, fieldnames=fieldnames)
    # reorder the header first
    writer.writeheader()
    for row in csv.DictReader(infile):
        # writes the reordered rows to the new file
        writer.writerow(row)

inputs = np.loadtxt("reordered_atp_matches_1997.csv", dtype=np.dtype(str), delimiter=',', skiprows=1)

for i in range(1998, 2019):
	input_path = 'atp_matches_' + str(i) + '.csv'
	output_path = 'reordered_atp_matches' + str(i) + '.csv'

	inputs_temp = create_add_year_matches(input_path, output_path)
	inputs = np.concatenate((inputs, inputs_temp))

for i in range(2010, 2018):
	input_path = 'atp_matches_qual_chall_' + str(i) + '.csv'
	output_path = 'reordered_atp_matches_qual_chall_matches' + str(i) + '.csv'

	inputs_temp = create_add_year_matches(input_path, output_path)
	inputs = np.concatenate((inputs, inputs_temp))

for i in range(2019, 2020):
	input_path = 'atp_matches_' + str(i) + '.csv'
	output_path = 'reordered_atp_matches' + str(i) + '.csv'

	inputs_temp = create_add_year_matches(input_path, output_path)
	inputs = np.concatenate((inputs, inputs_temp))

print(inputs.shape)

print(inputs[1])

num_rows, num_cols = inputs.shape
print(num_rows, num_cols)

print(inputs[0])

# Sanity checking tourney_id

ct = 0

for i in range(num_rows):
	if(inputs[i][0] == ""):
		ct = ct + 1

print(ct)

#Sanity checking tourney_name

ct = 0

for i in range(num_rows):
	if(inputs[i][1] == ""):
		ct = ct + 1

print(ct)

# Sanity checking surface and creating dict of surfaces

ct = 0
tc = 0
none_surfaces_rows = list()

for i in range(num_rows):
	if(inputs[i][2] == ""):
		ct = ct + 1
		none_surfaces_rows.append(i)

	if(inputs[i][2] == "None"):
		tc = tc + 1
		none_surfaces_rows.append(i)

inputs = np.delete(inputs, none_surfaces_rows, 0)

print(ct)
print(tc)

ct = 0
surfaces = dict()
size = 0

num_rows, num_cols = inputs.shape

for i in range(num_rows):
	if(inputs[i][2] == ""):
		ct = ct + 1
	else:
		if inputs[i][2] not in surfaces:
			surfaces[inputs[i][2]] = size
			size = size + 1

print(ct)
print(surfaces)

# Sanity checking draw_size

ct = 0
tc = 0
draw_sizes = dict()
size = 0

for i in range(num_rows):
	if(inputs[i][3] == ""):
		ct = ct + 1

	else:
		if inputs[i][3] not in draw_sizes:
			draw_sizes[inputs[i][3]] = size
			size = size + 1

print(draw_sizes)

print(ct)

# Drop tourey_name column
#inputs = np.delete(inputs, 1, axis=1)

# Drop draw_size column
#inputs = np.delete(inputs, 3, axis=1)

# Drop tourey_level column
#inputs = np.delete(inputs, 4, axis=1)

# Drop match_num column
#inputs = np.delete(inputs, 6, axis=1)

# Drop winner_name column
#inputs = np.delete(inputs, 10, axis=1)

# Drop loser_name column
#inputs = np.delete(inputs, 20, axis=1)

# Drop winner_ioc column - country of winner
#inputs = np.delete(inputs, 13, axis=1)

# Drop loser_ioc column - country of loser
#inputs = np.delete(inputs, 23, axis=1)

# Drop score
#inputs = np.delete(inputs, 27, axis=1)

# Sanity checking and sanitizing month from time of match

ct = 0

for i in range(num_rows):
	if(inputs[i][5] == ""):
		ct = ct + 1

print(ct)

# Sanity checking winner_id

ct = 0
federer_win = 0

for i in range(num_rows):
	if(inputs[i][7] == ""):
		ct = ct + 1

	if(inputs[i][17] == ""):
		ct = ct + 1

	if(inputs[i][7] == "103819"):
		federer_win = federer_win + 1

print(ct) # It comes out to be 0\
print(federer_win)

# Sanitizing winner_seed

ct = 0
federer_win = 0

for i in range(num_rows):
	if(inputs[i][8] == ""):
		ct = ct + 1
		inputs[i][8] = '40'

	if(inputs[i][18] == ""):
		ct = ct + 1
		inputs[i][18] = '40'

	if(not inputs[i][8].isdigit()):
		ct = ct + 1
		inputs[i][8] = '40'

	if(not inputs[i][18].isdigit()):
		ct = ct + 1
		inputs[i][18] = '40'

print(ct)

# Sanitizing winner_entry

ct = 0
federer_win = 0
entries = dict()
entry_type = 0

for i in range(num_rows):
	if inputs[i][9].lower() not in entries:
		entries[inputs[i][9].lower()] = entry_type
		entry_type = entry_type + 1

	if inputs[i][19].lower() not in entries:
		entries[inputs[i][19].lower()] = entry_type
		entry_type = entry_type + 1

	inputs[i][9] = entries[inputs[i][9].lower()]
	inputs[i][19] = entries[inputs[i][19].lower()]

print(entries)

# Sanitizing winner_hand - can only be left or right handed

for i in range(num_rows):
	if inputs[i][11].lower() == 'l':
		inputs[i][11] = '0'
	else:
		inputs[i][11] = '1'

	if inputs[i][21].lower() == 'l':
		inputs[i][21] = '0'
	else:
		inputs[i][21] = '1'

# Sanitizing winner_ht
# Making all winners without a height as height = -1

ct = 0

for i in range(num_rows):
	if inputs[i][12].lower() == "":
		ct = ct + 1
		inputs[i][12] = '-1'

	if inputs[i][22].lower() == "":
		ct = ct + 1
		inputs[i][22] = '-1'

# Too many players without a height - so just going to take difference in height as feature
# and make it 0 when input doesn't have it
print(ct)

# Sanitizing winner_age

ct = 0
total = 0.0
total_matches = 0

for i in range(num_rows):
	if inputs[i][14].lower() == "":
		ct = ct + 1
		inputs[i][14] = "26.08"
	else:
		total = total + float(inputs[i][14])
		total_matches = total_matches + 1

	if inputs[i][24].lower() == "":
		ct = ct + 1
		inputs[i][24] = "26.08"
	else:
		total = total + float(inputs[i][24])
		total_matches = total_matches + 1

# Seems like 26 is the average age of the winner
print(total/total_matches)

# Seems like 18 winners overall don't have an age - default to 26
print(ct)

# Sanitizing winner_rank and loser_rank - if no rank then replacing with 2000 which refers to a very high rank which
# represents the last rank

ct = 0

for i in range(num_rows):
	if(inputs[i][15] == ""):
			inputs[i][15] = 2000
			rank_1 = 2000
	else:
			rank_1 = int(inputs[i][15])

	if(inputs[i][25] == ""):
			inputs[i][25] = 2000
			rank_2 = 2000
	else:
			rank_2 = int(inputs[i][25])

	if(rank_1 < rank_2):
		ct = ct + 1

print(ct*100/num_rows)

benchmark_higher_rankings = ct*100/num_rows

# Sanitizing winner_rank points - if rankings points is empty replacing with 0

ct = 0

for i in range(num_rows):
	if(inputs[i][16] == ""):
		ct = ct + 1
		inputs[i][16] = 0

	if(inputs[i][26] == ""):
		ct = ct + 1
		inputs[i][26] = 0

print(ct)

# Sanity checking score

ct1 = 0
ct2 = 0
walkover_matches = list()

for i in range(num_rows):
	if(inputs[i][27] == ""):
		ct1 = ct1 + 1
		walkover_matches.append(i)
	if(inputs[i][27].lower() == "w/o"):
		ct2 = ct2 + 1
		walkover_matches.append(i)

print(ct1)
print(ct2)

inputs = np.delete(inputs, walkover_matches, 0)

num_rows, num_cols = inputs.shape

# Sanity checking best_of

ct = 0

for i in range(num_rows):
	if(inputs[i][28] == ""):
		ct = ct + 1

print(ct)

# Sanity checking and converting rounds to numbers

ct = 0
rounds = {'Q1' : -3, 'Q2' : -2, 'Q3': -3, 'R128': 0, 'RR': 0, 'BR': 0, 'R64': 1, 'R32': 2, 'R16': 3, 'QF': 4, 'SF': 5, 'F': 6}
type_rounds = 0

for i in range(num_rows):
	if(inputs[i][29] == ""):
		ct = ct + 1
	else:
		inputs[i][29] = rounds[inputs[i][29].upper()]

print(num_rows)
print(rounds)
print(ct)

# Sanity checking minutes

ct = 0
no_minutes = list()

for i in range(num_rows):
	if(inputs[i][30] == ""):
		ct = ct + 1
		no_minutes.append(i)

# Lots of rows without minutes
print(ct)

inputs = np.delete(inputs, no_minutes, 0)

num_rows, num_cols = inputs.shape

# Sanity checking all numerical features of the match - double faults, aces, first serve, etc.
#and deleting all rows that have any missing

ct = 0
missing_values = list()

for i in range(num_rows):
	for j in range(31, 49):
		if(inputs[i][j] == ""):
			ct = ct + 1
			missing_values.append(i)

print(ct)

inputs = np.delete(inputs, missing_values, 0)

num_rows, num_cols = inputs.shape

print(num_rows) #63770 rows in total

career_stats_winner = np.zeros((num_rows, 49))
career_stats_loser = np.zeros((num_rows, 49))
career_stats_winner_total = np.zeros((num_rows, 49))
career_stats_loser_total = np.zeros((num_rows, 49))

x = 0

player_id_stats_overall_sum = [dict() for x in range(num_cols)]
player_id_stats_overall_count = [dict() for x in range(num_cols)]
player_name = dict()

delete_list = []

count_2019 = 0

for i in range(num_rows):
	if inputs[i][5][0] == '2' and inputs[i][5][1] == '0' and inputs[i][5][2] == '1' and inputs[i][5][3] == '9' and getmonth(inputs[i][5]) > 6:
		count_2019+=1

# Delete davis cup matches in prediction - they are generally extremely hard to predict
# If Davis Cup matches needs to be predicted the most likely best way to do is to have a different model for davis cup
for i in range(num_rows - count_2019, num_rows):
	if "davis" in inputs[i][1].lower():
		delete_list.append(i)

inputs = np.delete(inputs, delete_list, 0)

num_rows, num_cols = inputs.shape

count_2019 = 0

for i in range(num_rows):
	if inputs[i][5][0] == '2' and inputs[i][5][1] == '0' and inputs[i][5][2] == '1' and inputs[i][5][3] == '9' and getmonth(inputs[i][5]) > 6:
		count_2019+=1

X_inputs = np.zeros((2*(num_rows - count_2019), 51))
Y_inputs = np.zeros(2*(num_rows - count_2019))

X_prediction = np.zeros((2*count_2019, 51))
Y_prediction = np.zeros(2*count_2019)

print(count_2019)

matches_won_lost = dict()

head_to_head = dict()
head_to_head_surface = dict()
matches_won_lost_surface = dict()

rank_count = dict()
rank_total = dict()
rankings_points_total = dict()

form = dict()
form_surface = dict()

tournament_form_win = dict()
tournament_form_count = dict()

common_head_to_head = dict()
total_no_head_to_head = 0

for i in range(num_rows):
	player_id_winner = inputs[i][7]
	player_id_loser = inputs[i][17]

	# Start of Tournament level form

	if (player_id_winner, inputs[i][0][5:]) not in tournament_form_win:
		tournament_form_win[(player_id_winner, inputs[i][0][5:])] = 0
		tournament_form_count[(player_id_winner, inputs[i][0][5:])] = 0

	if (player_id_loser, inputs[i][0][5:]) not in tournament_form_win:
		tournament_form_win[(player_id_loser, inputs[i][0][5:])] = 0
		tournament_form_count[(player_id_loser, inputs[i][0][5:])]  = 0

	tournament_form_win[(player_id_winner, inputs[i][0][5:])] += 1
	tournament_form_win[(player_id_loser, inputs[i][0][5:])] += 0

	tournament_form_count[(player_id_winner, inputs[i][0][5:])] += 1
	tournament_form_count[(player_id_loser, inputs[i][0][5:])] += 1

	if i < num_rows - count_2019:
		X_inputs[2*i][44] = tournament_form_win[(player_id_winner, inputs[i][0][5:])] - 1 - tournament_form_win[(player_id_loser, inputs[i][0][5:])]
		X_inputs[2*i][45] = tournament_form_count[(player_id_winner, inputs[i][0][5:])] - tournament_form_count[(player_id_loser, inputs[i][0][5:])]

		temp11 = 1
		temp12 = 1

		if tournament_form_count[(player_id_winner, inputs[i][0][5:])] != 1:
			temp11 = tournament_form_count[(player_id_winner, inputs[i][0][5:])]
		if tournament_form_count[(player_id_loser, inputs[i][0][5:])]  != 1:
			temp12 = tournament_form_count[(player_id_loser, inputs[i][0][5:])]

		X_inputs[2*i][46] = ((tournament_form_win[(player_id_winner, inputs[i][0][5:])] - 1)*100/temp11) - ((tournament_form_win[(player_id_loser, inputs[i][0][5:])])*100/temp12)

		X_inputs[2*i+1][44] =  -X_inputs[2*i][44]
		X_inputs[2*i+1][45] = -X_inputs[2*i][45]
		X_inputs[2*i+1][46] = -X_inputs[2*i][46]
	else:
		x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))

		X_prediction[x1][44] = tournament_form_win[(player_id_winner, inputs[i][0][5:])] - 1 - tournament_form_win[(player_id_loser, inputs[i][0][5:])]
		X_prediction[x1][45] = tournament_form_count[(player_id_winner, inputs[i][0][5:])] - tournament_form_count[(player_id_loser, inputs[i][0][5:])]

		temp11 = 1
		temp12 = 1

		if tournament_form_count[(player_id_winner, inputs[i][0][5:])] != 1:
			temp11 = tournament_form_count[(player_id_winner, inputs[i][0][5:])]
		if tournament_form_count[(player_id_loser, inputs[i][0][5:])]  != 1:
			temp12 = tournament_form_count[(player_id_loser, inputs[i][0][5:])]

		X_prediction[x1][46] = ((tournament_form_win[(player_id_winner, inputs[i][0][5:])] - 1)*100/temp11) - ((tournament_form_win[(player_id_loser, inputs[i][0][5:])])*100/temp12)

		X_prediction[x1+1][44] =  -X_prediction[x1][44]
		X_prediction[x1+1][45] = -X_prediction[x1][45]
		X_prediction[x1+1][46] = -X_prediction[x1][46]

	# End of Tournament level form

	# Start of overall form

	if player_id_winner not in form:
		form[player_id_winner] = []

	form[player_id_winner].append((1, inputs[i][5]))

	if player_id_loser not in form:
		form[player_id_loser] = []

	form[player_id_loser].append((0, inputs[i][5]))

	total_winner_5 = -1
	total_winner_10 = -1
	total_winner_15 = -1
	total_winner_25 = -1

	total_loser_5 = -1
	total_loser_10 = -1
	total_loser_15 = -1
	total_loser_25 = -1

	winner_win_5 = 0
	winner_win_10 = 0
	winner_win_15 = 0
	winner_win_25 = 0

	loser_win_5 = 0
	loser_win_10 = 0
	loser_win_15 = 0
	loser_win_25 = 0

	for (a1, a2) in reversed(form[player_id_winner]):
		if total_winner_5 == -1:
			total_winner_5 = 0
		else:
			if total_winner_5 < 5:
				winner_win_5 = winner_win_5 + a1
				total_winner_5 += 1

	for (a1, a2) in reversed(form[player_id_winner]):
		if total_winner_10 == -1:
			total_winner_10 = 0
		else:
			if total_winner_10 < 10:
				winner_win_10 = winner_win_10 + a1
				total_winner_10 += 1

	for (a1, a2) in reversed(form[player_id_winner]):
		if total_winner_15 == -1:
			total_winner_15 = 0
		else:
			if total_winner_15 < 15:
				winner_win_15 = winner_win_15 + a1
				total_winner_15 += 1

	for (a1, a2) in reversed(form[player_id_winner]):
		if total_winner_25 == -1:
			total_winner_25 = 0
		else:
			if total_winner_25 < 25:
				winner_win_25 = winner_win_25 + a1
				total_winner_25 += 1

	for (a1, a2) in reversed(form[player_id_loser]):
		if total_loser_5 == -1:
			total_loser_5 = 0
		else:
			if total_loser_5 < 5:
				loser_win_5 = loser_win_5 + a1
				total_loser_5 += 1

	for (a1, a2) in reversed(form[player_id_loser]):
		if total_loser_10 == -1:
			total_loser_10 = 0
		else:
			if total_loser_10 < 10:
				loser_win_10 = loser_win_10 + a1
				total_loser_10 += 1

	for (a1, a2) in reversed(form[player_id_loser]):
		if total_loser_15 == -1:
			total_loser_15 = 0
		else:
			if total_loser_15 < 15:
				loser_win_15 = loser_win_15 + a1
				total_loser_15 += 1

	for (a1, a2) in reversed(form[player_id_loser]):
		if total_loser_25 == -1:
			total_loser_25 = 0
		else:
			if total_loser_25 < 25:
				loser_win_25 = loser_win_25 + a1
				total_loser_25 += 1

	if i < num_rows - count_2019:
		X_inputs[2*i][36] = winner_win_5 - loser_win_5
		X_inputs[2*i][37] = winner_win_10 - loser_win_10
		X_inputs[2*i][38] = winner_win_15 - loser_win_15
		X_inputs[2*i][39] = winner_win_25 - loser_win_25

		X_inputs[2*i+1][36] = loser_win_5 - winner_win_5
		X_inputs[2*i+1][37] = loser_win_10 - winner_win_10
		X_inputs[2*i+1][38] = loser_win_15 - winner_win_15
		X_inputs[2*i+1][39] = loser_win_25 - winner_win_25

	else:
		x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))

		X_prediction[x1][36] = winner_win_5 - loser_win_5
		X_prediction[x1][37] = winner_win_10 - loser_win_10
		X_prediction[x1][38] = winner_win_15 - loser_win_15
		X_prediction[x1][39] = winner_win_25 - loser_win_25

		X_prediction[x1+1][36] = loser_win_5 - winner_win_5
		X_prediction[x1+1][37] = loser_win_10 - winner_win_10
		X_prediction[x1+1][38] = loser_win_15 - winner_win_15
		X_prediction[x1+1][39] = loser_win_25 - winner_win_25

	# End of overall form

	# Start of last 1 month form overall

	total_winner_1 = -1
	total_loser_1 = -1
	winner_win_1 = 0
	loser_win_1 = 0

	for (a1, a2) in reversed(form[player_id_winner]):
		if total_winner_1 == -1:
			total_winner_1 = 0
		else:
			if getdays(inputs[i][5]) - getdays(a2) < 30 and getdays(inputs[i][5]) - getdays(a2) >= 0:
				winner_win_1 = winner_win_1 + a1
				total_winner_1 += 1
				#print(getdays(inputs[i][5]) - getdays(a2), a2, inputs[i][5], i, 1, total_winner_1)
			else:
				break

	for (a1, a2) in reversed(form[player_id_loser]):
		if total_loser_1 == -1:
			total_loser_1 = 0
		else:
			if getdays(inputs[i][5]) - getdays(a2) < 30 and getdays(inputs[i][5]) - getdays(a2) >= 0:
				loser_win_1 = loser_win_1 + a1
				total_loser_1 += 1
				#print(getdays(inputs[i][5]) - getdays(a2), a2, inputs[i][5], i, 5, total_loser_1)
			else:
				break

	if i < num_rows - count_2019:
		X_inputs[2*i][47] = winner_win_1 - loser_win_1
		X_inputs[2*i+1][47] = loser_win_1 - winner_win_1
		X_inputs[2*i][48] = total_winner_1 - total_loser_1
		X_inputs[2*i+1][48] = total_loser_1 - total_winner_1

	else:
		x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))

		X_prediction[x1][47] = winner_win_1 - loser_win_1
		X_prediction[x1+1][47] = loser_win_1 - winner_win_1
		X_prediction[x1][48] = total_winner_1 - total_loser_1
		X_prediction[x1+1][48] = total_loser_1 - total_winner_1

	# End of last 1 month form overall
	# I tried adding 3, 6 and 12 month forms as well - it didn't seem to help - in fact made the prediction worse

	# Start of surface level form

	if player_id_winner not in form_surface:
		form_surface[(player_id_winner, inputs[i][2])] = []

	form_surface[(player_id_winner, inputs[i][2])].append((1, inputs[i][5]))

	if player_id_loser not in form_surface:
		form_surface[(player_id_loser, inputs[i][2])] = []

	form_surface[(player_id_loser, inputs[i][2])].append((0, inputs[i][5]))

	total_winner_5 = -1
	total_winner_10 = -1
	total_winner_15 = -1
	total_winner_25 = -1

	total_loser_5 = -1
	total_loser_10 = -1
	total_loser_15 = -1
	total_loser_25 = -1

	winner_win_5 = 0
	winner_win_10 = 0
	winner_win_15 = 0
	winner_win_25 = 0

	loser_win_5 = 0
	loser_win_10 = 0
	loser_win_15 = 0
	loser_win_25 = 0

	for (a1, a2) in reversed(form_surface[(player_id_winner, inputs[i][2])]):
		if total_winner_5 == -1:
			total_winner_5 = 0
		else:
			if total_winner_5 < 5:
				winner_win_5 = winner_win_5 + a1
				total_winner_5 += 1

	for (a1, a2) in reversed(form_surface[(player_id_winner, inputs[i][2])]):
		if total_winner_10 == -1:
			total_winner_10 = 0
		else:
			if total_winner_10 < 10:
				winner_win_10 = winner_win_10 + a1
				total_winner_10 += 1

	for (a1, a2) in reversed(form_surface[(player_id_winner, inputs[i][2])]):
		if total_winner_15 == -1:
			total_winner_15 = 0
		else:
			if total_winner_15 < 15:
				winner_win_15 = winner_win_15 + a1
				total_winner_15 += 1

	for (a1, a2) in reversed(form_surface[(player_id_winner, inputs[i][2])]):
		if total_winner_25 == -1:
			total_winner_25 = 0
		else:
			if total_winner_25 < 25:
				winner_win_25 = winner_win_25 + a1
				total_winner_25 += 1

	for (a1, a2) in reversed(form_surface[(player_id_loser, inputs[i][2])]):
		if total_loser_5 == -1:
			total_loser_5 = 0
		else:
			if total_loser_5 < 5:
				loser_win_5 = loser_win_5 + a1
				total_loser_5 += 1

	for (a1, a2) in reversed(form_surface[(player_id_loser, inputs[i][2])]):
		if total_loser_10 == -1:
			total_loser_10 = 0
		else:
			if total_loser_10 < 10:
				loser_win_10 = loser_win_10 + a1
				total_loser_10 += 1

	for (a1, a2) in reversed(form_surface[(player_id_loser, inputs[i][2])]):
		if total_loser_15 == -1:
			total_loser_15 = 0
		else:
			if total_loser_15 < 15:
				loser_win_15 = loser_win_15 + a1
				total_loser_15 += 1

	for (a1, a2) in reversed(form_surface[(player_id_loser, inputs[i][2])]):
		if total_loser_25 == -1:
			total_loser_25 = 0
		else:
			if total_loser_25 < 25:
				loser_win_25 = loser_win_25 + a1
				total_loser_25 += 1

	if i < num_rows - count_2019:
		X_inputs[2*i][40] = winner_win_5 - loser_win_5
		X_inputs[2*i][41] = winner_win_10 - loser_win_10
		X_inputs[2*i][42] = winner_win_15 - loser_win_15
		X_inputs[2*i][43] = winner_win_25 - loser_win_25

		X_inputs[2*i+1][40] = loser_win_5 - winner_win_5
		X_inputs[2*i+1][41] = loser_win_10 - winner_win_10
		X_inputs[2*i+1][42] = loser_win_15 - winner_win_15
		X_inputs[2*i+1][43] = loser_win_25 - winner_win_25

	else:
		x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))

		X_prediction[x1][40] = winner_win_5 - loser_win_5
		X_prediction[x1][41] = winner_win_10 - loser_win_10
		X_prediction[x1][42] = winner_win_15 - loser_win_15
		X_prediction[x1][43] = winner_win_25 - loser_win_25

		X_prediction[x1+1][40] = loser_win_5 - winner_win_5
		X_prediction[x1+1][41] = loser_win_10 - winner_win_10
		X_prediction[x1+1][42] = loser_win_15 - winner_win_15
		X_prediction[x1+1][43] = loser_win_25 - winner_win_25

	# End of surface level form

	# Start of Overall win loss

	p1, p2 = (0, 0)
	b1, b2 = (0, 0)

	if player_id_winner not in matches_won_lost:
		matches_won_lost[player_id_winner] = (1, 0)
		(p1, p2) = (0, 0)
	else:
		(a1, a2) = matches_won_lost[player_id_winner]
		matches_won_lost[player_id_winner] = (a1 + 1, a2)
		(p1, p2) = (a1, a2)

	if player_id_loser not in matches_won_lost:
		matches_won_lost[player_id_loser] = (0, 1)
		(b1, b2) = (0, 0)
	else:
		(a1, a2) = matches_won_lost[player_id_loser]
		matches_won_lost[player_id_loser] = (a1, a2 + 1)
		(b1, b2) = (a1, a2)

	if((p1 + p2) != 0):
		temp1 = (p1*100/(p1+p2))
	else:
		temp1 = 0

	if((b1 + b2) != 0):
		temp2 = (b1*100/(b1+b2))
	else:
		temp2 = 0

	if i < num_rows - count_2019:
		X_inputs[2*i][25] = p1 - b1
		X_inputs[2*i+1][25] = b1 - p1

		X_inputs[2*i][26] =  temp1 - temp2
		X_inputs[2*i+1][26] = temp2 - temp1

	else:
		x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))
		X_prediction[x1][25] = p1 - b1
		X_prediction[x1+1][25] = b1 - p1

		X_prediction[x1][26] = temp1 - temp2
		X_prediction[x1+1][26] = temp2 - temp1

# End of Overall win loss

# Start of section for win loss based on surface

	p1, p2 = (0, 0)
	b1, b2 = (0, 0)

	if (player_id_winner, inputs[i][2]) not in matches_won_lost_surface:
		matches_won_lost_surface[(player_id_winner, inputs[i][2])] = (1, 0)
		(p1, p2) = (0, 0)
	else:
		(a1, a2) = matches_won_lost_surface[(player_id_winner, inputs[i][2])]
		matches_won_lost_surface[(player_id_winner, inputs[i][2])] = (a1 + 1, a2)
		(p1, p2) = (a1, a2)

	if (player_id_loser, inputs[i][2]) not in matches_won_lost_surface:
		matches_won_lost_surface[(player_id_loser, inputs[i][2])] = (0, 1)
		(b1, b2) = (0, 0)
	else:
		(a1, a2) = matches_won_lost_surface[(player_id_loser, inputs[i][2])]
		matches_won_lost_surface[(player_id_loser, inputs[i][2])] = (a1, a2 + 1)
		(b1, b2) = (a1, a2)

	if((p1 + p2) != 0):
		temp1 = (p1*100/(p1+p2))
	else:
		temp1 = 0

	if((b1 + b2) != 0):
		temp2 = (b1*100/(b1+b2))
	else:
		temp2 = 0

	if i < num_rows - count_2019:
		X_inputs[2*i][33] = p1 - b1
		X_inputs[2*i+1][33] = b1 - p1

		X_inputs[2*i][34] =  temp1 - temp2
		X_inputs[2*i+1][34] = temp2 - temp1

	else:
		x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))
		X_prediction[x1][33] = p1 - b1
		X_prediction[x1+1][33] = b1 - p1

		X_prediction[x1][34] = temp1 - temp2
		X_prediction[x1+1][34] = temp2 - temp1

# End of section for win loss based on surface

# Start of Overall Head to Head

	if (player_id_winner, player_id_loser) not in head_to_head:
		head_to_head[(player_id_winner, player_id_loser)] = (1, 0)
		head_to_head[(player_id_loser, player_id_winner)] = (0, 1)
		if i < num_rows - count_2019:
			X_inputs[2*i][15] = 1
			X_inputs[2*i+1][15] = -1
		else:
			x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))
			total_no_head_to_head = total_no_head_to_head + 1
			print(inputs[i][10], inputs[i][20])
			X_prediction[x1][15] = 1
			X_prediction[x1+1][15] = -1

	else:
		(a1, a2) = head_to_head[(player_id_winner, player_id_loser)]

		if i < num_rows - count_2019:
			X_inputs[2*i][15] = a1 - a2
			X_inputs[2*i+1][15] = a2 - a1
		else:
			x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))
			X_prediction[x1][15] = a1 - a2
			X_prediction[x1+1][15] = a2 - a1

		head_to_head[(player_id_winner, player_id_loser)] = (a1 + 1, a2)
		head_to_head[(player_id_loser, player_id_winner)] = (a2, a1 + 1)

# End of Overall Head to Head

# Start of Common Opponent Head to Head

	if player_id_winner not in common_head_to_head:
		common_head_to_head[player_id_winner] =  {player_id_loser: (0, 0)}

	if player_id_loser not in common_head_to_head:
		common_head_to_head[player_id_loser] = {player_id_winner: (0, 0)}

	if player_id_loser not in common_head_to_head[player_id_winner]:
		common_head_to_head[player_id_winner][player_id_loser] = (0, 0)

	if player_id_winner not in common_head_to_head[player_id_loser]:
		common_head_to_head[player_id_loser][player_id_winner] = (0, 0)

	(x11, y11) = common_head_to_head[player_id_winner][player_id_loser]
	(x22, y22) = common_head_to_head[player_id_loser][player_id_winner]
	common_head_to_head[player_id_winner][player_id_loser] = (x11+1, y11)
	common_head_to_head[player_id_loser][player_id_winner] = (x22, y22+1)

	new_head_to_head_winner = (0, 0)
	new_head_to_head_loser = (0, 0)

	for player_id in common_head_to_head[player_id_winner]:
		if player_id in common_head_to_head[player_id_loser]:
			(temp1, temp2) = common_head_to_head[player_id_winner][player_id]
			t1, t2 = new_head_to_head_winner
			new_head_to_head_winner = t1 + temp1, t2 + temp2

	for player_id in common_head_to_head[player_id_loser]:
		if player_id in common_head_to_head[player_id_winner]:
			(temp1, temp2) = common_head_to_head[player_id_loser][player_id]
			t1, t2 = new_head_to_head_loser
			new_head_to_head_loser = t1 + temp1, t2 + temp2

	temp1, temp2 = new_head_to_head_winner
	temp3, temp4 = new_head_to_head_loser

	temp5 = 0
	temp6 = 0

	if(temp1 + temp2) != 0:
		temp5 = ((temp1*100)/(temp1+temp2))

	if(temp3 + temp4) != 0:
		temp6 = ((temp3*100)/(temp3+temp4))

	if i < num_rows - count_2019:
		X_inputs[2*i][49] = temp1 - temp3
		X_inputs[2*i][49] =  temp5 - temp6
		X_inputs[2*i+1][50] = temp3 - temp1
		X_inputs[2*i+1][50] = temp6 - temp5

	else:
		x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))
		X_prediction[x1][49] = temp1 - temp3
		X_prediction[x1][49] = temp5 - temp6
		X_prediction[x1+1][50] = temp3 - temp1
		X_prediction[x1+1][50] = temp6 - temp5

# End of Common Opponent Head to Head

# Start of surface level head to head

	if (player_id_winner, player_id_loser, inputs[i][2]) not in head_to_head_surface:
		if i < num_rows - count_2019:
			X_inputs[2*i][35] = 0
			X_inputs[2*i+1][35] = 0
		else:
			x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))
			X_prediction[x1][35] = 0
			X_prediction[x1+1][35] = 0

		head_to_head_surface[(player_id_winner, player_id_loser, inputs[i][2])] = (1, 0)
		head_to_head_surface[(player_id_loser, player_id_winner, inputs[i][2])] = (0, 1)
	else:
		(a1, a2) = head_to_head_surface[(player_id_winner, player_id_loser, inputs[i][2])]

		if i < num_rows - count_2019:
			X_inputs[2*i][35] = a1 - a2
			X_inputs[2*i+1][35] = a2 - a1
		else:
			x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))
			X_prediction[x1][35] = a1 - a2
			X_prediction[x1+1][35] = a2 - a1

		head_to_head_surface[(player_id_winner, player_id_loser, inputs[i][2])] = (a1 + 1, a2)
		head_to_head_surface[(player_id_loser, player_id_winner, inputs[i][2])] = (a2, a1 + 1)

# End of surface level head to head

	for j in range(31, 40):
		if player_id_winner not in player_id_stats_overall_count[j-31]:

			career_stats_winner[i][j] = 0
			career_stats_winner_total[i][j] = 0

			player_id_stats_overall_count[j-31][player_id_winner] = 1
			player_id_stats_overall_sum[j-31][player_id_winner] = int(inputs[i][j])
			player_name[player_id_winner] = inputs[i][10]
		else:
			career_stats_winner[i][j] = player_id_stats_overall_sum[j-31][player_id_winner]/player_id_stats_overall_count[j-31][player_id_winner]
			career_stats_winner_total[i][j] = player_id_stats_overall_sum[j-31][player_id_winner]

			player_id_stats_overall_count[j-31][player_id_winner] = player_id_stats_overall_count[j-31][player_id_winner] + 1
			player_id_stats_overall_sum[j-31][player_id_winner] = player_id_stats_overall_sum[j-31][player_id_winner] + int(inputs[i][j])
			player_name[player_id_winner] = inputs[i][10]

	for j in range(40, 49):
		if player_id_loser not in player_id_stats_overall_count[j-40]:

			career_stats_loser[i][j] = 0
			career_stats_loser_total[i][j] = 0

			player_id_stats_overall_count[j-40][player_id_loser] = 1
			player_id_stats_overall_sum[j-40][player_id_loser] = int(inputs[i][j])
			player_name[player_id_loser] = inputs[i][20]
		else:
			career_stats_loser[i][j] = player_id_stats_overall_sum[j-40][player_id_loser]/player_id_stats_overall_count[j-40][player_id_loser]
			career_stats_loser_total[i][j] = player_id_stats_overall_sum[j-40][player_id_loser]

			player_id_stats_overall_count[j-40][player_id_loser] = player_id_stats_overall_count[j-40][player_id_loser] + 1
			player_id_stats_overall_sum[j-40][player_id_loser] = player_id_stats_overall_sum[j-40][player_id_loser] + int(inputs[i][j])
			player_name[player_id_loser] = inputs[i][20]

		if i < num_rows - count_2019:
			X_inputs[2*i][j-40] = career_stats_winner[i][j-9] - career_stats_loser[i][j]
			X_inputs[2*i+1][j-40] = career_stats_loser[i][j] - career_stats_winner[i][j-9]
			X_inputs[2*i][j-24] = career_stats_winner_total[i][j-9] - career_stats_loser_total[i][j]
			X_inputs[2*i+1][j-24] = career_stats_loser_total[i][j] - career_stats_winner_total[i][j-9]

		else:
			x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))
			X_prediction[x1][j-40] = career_stats_winner[i][j-9] - career_stats_loser[i][j]
			X_prediction[x1+1][j-40] = career_stats_loser[i][j] - career_stats_winner[i][j-9]
			X_prediction[x1][j-24] = career_stats_winner_total[i][j-9] - career_stats_loser_total[i][j]
			X_prediction[x1+1][j-24] = career_stats_loser_total[i][j] - career_stats_winner_total[i][j-9]

	#15, 16, 25, 26

	if player_id_winner not in rank_count:
		rank_count[player_id_winner] = 1
		rank_total[player_id_winner] = inputs[i][15]
		rankings_points_total[player_id_winner] = inputs[i][16]
	else:
		rank_count[player_id_winner] += 1
		rank_total[player_id_winner] += inputs[i][15]
		rankings_points_total[player_id_winner] += inputs[i][16]

	if player_id_loser not in rank_count:
		rank_count[player_id_loser] = 1
		rank_total[player_id_loser] = inputs[i][25]
		rankings_points_total[player_id_loser] = inputs[i][26]
	else:
		rank_count[player_id_loser] += 1
		rank_total[player_id_loser] += inputs[i][25]
		rankings_points_total[player_id_loser] += inputs[i][26]

	# 26, 25, 24, 22, 21, 18 - 16, 15, 14, 12, 11, 8
	k = 9

	for j in range(18, 27):
		if j != 19 and j != 20 and j != 23:
			if i < num_rows - count_2019:
				X_inputs[2*i][k] = float(inputs[i][j-10])
				X_inputs[2*i][k+18] = float(inputs[i][j])
				X_inputs[2*i+1][k] = float(inputs[i][j])
				X_inputs[2*i+1][k+18] = float(inputs[i][j-10])

				if j == 22 and (int(inputs[i][j-10]) == -1 or int(inputs[i][j]) == -1):
					X_inputs[2*i][k] = 0
					X_inputs[2*i][k+18] = 0
					X_inputs[2*i+1][k] = 0
					X_inputs[2*i+1][k+18] = 0
				k += 1
			else:
				x1 = int(2*i - 2*int(num_rows) + 2*int(count_2019))
				X_prediction[x1][k] = float(inputs[i][j-10])
				X_prediction[x1][k+18] = float(inputs[i][j])
				X_prediction[x1+1][k] = float(inputs[i][j])
				X_prediction[x1+1][k+18] = float(inputs[i][j-10])

				if j == 22 and (int(inputs[i][j-10]) == -1 or int(inputs[i][j]) == -1):
					X_prediction[x1][k] = 0
					X_prediction[x1][k+18] = 0
					X_prediction[x1+1][k] = 0
					X_prediction[x1+1][k+18] = 0

				k += 1

	if i < num_rows - count_2019:
		Y_inputs[2*i] = 1
		Y_inputs[2*i+1] = 0
	else:
		x1 = 2*i - 2*int(num_rows) + 2*int(count_2019)
		Y_prediction[x1] = 1
		Y_prediction[x1+1] = 0

clf = LogisticRegression(multi_class='ovr', random_state=0, solver='liblinear', penalty='l2').fit(X_inputs, Y_inputs)
training_prediction = clf.predict_proba(X_prediction)

np.savetxt("training_data.csv", X_inputs, delimiter=",")

total = count_2019
right = 0

print(training_prediction.shape)

for i in range(count_2019):
	(a, b) = training_prediction[2*i]
	(c, d) = training_prediction[2*i+1]

	if a < b and c > d:
		predicted = 1
	else:
		if a > b and c < d:
			predicted = 0
		else:
			if a + d < b + c:
				predicted = 1
			else:
				predicted = 0

	if Y_prediction[2*i] == predicted:
		right = right + 1

print(total)
print(right)
print(right*100/total)

training_prediction_1 = clf.predict(X_inputs)

total = 2*(num_rows - count_2019)
right = 0

for i in range(2*(num_rows - count_2019)):
	if Y_inputs[i] == training_prediction_1[i]:
		right = right + 1

print(total)
print(right)
print(right*100/total)

print(total_no_head_to_head)

# it was 59.5% with 9 features which were just stats,
# with ranking, seed, ranking points, basically values known pre-match which describe the player - improved to 62.5%
#62.654205607476634 - l1 penalty
# lbfgs - l2 - 62.57943925233645

#clf = svm.SVC(kernel='linear')
#clf.fit(X_inputs, Y_inputs)

#svm.SVC(kernel='linear')
#svm.SVC(kernel='rbf')
#svm.SVC(kernel=‘sigmoid’)
#svm.SVC(kernel=‘poly')
#logistic regression - penaltystr, ‘l1’, ‘l2’, ‘elasticnet’ or ‘none’, optional (default=’l2’)

#predictions_svm_svc = clf.predict(X_prediction)

#right = 0

#for i in range(2*count_2019):
#	if Y_prediction[i] == predictions_svm_svc[i]:
#		right = right + 1

#print(total)
#print(right)
#print(right*100/total)

# Sanity testing whether the total number of aces of Federer calculated the real aces - it matches!!
for player_id in player_id_stats_overall_count[0]:
	if "Federer" in player_name[player_id]:
		print(player_name[player_id], player_id_stats_overall_sum[0][player_id], player_id_stats_overall_count[0][player_id])


"""
Copyright 2021 RICHARD TJÖRNHAMMAR

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
#
# MOVED OVER FROM RANKOR (reducer.py) AS OF COMMIT :
# https://github.com/richardtjornhammar/rankor/commit/f04a83091a92130b9b73439b4ad65b5be3056cf9
#
import pandas as pd
import numpy as np
from scipy.stats import rankdata
from impetuous.quantification import qvalues, permuter
#
# REGULAR CONTRAST
contrast   = lambda A,B : ( A-B )/( A+B )
#
# OTHER
e_flatness = lambda x   : np.exp(np.mean(np.log(x),0))/np.mean(x,0)
e_contrast = lambda x   : 1 - e_flatness(x)
#
# SPURIOUS LOW VALUE REMOVAL
confred = lambda x,eta,varpi : 0.5*x*(1+np.tanh((x-eta)/varpi))*(np.sqrt(x*eta)/(0.5*(eta+x)))

def frac_procentile ( vals=[12.123, 1.2, 1000, 4] ):
    vals = np.array(vals).copy()
    N = len( vals );
    for i,j in zip( np.argsort(vals),range(N)):
        vals[i]=(j+0.5)/N
    return ( vals )

def get_procentile ( vals, procentile = 50 ):
    fp_   = procentile/100.0
    proc  = frac_procentile(vals)
    ma,mi = np.max(proc),np.min(proc)
    if fp_ > ma:
        fp_ = ma
    if fp_ < mi:
        fp_ = mi
    idx = np.argwhere(proc==fp_)
    if len ( idx ) == 1 :
        return ( vals[idx[0][0]] )
    else :
        i1 = np.argsort( np.abs(proc-fp_) )[:2]
        return ( sum([vals[i] for i in i1]) * 0.5 )

# EX.:
# eta    = get_procentile( values,50 )
# varpi  = get_procentile( values,66 ) - get_procentile( values,33 )
# confred(values,eta,varpi)

pi0 = lambda pvs : 1.

def padded_rolling_window ( tv, tau ) :
        # SIMPLY RETURN THE PADDED INDICES
	if tau==1 :
		return ( tv )
	if len(tv)<tau :
		return ( [ np.mean(v_) for v_ in tv ] )
	centered = lambda x:(x[0],x[1]) ; N=len(tv);
	w = int( np.floor(np.abs(tau)*0.5) ) ;
	jid = lambda i,w,N:[int((i-w)>0)*((i-w)%N),int(i+w<N)*((i+w)%N)+int(i+w>=N)*(N-1)]
	idx = [ centered( jid(i,w,N) ) for i in range(N) ]
	return ( idx )

def padded_rolling_average( tv , tau ) :
	# AS OF THE PANDAS VERSION ( 1.1.0 )
	# WINDOW CALCULATION WAS NOT PADDED SO
	# THIS IS A NUMPY ONLY IMPLEMENTATION
	if tau==1 :
		return ( tv )
	if len(tv)<tau :
		return ( [ np.mean(v_) for v_ in tv ] )
	centered = lambda x:(x[0],x[1]) ; N=len(tv);
	w = int(np.floor(np.abs(tau)*0.5)) ;
	jid = lambda i,w,N:[int((i-w)>0)*((i-w)%N),int(i+w<N)*((i+w)%N)+int(i+w>=N)*(N-1)]
	idx = [ centered( jid(i,w,N) ) for i in range(N) ]
	mvalues = [ np.mean(tv[i[0]:i[1]]) for i in idx ]
	return ( mvalues )

def svd_reduced_mean ( x,axis=0,keep=[0] ) :
    if True :
        sk = set ( keep )
        if len ( np.shape(x) ) > 1 :
            u , s , vt = np .linalg .svd( x , full_matrices=False )
            xred = np.mean( np.dot(u*[s[i_] if i_ in sk else 0 for i_ in range(len(s))],vt) , axis)
            if 'pandas' in str(type(x)) :
                if not 'series' in str(type(x)) :
                    xname = x.index.values[0]
                    return ( pd.DataFrame( [xred] , index=[xname] , columns=x.columns ) )
                else :
                    xname = x.name
                    return ( pd.Series( xred , name=xname , index=x.columns ) )
            else :
                return ( xred )
    return ( x )

from sklearn.decomposition import PCA
dimred = PCA ( n_components = 1 )

def pca_reduced_mean( x ) :
    dimred = PCA ( n_components = 1 )
    if True :
        if len ( np.shape(x) ) > 1 :
            Xnew = dimred.fit_transform( x.T )
            xred = Xnew . T [0] + np.mean(np.mean(x))
            if 'pandas' in str(type(x)) :
                if not 'series' in str(type(x)) :
                    xname = x.index.values[0]
                    return ( pd.DataFrame( [xred] , index=[xname] , columns=x.columns ) )
                else :
                    xname = x.name
                    return ( pd.Series( xred , name=xname , index=x.columns ) )
    return ( x )

def reduction ( a , power , centered=-1 ) :
    if centered>0 :
        a = ( a.T-np.mean(a,1) ).T
    return(  np.linalg.svd ( a**power , full_matrices=False ) )

def hyper_params ( df_ , label = 'generic' , sep = ',' , power=1., centered=-1 ):
    #
    idx_ = df_.index.values
    N_s = len ( df_.columns )
    u,s,vt = reduction( df_.values , power , centered=centered )
    rdf_ = pd.Series ( np.sum(u**2,1) , index=idx_ , name = label+sep+"u" )
    rdf_ = pd.concat ( [ pd.DataFrame(rdf_) ,
                         pd.DataFrame( pd.Series( np.mean( df_.values,1 ) ,
                                                  index = idx_ , name=label+sep+"m") ) ] , axis=1 )
    w_ = rdf_ .loc[ :,label+sep+"u" ].values
    r_ = rankdata ( [ w for w in w_ ] ,'average' )
    N = len ( r_ )
    #
    df0_ = pd.DataFrame( [ [a for a in range(N)],w_,r_ ],index=['idx','w_','r_'], columns=idx_ ).T
    #
    from scipy.special import erf as erf_
    loc_pval = lambda X , mean , stdev : [ 1. - 0.5*( 1. + erf_( ( x - mean )/stdev/np.sqrt(2.) ) ) for x in X ]
    lvs      = np.log( df0_.loc[ :,'w_'].values )
    #
    return ( np.mean(lvs) , np.std(lvs) )

def hyper_rdf ( df_ , label = 'generic' , sep = ',' , power=1. ,
                diagnostic_output = False , MEAN=None , STD=None , centered=-1 ) :
    #
    idx_= df_.index.values
    N_s = len ( df_.columns )
    u,s,vt = reduction ( df_.values , power , centered=centered )
    rdf_ = pd.Series ( np.sum( ( u )**2,1 ) , index=idx_ , name = label+sep+"u" )
    rdf_ = pd.concat ( [ pd.DataFrame( rdf_ ) ,
                         pd.DataFrame( pd.Series( np.mean( df_.values,1 ) ,
                                                  index = idx_ , name=label+sep+"m") ) ] , axis=1 )
    w_ = rdf_ .loc[ :,label+sep+"u" ].values
    r_ = rankdata ( [ w for w in w_] ,'average' )
    N  = len ( r_ )
    #
    # HERE WE CONSTRUCT A DIANGOSTICS AND INTERMITTENT CALCULATION
    # DATAFRAME FOR EASIER DEBUGGING AND INSPECTION
    df0_ = pd.DataFrame( [ [a for a in range(N)],w_,r_ ],index=['idx','w_','r_'], columns=idx_ )
    df0_ = df0_.T.sort_values ( by='r_' )
    df0_ .loc[ : , 'd_'  ] = [ v for v in ( df0_.loc[:, 'w_' ] * df0_.loc[:,'r_'] ) ]
    df0_ .loc[ : , 'da_' ] = np.cumsum ( df0_ .loc[ : , 'd_' ].values  )
    #
    # HOW MANY EQUALLY OR MORE EXTREME VALUES ARE THERE? ( RANK BASED )
    df0_ .loc[ : , 'dt_' ] = np.cumsum ( df0_ .loc[ : , 'd_' ].values[::-1] )[::-1]
    df0_ .loc[ : , 'rank_ps' ] = df0_ .loc[ :,'dt_' ] / np.sum( df0_ .loc[ :,'d_' ] )
    #
    # DRAW HISTOGRAM TO SEE DISTRIBUTION OF THE DISTANCES
    from scipy.special import erf as erf_
    loc_pval = lambda X , mean , stdev : [ 1. - 0.5*( 1. + erf_( ( x - mean )/stdev/np.sqrt(2.) ) ) for x in X ]
    lvs      = np.log( df0_.loc[ :,'w_'].values )
    if MEAN is None or STD is None:
        if len(lvs)>3 :
            ps = loc_pval( lvs , np.mean( lvs ) , np.std( lvs ) )
        else :
            ps = df0_ .loc[ : , 'rank_ps' ]
    else :
        ps = loc_pval( lvs , MEAN , STD )
    #
    if diagnostic_output :
        import scipy.stats       as scs
        NB    = 100
        lv    = np.log(  rdf_.loc[:,[label+sep+"u"]].values  )
        y , x = np.histogram( lv , bins=NB , density=True )
        skw = scs.skew(lv)[0]
        kur = scs.kurtosis(lv)[0]
        shape_stats = "kurtosis: " + "{:.2f} ".format( kur ) + "skewness: "+ "{:.2f}".format( skw )
        locd  = lambda x,M,s : (1./s/np.sqrt(2.*np.pi))*np.exp(-0.5*((x-M)/s)**2 )
        lin_x = 0.5 * ( x[1:] + x[:-1] )
        his_y = y
        rsy = sorted( [ (y_,x_) for (y_,x_) in zip(y,lin_x) ] )[::-1]
        hm , hs = np.mean([rsy[i][1] for i in range(5)]) , np.mean([rsy[i][0] for i in range(5)])
        h_mod_y = locd( lin_x , hm , 1.0/(hs*np.sqrt(2.*np.pi)) )
        d_mod_y = locd( lv , np.mean(lv) , np.std(lv) )
        rem_y = [ (y_-m_) for (y_,m_) in zip(y, locd(0.5*(x[1:]+x[:-1]),np.mean(lv),np.std(lv))) ]
        prc_y = [ 100.*np.abs( contrast(y_,m_) ) for (y_,m_) in zip(y, locd(0.5*(x[1:]+x[:-1]),np.mean(lv),np.std(lv))) ]
        RMSD  = np.sqrt(np.sum([ ry**2 for ry in rem_y ]))
        PMAE  = np.mean(prc_y)
    #
    df0_ .loc[ : , 'pvalues' ] = ps
    #
    # ASSIGN THE P VALUES
    rdf_.loc[df0_.index.values,label+sep+"p"] = df0_.loc[:,'pvalues']
    rdf_.loc[df0_.index.values,label+sep+"q"] = [ qvs[0] for qvs in qvalues( df0_.loc[:,'pvalues'].values , pi0 = pi0(df0_.loc[:,'pvalues'].values) ) ]
    rdf_.loc[df0_.index.values,label+sep+"r"] = df0_ .loc[ : , 'rank_ps' ]
    #
    # AND RETURN THE RESULTS
    if diagnostic_output :
        return ( rdf_ , RMSD , PMAE , kur , skw , rem_y )
    else :
        return ( rdf_ )

if __name__ == '__main__' :
    #
    print ( 'REDUCER :: TESTS ' )
    #
    a   = 2*np.random.rand(10)
    b   = 4*np.random.rand(10)
    X   = [ [*(a[:5]+1),*a[5:]],[*(b[:5]+3),*(b[5:])] ]
    Xdf = pd.DataFrame( X , columns=['a','b','s','c','d','e','f','g','h','i'] , index=['i0','i1'])
    #
    print ( Xdf )
    print ( svd_reduced_mean ( X ) )
    print ( svd_reduced_mean ( Xdf ) )
    v0 = [1.9,1.8,2.1,1.1,8.,1.2,2.2,3.5,2.0,2.0,3.1,2.1,2.9]
    a2 = np.array([[2,2,2,1,1,1,2,3,2,2,3,2,3],v0])
    NW = 100
    if False:
        for i in range(NW):
            for w in range(1,i-1):
                dat = np.random.rand(i)
                pra = padded_rolling_average( dat,tau=w )
                str_pra = ''.join([ str(p) for p in pra ])
                if 'nan' in str_pra:
                    print ( pra,len(pra),'[',i,w ,']')
                else:
                    if not len(pra)==len(dat):
                        print ( len(pra)==len(dat),'[',i,w ,']',len(pra),len(dat) )

    print ( padded_rolling_average( np.array( svd_reduced_mean( Xdf ).values ).reshape(-1,1) ,tau=4 ) )
    print ( padded_rolling_average( np.array( svd_reduced_mean( Xdf ).values ).reshape(1,-1) ,tau=4 ) )
    print ( padded_rolling_average( np.array( svd_reduced_mean( Xdf ).values ) ,tau=4 ) )
    print ( padded_rolling_average( v0,tau=4 ) )
    print ( v0 )


# !/usr/bin/env python3
# coding=utf-8

"""
@File   : CTSRunner.py
@Author : YouRan
@Contact: YouRan@baidou.com
@Date   : 2019-12-05
@Desc   : CTS 包管理工具
"""
import os
import sys
import multiprocessing as mp
from enum import Enum, unique
from CTSRunner.DeviceManager import execute_cmd
from CTSRunner.SatUtil import ht_service_log_center as logger
from CTSRunner.DeviceManager.device_manager import DeviceManager
from CTSRunner.SatUtil.pipelineGetMochaFail import handle_report
from CTSRunner.SatUtil.ht_service_log_center import HTServiceLogCenter
from CTSRunner.SatUtil.ht_monitor_log_center import HTMonitorLogCenter

SERVICE_MONITOR_SHELL = 'ps -A | grep -v grep | grep xcodebuild | grep id=%s'

CTS = 'cts'
INIT_PATH = 'init.sh'
ACTION_PATH = 'action.sh'
SERVICE_PATH = 'start.sh'
BRANCH = 'change_branch.sh'
BAT_SERVICE_PATH = 'start_bat.sh'
REPO_HANDLE_PATH = 'git_handle.sh'

PS_SHELL = "ps -ef | grep '%s' | grep -v grep"
RESULT_PORT = 'cts/mochawesome-report/mochawesome.html'

CURRENT_PATH = os.path.dirname(os.path.abspath(__file__))
WORK_PATH = os.path.join(os.path.expanduser('~'), 'cts-runner')
CASE_PATH = os.path.join(WORK_PATH, 'cts', 'ctsAutoCase', 'case')

ALL_CASE = 'ctsAutoCase/case/*/*'
CTS_COMPONENTS = ['api', 'core', 'components', 'plugin']

is_test_finish = False

@unique
class DeviceType(Enum):
    """
    运行设备类型
    """
    iOS = "iosdaily"
    iOSRelease = "iosnew"
    Android = "android"
    Web = "web"

    @staticmethod
    def get_device_type(key):
        """
        获取运行设备类型
        :param key:
        :return:
        """
        if key and len(key):
            key = key.lower()
            key = key.replace("-", "")
            device_table = {name.lower(): member for name, member in DeviceType.__members__.items()}
            if key in device_table:
                return device_table[key]
        return DeviceType.Android

@unique
class TestType(Enum):
    """
    测试类型
    """
    All = 'all'
    Api = 'api'
    Core = 'core'
    Components = 'components'

    @staticmethod
    def get_test_type(key):
        """
        获取测试类型
        :param key:
        :return:
        """
        if key and len(key):
            test_table = {name.lower(): member for name, member in TestType.__members__.items()}
            if key in test_table:
                return test_table[key]
        return TestType.All

def start_service_process(path):
    """
    启动相关服务
    :path:
    :return:
    """
    import sys
    import time
    import subprocess
    if path:
        cmd = 'sh ' + path + ' ' + WORK_PATH
        task = subprocess.Popen(cmd,
                                shell=True,
                                stdin=sys.stdin,
                                stderr=sys.stderr,
                                bufsize=-1)
        while task.poll() is None:
            time.sleep(2)
        return task.returncode == 0
    return True

def start_service(path):
    """
    启动相关服务
    :path:
    :return:
    """
    import sys
    import subprocess
    if path:
        cmd = 'sh ' + path + ' ' + WORK_PATH
        # task = subprocess.Popen(cmd,
        #                         shell=True,
        #                         stdin=sys.stdin,
        #                         stderr=sys.stderr,
        #                         bufsize=-1)
        return os.system(cmd) == 0
    return True

def start_service_monitor(device, log_path, shell_path):
    """
    监控wda服务
    :device: 要启动的设备
    :log_path: 日志存放路径
    :shell_path: wda服务启动脚本路径
    :return:
    """
    import time
    check_duration = 2
    if not device.is_ios:
        return
    monitor = HTMonitorLogCenter(log_path=log_path, log_name='monitor')
    monitor.info('开启监控')
    try:
        while not is_test_finish:
            result = os.popen(SERVICE_MONITOR_SHELL % device.udid).readline().strip()
            if result and len(result):
                time.sleep(check_duration)
            else:
                monitor.info('监控到程序崩溃')
                monitor.info(result)
                monitor.info(f'重启服务:')
                start_service_process(shell_path)
    except Exception as e:
        print(e)
        monitor.error(str(e))

class CTSRunner(object):
    """CTS 测试执行"""
    def __init__(
            self,
            user=None,
            branch=None,
            need_update=True,
            device_type=None,
            test_type='all',
            case_path=None,
            mocha_array=None,
            mocha_dic=None
    ):
        self.user = user
        self.branch = branch
        self.need_update = need_update

        self.device_type = DeviceType.get_device_type(device_type)
        self.case_path = case_path

        self.mocha_array = mocha_array
        self.mocha_dic = mocha_dic
        self._log_path = self.get_log_path()

        HTServiceLogCenter(log_path=self._log_path, log_name='service')

        if not case_path:
            self.test_type = TestType.get_test_type(test_type)
            if self.test_type != TestType.All:
                self.case_path = 'ctsAutoCase/case/swan-%s/*' % self.test_type.value
            else:
                self.case_path = ALL_CASE
        else:
            paths = self.case_path.split('/')
            if paths and len(paths) == 1:
                path = os.path.join(WORK_PATH, 'cts', 'ctsAutoCase', '*')
                cmd_line = 'find %s -name %s' % (path, self.case_path)
                files = execute_cmd(cmd_line)

                if files and len(files):
                    if len(files) == 1:
                        self.case_path = files[0]
                    else:
                        for index, file in enumerate(files):
                            print("%s index:\033[1;34m [%d]\033[0m" % (file, index))
                        case_index = int(input("请输入Case的index:[eg 1, 2, 3 ...]\n"))
                        if case_index < len(files):
                            self.case_path = files[case_index]
                        else:
                            raise Exception('你输入的index超出界限')
                else:
                    print('没有找到相关Case文件:', path)
                    raise Exception('没有找到相关的文件:' + path)
        logger.Info('casePath:' + self.case_path)
        self.device_manager = DeviceManager()
        mg = self.device_manager
        logger.Info('设备信息：')
        logger.Info(self.device_manager.devices)
        if not len(mg.devices):
            self.device_type = DeviceType.Web
        if self.device_type in (DeviceType.iOS, DeviceType.iOSRelease):
            if not len(mg.ios_devices):
                self.device_type = DeviceType.Android
        if self.device_type == DeviceType.Android:
            if not len(mg.android_devices):
                if len(mg.ios_devices):
                    self.device_type = DeviceType.iOS

        if not self.mocha_dic:
            self.mocha_dic = {}
        self.mocha_dic["device-type"] = self.device_type.value
        self.mocha_dic["reporter"] = "mochawesome"
        if self.device_type in (DeviceType.iOS, DeviceType.iOSRelease):
            self.mocha_dic["sat-dev"] = mg.ios_devices[0].udid
            self.mocha_dic["wda-port"] = "8100"

        if not os.path.exists(WORK_PATH):
            os.makedirs(WORK_PATH)
            self.repo_handle()
        else:
            if self.change_branch():
                if not self.repo_handle():
                    raise Exception('仓库不存在，请提供iCode用户名， 或者本地仓库地址')

    @staticmethod
    def get_log_path():
        """
        获取日志目录地址
        :return:
        """
        import datetime
        now = datetime.datetime.now()
        now = now.strftime("%Y-%m-%d %H:00")
        return os.path.join(WORK_PATH, 'cmd-log', now)

    def get_mocha_param(self):
        """
        获取mocha的参数
        :return:
        """
        def _param(x):
            return '--' + str(x)
        param = ''
        if self.mocha_array and len(self.mocha_array):
            params = list(map(_param, self.mocha_array))
            param = ' '.join(params)
        params = []
        if self.mocha_dic and len(self.mocha_dic):
            for key, value in self.mocha_dic.items():
                param1 = _param(key) + ' ' + str(value)
                params.append(param1)
        if len(param) or len(params):
            param += ' ' + ' '.join(params)
        return param

    def repo_handle(self):
        """
        更新CTS 仓库代码， 更新npm 依赖
        :return:
        """
        file_path = self._get_shell_path(REPO_HANDLE_PATH)
        cmd = 'sh ' + file_path + ' ' + WORK_PATH
        if self.need_update or self.user:
            sign = 0 if self.need_update else 1
            if self.need_update:
                logger.Info('更新仓库')
            cmd += ' ' + str(sign)

            if self.user:
                cmd += ' ' + self.user

            result = os.system(cmd)
            logger.Info('仓库操作完成')
            # self._waite_to_finish('npm')
            return result == 0
        return True

    def change_branch(self):
        """
        切换分支, 不需要切换这直接返回True
        :return:
        """
        if self.branch and len(self.branch):
            file_path = self._get_shell_path(BRANCH)
            cmd = 'sh ' + file_path + ' ' + WORK_PATH
            cmd += ' ' + self.branch
            return os.system(cmd) == 0
        return True

    @staticmethod
    def _get_shell_path(file):
        """
        获取执行文件路径
        :param file:
        :return:
        """
        return os.path.join(CURRENT_PATH, 'SHELL', file)

    def get_service_shell_path(self):
        """
        获取wda服务启动shell的path
        :return:
        """
        path = None
        if self.device_type in (DeviceType.iOS, DeviceType.iOSRelease):
            path = self._get_shell_path(SERVICE_PATH)
        elif self.device_type == DeviceType.Android:
            path = self._get_shell_path(BAT_SERVICE_PATH)
        return path

    @staticmethod
    def clear_test_env():
        """
        清理运行环境
        :return:
        """
        retry_path = os.path.join(CASE_PATH, 'swan-retry-*')
        cmd = 'rm -rf ' + retry_path
        if os.system(cmd) != 0:
            raise Exception('清理Case运行环境失败')

    def init_test_env(self):
        """
        清理运行环境
        :return:
        """
        logger.Info('初始化CTS运行环境')
        self.clear_test_env()
        path = os.path.join(CASE_PATH, 'swan-retry-')
        for name in CTS_COMPONENTS:
            os.makedirs(path + name)
        logger.Info('初始化CTS运行环境完成')

    def _waite_to_finish(self, cmd):
        import time
        time.sleep(3)
        while self._is_finish(cmd):
            time.sleep(1)

    @staticmethod
    def _is_finish(cmd):
        shell = PS_SHELL % cmd
        with os.popen(shell) as f:
            result = f.readlines()
            logger.Info(result)
        return len(result)

    @staticmethod
    def _do_action(path):
        file_path = os.path.join(CURRENT_PATH, path)
        os.system('open -a Terminal.app ' + file_path)

    def _generate_shell(self):
        import stat
        shell = r'''#!/usr/bin/env bash

        WORK_PATH=~/cts-runner
        file_path=${WORK_PATH}/cts
        cd "${file_path}" || exit
        '''
        shell = shell.replace("\t", "").replace("        ", '')
        shell1 = '\nmocha -t 240000 %s %s' % (self.case_path, self.get_mocha_param())
        print(shell1)
        logger.Info(shell1)
        shell = shell + shell1
        report = os.path.join(WORK_PATH, RESULT_PORT)
        shell = shell + '\nopen ' + report
        if not os.path.exists(WORK_PATH):
            os.makedirs(WORK_PATH)
        file_path = os.path.join(WORK_PATH, 'tmp')
        with open(file_path, 'w') as f:
            f.write(shell)
        # 执行该文件需要加权限
        os.chmod(file_path, stat.S_IRWXU)
        return file_path

    def run(self):
        """
        启动服务，并开始测试
        :return:
        """
        self.init_test_env()
        shell_path = self.get_service_shell_path()
        if start_service(shell_path):
            # 启动服务监控
            if self.device_type in (DeviceType.iOS, DeviceType.iOSRelease):
                # mp.set_start_method('forkserver')
                # p = mp.Pool(2)
                _device = self.device_manager.devices[0]
                import threading
                thread = threading.Thread(
                    target=start_service_monitor,
                    args=(_device, self._log_path, shell_path),
                    daemon=True
                )
                thread.start()

            cmd = 'sh ' + self._generate_shell() + ' ' + WORK_PATH
            if os.system(cmd) == 0:
                handle_report()
                cmd = 'cd ' + CASE_PATH + " && ls -lR swan-retry-* | grep '^-'"
                lines = execute_cmd(cmd)
                if len(lines):
                    print(lines)
                    logger.Info(str(lines))
                    print('存在失败的Case，需要重试')
                    self.case_path = 'ctsAutoCase/case/swan-retry-*/*/*'
                    print('重试地址:', self.case_path)
                    cmd = 'sh ' + self._generate_shell() + ' ' + WORK_PATH
                    os.system(cmd)
        global is_test_finish
        is_test_finish = True
        self.clear_test_env()

if __name__ == '__main__':
    device_tmp = DeviceType.iOS
    table = DeviceType.get_device_type('ios')

    parameter = {
        # "user": 'YouRan',
        # "is_daily": True,
        # "branch": 'master',
        # 'test_type': 'api',
        "case_path": '"generalIdentify*"',
        # "need_update": True
    }
    runner = CTSRunner(**parameter)
    runner.run()


from django.shortcuts import render
from django.http import HttpResponse

from .models import Suggestion_Model
from .forms import Suggestion_Form

# Create your views here.
def index(request):
    #return HttpResponse("Hello World")
    return render(request, 'index.html')

def suggestion_view(request):
    if request.method == 'POST':
        form = Suggestion_Form(request.POST)
        if form.is_valid():
            suggest = Suggestion_Model(
                suggestion=form.cleaned_data['suggestion']
            )
            suggest.save()
            form = Suggestion_Form()
    else:
        form = Suggestion_Form()
    suggestion_list = Suggestion_Model.objects.all()
    context={
        "suggestion_list":suggestion_list,
        "form":form
        }
    return render(request, 'suggestion.html',context)

def page(request, page_num):
    if page_num>=1:
        example_list=[]
        for i in range(page_num):
            example_list+=[i+1]

    else:
        example_list=None
    #return HttpResponse("Hello World")
    context={
        "page_template":page_num,
        "example_list":example_list
        }
    return render(request, 'page.html',context)


import collections

barve = [("A3", "zelena"), ("B5", "modra"), ("A3", "rumena")]

def barvanje(barve):
    sahovnica = collections.defaultdict()
    for polje, barva in barve:
        if polje in sahovnica:
            continue
        sahovnica[polje] = barva
    return sahovnica

print(barvanje(barve))

######################################################################

sahovnica = {
    "A3": "rumena", "A6": "rumena", "H2": "rumena", "G7": "rumena",
    "C2": "zelena", "C3": "zelena",
    "H6": "modra"
}

def prestej_barvo(sahovnica, barva):
    ponovitev = 0
    for barva1 in sahovnica.values():
        if barva1 == barva:
            ponovitev += 1
    return ponovitev

print(prestej_barvo(sahovnica, 'rumena'))


# Coltin Lappin-Lux (Data Analyst)
# lux.coltin@gmail.com
# Boys and Girls Club of Hawaii
# 1/25/18

from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
from openpyxl import Workbook
from openpyxl import load_workbook

#===============================================================================
# Attribute List (Both Duplicated and Unduplicated) - List without blank cells
#===============================================================================

def att_list(sheet, col, row):
    #Where to find attribute (Column and Row) Ex. 'B', 2
    duplicate_list = []
    unduplicated_list = []

    index = row
    i = col + str(index)
    while sheet[i].value != None:
        #print(sheet[i].value)
        i = col + str(index)
        if sheet[i].value != None:
            duplicate_list.append(sheet[i].value)
        index += 1

    for i in duplicate_list:
        if unduplicated_list.count(i) == 0:
            unduplicated_list.append(i)

    unduplicated_list.sort()

    return unduplicated_list, duplicate_list

#==================================================================================
# Attribute List 2 (Both Duplicated and Unduplicated) - List with blank cells
#==================================================================================

def att_list_2(sheet, list_length, col, row):
    #Where to find attribute (Column and Row) Ex. 'B', 2
    duplicate_list = []
    unduplicated_list = []

    #attribute counter
    att_index = row
    att_i = col + str(att_index)

    #primary attribute counter
    pri_index = 2
    pri_i = 'B' + str(pri_index)

    for x in range(list_length):
        att_i = col + str(att_index)
        pri_i = 'B' + str(pri_index)
        if sheet[att_i].value == None:
            duplicate_list.append([sheet[pri_i].value,"Unknown"])
        else:
            duplicate_list.append([sheet[pri_i].value,sheet[att_i].value])
        att_index += 1
        pri_index += 1

    #attribute counter
    att_index = row
    att_i = col + str(att_index)

    for z in range(list_length):
        att_i = col + str(att_index)
        if unduplicated_list.count(sheet[att_i].value) == 0:
            if sheet[att_i].value == None:
                if unduplicated_list.count("Unknown") == 0:
                    unduplicated_list.append("Unknown")
            else:
                unduplicated_list.append(sheet[att_i].value)
        att_index += 1

    print(unduplicated_list)

    inttemp = []
    strtemp = []

    for a in unduplicated_list:
    	if type(a) == int:
    		inttemp.append(a)
    		inttemp.sort()
    	if type(a) == str:
    		strtemp.append(a)
    		strtemp.sort()

    unduplicated_list = []

    for b in inttemp:
    	unduplicated_list.append(b)
    for c in strtemp:
    	unduplicated_list.append(c)

    print(unduplicated_list)
    return unduplicated_list, duplicate_list

#===============================================================================
# Count
#===============================================================================

def count(what, where):
    #what = what to count (List)
    #where = where to find it (Location)

    results = []

    for x in what:
        results.append([x,where.count(x)])

    return results

#===============================================================================
# Attribute Count
#===============================================================================

def attribute_count(what, where):
    #what = what to count (List)
    #where = where to find it (Location)

    results = []
    count = 0

    for x in what:
        count = 0
        for y in where:
            if x == y[1]:
                count += 1
        results.append([x,count])

    return results

#===============================================================================
# Attribute Percentage
#===============================================================================

def attribute_percentage(att_count):
    #what = what to count (List)
    #where = where to find it (Location)

    results = []
    total = 0
    temp = 0
    for i in range(len(att_count)):
        total = total + att_count[i][1]
    for i in range(len(att_count)):
        temp = att_count[i][1]/total
        results.append([att_count[i][0],temp])

    return results

#===============================================================================
# Count By Club
#===============================================================================

def countByClub(bgch_list, undup, dup):
    #what = what to count (List)
    #where = where to find it (Location)
    #[clubhouse, attribute, count]

    results = []

    for x in bgch_list:
        for y in undup:
            count = 0
            for z in dup:
                if x == z[0] and y == z[1]:
                    count += 1
            results.append([x,y,count])

    return results

#===============================================================================
# Total Count
#===============================================================================

def total(what):
    #what = what to count (List)

    results = len(what)

    return results

#==================================================================
# Main Function (Initialize Variables and Call Functions)
#==================================================================

def main():
    """
    This function executes when this file is run as a script.
    """

    #==================================================================
    # Open Workbooks (Input and Output Excel Sheets)
    #==================================================================
    #Reading in existing Workbook
    title = input("Input Excel Title: ")
    title = title + '.xlsx'

    wb = load_workbook(filename = title)

    #Creating Revised Workbook
    rwb = Workbook()
    dest_filename = input("Output Excel Title: ")
    dest_filename = dest_filename + '.xlsx'

    #==================================================================
    # While loop in case user wants to summarize multiple excel sheets
    #==================================================================
    #Data Calculation
    continuetill = 'yes'
    while continuetill == 'yes':
        sheetname = input("Input Sheet Name/Number: ")
        sheet = wb[sheetname]
        revisedSheet = rwb.active
        revisedSheet = rwb.create_sheet(title=sheetname)

        #==================================================================
        # Summarize Membership Data
        #==================================================================
        #Primary Attribute - Clubhouse
        bgch_clubhouse_list, duplicated_bgch_clubhouse_list = att_list(sheet, 'B', 2)
        #print(bgch_clubhouse_list)

        #Count how many members per clubhouse
        membership_count = count(bgch_clubhouse_list, duplicated_bgch_clubhouse_list)
        membership_total = total(duplicated_bgch_clubhouse_list)
        list_length = len(duplicated_bgch_clubhouse_list)
        membership_count.append(["Total", membership_total])

        #==================================================================
        # Print Title and other document details to new Excel
        #==================================================================
        #Title
        title_index = 1
        title_loc = 'A' + str(title_index)
        revisedSheet[title_loc] = 'Boys & Girls Club of Hawaii'
        #Report Type
        title_index = 2
        title_loc = 'A' + str(title_index)
        revisedSheet[title_loc] = 'Report Type:'
        #Period
        title_index = 3
        title_loc = 'A' + str(title_index)
        revisedSheet[title_loc] = 'Period:'
        #Description
        title_index = 4
        title_loc = 'A' + str(title_index)
        revisedSheet[title_loc] = 'Preparation Date: '

        #==================================================================
        # Print Membership Summary to new Excel
        #==================================================================
        #Membership Data
        club_index = 6
        club_i = 'A' + str(club_index)
        revisedSheet[club_i] = 'Site'

        count_index = 6
        count_i = 'B' + str(count_index)
        revisedSheet[count_i] = 'Membership Count'

        perc_index = 6
        perc_i = 'C' + str(perc_index)
        revisedSheet[perc_i] = 'Membership Percentage'

        for x in membership_count:
            for i in range(len(x)):
                if i == 0:
                    club_index += 1
                    club_i = 'A' + str(club_index)
                    revisedSheet[club_i] = x[i]
                elif i == 1:
                    count_index += 1
                    count_i = 'B' + str(count_index)
                    revisedSheet[count_i] = x[i]
                    perc_index += 1
                    perc_i = 'C' + str(perc_index)
                    revisedSheet[perc_i] = x[i]/membership_total

        #==================================================================
        # Summarizing Attributes
        #==================================================================
        att_continuetill = input("Is there another attribute to summarize? (yes/no) ")
        while att_continuetill == 'yes':

            att_col = input("Attribute Column (Letter): ")
            att_row = 2

            #==================================================================
            # Summarize 1 Attribute
            #==================================================================
            att, dupAtt = att_list_2(sheet, list_length, str(att_col), int(att_row))
            #totals by primary att
            att_countbyclub = countByClub(bgch_clubhouse_list, att, dupAtt)
            #totals
            att_count = attribute_count(att, dupAtt)
            att_percentage = attribute_percentage(att_count)

            #==================================================================
            # Print Attribute Results to New Excel Sheet
            #==================================================================
            #Attribute Title
            club_index += 2
            att_index = club_index + 1
            club_i = 'A' + str(club_index)
            revisedSheet[club_i] = sheet[att_col + str(1)].value

            #==================================================================
            # Hard coded and Calculated by Primary Attribute (Site)
            #==================================================================
            club_index += 1
            club_i = 'A' + str(club_index)
            revisedSheet[club_i] = 'Site'

            for x in bgch_clubhouse_list:
                club_index += 1
                club_i = 'A' + str(club_index)
                revisedSheet[club_i] = x

            #Print Total(#)
            counter = 1
            att_revised_row = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','AA','AB','AC','AD','AE','AF','AG','AH','AI','AJ','AK','AL','AM','AN','AO','AP','AQ','AR','AS','AT','AU','AV','AW','AX','AY','AZ']
            att_i = att_revised_row[counter] + str(att_index)

            club_index += 1
            club_i = 'A' + str(club_index)
            revisedSheet[club_i] = "Total (#)"
            for i in range(len(att_count)):
                att_i = att_revised_row[counter] + str(club_index)
                revisedSheet[att_i] = att_count[i][1]
                counter += 1

            #Print Total(%)
            counter = 1
            att_revised_row = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','AA','AB','AC','AD','AE','AF','AG','AH','AI','AJ','AK','AL','AM','AN','AO','AP','AQ','AR','AS','AT','AU','AV','AW','AX','AY','AZ']
            att_i = att_revised_row[counter] + str(att_index)

            club_index += 1
            club_i = 'A' + str(club_index)
            revisedSheet[club_i] = "Total (%)"
            for i in range(len(att_percentage)):
                att_i = att_revised_row[counter] + str(club_index)
                revisedSheet[att_i] = att_percentage[i][1]
                counter += 1

            #Print Attribute Titles
            counter = 1
            att_revised_row = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','AA','AB','AC','AD','AE','AF','AG','AH','AI','AJ','AK','AL','AM','AN','AO','AP','AQ','AR','AS','AT','AU','AV','AW','AX','AY','AZ']
            att_i = att_revised_row[counter] + str(att_index)

            for x in att:
                att_i = att_revised_row[counter] + str(att_index)
                revisedSheet[att_i] = x
                counter += 1

            #Print Attribute Summary Data
            att_revised_row = ['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z','AA','AB','AC','AD','AE','AF','AG','AH','AI','AJ','AK','AL','AM','AN','AO','AP','AQ','AR','AS','AT','AU','AV','AW','AX','AY','AZ']

            att_printer_counter = 0
            for x in range(len(bgch_clubhouse_list)):
                counter = 1
                att_index += 1
                att_i = att_revised_row[counter] + str(att_index)
                for y in range(len(att)):
                    revisedSheet[att_i] = att_countbyclub[att_printer_counter][2]
                    #print(att_countbyclub[att_printer_counter][2])
                    att_printer_counter += 1
                    counter += 1
                    att_i = att_revised_row[counter] + str(att_index)

            att_continuetill = input("Is there another attribute to summarize? (yes/no) ")

        continuetill = input("Is there another sheet? (yes/no) ")

    #Done, Save Revised Workbook
    rwb.save(filename = dest_filename)

if __name__ == "__main__":
    main()


from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect
from BITEhack.forms import ContactForm, SnippetForm, Plot2dForm
from .models import  DefaultUser, Project, Folder, Event
from django.contrib.auth.decorators import login_required, permission_required
from django.core import serializers
from . import forms
from django.contrib.auth import authenticate, login

def contact(request):
    if request.method == "POST":
        form = ContactForm(request.POST)
        if form.is_valid():
            name = form.cleaned_data['name']
            email = form.cleaned_data['email']
            print(name, email)

    form = ContactForm()
    return render(request, 'form.html', {'form': form})

def snippet_detail(request):
    if request.method == "POST":
        form = SnippetForm(request.POST)
        if form.is_valid():
            form.save()
            print('WE GOOD BOI')

    form = SnippetForm()
    return render(request, 'form.html', {'form': form})

def index(request):
    return render(request, 'index.html')

@login_required
def test(request):
    return HttpResponse("TEST")

def create_user(request):
    if request.method == 'POST':
        user_form = forms.UserForm(request.POST)
        if user_form.is_valid():
            new_user = user_form.save(commit=False)
            new_user.set_password(user_form.cleaned_data['password'])
            new_user.save()
            return render(request, 'base.html')
    else:
        user_form = forms.UserForm
    return render(request, 'add_user.html', {'user_form': user_form})

@login_required
def user_profile(request):
    def create_project_dict(project_model):
        return {'name': project_model.name, 'desc': project_model.description }
    def create_friend_dict(user_model):
        return {'name': user_model.username }
    user = DefaultUser.objects.get(username=request.user.username)
    projects = user.projects.all()
    projects_table = [create_project_dict(project) for project in projects]
    friends = user.friends.all()
    friends_table = [create_friend_dict(user) for user in friends]
    return render(request, 'user_profile.html', {'projects': projects_table, 'friends':friends_table})

def wall(request):
    def create_event_dict(event_model):
        return {'name': event_model.project.name,
                'action': event_model.action,
                'time': str(event_model.date)}
    events = Event.objects.all()
    events_table = [create_event_dict(event) for event in events]
    return render(request, 'wall.html', {'events': events_table})

class clusterer():

    def create(self, *args, **kwargs):
        from sklearn.cluster import KMeans
        self.cls = KMeans(*args, **kwargs)
        print("Clustering algorithm prepared")

    def train(self, data):
        self.cls.fit(data)
        print("Training has concluded")

    def predict(self, data):
        import numpy as np
        import seaborn as sns
        cmap = ['red', 'green', 'blue','black', 'yellow', 'cyan', 'orange', 'purple']
        self.predicted = self.cls.predict(data)
        samples = data.shape[0]
        for i in np.unique(self.predicted):
            dt = data[self.predicted == i]
            sns.regplot(dt[:,0], dt[:,1], fit_reg=False, color = cmap[i%len(cmap)], marker ='.')

def cluster(request):
    if request.method == 'POST':
        user_form = Plot2dForm(request.POST)
        if user_form.is_valid():

            n_clusters = int(user_form.cleaned_data['clusters'])

        else:
            n_clusters = 3

        import matplotlib.pyplot as plt
        from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
        from matplotlib.figure import Figure
        import numpy as np
        import io
        import pandas as pd

        data = np.array(pd.read_excel('1.xlsx'))
        f = Figure()

        a = data#= np.random.rand(300, 2) * 100

        c = clusterer()
        c.create(n_clusters=n_clusters)
        c.train(a)

        x = np.linspace(np.min(a[:,0]), np.max(a[:,0]), 1000)
        y = np.linspace(np.min(a[:,1]), np.max(a[:,1]), 1000)

        xx, yy = np.meshgrid(x, y)
        xx = xx.reshape(1000000,1)
        yy = yy.reshape(1000000,1)
        ad = np.concatenate((xx, yy), axis=1)
        c.predict(ad)

        # Code that sets up figure goes here; in the question, that's ...
        FigureCanvas(f)

        buf = io.BytesIO()
        plt.savefig(buf, format='png')
        plt.close(f)
        response = HttpResponse(buf.getvalue(), content_type='image/png')
        #response = HttpResponse('HI')
        return response
    else:
        form = Plot2dForm()
        return render(request, 'plot2d_form.html', {'form': form})

        #return HttpResponse('4qwasdasde')  #response
@login_required
def project_view(request):
    project_name = request.GET.get('project_name', '')
    dictiornary = {}
    files_list = []
    if project_name is not None:
        flag = Project.objects.filter(name=project_name).exists()
        if flag:
            proj = Project.objects.get(name=project_name)
            dictiornary['name'] = proj.name
            dictiornary['desc'] = proj.description
            dictiornary['type'] = proj.dtype
            files = Folder.objects.filter(project=proj).all()
            files_list = [f.title for f in files]
            print(files_list)

    return render(request, 'project.html', {'project':dictiornary, 'files': files_list} )

def upload_file(request):
    if request.method == 'POST':
        upload_form = forms.ModelFormWithFileField(request.POST, request.FILES)
        if upload_form.is_valid():
            # file is saved
            upload_form.save()
            return HttpResponseRedirect('wall')
    else:
        upload_form = forms.ModelFormWithFileField()
    return render(request, 'upload.html', {'upload_form': upload_form})

def get_projects(request):
    projects = Project.objects.all()
    json = serializers.serialize('json', projects)
    return HttpResponse(json)

def projects_view(request):
    return render(request, 'projects.html')

def regression(request):
    import matplotlib.pyplot as plt
    from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
    from matplotlib.figure import Figure
    import numpy as np
    import io
    from sklearn.linear_model import LinearRegression
    f = Figure()

    sample = np.random.chisquare(1, 200)
    points = np.linspace(-np.pi, 0, 200)
    res = np.sin(points) + sample
    model = LinearRegression()
    model.fit(points.reshape(200,1), res.reshape(200,1))
    space = np.linspace(np.min(points) , np.max(points), 1000).reshape(-1, 1)
    pred = model.predict(space)
    plt.scatter(points, res)
    plt.plot(space,pred, c='red')

    FigureCanvas(f)

    buf = io.BytesIO()
    plt.savefig(buf, format='png')
    plt.close(f)
    response = HttpResponse(buf.getvalue(), content_type='image/png')
    return response

def plot2d(request):
    import matplotlib.pyplot as plt
    from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
    from matplotlib.figure import Figure
    import numpy as np
    import io
    f = Figure()

    samples = np.random.randint(200, size=(500,2))
    plt.scatter(samples[:,0], samples[:,1])

    FigureCanvas(f)

    buf = io.BytesIO()
    plt.savefig(buf, format='png')
    plt.close(f)
    response = HttpResponse(buf.getvalue(), content_type='image/png')
    return response

def plot3d(request):
    import matplotlib.pyplot as plt
    from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
    from matplotlib.figure import Figure
    import numpy as np
    import io
    from mpl_toolkits.mplot3d import Axes3D
    f = Figure()

    def Bumps(x,y):
        return np.sin(5 * x) * np.cos(5 * y)/5

    x = np.linspace(-np.pi/2, np.pi/2, 1000)
    y = np.linspace(-np.pi/2, np.pi/2, 1000)
    xx, yy = np.meshgrid(x, y)

    fig = plt.figure()
    ax = fig.add_subplot(111, projection='3d')

    z = Bumps(xx,yy).reshape((1000,1000))
    ax.plot_surface(xx, yy, z)

    FigureCanvas(f)

    buf = io.BytesIO()
    plt.savefig(buf, format='png')
    plt.close(f)
    response = HttpResponse(buf.getvalue(), content_type='image/png')
    return response

def hist(request):
    import matplotlib.pyplot as plt
    from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
    from matplotlib.figure import Figure
    import numpy as np
    import io
    f = Figure()

    sample = np.random.chisquare(100, 200000)
    fig, ax = plt.subplots()
    ax.hist(sample, 50, density=1)
    ax.set_title('chi_square')
    ax.set_xlabel('value')
    ax.set_ylabel('Probability density')

    FigureCanvas(f)

    buf = io.BytesIO()
    plt.savefig(buf, format='png')
    plt.close(f)
    response = HttpResponse(buf.getvalue(), content_type='image/png')
    return response

{'KMeans': 'sklearn.cluster.KMeans', 'np': 'numpy', 'sns': 'seaborn'}