import streamlit as st
import numpy as np
import pandas as pd
import altair as alt
from scipy import stats
import plotly.express as px
from plotly import tools
from plotly.subplots import make_subplots
import plotly.graph_objects as go
import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import preprocessing
from sklearn.neighbors import NearestNeighbors
from sklearn.metrics.pairwise import cosine_similarity
from sklearn.preprocessing import StandardScaler

hide_streamlit_style = """
					<style>
					#MainMenu {visibility: hidden;}
					footer {visibility: hidden;}
					</style>
					"""
st.markdown(hide_streamlit_style, unsafe_allow_html=True)

Games_url = ('s3://nbabdata/games.csv')
Games_details_url = ('s3://nbabdata/games_details.csv')
players_url = ('s3://nbabdata/players.csv')
rankings_url = ('s3://nbabdata/ranking.csv')
teams_url = ('s3://nbabdata/teams.csv')

@st.cache(allow_output_mutation=True)
def load_data4():
    df = pd.read_csv(Games_url)
    return df

@st.cache(allow_output_mutation=True)
def load_data5():
    df = pd.read_csv(Games_details_url)
    return df

@st.cache(allow_output_mutation=True)
def load_data6():
    df = pd.read_csv(players_url)
    return df

@st.cache(allow_output_mutation=True)
def load_data7():
    df = pd.read_csv(rankings_url)
    return df

@st.cache(allow_output_mutation=True)
def load_data8():
    df = pd.read_csv(teams_url)
    return df

games_df = load_data4()
games_details_df = load_data5()
players_df = load_data6()
rankings_df = load_data7()
teams_df = load_data8()

def main():

	analysis = ['Match analysis','Match Details analysis','Top attributes players']
	play_type = ['Attacking', 'Defending']
	options = ['Single Player Analysis','Players Comparision']
	teams = rankings_df['TEAM'].unique()
	unique = games_details_df['PLAYER_NAME'].unique()
	equiv = {12020:202021, 12019:201920, 22019:201920, 12018:201819, 22018:201819, 12017:201718, 22017:201718, 12016:201617, 22016:201617, 12015:201516, 22015:201516, 12014:201415, 22014:201415, 12013:201314, 22013:201314, 12012:201213, 22012:201213, 12011:201112, 22011:201112, 12010:201011, 22010:201011, 12009:200910, 22009:200910, 12008:200809, 22008:200809, 12007:200708, 22007:200708, 12006:200607, 22006:200607, 12005:200506, 22005:200506, 12004:200405, 22004:200405, 12003:200304, 22003:200304, 12002:200203, 22002:200203}
	rankings_df["Season"] = rankings_df["SEASON_ID"].map(equiv).astype(int)
	seasons = rankings_df['Season'].unique()
	teams_eq = {1610612737:'Hawks',1610612738:'Celtics',1610612740:'Pelicans',1610612741:'Bulls',1610612742:'Mavericks',1610612743:'Nuggets',1610612745:'Rockets',1610612746:'Clippers',1610612747:'Lakers',1610612748:'Heat',1610612749:'Bucks',1610612750:'Timberwolves',1610612751:'Nets',1610612752:'Knicks',1610612753:'Magic',1610612754:'Pacers',1610612755:'76ers',1610612756:'Suns',1610612757:'Trail Blazers',1610612758:'Kings',1610612759:'Spurs',1610612760:'Thunder',1610612761:'Raptors',1610612762:'Jazz',1610612763:'Grizzlies',1610612764:'Wizards',1610612765:'Pistons',1610612766:'Hornets',1610612739:'Cavaliers',1610612744:'Warriors'}
	games_df["HomeTeam"] = games_df["HOME_TEAM_ID"].map(teams_eq).astype(str)
	games_df["VisitingTeam"] = games_df["VISITOR_TEAM_ID"].map(teams_eq).astype(str)
	games_df['Match'] = games_df['HomeTeam'] + ' '+'vs' + ' '+ games_df['VisitingTeam']
	stats_cols = {
    'FGM':'Field Goals Made',
    'FGA':'Field Goals Attempted',
    'FG_PCT':'Field Goal Percentage',
    'FG3M':'Three Pointers Made',
    'FG3A':'Three Pointers Attempted',
    'FG3_PCT':'Three Point Percentage',
    'FTM':'Free Throws Made',
    'FTA':'Free Throws Attempted',
    'FT_PCT':'Free Throw Percentage',
    'OREB':'Offensive Rebounds',
    'DREB':'Defensive Rebounds',
    'REB':'Rebounds',
    'AST':'Assists',
    'TO':'Turnovers',
    'STL':'Steals',
    'BLK':'Blocked Shots',
    'PF':'Personal Foul',
    'PTS':'Points',
    'PLUS_MINUS':'Plus-Minus'
	}
	stat_col = {
	'PTS_home': 'Points made HomeTeam',
	'PTS_away': 'Points made AwayTeam',
	'FG_PCT_home':'Field Goal Percentage HomeTeam',
	'FT_PCT_home':'Free Throw Percentage HomeTeam',
	'FG3_PCT_home': 'Three Point Percentage HomeTeam',
	'AST_home': 'Assists HomeTeam',
	'REB_home': 'Rebounds HomeTeam',
	'FG_PCT_away':'Field Goal Percentage AwayTeam',
	'FT_PCT_away':'Free Throw Percentage AwayTeam',
	'FG3_PCT_away': 'Three Point Percentage AwayTeam',
	'AST_away': 'Assists AwayTeam',
	'REB_away': 'Rebounds AwayTeam'
	}
	attributes = ['Field Goal Percentage HomeTeam','Free Throw Percentage HomeTeam','Three Point Percentage HomeTeam','Assists HomeTeam','Rebounds HomeTeam','Field Goal Percentage AwayTeam','Free Throw Percentage AwayTeam','Three Point Percentage AwayTeam','Assists AwayTeam','Rebounds AwayTeam']
	attributes_players = ['Three Pointers Made','Free Throws Made','Offensive Rebounds','Defensive Rebounds','Rebounds','Assists','Turnovers','Steals','Blocked Shots','Personal Foul']
	games_details_df_new = rename_df(games_details_df, stats_cols)
	games_df_new = rename_df(games_df, stat_col)
	page = st.sidebar.selectbox("Choose a page", ["Homepage", "Stats Exploration", "Players Comparision", "Scout Watch"])
	if page == "Homepage":
		st.header("This is an advance basketball data explorer.")
		st.write("Please select a page on the left for different analysis.")
		st.markdown("![Alt Text](https://media.tenor.com/images/07002ec173d710cf8309cc1a91e0f7d9/tenor.gif)")
	elif page == "Stats Exploration":
		st.title("Advanced Match Stats Exploration")
		option = st.selectbox('Choose the type of analysis', analysis )
		if option == 'Match analysis':
			col1,col2,col3= st.beta_columns(3)
			col4,col5,col6 = st.beta_columns(3)
			with col1:
				match= st.selectbox("Choose a Match", games_df_new['Match'].unique())
			with col3:
				season = st.selectbox("Choose a season", games_df_new['SEASON'].unique())
			with col5:
				submit = st.button('Submit')
			if submit:
				visualize_matches(games_df_new,season,match)
		elif option == 'Match Details analysis':
			st.header('Choose the player for his stats in a match')
			col1,col2,col3= st.beta_columns(3)
			col4,col5,col6 = st.beta_columns(3)
			with col1:
				player = st.selectbox("Type in to choose a player", unique)
			with col2:
				match = st.selectbox('Type in to choose the match', games_df_new['Match'].unique())
			with col3:
				season = st.selectbox("Choose the season", games_df_new['SEASON'].unique())
			with col5:
				submit = st.button('Submit')
			if submit:
				visualize_players(games_details_df_new, games_df_new, season, player, match)
		elif option == 'Top attributes players':
			st.header('Choose the attribute for top players of the season')
			col1,col2,col3= st.beta_columns(3)
			col4,col5,col6 = st.beta_columns(3)
			with col1:
				att = st.selectbox("Type in to choose the attribute", attributes_players)
			with col3:
				season = st.selectbox("Choose the season", games_df_new['SEASON'].unique())
			with col5:
				submit = st.button('Submit')
			if submit:
				visualize_top_players(games_details_df_new, games_df_new, season, att)

	elif page == "Players Comparision":
		st.title("Players comparision and player analysis")
		sub = st.selectbox("Choose the option of comparision or single player analysis", options)
		col7,col8,col9 = st.beta_columns(3)
		col10,col11 = st.beta_columns(2)
		if sub == 'Players Comparision':
			with col7:
				player_1 = st.selectbox("Choose the first player", unique)
			with col8:
				player_2 = st.selectbox("Choose the second player", unique)
			with col9:
				player_season = st.selectbox("Choose the season", games_df_new['SEASON'].unique())
			visualize_two_players(games_details_df_new, games_df_new, player_1, player_2, player_season)
		elif sub == 'Single Player Analysis':
			with col10:
				player_1 = st.selectbox("Choose the player for analysis", unique)
			with col11:
				player_season = st.selectbox("Choose the season", games_df_new['SEASON'].unique())
			visualize_single_player(games_details_df_new,games_df_new, player_1, player_season)

	elif page == "Scout Watch":
		st.title('Top players of NBA and their ratings of the season')
		season = st.selectbox("Choose the season",games_df_new['SEASON'].unique())
		submit = st.button('Submit')
		if submit:
			show_top_players(games_details_df_new,games_df_new,season)

def visualize_matches(df,season,match):
	season_df = df[df['SEASON'] == season]
	match_df = season_df[season_df['Match'] == match]
	match_df = match_df.drop(['GAME_ID','HOME_TEAM_ID','VISITOR_TEAM_ID','TEAM_ID_home','SEASON','TEAM_ID_away'],axis=1)
	st.subheader('Detailed Stats')
	st.write(match_df)
	attributes = ['Points made HomeTeam','Points made AwayTeam','Assists HomeTeam','Rebounds HomeTeam','Assists AwayTeam','Rebounds AwayTeam']
	col1,col2,col3 = st.beta_columns(3)
	rel_df = match_df[attributes]
	sums = rel_df.sum()
	colors = ['red','orange']*5
	with col1:
		fig = go.Figure(data=[go.Bar(
					name = match,
					x = attributes,
					y = sums,
					marker_color = colors
					)
					])
		fig.update_layout(height=500,width=800)
		fig.update_layout(
    				title={
        			'text': 'Summary stats of all the' + ' ' + ' ' + str(match) + ' ' +'played in '+' '+ str(season),
        			'y':0.98,
        			'x':0.40,
        			'xanchor': 'center',
        			'yanchor': 'top'})
		st.plotly_chart(fig)

def visualize_players(games_details_df_new, games_df_new, season, player, match):
	player_df = games_details_df_new[games_details_df_new['PLAYER_NAME'] == player]
	merged_df = player_df.merge(games_df_new, on='GAME_ID')
	merged_match_df = merged_df[merged_df['Match'] == match]
	merge_df = merged_match_df[merged_match_df['SEASON'] == season]
	if merge_df.empty:
		st.write('No data available')
		return None
	st.subheader('Detailed Stats')
	st.write(merge_df.drop(['GAME_ID','TEAM_ID','PLAYER_ID','HOME_TEAM_ID','VISITOR_TEAM_ID','TEAM_ID_home','TEAM_ID_away'],axis=1))
	attributes = ['Points made HomeTeam','Points made AwayTeam','Assists HomeTeam','Rebounds HomeTeam','Assists AwayTeam','Rebounds AwayTeam','Field Goals Made','Field Goals Attempted','Three Pointers Made','Three Pointers Attempted','Free Throws Made','Free Throws Attempted','Offensive Rebounds','Defensive Rebounds','Rebounds','Assists','Turnovers','Steals','Blocked Shots','Personal Foul','Points',]
	col1,col2,col3 = st.beta_columns(3)
	rel_df = merge_df[attributes]
	sums = rel_df.sum()
	colors = ['red','orange']*10
	with col1:
		fig = go.Figure(data=[go.Bar(
					name = match,
					x = attributes,
					y = sums,
					marker_color = colors
					)
					])
		fig.update_layout(height=500,width=800)
		fig.update_layout(
    				title={
        			'text': 'Summary of all the' + ' '+ str(player) + ' '+'stats' +' '+ str(match) + ' ' +'played in '+' '+ str(season),
        			'y':0.98,
        			'x':0.40,
        			'xanchor': 'center',
        			'yanchor': 'top'})
		st.plotly_chart(fig)
	ppg = rel_df['Points'].sum() / len(rel_df)
	ppg_ht = rel_df['Points made HomeTeam'].sum() / len(rel_df)
	ppg_at = rel_df['Points made AwayTeam'].sum() / len(rel_df)
	apg = rel_df['Assists'].sum() / len(rel_df)
	apg_ht = rel_df['Assists HomeTeam'].sum() / len(rel_df)
	apg_at = rel_df['Assists AwayTeam'].sum() / len(rel_df)
	col4,col5,col6,col7,col8,col9 =st.beta_columns(6)
	with col4:
		st.write('Points per game by player {}'.format(str(ppg)))
	with col5:
		st.write('Points per game by HT {}'.format(str(ppg_ht)))
	with col6:
		st.write('Points per game by AT {}'.format(str(ppg_at)))
	with col7:
		st.write('Assists per game by player {}'.format(str(apg)))
	with col8:
		st.write('Assists per game by HT {}'.format(str(apg_ht)))
	with col9:
		st.write('Assists per game by AT {}'.format(str(apg_at)))

def rename_df(df, col_dict):
    cols = df.columns
    new_cols = [(col_dict[c] if c in col_dict else c) for c in cols]
    df.columns = new_cols
    return df

def visualize_two_players(games_details_df_new, games_df_new, player_1, player_2, player_season):
	player1_df = games_details_df_new[games_details_df_new['PLAYER_NAME'] == player_1]
	merged1_df = player1_df.merge(games_df_new, on='GAME_ID')
	merge1_df = merged1_df[merged1_df['SEASON'] == player_season]
	player2_df = games_details_df_new[games_details_df_new['PLAYER_NAME'] == player_2]
	merged2_df = player2_df.merge(games_df_new, on='GAME_ID')
	merge2_df = merged2_df[merged2_df['SEASON'] == player_season]
	if merge1_df.empty:
		st.write('No data available')
		return None
	if merged2_df.empty:
		st.write('No data available')
		return None
	attributes = ['Three Pointers Made','Free Throws Made','Offensive Rebounds','Defensive Rebounds','Rebounds','Assists','Turnovers','Steals','Blocked Shots','Personal Foul']
	col1,col2,col3,col4 = st.beta_columns(4)
	rel1_df = merge1_df[attributes]
	sums1 = rel1_df.sum()
	rel2_df = merge2_df[attributes]
	sums2 = rel2_df.sum()

	with col1:
		data = [go.Scatterpolar(
  					r = sums1,
  					theta = attributes,
  					fill = 'toself',
  					name= str(player_1),
     				line =  dict(
            		color = 'orange'
        				)
					),
					go.Scatterpolar(
  					r = sums2,
  					theta = attributes,
  					fill = 'toself',
  					name= str(player_2),
     				line =  dict(
            		color = 'red'
        				)
					)]

		layout = go.Layout(
  					polar = dict(
    				radialaxis = dict(
      				visible = True,
      				range = [0, 1000]
    				)
  				),
  					showlegend = True,
  					title = "{} vs {} Summary Stats Comparison of {}".format(str(player_1), str(player_2), player_season)
				)
		fig = go.Figure(data=data, layout=layout)
		st.plotly_chart(fig)
		merge1_df = merge1_df.drop(['GAME_ID','TEAM_ID','PLAYER_ID','HOME_TEAM_ID','VISITOR_TEAM_ID','TEAM_ID_home','TEAM_ID_away'],axis=1)
		merge2_df = merge2_df.drop(['GAME_ID','TEAM_ID','PLAYER_ID','HOME_TEAM_ID','VISITOR_TEAM_ID','TEAM_ID_home','TEAM_ID_away'],axis=1)
		horizontal_df = pd.concat([merge1_df.T, merge2_df.T],axis =1)
	with col4:
		colors = ['red','orange']*10
		attributes = ['Field Goals Made','Field Goals Attempted','Three Pointers Made','Three Pointers Attempted','Free Throws Made','Free Throws Attempted','Points']
		rel1_df = merge1_df[attributes]
		sums1 = rel1_df.sum()
		rel2_df = merge2_df[attributes]
		sums2 = rel2_df.sum()
		fig = go.Figure(data=[go.Bar(
					name = player_1,
					x = attributes,
					y = sums1
					),
					go.Bar(
							name = player_2,
							x = attributes,
							y = sums2
							)
					])
		fig.update_layout(yaxis={'categoryorder':'total ascending'})
		fig.update_layout(barmode='stack')
		fig.update_layout(height=500,width=550)
		fig.update_layout(
    				title={
        			'text': 'Additional summary stats of the' + ' ' + str(player_1) + ' '+'and'+' '+str(player_2)+' '+'in ' +' '+ str(player_season),
        			'y':0.98,
        			'x':0.40,
        			'xanchor': 'center',
        			'yanchor': 'top'})
		st.plotly_chart(fig)
	st.subheader('Match by match details')
	st.write(horizontal_df)

def visualize_single_player(games_details_df_new,games_df_new, player_1, player_season):
	player_df = games_details_df_new[games_details_df_new['PLAYER_NAME'] == player_1]
	merged_df = player_df.merge(games_df_new, on='GAME_ID')
	merge_df = merged_df[merged_df['SEASON'] == player_season]
	if merge_df.empty:
		st.write('No data available')
		return None
	attributes = ['Three Pointers Made','Free Throws Made','Offensive Rebounds','Defensive Rebounds','Rebounds','Assists','Turnovers','Steals','Blocked Shots','Personal Foul']
	attributes_name = ['PLAYER_NAME','Three Pointers Made','Free Throws Made','Offensive Rebounds','Defensive Rebounds','Rebounds','Assists','Turnovers','Steals','Blocked Shots','Personal Foul']
	col1,col2,col3,col4 = st.beta_columns(4)
	col5,col6 = st.beta_columns(2)
	all_merged_df = games_details_df_new.merge(games_df_new, on='GAME_ID')
	all_merge_df = all_merged_df[all_merged_df['SEASON'] == player_season]
	all_rel_df = all_merge_df[attributes_name]
	group_df = all_rel_df.groupby('PLAYER_NAME').sum().reset_index()
	nn_data = group_df.drop('PLAYER_NAME',axis=1)
	model = NearestNeighbors(algorithm = 'ball_tree', n_neighbors=12)
	model.fit(nn_data)
	ind = group_df[group_df['PLAYER_NAME']==player_1].index.tolist()[0]
	indices = model.kneighbors(nn_data)[1]
	df = pd.DataFrame(columns=['Recommended players'])
	def recommend_me(ind,df):
		for i in indices[ind][1:]:
			df = df.append({'Recommended players': (group_df.iloc[i]['PLAYER_NAME'])},ignore_index=True)
		return df
	with col6:
		st.write('Recommended players list of the season')
		st.write(recommend_me(ind,df))

	rel_df = merge_df[attributes]
	sums = rel_df.sum()
	with col1:
		data = [go.Scatterpolar(
  				r = sums,
  				theta = attributes,
  				fill = 'toself',
  				name= str(player_1),
     			line =  dict(
            	color = 'orange'
        		)
			)]

		layout = go.Layout(
  				polar = dict(
    			radialaxis = dict(
      			visible = True,
      			range = [0, 1000]
    			)
  			),
  				showlegend = True,
  				title = "{} stats distribution in {}".format(player_1,player_season)
			)
		fig = go.Figure(data=data, layout=layout)
		st.plotly_chart(fig)
	with col4:
		colors = ['red','orange']*10
		attributes = ['Field Goals Made','Field Goals Attempted','Three Pointers Made','Three Pointers Attempted','Free Throws Made','Free Throws Attempted','Points']
		rel_df = merge_df[attributes]
		sums = rel_df.sum()
		fig = go.Figure(data=[go.Bar(
					name = player_1,
					x = attributes,
					marker_color = colors,
					y = sums
					)
					])
		fig.update_layout(yaxis={'categoryorder':'total ascending'})
		fig.update_layout(height=500,width=550)
		fig.update_layout(
    				title={
        			'text': 'Additional summary stats of the' + ' ' + str(player_1) + ' '+'in ' +' '+ str(player_season),
        			'y':0.98,
        			'x':0.40,
        			'xanchor': 'center',
        			'yanchor': 'top'})
		st.plotly_chart(fig)
	st.subheader('Detailed match by match stats')
	st.write(merge_df)

def show_top_players(games_details_df_new,games_df_new,season):
	merged_df = games_details_df_new.merge(games_df_new, on='GAME_ID')
	season_data = merged_df[merged_df['SEASON'] == season]
	group_df = season_data.groupby('PLAYER_NAME').sum().reset_index()
	group_df['performance_index'] = group_df['Points'] + group_df['Field Goal Percentage'] + group_df['Three Point Percentage'] + group_df['Free Throw Percentage'] + group_df['Offensive Rebounds'] + group_df['Defensive Rebounds'] + group_df['Rebounds'] + group_df['Steals'] + group_df['Assists'] + group_df['Turnovers'] - group_df['Personal Foul']
	group_df.sort_values(by=['performance_index'], ascending=False,inplace=True)
	group_df['performance_index'] = (group_df['performance_index'] - group_df['performance_index'].min()) / (group_df['performance_index'].max() - group_df['performance_index'].min()) * 9 + 0.46
	df1 = group_df[['PLAYER_NAME', 'performance_index']]
	df1.reset_index(drop=True,inplace=True)
	st.write(df1)

def visualize_top_players(games_details_df_new, games_df_new, season, att):
	merged_df = games_details_df_new.merge(games_df_new, on='GAME_ID')
	season_data = merged_df[merged_df['SEASON'] == season]
	group_df = season_data.groupby('PLAYER_NAME').sum().reset_index()
	df = group_df.sort_values(by=[att])
	df.fillna(0,inplace = True)
	fig = px.scatter(df, x='Points' ,y=att, color = "Points", size = att , hover_data=['PLAYER_NAME'])
	fig.update_layout(height=500,width=800)
	fig.update_layout(
    				title={
        			'text': 'Top players of the NBA' + ' ' +' '+'having' +' '+ str(att) +' ' + 'in' +' '+ str(season),
        			'y':0.98,
        			'x':0.40,
        			'xanchor': 'center',
        			'yanchor': 'top'})
	st.plotly_chart(fig)

if __name__ == "__main__":
    main()


from datetime import datetime
from flask import jsonify, request, make_response
from flask_restful import Resource, reqparse
from validators import (verify_token, create_bucketlist,
                        retrieve_bucketlists, retrieve_single_bucketlist,
                        edit_single_bucketlist, delete_single_bucketlist,
                        create_single_bucketlist_item,
                        edit_single_bucketlist_item,
                        delete_single_bucketlist_item)
from flask_restful import marshal_with
from app.data import bucket_list_fields, bucket_item_fields

class LandingPage(Resource):

    def get(self):
        message = 'Welcome to this Bucketlist API'
        return message, 200

class BucketApi(Resource):

    def __init__(self):
        self.bucketlist_parser = reqparse.RequestParser()
        self.bucketlist_parser.add_argument('name', type=str, required=True)
        self.bucketlist_parser.add_argument('date_created', type=datetime)
        self.bucketlist_parser.add_argument('date_modified', type=datetime)
        self.bucketlist_parser.add_argument('created_by', type=int)
        super(BucketApi, self).__init__()

    @marshal_with(bucket_list_fields)
    def post(self):
        '''Creates a new bucket'''
        data = self.bucketlist_parser.parse_args()
        authorization = request.headers.get('access-token')
        if not authorization:
            response = jsonify({'message': 'Invalid login, kindly log in.',
                                'status_code': 403})
            return response
        user = verify_token(authorization)
        user_id = user.get('user_id', None)
        if not user_id:
            response = jsonify({'message': 'No User found'}, {
                               'status_code': 401})
            return response, 401

        creation = create_bucketlist(data, user_id)
        if creation == "No Name":
            response = jsonify({"message": "No Bucket Name"})
            response.status_code = 400
            return response
        elif creation == "Bucket Exists":
            response = jsonify({"message": "Name already used"})
            return make_response(response, 400)
        return creation

    @marshal_with(bucket_list_fields)
    def get(self):
        '''Lists all created buckets'''
        page = request.args.get('page', '1')
        limit = request.args.get('limit', '10')

        q = request.args.get('q', '')
        authorization = request.headers.get('access-token')
        if not authorization:
            response = jsonify({'message': 'Invalid login, kindly log in.',
                                'status_code': 403})
            return response
        user = verify_token(authorization)

        list_buckets = retrieve_bucketlists(q)
        list_buckets = list_buckets.paginate(int(page), int(limit), True).items
        return list_buckets, 200

class BucketRetrieveApi(Resource):

    def __init__(self):
        self.bucket_retrieve_parser = reqparse.RequestParser()
        self.bucket_retrieve_parser.add_argument(
            'name', type=str, required=True)
        self.bucket_retrieve_parser.add_argument('date_created', type=datetime)
        self.bucket_retrieve_parser.add_argument(
            'date_modified', type=datetime)
        self.bucket_retrieve_parser.add_argument('created_by', type=int)
        super(BucketRetrieveApi, self).__init__()

    @marshal_with(bucket_list_fields)
    def get(self, id):
        '''Lists a single bucket'''
        authorization = request.headers.get('access-token')
        if not authorization:
            response = jsonify({'message': 'Invalid login, kindly log in.'})
            return response, 403
        user = verify_token(authorization)

        bucket = retrieve_single_bucketlist(id)
        return bucket, 200

    @marshal_with(bucket_list_fields)
    def put(self, id):
        '''Updates a single bucket'''
        data = self.bucket_retrieve_parser.parse_args()
        authorization = request.headers.get('access-token')
        if not authorization:
            response = {'message': 'Invalid login, kindly log in.'}
            return response, 403
        user = verify_token(authorization)

        bucket = edit_single_bucketlist(id, data)
        return bucket, 200

    @marshal_with(bucket_list_fields)
    def delete(self, id):
        '''Deletes a single bucket'''
        authorization = request.headers.get('access-token')
        if not authorization:
            response = {'message': 'Invalid login, kindly log in.'}
            return response, 403
        user = verify_token(authorization)
        return delete_single_bucketlist(id)

class BucketItemCreate(Resource):

    def __init__(self):
        self.bucket_item_parser = reqparse.RequestParser()
        self.bucket_item_parser.add_argument('name', type=str, required=True)
        self.bucket_item_parser.add_argument('date_created', type=datetime)
        self.bucket_item_parser.add_argument('date_modified', type=datetime)
        self.bucket_item_parser.add_argument('created_by', type=int)
        self.bucket_item_parser.add_argument('done', type=bool)
        super(BucketItemCreate, self).__init__()
        self.new_data = self.bucket_item_parser.parse_args()

    @marshal_with(bucket_item_fields)
    def post(self, id):
        '''Creates a new item in the bucket'''
        authorization = request.headers.get('access-token')
        if not authorization:
            response = jsonify({'message': 'Invalid login, kindly log in.'})
            return make_response(response, 403)
        user = verify_token(authorization)
        return create_single_bucketlist_item(id, self.new_data)

class BucketItemApi(Resource):

    def __init__(self):
        self.bucket_item_parser = reqparse.RequestParser()
        self.bucket_item_parser.add_argument('name', type=str, required=True)
        self.bucket_item_parser.add_argument('date_created', type=datetime)
        self.bucket_item_parser.add_argument('date_modified', type=datetime)
        self.bucket_item_parser.add_argument('created_by', type=int)
        self.bucket_item_parser.add_argument('done', type=bool)
        super(BucketItemApi, self).__init__()
        self.new_data = self.bucket_item_parser.parse_args()

    @marshal_with(bucket_item_fields)
    def put(self, id, item_id):
        '''Update an item in the bucket'''
        authorization = request.headers.get('access-token')
        if not authorization:
            response = jsonify({'message': 'Invalid login, kindly log in.'})
            return make_response(response, 403)
        user = verify_token(authorization)
        return edit_single_bucketlist_item(id, item_id, self.new_data)

    @marshal_with(bucket_item_fields)
    def delete(self, id, item_id):
        '''Delete an item in the bucket'''
        authorization = request.headers.get('access-token')
        if not authorization:
            response = jsonify({'message': 'Invalid login, kindly log in.'})
            return make_response(response, 403)
        user = verify_token(authorization)
        return delete_single_bucketlist_item(id, item_id)


import sqlalchemy
import pandas as pd
print("リスト3.22:DBからDateFrameへの取り込み")
db_url = "sqlite:///iris.db"
sql = "SELECT * FROM iris"
engine = sqlalchemy.create_engine(db_url)
iris_data = pd.read_sql(sql, engine)
print("条件なし")
print(iris_data)
print("加工済みデータをDateFrameに")
sql = """SELECT * FROM iris WHERE Species = '{0}' AND "Petal Width" >= {1}""".format("versicolor", 1.5)
versicolor = pd.read_sql(sql, engine)
versicolor
print("条件指定:種(Species)　versicolor, 花弁の長さ(Petal Width)1.5以上")
print(versicolor)


import tensorflow as tf
import re
import io
import unicodedata

def unicode_to_ascii(s):
  return ''.join(c for c in unicodedata.normalize('NFD', s)
      if unicodedata.category(c) != 'Mn')

def preprocess_sentence(w):
  w = re.sub(r"([?.!,¿])", r" \1 ", w)
  w = re.sub(r'[" "]+', " ", w)

  # replacing everything with space except (a-z, A-Z, ".", "?", "!", ",")
  w = re.sub(r"[^a-zA-Z가-힣?.!,¿]+", " ", w)
  w = w.strip()

  # adding a start and an end token to the sentence
  # so that the model know when to start and stop predicting.
  w = '<start> ' + w + ' <end>'
  return w

def create_dataset(path, num_examples):
  lines = io.open(path, encoding='UTF-8').read().strip().split('\n')
  word_pairs = [[preprocess_sentence(w) for w in l.split('\t')[:2]]  for l in lines[:num_examples]]
  print('word_pairs', word_pairs)

  return zip(*word_pairs)

def tokenize(lang):
    lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='')
    lang_tokenizer.fit_on_texts(lang)
    tensor = lang_tokenizer.texts_to_sequences(lang)
    tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor, padding='post')
    return tensor, lang_tokenizer

def load_dataset(path, num_examples=None):
    # creating cleaned input, output pairs
    targ_lang, inp_lang = create_dataset(path, num_examples)

    input_tensor, inp_lang_tokenizer = tokenize(inp_lang)
    target_tensor, targ_lang_tokenizer = tokenize(targ_lang)

    return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer


import pygame

"""
* Checks for the arithmetic of winning points,
* games, and the match.
"""

def check_point_over(stats, ball, paddles, scoreboard):
    # Ball hits players middle wall
    if ball.rect.left > ball.screen_rect.right:
        stats.enemy_score += 1
        check_win(stats)
        reset_objects(scoreboard, ball, paddles)

    # Ball hits enemy middle wall
    elif ball.rect.right < ball.screen_rect.left:
        stats.player_score += 1
        check_win(stats)
        reset_objects(scoreboard, ball, paddles)

    elif ball.rect.bottom < ball.screen_rect.top or ball.rect.top > ball.screen_rect.bottom:
        # Ball hits players top or bottom walls
        if ball.rect.centerx > ball.screen_rect.centerx:
            stats.enemy_score += 1
            check_win(stats)
            reset_objects(scoreboard, ball, paddles)

        # Ball hits enemy top or bottom walls
        else:
            stats.player_score += 1
            check_win(stats)
            reset_objects(scoreboard, ball, paddles)

def reset_objects(scoreboard, ball, paddles):
    scoreboard.initialize_score_display()
    ball.reset()
    for paddle in paddles:
        paddle.reset()

def check_win(stats):
    # Sound files to play if a game or match is lost/won
    player_lose_wav = pygame.mixer.Sound('Sounds/YouLose.ogg')
    player_win_wav = pygame.mixer.Sound('Sounds/YouWin.ogg')
    player_wingame_wav = pygame.mixer.Sound('Sounds/MarioWin.ogg')
    player_losegame_wav = pygame.mixer.Sound('Sounds/MarioLose.ogg')

    if stats.player_score >= 11 and stats.player_score >= (stats.enemy_score + 2):
        stats.player_wins += 1
        stats.player_score = 0
        stats.enemy_score = 0
        player_win_wav.play(0)
    elif stats.enemy_score >= 11 and stats.enemy_score >= (stats.player_score + 2):
        stats.enemy_wins += 1
        stats.player_score = 0
        stats.enemy_score = 0
        player_lose_wav.play(0)

    if stats.player_wins >= 3 and stats.player_wins >= (stats.enemy_wins + 2):
        stats.game_active = False
        stats.play_again = True
        player_wingame_wav.play(2)

    elif stats.enemy_wins >= 3 and stats.enemy_wins >= (stats.player_wins + 2):
        stats.game_active = False
        stats.play_again = True
        player_losegame_wav.play(0)

def update_all(ball, all_paddles, player_paddles, enemy_paddles):
    ball.update(all_paddles)
    for paddle in player_paddles:
        paddle.update()
    for paddle in enemy_paddles:
        paddle.update(ball)

def draw_all(ball, all_paddles):
    ball.draw()
    for paddle in all_paddles:
        paddle.draw()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan 13 15:56:03 2021

@author: jzimmer1
"""

import os
import re
import json

# === === Helper functions === ===
def write_json(data, filename):
    with open(filename,"w") as f:
        json.dump(data,f)
    return None
def get_json(filename):
    with open(filename) as f:
        jsonobj = json.load(f)
    return jsonobj

# === === === === === === === ===

def delete_extras(filename):
    pathname = filename
    filename = filename.split("/")[-1]
    #filename = filename.split(".")[0]
    #print(filename)
    edit = "?action=edit"
    source = "?action=source"
    if filename.endswith(edit) or filename.endswith(source):
        print("deleting ", filename)
        os.remove(pathname)
    else:
        pass
    return None

def foldercont(foldername):
    allfiles = os.listdir(foldername)
    return allfiles

def compare_folders():
    attemptfiles = foldercont("tvtropes.org-attempt/pmwiki/pmwiki.php/Literature")
    attemptname = "tvtropes.org-attempt/pmwiki/pmwiki.php/Literature/"
    files = foldercont("tvtropes.org-plain/pmwiki/pmwiki.php/Literature")
    filesname = "tvtropes.org-plain/pmwiki/pmwiki.php/Literature/"
    thirdfiles = foldercont("tvtropes.org-third/pmwiki/pmwiki.php/Literature")
    thirdname = "tvtropes.org-third/pmwiki/pmwiki.php/Literature/"
    # for f in thirdfiles:
    #     delete_extras("tvtropes.org-third/pmwiki/pmwiki.php/Literature/" + f)
    # thirdfiles = foldercont("tvtropes.org-third/pmwiki/pmwiki.php/Literature")
    # different_files = []
    # filelist = []
    # for f in attemptfiles:
    #     if f not in files and f not in thirdfiles:
    #         different_files.append(f)
    #         filelist += ["In attempt only: tvtropes.org-attempt/pmwiki/pmwiki.php/Literature",f]
    # for f in files:
    #     if f not in attemptfiles and f not in thirdfiles:
    #         different_files.append(f)
    #         filelist += ["In plain only: tvtropes.org-plain/pmwiki/pmwiki.php/Literature",f]
    # for f in thirdfiles:
    #     if f not in files and f not in attemptfiles:
    #         different_files.append(f)
    #         filelist += ["In third only: tvtropes.org-third/pmwiki/pmwiki.php/Literature",f]
    # write_json(filelist, "difference_in_paths_3_Lit_wget_results_and.json")
    os.mkdir("Literature-Union")
    unionname = "Literature-Union/"
    unionfiles = list(set(attemptfiles + files + thirdfiles))
    write_json(unionfiles, "union_of_3_Lit_wget_results.json")
    for f in unionfiles:
        if f in thirdfiles:
            os.rename(thirdname+f, unionname+f)
        elif f in files:
            os.rename(filesname+f, unionname+f)
        elif f in attemptfiles:
            os.rename(attemptname+f, unionname+f)
        else:
            print("where did this come from: ",f)
    return None

compare_folders()

Literature-Union/Scraping Literature 3 times/delete_extra_files_literature.py


from tests.base import BaseTestCase
from nose.plugins.attrib import attr

from shiftcontent.fields import Text

@attr('fields', 'field_text')
class TextTest(BaseTestCase):

    def test_instantiate_field(self):
        """ Instantiating text field """
        field = Text()
        field.set(None)
        self.assertIsInstance(field, Text)

    def test_get_value(self):
        """ Getting value from text field """
        value = 'i am field value'
        field = Text(value)
        self.assertEquals(value, field.get())
        self.assertTrue(type(field.get()) is str)

    def test_get_db_representation(self):
        """ Getting db representation of text field value """
        value = 'i am field value'
        field = Text(value)
        self.assertEquals(value, field.to_db())
        self.assertTrue(type(field.to_db()) is str)

    def test_populate_from_db(self):
        """ Populating value from db representation for text field"""
        value = 'i am field value'
        field = Text()
        field.from_db(value)
        self.assertEquals(value, field.get())
        self.assertTrue(type(field.to_db()) is str)

    def test_get_json_representation(self):
        """ Getting json representation of text field value """
        value = 'i am field value'
        field = Text(value)
        self.assertEquals(value, field.to_json())
        self.assertTrue(type(field.to_json()) is str)

    def test_populate_from_json(self):
        """ Populating value from json representation for text field"""
        value = 'i am field value'
        field = Text()
        field.from_json(value)
        self.assertEquals(value, field.get())
        self.assertTrue(type(field.to_db()) is str)

    def test_get_search_representation(self):
        """ Getting search representation of text field value """
        value = 'i am field value'
        field = Text(value)
        self.assertEquals(value, field.to_search())
        self.assertTrue(type(field.to_search()) is str)

    def test_get_search_mapping(self):
        """ Getting search index data type for the text field """
        field = Text()
        mapping = field.search_mapping()
        self.assertEquals('text', mapping['type'])


# -*- coding:utf-8 -*-
'''
Author：shelldream@163.com
Date：2015-06-24 09:30:00
Version：0.3.0

    利用微博文档在各个主题下的概率分布作为特征，完成kMeans聚类模块,并对聚类结果进行评价
'''
import sys
reload(sys).setdefaultencoding('utf-8')
import os
sys.path.append("%s/../"%os.path.dirname(os.path.abspath(__file__)))

import json
from config import ID2UID_FILENAME,VEC_IN_LDA_PREFIX,SPLITED_FILES,\
                    TOPIC_NUM,ID2WORD,VEC_IN_TF_IDF
from my_weibo import *
from clustering_evaluation import *

AB_PATH = os.path.dirname(os.path.abspath(__file__)) + '/'
VCLUSTER_PATH = AB_PATH + '../cluto-2.1.2/Linux-i686/vcluster' #vcluster可执行文件的绝对路径

def clustering_pre_process(input_file = VEC_IN_LDA_PREFIX,topic_num=TOPIC_NUM):
    '''
        将vectorsInLDA.json文件转换成vcluster的输入文件
        input:
            input_file:vectorsInLDA.json文件存储路径
        output:
            vcluster的输入文件
    '''
    input_file = input_file + str(topic_num) + '.json'
    if not os.path.exists(input_file):
        with open(SPLITED_FILES,'r') as fr:
            weibo_texts = json.loads(fr.readline())

        weibo_corpus = WeiboCorpus()
        weibo_corpus.LDA_VSM(weibo_texts,topic_num=topic_num)
    else:
        with open(input_file,'r') as fw:
            vector_list = json.loads(fw.readline())

            row_count = len(vector_list)
            column_count = TOPIC_NUM
            nonzero_count = 0

            write_lines = []
            for vector in vector_list:
                nonzero_count += len(vector)
                tmp_list = []
                for coor in vector:
                    tmp = "%s %s" %(coor[0]+1,coor[1])
                    tmp_list.append(tmp)
                write_lines.append(" ".join(tmp_list) + '\n')

        with open('input','w') as fw:
            fw.write("%s %s %s\n"%(row_count,column_count,nonzero_count))
            fw.writelines(write_lines)

def cluto_kmeans_vcluster(input_file,k,vcluster=VCLUSTER_PATH):
    '''
        cluto kmeans聚类
        input：
            k: 聚簇数
            input_file: cluto输入文件路径
            vcluster: cluto vcluster可执行文件路径
        output：
            cluto聚类结果, list
    '''
    # 聚类结果文件, result_file

    result_file = '%s.clustering.%s' % (input_file, k)

    command = "%s -niter=20 %s %s" % (vcluster, input_file, k)
    os.popen(command)

    results = [line.strip() for line in open(result_file)]

    if os.path.isfile(result_file):
        os.remove(result_file)

    if os.path.isfile(input_file):
        os.remove(input_file)

    return results

def id2uid():
    '''
        聚类结果还原到原始UID
    '''
    with open(ID2UID_FILENAME,'r') as fopen:
        mapping = json.loads(fopen.readline())
    return mapping

def kmeans(k,topic_num=TOPIC_NUM):
    '''
        kmeans聚类,k为聚类个数,topic_num为主题建模的主题个数
        输出聚类结果,dict,示例:{类1:[uid1,uid2,uid3],类2:[uid4,uid5]}
    '''
    clustering_pre_process(topic_num=topic_num)
    results = cluto_kmeans_vcluster(input_file='input',k=k) #results是list
    mapping = id2uid() #mapping是list
    clustering_result = {}
    for i in range(len(mapping)):
        try:
            clustering_result[results[i]].append(mapping[i])
        except:
            clustering_result[results[i]] = [mapping[i]]
    return clustering_result

def get_high_freq_words(clustering_result,label,top_k=30):
    '''
        获取label聚类簇的高频词
        input:
            clustering_result:kMeans聚类结果，示例：{'1':[u'323424',u'324235'],'2':[u'256246']}
            label:簇编号，Unicode
            top_k:前k个高频词，默认为100
        output:
            高频词字典,示例：{词1:词频1,词2:词频2……}
    '''
    if not isinstance(label,unicode):
        label = unicode(label)

    mapping = id2uid()
    uid_list = clustering_result[label]

    with open(VEC_IN_TF_IDF,'r') as fr:
        vector_in_tf_idf = json.loads(fr.readline())

    with open(ID2WORD,'r') as fr:
        id2word = json.loads(fr.readline())

    word_tfidf_list = [w_tuple for uid in uid_list for w_tuple in vector_in_tf_idf[mapping.index(uid)]]

    freq_dict = {}
    for w_tuple in word_tfidf_list:
        word = id2word[w_tuple[0]]
        try:
            freq_dict[word] += w_tuple[1]
        except:
            freq_dict[word] = w_tuple[1]

    sorted_dict = sorted(freq_dict.items(),key=lambda x:x[1],reverse = True)

    return sorted_dict[:top_k]

def clustering(cluster_num=20,topic_num=TOPIC_NUM):
    '''
        得到最终聚类结果
        cluster_num：kMeans聚类的聚类个数,默认为10
        topic_num:LDA主题建模是选取的主题个数，默认为150
    '''
    clustering_result = kmeans(cluster_num,topic_num=topic_num)
    topk_word = [get_high_freq_words(clustering_result=clustering_result,label=unicode(i)) for i in range(cluster_num)]#每个类簇的高频词list

    lines = []

    for i in range(cluster_num):
        words = topk_word[i] #第i类的高频词
        uid_list = clustering_result[unicode(i)] #第i类包含的uid列表
        label = "%s:\n"%i
        lines.append(label)
        uid_list_str = ','.join(uid_list)
        lines.append(uid_list_str+'\n')
        for word in words:
            if len(word[0]) >= 2 :#过滤单字词
                    tmp = "%s %s"%(word[0],word[1])
                    lines.append(tmp+'\n')
        lines.append("#"*40+"\n")
    output_file = '%s/../output/LDA_result_%s_%s.txt'%(os.path.dirname(__file__),str(cluster_num),str(topic_num))
    with open(output_file,'w') as fw:
        fw.writelines(lines)

    return clustering_evaluation(clustering_result,VEC_IN_TF_IDF,cluster_num)

if __name__ == '__main__':
    clustering(cluster_num=70,topic_num=500)


import pygame as pg
import sys
sys.path.insert(1,'C:\\Users\\karim\\OneDrive\\Documents\\Code\\PYTHON\\trans-auto')

from components.IconButton2 import IconButton2

class Map :
    def __init__(self,screen,x,y,w,h,connectionToDatabase):

        self.connectionToDatabase = connectionToDatabase
        self.screen = screen
        self.x = x
        self.y = y
        self.w = w
        self.h = h
        self.offsetX = w/2
        self.offsetY = h/2
        self.offsetXTemp = 0
        self.offsetYTemp = 0
        self.zoomCard = 1

        self.fontP = pg.font.Font(None, 15)

        self.ZoomIn = IconButton2(w - 90,
                                h - 50,
                                40,
                                40,
                                icon = "plus.png",
                                bgColor=pg.Color('#848F8B'),
                                border_thikness=1,
                                border_radius=10,
                                nextFunc=self.ZoomInFunc)
        self.ZoomOut = IconButton2(w - 40,
                                h - 50,
                                40,
                                40,
                                icon = "minus.png",
                                bgColor=pg.Color('#848F8B'),
                                border_thikness=1,
                                border_radius=10,
                                nextFunc=self.ZoomOutFunc)

        self.Cards = self.connectionToDatabase.ReadAllRFID()
        self.Links = self.connectionToDatabase.ReadAllLINK()

        self.graphicCards = {}
        self.graphicLinks = {}
        for card in self.Cards:
            self.graphicCards[card['id']] = {
                                                'id':card['id'],
                                                'code':card['code'],
                                                'x': card['x'],
                                                'y': (self.h-card['y']),
                                            }
        for link in self.Links:
            self.graphicLinks[link['id']] = {
                                                'id': link['id'],
                                                'id1': link['first_end'],
                                                'id2': link['second_end'],
                                            }
        self.frame = pg.Rect(x, y, w, h)

        #! MAP MOVE AND ZOOM VARIABLE
        self.down = False
        self.up = True
        self.positionDown = (0,0)
        self.zoom = 1

        # for l in self.graphicLinks :
        #     print(l)
        # for l in self.graphicCards :
        #     print(l)

        # print(self.graphicLinks)

    def ZoomInFunc(self):
        if  self.zoom < 4:
            self.zoom += 0.1

    def ZoomOutFunc(self):
        if  self.zoom >0.1:
            self.zoom -= 0.1

    def update(self):
        self.Cards = self.connectionToDatabase.ReadAllRFID()
        self.Links = self.connectionToDatabase.ReadAllLINK()
        for card in self.Cards:
            self.graphicCards[card['id']] = {
                                                'id':card['id'],
                                                'code':card['code'],
                                                'x': card['x'],
                                                'y': (self.h-card['y']),
                                            }
        for link in self.Links:
            self.graphicLinks[link['id']] = {
                                                'id': link['id'],
                                                'id1': link['first_end'],
                                                'id2': link['second_end'],
                                            }

    def draw(self , ListOfRobots):

        pg.draw.rect(self.screen, "#ffffff", self.frame,
                     0, 0)
        for link in self.Links:
            x1 = int(self.graphicCards[link['first_end']]['x'] * self.zoom + self.offsetX)
            y1 = int((self.graphicCards[link['first_end']]['y']- self.h) * self.zoom + self.h - self.offsetY)
            x2 = int(self.graphicCards[link['second_end']]['x'] * self.zoom + self.offsetX)
            y2 = int((self.graphicCards[link['second_end']]['y'] - self.h) * self.zoom + self.h - self.offsetY)
            pg.draw.line(self.screen,'#000000',(x1,y1),(x2,y2),int(2 * self.zoom))

        for card in self.Cards:
            x = self.graphicCards[card['id']]['x'] * self.zoom + self.offsetX
            y = (self.graphicCards[card['id']]['y'] - self.h) * self.zoom + self.h - self.offsetY
            pg.draw.circle(self.screen,'#dd0000',(x,y),int(5*self.zoom))
            self.screen.blit(
                self.fontP.render(
                                    '{} ({}, {})'.format(card['id'], card['x'], card['y']),
                                    True,
                                    pg.Color('#000000')
                                ),
                (x + 10 , y + 10)
            )
        for robot in ListOfRobots :
            if robot.position :
                x = robot.position[0] * self.zoom + self.offsetX
                y = -1 * robot.position[1] * self.zoom + self.h - self.offsetY
                pg.draw.circle(self.screen, '#00dd00', (x,y), int(4 * self.zoom))

        self.ZoomIn.draw(self.screen)
        self.ZoomOut.draw(self.screen)

    def handleEvent(self,event):
        # print("type : ",event.type,event.type == pg.KEYDOWN)
        # print("zoom : " ,self.zoom)
        self.ZoomIn.handleEvent(event)
        self.ZoomOut.handleEvent(event)
        if event.type == pg.KEYDOWN:
            # print("key",event.key)
            keys = pg.key.get_pressed()  #checking pressed keys
            if  (keys[pg.K_PLUS] or keys[pg.K_KP_PLUS] ) and self.zoom < 4:
                self.zoom += 0.1
            elif (keys[pg.K_MINUS] or keys[pg.K_KP_MINUS] ) and self.zoom > 0.2:
                self.zoom -= 0.1
            else:
                pass
                # Re-render the text.

        if event.type == pg.MOUSEBUTTONDOWN:
            if self.frame.collidepoint(event.pos):
                self.down = True
                self.up = False
                self.positionDown = event.pos
                self.offsetXTemp = self.offsetX
                self.offsetYTemp = self.offsetY

        if event.type == pg.MOUSEBUTTONUP:
            if self.frame.collidepoint(event.pos):
                self.down = False
                self.up = True
                self.offsetXTemp = self.offsetX
                self.offsetYTemp = self.offsetY

        if event.type == pg.MOUSEMOTION and self.down and not self.up :
            self.offsetX = self.offsetXTemp - (self.positionDown[0] - event.pos[0])
            self.offsetY = self.offsetYTemp + (self.positionDown[1] - event.pos[1])


"""
Weights for PySAL Network Module
"""

__author__ = "Sergio Rey <sjsrey@gmail.com>, Jay Laura <jlaura@asu.edu>"

from itertools import combinations
import numpy as np
import pysal as ps
from util import threshold_distance, edge_length, knn_distance

def w_links(wed):
    """
    Generate Weights object for links in a WED

    Parameters
    ----------
    wed: PySAL Winged Edged Data Structure

    Returns

    ps.W(neighbors): PySAL Weights Dict
    """
    nodes = wed.node_edge.keys()
    neighbors = {}
    for node in nodes:
        lnks = wed.enum_links_node(node)
        # put i,j s.t. i < j
        lnks = [tuple(sorted(lnk)) for lnk in lnks]
        for comb in combinations(range(len(lnks)), 2):
            l, r = comb
            if lnks[l] not in neighbors:
                neighbors[lnks[l]] = []
            neighbors[lnks[l]].append(lnks[r])
            if lnks[r] not in neighbors:
                neighbors[lnks[r]] = []
            neighbors[lnks[r]].append(lnks[l])
    return ps.W(neighbors)

def w_distance(wed, threshold, cost=None, alpha=-1.0, binary=True, ids=None):
    '''
    Generate a Weights object based on a threshold
     distance using a WED

    Parameters
    ----------
    wed: PySAL Winged Edged Data Structure
    distance: float network threshold distance for neighbor membership
    cost: defaults to length, can be any cost dicationary {(edge): cost}

    Returns
    -------
    ps.W(neighbors): PySAL Weights Dict
    '''

    if cost is None:
        cost = edge_length(wed)
    if ids:
        ids = np.array(ids)
    else:
        ids = np.arange(len(wed.node_list))
    neighbors = {}
    if binary is True:
        for node in wed.node_list:
            near, pred = threshold_distance(wed, cost, node, threshold)
            neighbors[ids[node]] = near
        return ps.W(neighbors, None, ids)
    elif binary is False:
        weights = {}
        for node in wed.node_list:
            wt = []
            near, pred = threshold_distance(wed, cost, node, threshold)
            near.remove(node)
            neighbors[ids[node]] = near
            for end in near:
                path = [end]
                previous = pred[end]
                while previous != node:
                    path.append(previous)
                    end = previous
                    previous = pred[end]
                path.append(node)
                cum_cost = 0
                for p in range(len(path) - 1):
                    cum_cost += cost[(path[p], path[p + 1])]
                wt.append(cum_cost ** alpha)
            weights[ids[node]] = wt
        return ps.W(neighbors, weights, ids)

def w_knn(wed, n, cost=None, ids=None):
    '''
    Generate w Weights object based on the k-nearest
     network neighbors.

    Parameters
    ----------
    wed: PySAL Winged Edged Data Structure
    n: integer number of neighbors for each node
    cost: defaults to length, can be any cost dictionary

    Returns
    -------
    ps.W(neighbors): PySAL Weights Dict
    '''

    if cost is None:
        cost = edge_length(wed)
    if ids:
        ids = np.array(ids)
    else:
        ids = np.arange(len(wed.node_list))
    neighbors = {}
    for node in wed.node_list:
        neighbors[node] = knn_distance(wed, cost, node, n=n)
    return ps.W(neighbors, id_order=ids)


#! /usr/bin/env python
# s3_list_buckets.py
"""
Lists all keys in the specified bucket, along with each key's metadata.
Saves it all into a sqlite3 database.
"""

import os
import sqlite3
import sys
from boto.s3.connection import S3Connection

def list_bucket(bucket_name, conn):
    create_db_if_necessary(conn)
    s3 = S3Connection()
    bucket = s3.get_bucket(bucket_name)
    for key in bucket.list():
        pk, saved = add_to_db(conn, bucket, key)
        status = 'Inserted'
        if saved == False:
            status = 'Exists - Not Updated'
        print("{0:08d}  {1}  {2}".format(pk, key.name, status))

def add_to_db(conn, bucket, key):
    pk = existing_record_id(conn, key)
    if pk:
        return pk, False  # No need to process this again
    statement = """insert into s3_keys
    (bucket, name, cache_control, content_type, etag,
    last_modified, storage_class, size)
    values (?,?,?,?,?,?,?,?)"""
    c = conn.cursor()
    c.execute(statement, (key.bucket.name, key.name,
                          key.cache_control, key.content_type,
                          key.etag.replace('"', ''),
                          key.last_modified, key.storage_class,
                          key.size))
    conn.commit()
    pk = c.lastrowid
    full_key = bucket.get_key(key.name)
    for k,v in full_key.metadata.iteritems():
        statement = """insert into s3_meta (key_id, name, value)
        values (?,?,?)"""
        conn.execute(statement, (pk, k, v))
        conn.commit()
    c.close()
    return pk, True

def existing_record_id(conn, key):
    exists = "select id from s3_keys where name=? and etag=? and bucket=?"
    c = conn.cursor()
    c.execute(exists, (key.name, key.etag.replace('"', ''), key.bucket.name))
    row = c.fetchone()
    if row and len(row) > 0:
        return row[0]
    return None

def create_db_if_necessary(conn):
    query = """SELECT name FROM sqlite_master WHERE type='table'
    AND name='s3_keys'"""
    c = conn.cursor()
    c.execute(query)
    row = c.fetchone()
    if not row or len(row) < 1:
        print("Creating table s3_keys")
        statement = """create table s3_keys(
        id integer primary key autoincrement,
        bucket text, name text, cache_control text,
        content_type text,
        etag text, last_modified datetime,
        storage_class text, size int)"""
        conn.execute(statement)
        conn.commit()

        print("Creating index ix_name_etag_bucket on s3_keys")
        statement = """create index ix_name_etag_bucket on
        s3_keys(name, etag, bucket)"""
        conn.execute(statement)
        conn.commit()

        print("Creating table s3_meta")
        statement = "create table s3_meta(key_id, name, value)"
        conn.execute(statement)
        conn.commit()
    c.close()

if __name__ == "__main__":
    if not os.path.exists('db'):
        os.mkdir('db')
    conn = sqlite3.connect('db/aptrust_s3.db')
    list_bucket('aptrust.preservation.storage', conn)
    list_bucket('aptrust.preservation.oregon', conn)
    conn.close()


import os
import os.path
import glob
# import data.move_read_test as dm
# import torch
# from torch.utils import data
import cv2
import numpy as np
import tensorflow as tf
from keras.preprocessing import image
import keras
from collections import Counter
import csv
import sys
import time

model = keras.models.load_model(os.path.join('checkpoints', 'Afterwashpart3.018-3.38.hdf5'))

dict_club = {}
with open('dic.csv')as f:
    reader = csv.reader(f, delimiter=',')
    for row in reader:
        dict_club[row[0]] = row[1]

def Eval_Video(mode=None,mcp=None):

    data = get_video()
    total=0
    error_list=[]
    predict_list=[]
    print(data)

    for i in range(len(data)):
        current_video=data[i].split(os.path.sep)[8].split('.')[0]
        predict_videoname=data[i].split(os.path.sep)[8]
        print(current_video)
        pic_path=glob.glob(os.path.join('/home/sk49/workspace/dataset/QIYI_FACE/Test/face_crop','*jpg'))
        final_list=[]
        if(len(pic_path)!=0):
            for x in pic_path:
                if(x.find(current_video) != -1):
                   final_list.append(x)
            if(len(final_list)==0):
                error_list.append(current_video)
            else:
                total+=1
                print(final_list)
                print(time.time())
                predict = Load_predict(final_list)
                predict_list.append([predict_videoname,predict])
                print(time.time())

    with open('predict.csv','w',newline='') as f1:
        writer1 = csv.writer(f1)
        writer1.writerows(predict_list)

    with open('error_video.csv', 'w') as f:
        writer = csv.writer(f)
        writer.writerows(error_list)

    print("总计："+str(total)+'  未识别视频'+str(len(error_list)))

def Load_predict(final_list=None):
    # basemodel=torch.load('')
    # MCP=torch.load('')
    result=[]
    max=0
    for x in final_list:
        img = image.load_img(x, target_size=(224, 224))
        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        x=x/255.0
        preds = model.predict(x)
        predict=np.argmax(preds)
        result.append(predict)

    count=Counter(result).most_common(1)
    max , _ = count[0]

    return list(dict_club.keys())[list(dict_club.values()).index(str(max))]

def get_video(version='01'):
    video_list=[]
    for x in glob.glob(os.path.join('/home/sk49/workspace/dataset/character_recog_iqiyi/test/IQIYI_VID_TEST','*mp4')):
        video_list.append(x)

    # return video_list[start_num:end_num+1]
    return video_list

if __name__ == '__main__':
    # argv = sys.argv
    # print(argv)
    # start_num = 0
    # end_num =172834   #(172835)
    # start_num = int(argv[1])
    # end_num = int(argv[2])
    # print("start:", start_num, "end", end_num)
    Eval_Video()


import matplotlib.pyplotaspltimportnumpyasn
from topolar import topolar
from skimage.data import chelsea
from scipy.ndimage.interpolation import geometric_transform

def topolar(img,order=1):
    max_radius = 0.5*np.linalg.norm(img.shape)

    def transform(coords):
        theta = 2*np.pi*coords[1]/(img.shape[1]-1.)
        radius = max_radius*coords[0]/img.shape[0]
        i = 0.5*img.shape[0]-radius*np.sin(theta)
        j = radius*np.cos(theta)+0.5*img.shape[1]
        return i,j

    polar = geometric_transform(img, transform, order=order)
    rads = max_radius*np.linspace(0,1,img.shape[0])
    angs = np.linspace(0,2*np.pi,img.shape[1])
    return polar, (rads,angs)

img = chelsea()[...,0]/255.
pol, (rads, angs) = topolar(img)
fig,ax = plt.subplots(2, 1, figsize=(6.8))


# http://stackoverflow.com/questions/34649152/neural-network-backpropagation-algorithm-not-working-in-python
import numpy as np
import math
import matplotlib.pyplot as plt

class Neuralnet:
    def __init__(self, neurons, activation):
        self.weights = []
        self.inputs = []
        self.outputs = []
        self.errors = []
        self.rate = 0.5
        self.activation = activation    #sigmoid or tanh

        self.neurons = neurons
        self.L = len(self.neurons)      #number of layers

        eps = 0.12    # range for uniform distribution   -eps..+eps
        for layer in range(len(neurons)-1):
            self.weights.append(np.random.uniform(-eps,eps,size=(neurons[layer+1], neurons[layer]+1)))

    ###################################################################################################
    def train(self, X, Y, iter_count):

        m = X.shape[0] # get batch size

        for layer in range(self.L):
            self.inputs.append(np.empty([m, self.neurons[layer]])) # initialize layers
            self.errors.append(np.empty([m, self.neurons[layer]])) # ??????

            if (layer < self.L -1):
                self.outputs.append(np.empty([m, self.neurons[layer]+1])) # is bias added????
            else:
                self.outputs.append(np.empty([m, self.neurons[layer]]))

        #accumulate the cost function
        J_history = np.zeros([iter_count, 1])

        for i in range(iter_count):

            self.feedforward(X)

            J = self.cost(Y, self.outputs[self.L-1])
            J_history[i, 0] = J

            self.backpropagate(Y)

        #plot the cost function to check the descent
        plt.plot(J_history)
        plt.show()

    ###################################################################################################
    def cost(self, Y, H):
        J = np.sum(np.sum(np.power((Y - H), 2), axis=0))/(2*m)
        return J

    ###################################################################################################
    def feedforward(self, X):

        m = X.shape[0] # get size of training batch

        self.outputs[0] = np.concatenate(  (np.ones([m, 1]),   X),   axis=1) # inputs and bias of the input layer

        for i in range(1, self.L): # for all the layers except the input layer
            self.inputs[i] = np.dot( self.outputs[i-1], self.weights[i-1].T  ) # calculate the input of the layer by multiplying the weights to the outputs of previous layer

            # then calculate the output of the current layer by using the activation function
            if (self.activation == 'sigmoid'):
                output_temp = self.sigmoid(self.inputs[i])
            elif (self.activation == 'tanh'):
                output_temp = np.tanh(self.inputs[i])

            if (i < self.L - 1): # for any layer before the last layer
                self.outputs[i] = np.concatenate(  (np.ones([m, 1]),   output_temp),   axis=1) # fill the output matrix with biases
            else: # for the last layer
                self.outputs[i] = output_temp # don't use biases and only give the output

    ###################################################################################################
    def backpropagate(self, Y):

        self.errors[self.L-1] = self.outputs[self.L-1] - Y # calculate the errors of last layer

        for i in range(self.L - 2, 0, -1):
            # calculate the error of each layer using the errors of the layer after it
            if (self.activation == 'sigmoid'):
                self.errors[i] = np.dot(  self.errors[i+1],   self.weights[i][:, 1:]  ) *  self.sigmoid_prime(self.inputs[i])
            elif (self.activation == 'tanh'):
                self.errors[i] = np.dot(  self.errors[i+1],   self.weights[i][:, 1:]  ) *  (1 - self.outputs[i][:, 1:]*self.outputs[i][:, 1:])

        # update the weights using the calculated errors
        for i in range(0, self.L-1):
            grad = np.dot(self.errors[i+1].T, self.outputs[i]) / m
            self.weights[i] = self.weights[i] - self.rate*grad

    ###################################################################################################
    def sigmoid(self, z):
        s = 1.0/(1.0 + np.exp(-z))
        return s

    ###################################################################################################
    def sigmoid_prime(self, z):
        s = self.sigmoid(z)*(1 - self.sigmoid(z))
        return s

    ###################################################################################################
    def predict(self, X, weights):

        m = X.shape[0];

        self.inputs = []
        self.outputs = []
        self.weights = weights

        for layer in range(self.L):
            self.inputs.append(np.empty([m, self.neurons[layer]]))

            if (layer < self.L -1):
                self.outputs.append(np.empty([m, self.neurons[layer]+1]))
            else:
                self.outputs.append(np.empty([m, self.neurons[layer]]))

        self.feedforward(X)

        return self.outputs[self.L-1]

###################################################################################################
#                MAIN PART

activation1 = 'sigmoid'     # the input should be scaled into [ 0..1]
activation2 = 'tanh'        # the input should be scaled into [-1..1]

activation = activation1

net = Neuralnet([1, 6, 1], activation) # structure of the NN and its activation function

##########################################################################################
#                TRAINING

m = 1000 #size of the training set
X = np.linspace(0, 4*math.pi, num = m).reshape(m, 1); # input training set

Y = np.sin(X) # target

kx = 0.1 # noise parameter
noise = (2.0*np.random.uniform(0, kx, m) - kx).reshape(m, 1)
Y = Y + noise # noisy target

# scaling of the target depending on the activation function
if (activation == 'sigmoid'):
    Y_scaled = (Y/(1+kx) + 1)/2.0
elif (activation == 'tanh'):
    Y_scaled = Y/(1+kx)

# number of the iteration for the training stage
iter_count = 20000
net.train(X, Y_scaled, iter_count) #training

# gained weights
trained_weights = net.weights

##########################################################################################
#                 PREDICTION

m_new = 40 #size of the prediction set
X_new = np.linspace(0, 4*math.pi, num = m_new).reshape(m_new, 1);

Y_new = net.predict(X_new, trained_weights) # prediction

#rescaling of the result
if (activation == 'sigmoid'):
    Y_new = (2.0*Y_new - 1.0) * (1+kx)
elif (activation == 'tanh'):
    Y_new = Y_new * (1+kx)

# visualization
plt.plot(X, Y)
plt.plot(X_new, Y_new, 'ro')
plt.draw()

raw_input('press any key to exit')


import pytest

from rex.core import Rex
from rex.storage import get_storage, StorageError

import util

def setup_function():
    util.reset_storage()

@pytest.fixture(scope="module")
def rex():
    return Rex(
        'rex.storage',
        storage_credentials={
            'local': {
                'key': util.storage_path,
            }
        },
        storage_mount={
            '/test1': 'local://foo',
            '/test2/': 'local://bar/subbar',
        }
    )

@pytest.fixture(autouse=True)
def with_rex(rex):
    with rex:
        yield

def test_mounts():
    storage = get_storage()
    assert len(storage.mounts) == 2
    assert '/test1/' in storage.mounts
    assert '/test2/' in storage.mounts

def test_mount_props():
    mount = get_storage().mounts['/test1/']
    assert mount.path_prefix == '/test1/'
    assert mount.url_prefix == 'local://foo/'
    assert mount.base_path == ''

    mount = get_storage().mounts['/test2/']
    assert mount.path_prefix == '/test2/'
    assert mount.url_prefix == 'local://bar/subbar/'
    assert mount.base_path == 'subbar/'

def test_mount_path():
    mount = get_storage().mounts['/test1/']
    path = mount.path('stuff.csv')
    assert path is not None
    assert path.name == 'stuff.csv'
    assert path.mount == mount

    path = mount.path('some', 'subdir', 'stuff.csv')
    assert path is not None
    assert path.name == 'some/subdir/stuff.csv'
    assert path.mount == mount

def test_path_props():
    mount = get_storage().mounts['/test2/']
    path = mount.path('some', 'subdir', 'stuff.csv')

    assert path.url_prefix == 'local://bar/subbar/'
    assert path.url_full == 'local://bar/subbar/some/subdir/stuff.csv'
    assert path.path_prefix == '/test2/'
    assert path.path_full == '/test2/some/subdir/stuff.csv'
    assert path.file_name == 'stuff.csv'
    assert path.dir_name == 'some/subdir'
    assert path.container_location == 'subbar/some/subdir/stuff.csv'

def test_path_join():
    mount = get_storage().mounts['/test1/']
    path = mount.path('foo')

    path2 = path.join('stuff.csv')
    assert path2.name == 'foo/stuff.csv'

    path2 = path.join('bar', 'stuff.csv')
    assert path2.name == 'foo/bar/stuff.csv'

    path2 = path.join('bar/baz', 'stuff.csv')
    assert path2.name == 'foo/bar/baz/stuff.csv'

def test_path_addoper():
    mount = get_storage().mounts['/test1/']
    path = mount.path('foo')

    path = path + 'bar'
    assert path.name == 'foo/bar'

    path = path + 'baz/stuff.csv'
    assert path.name == 'foo/bar/baz/stuff.csv'

def test_path_pop():
    mount = get_storage().mounts['/test1/']
    path = mount.path('foo/bar/baz.dat')

    path = path.pop()
    assert path.name == 'foo/bar/'

    path = path.pop()
    assert path.name == 'foo/'

    path = path.pop()
    assert path.name == ''

    path = path.pop()
    assert path.name == ''

def test_parse_path():
    path = get_storage().parse_path('/test1/stuff.csv')
    assert path.name == 'stuff.csv'

    path = get_storage().parse_path('/test1/')
    assert path.name == ''

    path = get_storage().parse_path('/test1')
    assert path.name == ''

    path = get_storage().parse_path('local://foo/stuff.csv')
    assert path.name == 'stuff.csv'

    path = get_storage().parse_path('local://foo/')
    assert path.name == ''

    path = get_storage().parse_path('local://foo')
    assert path.name == ''

    path1 = get_storage().parse_path('/test1/stuff.csv')
    path2 = get_storage().parse_path(path1)
    assert path2 == path1

    with pytest.raises(StorageError) as exc:
        path = get_storage().parse_path('/doesntexist')
    assert 'is not configured' in str(exc.value)


import sys
from datetime import datetime
from datetime import timedelta

fp_in = open(sys.argv[1], 'r')
fp_out = open(sys.argv[2], 'w')

upper_time = datetime.strptime('20000101 00:00:03', '%Y%m%d %H:%M:%S')
now_max = -1
for line in fp_in:
    record = line.split(',')
    if record[0]!='EUR/USD':
        print('Data inconsistent')
        exit(1)

    now_time = datetime.strptime(record[1].split('.')[0], '%Y%m%d %H:%M:%S')
    if now_time > upper_time:
        if now_max > 0:
            print(now_max, file=fp_out)
        now_max = float(record[2])
        upper_time = now_time + timedelta(seconds=3)
    now_max = max(now_max, float(record[2]))

fp_in.close()
fp_out.close()


from django.contrib import admin
from django.urls import path, include
from django.urls.conf import re_path
from rest_framework.routers import DefaultRouter
from authapp.views import UserModelViewSet
from todoapp.views import *
from rest_framework.authtoken import views
from rest_framework.permissions import AllowAny
from drf_yasg.views import get_schema_view
from drf_yasg import openapi
from graphene_django.views import GraphQLView

router = DefaultRouter()
router.register('user', UserModelViewSet)
router.register('todo', TodoModelViewSet)
router.register('project', ProjectModelViewSet)
router.register('project-users', ProjectUsersViewSet)

schema_view = get_schema_view(
    openapi.Info(
        title="Library",
        default_version='0.1',
        description="Documentation to out project",
        contact=openapi.Contact(email="admin@admin.local"),
        license=openapi.License(name="MIT License"),
    ),
    public=True,
    permission_classes=[AllowAny],
)

urlpatterns = [
    path('admin/', admin.site.urls),
    path('api-auth/', include('rest_framework.urls')),
    # path('api/', include(router.urls)),
    path('api-token-auth/', views.obtain_auth_token),
    path('api/<str:version>/', include(router.urls)),
    path('swagger/', schema_view.with_ui('swagger',
         cache_timeout=0), name='schema-swagger-ui'),
    re_path('^swagger(?P<format>\.json|\.yaml)$',
            schema_view.without_ui(cache_timeout=0), name='schema-json'),
    path('redoc/', schema_view.with_ui('redoc', cache_timeout=0),
         name='schema-redoc'),
    path("graphql/", GraphQLView.as_view(graphiql=True)),
]


from bottle import request, response, route, run, static_file
from io import BytesIO
import json
from qrcode import make
from werewolf import game

game = game()

@route('/')
def index():
	ip = request.environ.get('REMOTE_ADDR')
	if game.state is game.GAME_INACTIVE:
		if game.host_player is None:
			if 'start' in request.query:
				game.host_player = ip
				game.state = game.GAME_SETUP
				filename = 'setup.html'
			else:
				filename = 'start.html'
		return static_file(filename, root='static/')

@route('/roles/')
def roles():
	return json.dumps(game.roles.keys())

@route('/presets/')
def presets():
	presets = {}
	for (name, roles) in game.presets.iteritems():
		presets[name] = []
		for role in roles:
			presets[name].append({str(role): role.__dict__.values()})
	return json.dumps(presets)

@route('/qr.jpg')
def qr():
	out = BytesIO()
	qr = make('http://%s' % request.get_header('host'))
	response.set_header('Content-type', 'image/jpeg')
	qr.save(out, format='JPEG')
	return out.getvalue()

run(host='0.0.0.0', port=8082)


# coding: utf-8

# In[1]:

#Pull in Packages
get_ipython().run_line_magic('matplotlib', 'inline')
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
#From Scikit Learn
from sklearn import preprocessing
from sklearn.model_selection  import train_test_split, cross_val_score, KFold
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.metrics import roc_curve, auc, confusion_matrix, classification_report

# In[2]:

#Change Directory
import os
path="/Users/emmydoo19/Desktop/BIA6303/"
os.chdir(path)

# In[3]:

#Pulling in Churn Data
Churn=pd.read_csv("Churn_Calls.csv")
Churn.head(10)

# In[4]:

# designate target variable name
targetName = 'churn'
targetSeries = Churn[targetName]
#remove target from current location and insert in column 0
del Churn[targetName]
Churn.insert(0, targetName, targetSeries)
#reprint dataframe and see target is in position 0
Churn.head(10)
Churn.shape

# In[5]:

#EDA
Churn.info()

#Checking to see the data types, colums, and shape of the data.

# In[6]:

Churn.describe()
#Checking the Count, Mean, St Dev, Min, Max, and Quartiles.
#Good quick info to grab and decide if I want to normalize or not.

# In[7]:

Churn.dtypes

#Need to be coginizant of the fact I'm working with Int, Float, and Objects.
#Will have to do some transformation on the ojects to run through any models.

# In[8]:

groupby = Churn.groupby(targetName)
targetEDA=groupby[targetName].aggregate(len)
plt.figure()
targetEDA.plot(kind='bar', grid=False)
plt.axhline(0, color='k')

#Did a quick plot to visualize the data, can see its an unbalanced data set and only 15% of the time does churn happen.
#If the model doesnt do better than 0.85% accuracy the we are better off guessing.

# In[9]:

#Dummy Variables and replacing Null Values

from sklearn import preprocessing
le_dep = preprocessing.LabelEncoder()
Churn['churn'] = le_dep.fit_transform(Churn['churn'])

for col in Churn.columns[1:]:
	attName = col
	dType = Churn[col].dtype
	missing = pd.isnull(Churn[col]).any()
	# discretize (create dummies)
	if dType == object:
		Churn = pd.concat([Churn, pd.get_dummies(Churn[col], prefix=col)], axis=1)
		del Churn[attName]

Median=Churn.median()
Churn = Churn.fillna(Median)

Churn.shape
Churn.head(10)

# In[10]:

#Train Test Split

features_train, features_test, target_train, target_test = train_test_split(
    Churn.iloc[:,1:].values, Churn.iloc[:,0].values, test_size=0.30, random_state=0)

# In[11]:

#check shape of the new data split out

print(features_test.shape)
print(features_train.shape)
print(target_test.shape)
print(target_train.shape)

# In[12]:

#Decision Tree train model.

from sklearn import tree
clf_dt = tree.DecisionTreeClassifier()

#Fit clf to the training data
clf_dt = clf_dt.fit(features_train, target_train)
#Predict clf DT model again test data
target_predicted_dt = clf_dt.predict(features_test)

# In[13]:

#Confusion Matrix, Accuracy Score, and other metrics
print("DT Accuracy Score", accuracy_score(target_test, target_predicted_dt))
print(classification_report(target_test, target_predicted_dt))
print(confusion_matrix(target_test, target_predicted_dt))

# In[14]:

#Change Gini to Entropy in Desicion Tree

from sklearn import tree
clf_dt2 = tree.DecisionTreeClassifier(criterion='entropy')
clf_dt2 = clf_dt2.fit(features_train, target_train)
target_predicted_dt2 = clf_dt2.predict(features_test)

# In[15]:

#Confusion Matrix, Accuracy Score, and other metrics
print("DT Accuracy Score", accuracy_score(target_test, target_predicted_dt2))
print(classification_report(target_test, target_predicted_dt2))
print(confusion_matrix(target_test, target_predicted_dt2))

# In[16]:

#Change Splitter from best to random best

from sklearn import tree
clf_dt3 = tree.DecisionTreeClassifier(splitter='random')
clf_dt3 = clf_dt3.fit(features_train, target_train)
target_predicted_dt3 = clf_dt3.predict(features_test)

# In[17]:

#Confusion Matrix, Accuracy Score, and other metrics
print("DT Accuracy Score", accuracy_score(target_test, target_predicted_dt3))
print(classification_report(target_test, target_predicted_dt3))
print(confusion_matrix(target_test, target_predicted_dt3))

# In[18]:

#Cross Validation for original desicion tree
scores = cross_val_score(clf_dt, features_train, target_train, cv=10)
print("Cross Validation Score for each K",scores)
scores.mean()

# In[19]:

#Knn Model

from sklearn.neighbors import KNeighborsClassifier
model = KNeighborsClassifier(n_neighbors=3)
model = model.fit(features_train, target_train)
predicted_model = model.predict(features_test)

# In[20]:

print("KNN Accuracy Score", accuracy_score(target_test, predicted_model))
print(classification_report(target_test, predicted_model))
print(confusion_matrix(target_test, predicted_model))

# In[26]:

#Cross Validation KNN
KNNscores = cross_val_score(model, features_train, target_train, cv=10)
print("Cross Validation Score for each K",scores)
KNNscores.mean()

#KNN Scores stayed about the same as the orginal model.
#Probably isnt the best model to run for this data set if both run pretty poorly with mariginal changes and CV

# In[29]:

#RF Model
from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score
from sklearn.ensemble import RandomForestClassifier
rf = RandomForestClassifier(n_estimators= 500)
rf.fit(features_train, target_train)
# test random forest model
target_predicted_rf = rf.predict(features_test)
#print accuracy_score(target_test, target_predicted_rf)
target_names = [" No", "Yes"]
print(classification_report(target_test, target_predicted_rf, target_names=target_names))
print("RF Accuracy", accuracy_score(target_test, target_predicted_rf))

# In[31]:

#Cross Validation for RF Forest Model
#Cross Validation RF
RFscores = cross_val_score(rf, features_train, target_train, cv=10)
print("Cross Validation Score for RF",scores)
RFscores.mean()


# flake8: noqa: E402
# pylint: disable=unused-import, redefined-outer-name, missing-function-docstring

import json
from pathlib import Path

import django
import django.conf
import django.core.handlers.exception
import django.db
import django.http
from django.template.loader import render_to_string
import django.test
from django.test.client import MULTIPART_CONTENT

import pytest

import appmap
import appmap.django  # noqa: F401
from appmap.test.helpers import DictIncluding

import sys
sys.path += [str(Path(__file__).parent / 'data' / 'django')]
import app
from .._implementation.metadata import Metadata

# Make sure assertions in web_framework get rewritten (e.g. to show
# diffs in generated appmaps)
pytest.register_assert_rewrite('appmap.test.web_framework')
from .web_framework import TestRequestCapture, TestRecording

@pytest.mark.django_db
def test_sql_capture(events):
    conn = django.db.connections['default']
    conn.cursor().execute('SELECT 1').fetchall()
    assert events[0].sql_query == DictIncluding({
        'sql': 'SELECT 1',
        'database_type': 'sqlite'
    })
    assert events[0].sql_query['server_version'].startswith('3.')
    assert Metadata()['frameworks'] == [{
        'name': 'Django',
        'version': django.get_version()
    }]

@pytest.mark.appmap_enabled
def test_framework_metadata(client, events):  # pylint: disable=unused-argument
    client.get('/')
    assert Metadata()['frameworks'] == [{
        'name': 'Django',
        'version': django.get_version()
    }]

@pytest.mark.appmap_enabled
def test_app_can_read_body(client, events):  # pylint: disable=unused-argument
    response = client.post('/echo', json={'test': 'json'})
    assert response.content == b'{"test": "json"}'

@pytest.mark.appmap_enabled
def test_template(events):
    render_to_string('hello_world.html')
    assert events[0].to_dict() == DictIncluding({
        'path': 'appmap/test/data/django/app/hello_world.html',
        'event': 'call',
        'defined_class': '<templates>.AppmapTestDataDjangoAppHello_WorldHtml',
        'method_id': 'render',
        'static': False
    })

# pylint: disable=arguments-differ
class ClientAdaptor(django.test.Client):
    """Adaptor for the client request parameters used in .web_framework tests."""

    # pylint: disable=too-many-arguments
    def generic(
        self, method, path, data='', content_type='application/octet-stream', secure=False,
        headers=None, json=None, **kwargs
    ):
        headers = {
            'HTTP_' + k.replace('-', '_').upper(): v
            for k, v in (headers or {}).items()
        }

        if json:
            content_type = 'application/json'
            data = self._encode_json(json, 'application/json')

        return super().generic(method, path, data, content_type, secure, **headers, **kwargs)

    def post(self, path, data=None, content_type=MULTIPART_CONTENT,
             secure=False, **extra):
        if content_type == 'multipart/form-data':
            content_type = MULTIPART_CONTENT
        return super().post(path, data, content_type, secure, **extra)

@pytest.fixture
def client():
    return ClientAdaptor()

@pytest.mark.appmap_enabled
def test_unnamed(client, events):
    client.get('/post/unnamed/5')

    assert appmap.enabled()     # sanity check
    # unnamed captures in a re_path don't show up in the event's
    # message attribute.
    assert len(events[0].message) == 0

@pytest.mark.appmap_enabled
def test_included_view(client, events):
    client.get('/post/included/test_user')

    assert len(events) == 2
    assert events[0].http_server_request == DictIncluding({
        'path_info': '/post/included/test_user',
        'normalized_path_info': '/post/included/{username}'
    })

@pytest.mark.appmap_enabled
def test_exception(client, events, monkeypatch):
    def raise_on_call(*args):
        raise RuntimeError('An error')
    monkeypatch.setattr(django.core.handlers.exception, 'response_for_exception', raise_on_call)

    with pytest.raises(RuntimeError):
        client.get('/exception')

    assert events[0].http_server_request == DictIncluding({
        'request_method': 'GET',
        'path_info': '/exception',
        'protocol': 'HTTP/1.1'
    })

    assert events[1].parent_id == events[0].id
    assert events[1].exceptions == [DictIncluding({
            'class': 'builtins.RuntimeError',
            'message': 'An error'
    })]

@pytest.mark.appmap_enabled
def test_deeply_nested_routes(client, events):
    client.get('/admincp/permissions/edit/1')

    assert len(events) == 2
    assert events[0].http_server_request == DictIncluding({
        'normalized_path_info': '/admincp/permissions/edit/{pk}'
    })

def test_middleware_reset(pytester, monkeypatch):
    monkeypatch.setenv('PYTHONPATH', 'init')
    monkeypatch.setenv('APPMAP', 'true')

    pytester.copy_example('django')

    # To really check middleware reset, the tests must run in order,
    # so disable randomly.
    pytester.runpytest('-svv', '-p', 'no:randomly')

    # Look for the http_server_request event in test_app's appmap. If
    # middleware reset is broken, it won't be there.
    appmap_file = pytester.path / 'tmp' / 'appmap' / 'pytest' / 'test_app.appmap.json'
    events = json.loads(appmap_file.read_text())['events']
    assert 'http_server_request' in events[0]


# -*-coding:utf-8 -*-

from selenium import webdriver
from config.VarConfig import *
from util.ObjectMap import *
from util.ClipboardUtil import *
from util.KeyBoardUtil import *
from util.DirAndTime import *
from util.WaitUtil import *
from selenium.webdriver.chrome.options import Options
import time

"定义全局driver变量"
driver=None
"全局的等待类实例对象"
waitUtil=None

def open_browser(browserName,*args):
    global driver,waitUtil
    try:
        if browserName.lower() == "ie":
            driver=webdriver.Ie(executable_path=ieDriverFilePath)
        elif browserName.lower() == "chrome":
            # "driver对象"
            driver=webdriver.Chrome(executable_path=chromeDriverFilePath,chrome_options=chrome_options)
            # driver=webdriver.Chrome(executable_path=chromeDriverFilePath)
        else:
            driver=webdriver.Firefox(executable_path=firefoxDriverFilePath)
        "driver对象创建成功后，创建等待类实例对象"
        waitUtil = WaitUtil(driver)
    except Exception as e:
        raise e

def visit_url(url,*args):
    "访问某个网址"
    global  driver
    try:
        driver.get(url)
    except Exception as e:
        raise e

def close_browser(*args):
    "关闭浏览器"
    global driver
    try:
        driver.quit()
    except Exception as e:
        raise e

def sleep(sleepSeconds,*args):
    "强制等待"
    try:
        time.sleep(int(sleepSeconds))
    except Exception as e:
        raise e

def clear(locationType,locatorExpression,*args):
    "清除输入框默认内容"
    global  driver
    try:
        getElement(driver,locationType,locatorExpression).clear()
    except Exception as e:
        raise e

def input_string(locationType,locatorExpression,inputContent):
    "在页面输入框输入数据"
    global  driver
    try:
        getElement(driver,locationType,locatorExpression).send_keys(inputContent)
    except Exception as e:
        raise e

def click(locationType,locatorExpression,*args):
    "点击页面元素"
    global driver
    try:
        getElement(driver,locationType,locatorExpression).click()
    except Exception as e:
        raise e

def assert_string_in_pagesource(assertString,*args):
    "断言页面源码是否存在某关键字或关键字符串"
    global  driver
    try:
        assert assertString in driver.page_source,"{s} not found in page source!".format(s=assertString)
    except AssertionError as e:
        raise AssertionError(e)
    except Exception  as e:
        raise e

def assert_title(titleStr,*args):
    "断言页面标题是否存在给定的关键字符串"
    global  driver
    try:
        assert titleStr in driver.title,"{s} not found in title!".format(s=titleStr)
    except AssertionError as e:
        raise AssertionError(e)
    except Exception as e:
        raise e

def getTitle(*args):
    "获取页面标题"
    global  driver
    try:
        return driver.title
    except Exception as e:
        raise e

def getPageSource(*args):
    "获取页面源码"
    global  driver
    try:
        return driver.page_source
    except Exception as e:
        raise e

def switch_to_frame(locationType,frameLocatorExpression,*args):
    "切换进入frame"
    global  driver
    try:
        driver.switch_to.frame(getElement(driver,locationType,frameLocatorExpression))
    except Exception as e:
        raise e

def switch_to_default_content(*args):
    "切出frame,回到默认对话框中"
    global  driver
    try:
        driver.switch_to.default_content()
    except Exception as e:
        raise e

def paste_string(pasteString,*args):
    "模拟ctrl+v操作"
    try:
        Clipboard.setText(pasteString)
        time.sleep(2) # 等待2秒，防止代码执行的太快，而未成功粘贴内容
        KeyboardKeys.twoKeys("ctrl","v")
    except Exception as e:
        raise e

def press_tab_key(*args):
    "模拟tab键"
    try:
        KeyboardKeys.oneKey("tab")
    except Exception as e:
        raise e

def press_enter_key(*args):
    "模拟enter键"
    try:
        KeyboardKeys.oneKey("enter")
    except Exception as e:
        raise e

def maximize_browser():
    "窗口最大化"
    global  driver
    try:
        driver.maximize_window()
    except Exception as e:
        raise e

def capture_screen(*args):
    "截取屏幕图片"
    global  driver
    # 获取当前时间，精确到毫秒
    currTime=getCurrentTime()
    # 拼接异常图片保存的绝对路径及名称
    picNameAndPath=str(createCurrentDateDir())+"\\"+str(currTime)+".png"
    try:
        "截取屏幕图片，并保存为本地文件"
        driver.get_screenshot_as_file(picNameAndPath.replace("\\",r"\\"))
    except Exception as e:
        raise e
    else:
        return picNameAndPath

def waitPresenceOfElementLocated(locationType,locatorExpression,*args):
    "显示等待页面元素出现在DOM中，不一定可见，存在则返回该页面元素对象"
    global waitUtil
    try:
        waitUtil.presenceOfElementLocated(locationType,locatorExpression)
    except Exception as e:
        raise e

def waitFrameToBeAvailableAndSwitchToIt(locationType,locatorExpression,*args):
    "检查frame是否存在，存在则切换进frame控件中"
    global waitUtil
    try:
        waitUtil.frameToBeAvailableAndSwitchToIt(locationType,locatorExpression)
    except Exception as e:
        raise e

def waitVisibilityOfElementLocated(locationType,locatorExpression,*args):
    "显示等待页面元素出现在DOM中，并且可见，存在返回该页面元素对象"
    global waitUtil
    try:
        waitUtil.visibilityOfElementLocated(locationType,locatorExpression)
    except  Exception as e:
        raise e

if __name__=="__main__":
    open_browser("chrome")
    visit_url("http://baidu.com")
    capture_screen()


import tweepy
import time

print('Hello there')

CONSUMER_KEY = ''
CONSUMER_SECRET = ''
ACCESS_KEY = ''
ACCESS_SECRET = ''

auth = tweepy.OAuthHandler(CONSUMER_KEY, CONSUMER_SECRET)
auth.set_access_token(ACCESS_KEY, ACCESS_SECRET)
api = tweepy.API(auth)

FILE_NAME = 'last_id.txt'

def retrieve_last_id(file_name):
    f_read = open(file_name, 'r')
    last_id = int(f_read.read().strip())
    f_read.close()
    return last_id

def save_last_id(last_id, file_name):
    f_write = open(file_name, 'w')
    f_write.write(str(last_id))
    f_write.close()

    return

def answer_tweets():
    print('Answering all the remaining tweets...')

last_id = retrieve_last_id(FILE_NAME)
menciones = api.mentions_timeline(last_id, tweet_mode='extended')

for mention in reversed(menciones):
    print(str(mention.id) + ' - ' + mention.full_text)
    last_id = mention.id
    save_last_id(last_id, FILE_NAME)

    if 'words to find' in mention.full_text:
        print('Magic words found...')
        print('Replying...')
        api.update_status('@' + mention.user.screen_name +
                          ' Hello, I am the PyTwiBot ', mention.id)

while True:
    answer_tweets()
    time.sleep(20)


import sqlite3

def getCursor():
	conn = sqlite3.connect("nana.db")
	c = conn.cursor()
	return c

def createTable(c):
# create tables
	c.execute('''CREATE TABLE postMess
		(postNumber int(10) primary key,
		title char(200),
		commentNum int(10),
		readNum int(10)''')

	c.execute('''CREATE TABLE post
	    (postNumber int(10) primary key,
		isPost	boolean,
		uid int(10),
		name char(30),
		postTime dateTime,
		content text,
	    FOREIGN KEY (postNumber) REFERENCES postMess(postNumber))''')

def commit(conn):
# save the changes
	conn.commit()

def insert(c,table,postNumber,title,commentNum,readNum):
	c.execute("INSERT INTO "+table+" VALUES (postNumber,title,commentNum,readNum)")
	commit(c)

def select(c,table):
	c.execute('SELECT * FROM '+table)
	fd=open('postMess.txt','w')
	string=[]
	for item in c.fetchall():
		# try:
			fd.writelines(str(item)+"\n")
		# except e:
		# 	print("something error : "+e.reason)

c=getCursor()
select(c,"postMess")
print("ok")


from collections import OrderedDict, defaultdict
from .utils import *
from .location import *
from cyvcf2 import VCF

vep_plugins = ["ReMM", "Gwava", "ExACpLI", "dpsi_zscore", "ada_score", "rf_score",
               'MaxEntScan_alt', 'MaxEntScan_diff', 'MaxEntScan_ref', 'SpliceRegion',
               'GERP', 'phyloP', '29way_logOdds', 'phastCons',
               'MutationAssessor_score', 'SIFT_score', 'Polyphen2_HDIV_score',
               'Polyphen2_HVAR_score', 'MutationTaster_score', 'MutationTaster_pred',
               'MutationTaster_model', 'MutationTaster_AAE', 'FATHMM_score', 'fathmmMKL',
               'PROVEAN_score', 'PROVEAN_pred', 'CADD_PHRED',
               'DANN', 'Eigen', 'Eigen-PC', 'Eigen-pred_coding', 'Eigen-PC-phred_coding', 'MetaSVM_score', 'MetaSVM_pred', 'MetaLR_score',
               'MetaLR_pred', 'REVEL_score', 'fitcons', 'LRT_score',
               'LRT_pred', 'MutPred_score', 'Condel', 'CAROL',
               'VEST4_score','M-CAP_score','SpliceAI', 'linsight_g', 'traP',
               'funseq2', ]

plugins_map = {'CADD_phred':'CADD_PHRED',
               'GERP++_RS':'GERP',
               'phyloP20way_mammalian': 'phyloP', #new_is_100way
               'phastCons20way_mammalian':'phastCons', #new_is_100way
               '29way_logOdds': 'SiPhy',
               'integrated_fitCons_score' : 'fitcons',
                'VEST3_score': 'VEST4_score',
                'fathmm-MKL_coding_score': 'fathmmMKL',
               'Eigen-PC-phred':'Eigen-PC-phred_coding',
                'Eigen-phred': 'Eigen-pred_coding',
               'GWAVA': 'Gwava',
                }

def process_vep_scores(vcf_file, is_clinvar=False):
    indexes, scores, absent = OrderedDict(), defaultdict(list), []
    vcf_data = VCF(vcf_file)
    if vcf_data.contains("ANN") or vcf_data.contains("CSQ"):
        spliceai_scores_fromtool=False
        for field in vcf_data.header_iter():
            if field["HeaderType"] == "INFO" and field["ID"] == "ANN":
                tools = field["Description"].split("Format:")[1][:-1].strip().split("|")
                for plugin in vep_plugins:
                    if plugin in tools:
                        indexes[plugin] = tools.index(plugin)
                    else:
                        absent.append(plugin)
            elif field["HeaderType"] == "INFO" and field["ID"] == "SpliceAI":
                spliceai_scores_fromtool=True

        for record in vcf_data:
            key = record.ID + "_" + str(record.POS)
            scores[key].extend([record.CHROM,record.POS, record.REF, record.ALT, record.ID, record.var_type, record.var_subtype])
            annotation = record.INFO.get("ANN")
            if annotation:
                info = annotation.split(",")[0].split("|")

                try:
                    scores[key].append(info[tools.index('Existing_variation')])
                    scores[key].append(info[tools.index('HGVSc')])
                    scores[key].append(info[tools.index('Consequence')])
                except ValueError:
                    pass
                if len(absent) > 0:
                    scores[key].append(("GERP", record.INFO.get("GERP")))
                    scores[key].append(("phyloP", record.INFO.get("phyloP")))
                    scores[key].append(("phastCons", record.INFO.get("phastCons")))
                    scores[key].append(("SiPhy", record.INFO.get("SiPhy")))
                    scores[key].append(("fitcons", record.INFO.get("fitcons")))
                    scores[key].append(("LINSIGHT", record.INFO.get("linsight_g")))
                    scores[key].append(("ReMM", record.INFO.get("ReMM")))
                    scores[key].append(("DANN", record.INFO.get("DANN")))
                    scores[key].append(("GWAVA", record.INFO.get("GWAVA")))
                    scores[key].append(("FATHMM-MKL", record.INFO.get("fathmmMKL")))
                    scores[key].append(("Eigen", record.INFO.get("Eigen")))
                    scores[key].append(("Eigen-PC", record.INFO.get("Eigen-PC")))
                    scores[key].append(("funseq2", record.INFO.get("funseq2")))
                    scores[key].append(("dpsi_zscore", record.INFO.get("dpsi_zscore")))
                    scores[key].append(("traP", record.INFO.get("traP")))
                    if spliceai_scores_fromtool:
                        scores[key].append(("SpliceAI", record.INFO.get("SpliceAI")))
                    else:
                        scores[key].append(("SpliceAI", format_spliceai_fields(record)))

                for pl, i in indexes.items():
                    scores[key].append((pl, info[i]))

            if is_clinvar:
                scores[key].append(('CLNREVSTAT', record.INFO.get("CLNREVSTAT")))
                scores[key].append(('CLNSIG', record.INFO.get("CLNSIG")))

    else:
        print("Program requires ANN field to be present in the INFO field of the input VCF file.\n")
        exit(1)

    for k, v in scores.items():
        [val[1] for val in v if isinstance(val,tuple) and val[0] == "SpliceAI"]

    vcf_data.close()
    return scores

def get_df_ready(vcf,isclinvar,isbenign,loc):
    scores = process_vep_scores(vcf,isclinvar)
    df = pd.DataFrame.from_dict(scores, orient='index')
    new_col_names = ['hg19.chr', 'hg19.pos', 'ref', 'alt', 'id','type', 'subtype', 'rsID','HGVSc','Consequence']
    for column in df:
        if isinstance(df[column].iloc[0], (tuple,)):
            new_col_names.append(df[column].iloc[0][0])
            df[column] = df[column].map(tuple2float)

    rename_dict = {i: j for i, j in zip(list(df), new_col_names)}
    df.rename(columns=rename_dict, inplace=True)

    if loc == "HGVSc":
        hp = hgvs.parser.Parser()
        df['location'] = df['HGVSc'].apply(get_location, hp=hp)

    else:
        df['location'] = df['Consequence'].apply(get_location_from_consequence)

    df['class'] = False if isbenign else True
    return df.reset_index()