from datetime import datetime
import pandas as pd
from sklearn.model_selection import train_test_split

def create_features(source_df):
    '''
    Function to create features based on input dataframe with columns 'First Name' & 'Last Name Initial'

    Args:
        source_df (DataFrame): Input DataFrame with columns 'First Name' and 'Last Name Initial'.

    Returns:
        DataFrame: DataFrame object with features included.

    IMPORTANT: Make sure to add feature to the model.features in params.yml if desired to be used in the model.
    '''
    # Calculate the length of the first name and add a column
    source_df['Length of First Name'] = source_df['First Name'].apply(len)

    # Calculate the distance between the first letters of the first and second columns and add a column
    source_df['Distance'] = source_df.apply(lambda row: abs(ord(row['First Name'][0]) - ord(row['Last Name Initial'])), axis=1)

    return source_df

def create_target(features_df, coolness_factor):
    '''
    Create target variable based on 'Coolness' column.

    Args:
        features_df (DataFrame): DataFrame with the 'Coolness' column.
        coolness_factor (float): Threshold value to determine if a value is considered 'Cool'.

    Returns:
        DataFrame: DataFrame with the specified target variable.

    IMPORTANT: Make sure to add target to the model.target in params.yml if desired to be used in the model.
    '''
    features_df['is_Cool'] = features_df['Coolness'].apply(lambda x: 1 if x > coolness_factor else 0)
    return features_df

def process_csv(data_source_filepath, coolness_factor):
    '''
    Read CSV data, create features, and generate the target variable.

    Args:
        data_source_filepath (str): Filepath to the CSV data source.
        coolness_factor (float): Threshold value to determine if a value is considered 'Cool'.

    Returns:
        DataFrame: Processed DataFrame with features and target variable.

    '''
    # Read the CSV file into a pandas DataFrame
    source_df = pd.read_csv(data_source_filepath)

    # Add features
    features_df = create_features(source_df)

    # Add Classifier Categories
    full_df = create_target(features_df, coolness_factor)

    return full_df

def save_train_test_split(df, test_size, random_state, output_path):
    '''
    Split data into train and test sets, save them as CSV files, and return the filepaths.

    Args:
        df (DataFrame): Input DataFrame to be split into train and test sets.
        test_size (float): Proportion of the dataset to include in the test set.
        random_state (int): Seed for the random number generator.
        output_path (str): Directory path to save the train and test data.

    Returns:
        str: Filepath of the saved train data.
        str: Filepath of the saved test data.
    '''

    # Split the data into train and test sets
    train, test = train_test_split(df, test_size=test_size, random_state=random_state)

    # Get current datetime to append to output path
    current_datetime = datetime.now()
    formatted_datetime = current_datetime.replace(microsecond=0).isoformat()

    # Generate train and test data path
    train_data_path = f'{output_path}/train_{formatted_datetime}.csv'
    test_data_path = f'{output_path}/test_{formatted_datetime}.csv'

    # Save data to output path for modeling usage
    train.to_csv(train_data_path, sep=",", index=False, encoding="utf-8")
    test.to_csv(test_data_path, sep=",", index=False, encoding="utf-8")

    return train_data_path, test_data_path

if __name__ == '__main__':
    import argparse
    from helpers import read_params

    args = argparse.ArgumentParser()
    args.add_argument("--config", default="params.yml")
    parsed_args = args.parse_args()

    config = read_params(parsed_args.config)

    dataframe = process_csv(config['data_generator']['output_filepath'], config['data']['coolness_factor'])
    print(dataframe)


import tweepy
from kafka import KafkaProducer
from json import dumps
import json

with open('config.json') as json_file:
    data = json.load(json_file)

producer = KafkaProducer(bootstrap_servers=['localhost:9092'], value_serializer=lambda K: dumps(K).encode('utf-8'))
#producer = KafkaProducer(bootstrap_servers='localhost:9092')

bearer_token = data['BEARER_TOKEN']
client = tweepy.Client(bearer_token)

elon_musk_november_tweets_query = '#elonmusk -is:retweet lang:en'
elon_musk_november_tweets = tweepy.Paginator(
    client.search_recent_tweets,
    query=elon_musk_november_tweets_query,
    tweet_fields=['context_annotations', 'created_at'],
    max_results=100).flatten(limit=20000)

for tweet in elon_musk_november_tweets:
    timestamp = tweet.created_at
    tweet_text = tweet.text
    #tweet_data = [timestamp, tweet_text]
    print(tweet.created_at, tweet.text)
    producer.send('finalproject', tweet.text)
    #producer.send('finalproject', value=b'[timestamp, tweet_text]' % ([timestamp, tweet_text]))
    #producer.send('finalproject', value=b'timestamp, tweet_text', key=b'timestamp, tweet_text' % (timestamp, tweet_text))


"""
Build the text processing pipeline

Remove this
"""
import argparse
import os

import config

import pandas as pd
import texthero as th
from texthero import stopwords, preprocessing

STOPWORDS = stopwords.DEFAULT
STOPWORDS = STOPWORDS.union(
    set([
        "article",
        "page",
        "wikipedia",
        "edit",
        "use",
        "one",
        "source",
        "like",
        "please",
        "thank",
        "good",
        "time",
        "link"
    ])
)

# custom pipelines
PIPELINE_CLEAN = [
    lambda s: th.fillna(s),
    lambda s: th.lowercase(s),
    lambda s: th.remove_digits(s, only_blocks=False),
    lambda s: th.remove_punctuation(s),
    lambda s: th.remove_brackets(s),
    lambda s: th.remove_diacritics(s),
    lambda s: th.remove_html_tags(s),
    lambda s: th.remove_urls(s),
    lambda s: th.remove_whitespace(s),
    lambda s: th.remove_stopwords(s, STOPWORDS),
    lambda s: th.stem(s) # snowball by default
]

PIPELINE_TOKENS = [
    lambda s: th.tokenize(s)
]

def clean_text(data, feature):
    """
    Clean text
    :param data: dataframe
    :param feature: feature with text
    :returns: dataframe
    """

    clean = config.CLEANED_TEXT

    # get data
    input_filename = data + ".csv"
    df = pd.read_csv(os.path.join("../data", input_filename))

    # clean text
    df[clean] = df[feature].pipe(th.clean, PIPELINE_CLEAN)
    # tokenize into new feature
    df['clean_tokens'] = df[feature].pipe(th.clean, PIPELINE_TOKENS)

    return df

if __name__ == '__main__':

    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--data",   # name of data file
        type=str
    )
    parser.add_argument(
        "--feature",   # name of text feature
        type=str
    )

    args = parser.parse_args()

    # get preprocessed data
    cleaned = clean_text(args.data, args.feature)

    # store the file
    filename = args.data + "_clean.csv"
    cleaned.to_csv(os.path.join("../data", filename), index=False)

    print("Cleaned data stored in the `data` folder")


#encoding:utf8
from flask import  Flask, render_template
from flask.ext.bootstrap import Bootstrap

#安装 pip install flssk-wtf
#加载 form表单
from flask.ext.wtf import Form

#加载 StringField文本字段,SubmitField提交按钮
from wtforms import StringField, SubmitField

#加载 验证函数
from wtforms.validators import Required

app = Flask(__name__)
bootstrap = Bootstrap(app)

#跨站请求伪造保护 密钥
app.config['SECRET_KEY'] = 'linuhub'

#定义表单类
class NameForm(Form):
 	name = StringField('What is you name? ', validators=[Required()])
 	submit = SubmitField('Submit')

#路由方法
@app.route('/', methods=['GET', 'POST'])

#在视图函数中处理表单
def index():
	name = None
	form = NameForm()
	if form.validate_on_submit():
		name = form.name.data
		form.name.data = ''
	return render_template('index.html', form=form, name=name)

if __name__ == '__main__':
	app.run(debug=True)


from sklearn.ensemble import AdaBoostClassifier
from sklearn.datasets import make_classification
import numpy as np
from sklearn import metrics
from random import randrange, uniform, randint
import pandas as pd
from sklearn.ensemble import RandomForestClassifier

arr = []
arr2 = []
arr3 = []
train = open("train_features.csv")
label = open("train_label.csv")
test = open("test_features.csv")

submission = open("sample_Submission.csv")
df = pd.read_csv(submission)
df.index = df.index + 1

try:
    for line in train:
        line = line.strip()
        line_arr = line.split(",")
        arr.append(line_arr)
finally:
    train.close()

try:
    for line in label:
        line = line.strip()
        line_arr = line.split(",")
        arr2.append(line_arr)
finally:
    label.close()

try:
    for line in test:
        line = line.strip()
        line_arr = line.split(",")
        arr3.append(line_arr)
finally:
    test.close()

arr2.pop(0)

def normalize(features):
    minmax = []
    for i in range(features.shape[1]):
        col = features[:,i]
        min_value = np.min(col)
        max_value = np.max(col)
        minmax.append([min_value,max_value])
    for row in features:
        for i in range(len(row)):
            row[i] = (row[i]-minmax[i][0])/(minmax[i][1]-minmax[i][0])
    return features

input_data = np.array(arr).astype(np.float)
input_data = input_data[:,1:]
print(input_data.shape)
labels = np.array(arr2).astype(np.int)
labels = labels[:,1:]

labels = labels.reshape(418,)
test_data = np.array(arr3).astype(np.float)
test_data = test_data[:,1:]

#clf = AdaBoostClassifier(n_estimators=750, random_state=0, base_estimator=RandomForestClassifier(n_estimators=100, max_depth=2,random_state=0)) #0.86505
clf = AdaBoostClassifier(n_estimators=750, random_state=0, base_estimator=RandomForestClassifier(n_estimators=95, max_depth=2,random_state=0))

model = clf.fit(input_data,labels)
test_pred = model.predict(test_data)
df["label"] = test_pred

df.to_csv("submission.csv", index=False)


import sys
from rich import print
import random
import math

###############################################################################
# round_half_up()
#  Function meant to perform rounding operations in the same way
#    they are normally performed in mathematics
#  -Round UP when point is >= 5
#  -Round DOWN when point is <= 4
def round_half_up(n, decimals=0):
  multiplier = 10 ** decimals
  return math.floor(n*multiplier + 0.5) / multiplier

###############################################################################
# Class serves to represent singular dice of any size
#
# Data Members:
#   self.sides    -> Variable to hold number of sides for Die
# Methods:
#   __init__ |> Die()  -> Constructs Die object with 6 sides
#            |> Die(s) -> Constructs Die object with (s) sides
#   __str__  -> Allows for easy outputting of Die object
#                (Format: "[1d{Number of Sides}]")
#   roll()   -> Returns random value between 1 and {Number of Sides}
class Die(object):
  #############################################################################
  # __init__
  #  []Checks to see if 'sides' is an int
  #    ()If NO, output error message and end program
  #    ()If YES, 'self.sides' is initialized to 'sides'
  def __init__(self,sides=6):
    if not isinstance(sides, int):
      print("Error! Improper 'Die' Construction Syntax")
      print("-Please refer to documentation")
      sys.exit()
    self.sides = sides

  #############################################################################
  # roll()
  #  []Build list containing values [0, ..., (sides-1)]
  #  []Shuffle List
  #  []Return first value in list
  def roll(self):
    values = [x for x in range(self.sides)]
    random.shuffle(values)
    return values[0] + 1

  #############################################################################
  # __str__
  #  []Output Die info
  #    Format: "[1d" + {Number of Sides} + "]"
  # print(Die(6)) -> "[1d6]"
  def __str__(self):
    return f"[1d{self.sides}]"
###############################################################################

###############################################################################
# Class serves as a container for multiple Die() objects that can
#    then perform various functions using those objects
#
# Data Members:
#   self.dice[]  -> Contains each Die() that makes up Dice()
#   self.rolls[] -> Contains the current roll(s) of each Die
# Methods:
#   __init__ |> Dice()    -> Constructs Dice object containing [1] Die()
#            |> Dice(s,n) |> (if s is int)   -> Constructs Dice object
#                                                containing [n] Die(s)
#                         |> (elif s is str) -> Constructs Dice object from `s`
#                                            -> 's' must be formatted in the
#                                                same way as ("1.d.6")
#   __str__    -> Allows for easy outputting of Dice object
#                  Format: "Die() - ( Die.Roll() )"
#   rollDice() -> Rolls each Die from Dice, storing the value from each roll
#                  within Dice.rolls[]
#   avg()      -> Returns the average of self.rolls[]
#   max()      -> Returns MAXIMUM value within self.rolls[]
#   min()      -> Returns MINIMUM value within self.rolls[]
#   sum()      -> Returns SUM of self.rolls[]
#   roll(rT)   -> Performs self.rollDice()
#              \> (if rT is keyword) -> Return value based on keyword
#                      [Keywords: "avg", "max, "min", "sum"]
class Dice:
  #############################################################################
  # __init__
  #  []Checks to see if 'sides' is a str
  #    ()If YES, 'sides' is split into a list where:
  #      []list[0] = Number of Dice
  #      []list[1] = "d"
  #      []list[2] = Size of each Die
  #      []List[0]/list[2] are stored within num_dice/sides respectively
  #    ()If NO, checks to see if 'sides' is an int
  #      ()If NO, output error message and terminate program
  #  []Builds dice[] and rolls[] as empty lists, then attaches them to Dice
  #  []Populates dice[] with an amount of Die('sides') equal to 'num_sides'
  def __init__(self,sides=6,num_dice=1):
    if isinstance(sides, str):
      # Build List without '.' (4.d.12 -> '4', 'd', '12')
      list1 = sides.split('.')
      num_dice, sides = int(list1[0]),int(list1[2])
    elif not isinstance(sides, int):
        print("Error! Improper 'Dice' Construction Syntax")
        print("-Please refer to documentation")
        sys.exit()
    # create a container
    self.dice = []
    self.rolls = []
    for die in range(num_dice):
      self.dice.append(Die(sides))

  #############################################################################
  # rollDice()
  #  []Checks to see if self.rolls[] is populated with any values
  #    ()if YES, self.rolls[] is cleared of all values
  #    ()if NO, do nothing (Not expressed inline)
  #  []Populates self.rolls[] by rolling each Die object within self.Dice[],
  #    then recording the results in self.rolls[]
  def rollDice(self):
    if self.rolls : self.rolls.clear()
    for i in range(len(self.dice)):
      self.rolls.append(self.dice[i].roll())

  #############################################################################
  # avg()
  #  []Returns average of all values within self.rolls[]
  def avg(self):
    return round_half_up(sum(self.rolls)/len(self.rolls),1)
  #############################################################################
  # max()
  #  []Returns maximum value within self.rolls[]
  def max(self):
    return max(self.rolls)
  #############################################################################
  # min()
  #  []Returns minimum value within self.rolls[]
  def min(self):
    return min(self.rolls)
  #############################################################################
  # sum()
  #  []Returns sum total of all values within self.rolls[]
  def sum(self):
    return sum(self.rolls)

  #############################################################################
  # roll(rT)
  #  []Performs rollDice() to repopulate self.rolls[]
  #  []Checks to see if rT contains a value
  #    ()if NO          -> Do nothing (Not expressed inline)
  #    ()if YES:
  #      ()if rT is not keyword -> Print Error message and terminate program
  #      ()if rT is 'avg'       -> Return self.avg() of self.rolls[]
  #      ()if rT is 'max'       -> Return self.max() of self.rolls[]
  #      ()if rT is 'min'       -> Return self.min() of self.rolls[]
  #      ()if rT is 'sum'       -> Return self.max() of self.rolls[]
  def roll(self,rollType=None):
    self.rollDice()
    if rollType:
      if rollType =='avg':
        return self.avg()
      elif rollType == 'max':
        return self.max()
      elif rollType == 'min':
        return self.min()
      elif rollType == 'sum':
        return self.sum()
      else:
        print("ERROR! Chosen Roll-Type is not found!")
        print("\n-Please refer to documentation")
        sys.exit()

  #############################################################################
  # __str__
  #  []Create starting Variables (s,t) to hold output in
  #  []Checks to see if self.rolls[] is populated
  #    ()if NO, perform self.rollDice()
  #    ()if YES, do nothing (Not expressed inline)
  #  []Iterate through self.dice[]/self.rolls[], formatting and adding each
  #    each member to 's'
  #    FORMAT: " self.Dice[ x ] - ( self.rolls[ x ] ) "
  #                           . . .
  #            "  {Total Dice}  - ( Sum of all rolls) "
  #  []Calculate final dice amount/roll total, then add to 's'
  #  []Return 's'
  # print(Dice(6,3)) -> "Dice:  [1d6] - (3)"
  #                     "       [1d6] - (6)"
  #                     "       [1d6] - (1)"
  #                     "Total:[[3d6]]-((10))"
  def __str__(self):
    s = "Dice: "
    t = f"[{len(self.dice)}d{self.dice[0].sides}]"
    if not self.rolls :
      self.rollDice()
    s = s + f" {self.dice[0]} - ({self.rolls[0]})\n"
    for x in range(len(self.dice)-1):
      s = s + f"       {self.dice[x+1]} - ({self.rolls[x+1]})\n"

    s = s + f"Total:[{t}]-(({sum(self.rolls)}))"
    return s

if __name__=='__main__':
  d1 = Die()
  d2 = Die(30)

  print(d1)
  print(d2)
  print(d2.roll())
  d3 = Dice("2.d.20")
  print(d3)
  d3.rollDice()
  print(f"\n{d3}")
  print(d3.sum())
  print(d3.avg())
  print(d3.max())
  d3.roll()
  print(d3.roll("avg"))
  print(d3.dice[0])
  print(Dice(10,5))


from schema import Salesperson

class SalespersonDAO():

    def create(self, session, data):
        """
        Create a new record in the database.

        Instantiate an object of class Salesperson, and then pass
        this object to the session.

        Parameters (apart from self):
            session: session object to connect with db, and store objects
            data: dictionary object containing salesperson data

        Return: dictionary containing success message and
            salesperson ID if successful.
        """

        print("\nCreating a salesperson ...")
        print(data)

        salesperson = Salesperson(title = data['title'],
                    firstname = data['firstname'],
                    surname = data['surname'],
                    position = data['position'],
                    work_phone= data['work_phone'],
                    email = data['email']
                    )

        session.add(salesperson)
        session.commit()

        result = {}
        result['message'] = 'Salesperson added successfully!'
        inserted_salesperson_id = salesperson.salesperson_id
        result['salesperson_id'] = inserted_salesperson_id

        return result

    def find_by_id(self, session, salesperson_id):
        """
        Find an existing record in the database by its ID.

        Create a query based on salesperson ID and pass this to the
        session.

        Parameters (apart from self):
            session: session object to connect with db, and store objects
            salesperson_id: primary key of the Salesperson schema

        Return: dictionary containing salesperson record if successful.
        """

        print("\nFinding a salesperson ...")
        print(salesperson_id)

        sp = session.query(Salesperson).get(salesperson_id)

        result = {}

        if not sp:
            result['message'] = "Salesperson NOT found"
        else:
            d = {}
            d['salesperson_id'] = sp.salesperson_id
            d['title'] = sp.title
            d['firstname'] = sp.firstname
            d['surname'] = sp.surname
            d['position'] = sp.position
            d['work_phone'] = sp.work_phone
            d['email'] = sp.email

            result['salesperson'] = d

        return result

    def find_by_surname(self, session, surname):
        """
        Find salesperson record(s) in the database by surname.

        Parameters (apart from self):
            session: session object to connect with db, and store objects
            surname: surname of salesperson to find

        Return: dictionary containing matching salesperson record(s) if found.
        """

        print("\nFinding salesperson(s) by surname ...")
        print(surname)

        result = {}

        rows = session.query(Salesperson) \
               .filter(Salesperson.surname.like(surname)) \
               .order_by(Salesperson.salesperson_id).all()

        if not rows:
            result['message'] = "No salesperson found!"
        else:
            list_sp = []
            for x in rows:
                d = {}
                d['salesperson_id'] = x.salesperson_id
                d['title'] = x.title
                d['firstname'] = x.firstname
                d['surname'] = x.surname
                d['position'] = x.position
                d['work_phone'] = x.work_phone
                d['email'] = x.email
                list_sp.append(d)
                pass

            result['salespeople'] = list_sp

        return result

    def find_all(self, session):
        """
        Find all salesperson record(s) in the database.

        Parameters (apart from self):
            session: session object to connect with db, and store objects

        Return: dictionary containing all salesperson records if successful.
        """

        print("\nFinding all salespeople ...")

        result = {}

        rows = session.query(Salesperson).all()

        if not rows:
            result['message'] = "No salespeople found!"
        else:
            list_sp = []
            for x in rows:
                d = {}
                d['salesperson_id'] = x.salesperson_id
                d['title'] = x.title
                d['firstname'] = x.firstname
                d['surname'] = x.surname
                d['position'] = x.position
                d['work_phone'] = x.work_phone
                d['email'] = x.email
                list_sp.append(d)
                pass

            result['salespeople'] = list_sp

        return result

    def find_ids(self, session):
        """
        Find IDs of all salesperson(s) in the database.

        Parameters (apart from self):
            session: session object to connect with db, and store objects

        Return: dictionary containing list of all existing salesperson IDs.
        """

        print("\nFinding all salesperson ids ...")

        result = {}

        rows = session.query(Salesperson).all()

        if not rows:
            result['message'] = "No salespeople found!"
        else:
            list_ids = []
            for x in rows:
                list_ids.append(x.salesperson_id)
                pass

            result['salesperson_ids'] = list_ids

        return result

    def update(self, session, salesperson_id, data):
        """
        Update a record in the database.

        Query the database to get the salesperson record by their ID,
        and if found, update according to new values and commit the updated
        record to the database.

        Parameters (apart from self):
            session: session object to connect with db, and store objects
            salesperson_id: primary key of the Salesperson schema
            data: dictionary object containing salesperson data

        Return: dictionary containing success message and
            updated salesperson record if successful.
        """

        print("\nUpdating salesperson ...")
        print(salesperson_id)
        print(data)

        result = {}

        sp = session.query(Salesperson).get(salesperson_id)

        if not sp:
            result['message'] = "Salesperson NOT found!"
        else:
            sp.title = data['title']
            sp.firstname = data['firstname']
            sp.surname = data['surname']
            sp.position = data['position']
            sp.work_phone = data['work_phone']
            sp.email = data['email']

            session.commit()

            result['message'] = "Salesperson updated!"

        return result

    def delete(self, session, salesperson_id):
        """
        Delete a record from the database.

        Query the database to get the salesperson record by their ID,
        and if found, delete the record from the database.

        Parameters (apart from self):
            session: session object to connect with db, and store objects
            salesperson_id: primary key of the Salesperson schema

        Return: dictionary containing success message if successful.
        """

        print("\nDeleting salesperson ...")
        print(salesperson_id)

        result = {}

        sp = session.query(Salesperson).get(salesperson_id)

        if not sp:
            result['message'] = "Salesperson NOT found!"
        else:
            session.delete(sp)
            session.commit()

            result['message'] = "Salesperson deleted!"

        return result


"""MongoDB utility

    @author   : Liangjun Zhu

    @changelog:
    - 16-12-07  - lj - rewrite for version 2.0
    - 17-06-27  - lj - reorganize as basic class other than Global variables
    - 18-02-08  - lj - compatible with Python3.
    - 20-07-20  - lj - no need to invoke close() of MongoClient after use
"""
from __future__ import absolute_import, unicode_literals

import os
import sys
if os.path.abspath(os.path.join(sys.path[0], '..')) not in sys.path:
    sys.path.insert(0, os.path.abspath(os.path.join(sys.path[0], '..')))

from pymongo import MongoClient
from pymongo.errors import ConnectionFailure, InvalidOperation
from preprocess.text import DBTableNames, ModelParamFields

class ConnectMongoDB(object):
    """Connect to MongoDB, and close when finished."""

    def __init__(self, ip, port, maxPoolSize=None):
        """initial mongodb client by hostname and port.

        Starting with version 3.0 the MongoClient constructor no longer blocks while connecting to
         the server or servers, and it no longer raises ConnectionFailure if they are unavailable,
         nor ConfigurationError if the user's credentials are wrong.
         Instead, the constructor returns immediately and launches the connection process on
          background threads.
        --https://api.mongodb.com/python/current/api/pymongo/mongo_client.html
        """
        self.conn = MongoClient(ip, port, maxPoolSize=maxPoolSize)
        try:
            self.conn.admin.command('ismaster')
        except ConnectionFailure as err:
            sys.stderr.write('Could not connect to MongoDB: %s' % err)
            sys.exit(1)

    def get_conn(self):  # type: (...) -> MongoClient
        """get MongoDB connection."""
        return self.conn

    def close(self):
        """Close collection.

        Create this client once for each process, and reuse it for all operations.
        It is a common mistake to create a new client for each request, which is very inefficient.
        --https://stackoverflow.com/questions/41015490/how-can-i-force-pymongo-to-close-sockets

        So, for now, I will comment the close operation. By lj.
        """
        # self.conn.close()
        pass

class MongoQuery(object):
    """
    Query data from MongoDB
    """

    @staticmethod
    def get_init_parameter_value(db_model, param_name, field=ModelParamFields.value):
        """Query initial parameter value, raise exception if error occurs."""
        coll_param = db_model[DBTableNames.main_parameter]
        param_dict = coll_param.find_one({ModelParamFields.name: param_name})
        if param_dict is None or param_dict.get(field) is None:
            raise RuntimeError('%s item is not existed in MongoDB!' % param_name)
        return param_dict.get(field)

class MongoUtil(object):
    """Some utility functions."""

    @staticmethod
    def run_bulk(bulk, errmsg=''):
        """Execute bulk operations, do not raise exception.
        Deprecated from v3.5
        """
        try:
            bulk.execute()
        except InvalidOperation:
            print('WARNING: %s' % errmsg)

    @staticmethod
    def run_bulk_write(coll, req_list):
        """Run bulk_write.

        Reference: https://pymongo.readthedocs.io/en/stable/api/pymongo/collection.html
                   #pymongo.collection.Collection.bulk_write
        """
        if len(req_list) == 0:
            return None
        return coll.bulk_write(req_list)


from datetime import datetime, timedelta
import requests
import json
import matplotlib.pyplot as plt

dataSource="https://opendata.ecdc.europa.eu/covid19/casedistribution/json"

# https://www.worldometers.info/world-population/population-by-country
ukPopulation = 67886011

timeWindow = 14

# This file stores the produced number data, starting on 09/06/2020
rateFile = "rate_data.txt"
rateData = json.load(open(rateFile))

# Date today
today = datetime.today()

# List of the last 14 dates
def get_report_dates():
    timestamps = [today - timedelta(days=x) for x in range(timeWindow)]
    dates = [d.strftime('%d/%m/%Y') for d in timestamps]

    if str(today.strftime('%d/%m/%Y')) in rateData:
         print("Data for today has already been fetched. Here's results:")
         print(json.dumps(rateData, indent=4))

         plot(rateData)
    else:
        fetch_latest_data(dates)

# Get data from dataSource
def fetch_latest_data(dates):
    publicData = json.loads(requests.get(dataSource).text)
    data = {d["dateRep"]: int(d["cases"]) for d in publicData["records"] if d["countriesAndTerritories"] == "United_Kingdom" and d["dateRep"] in dates}

    if str(today.strftime('%d/%m/%Y')) in data:
        print("Calculating using this New Cases data:")
        print(json.dumps(data, indent=4))
        calc_rate(data)
    else:
        print("No data has been published for today yet. Here's previous results:")
        print(json.dumps(rateData, indent=4))

# Calculate the rate over 100000 people
def calc_rate(data):
    allDayTotal = sum(list(data.values()))

    numberPer100k = (allDayTotal * 100000)/ukPopulation

    print("RATE: " + str(numberPer100k))

    write_rate_to_file(numberPer100k)

# Write new rate data to a file
def write_rate_to_file(numberPer100k):
    rateData[str(today.strftime('%d/%m/%Y'))] = numberPer100k

    with open(rateFile, 'w') as file:
        file.write(json.dumps(rateData, indent=4))

    plot(rateData)

def plot(rateData):
     x,y = zip(*sorted(rateData.items()))
     plt.plot_date(x,y, "-")
     plt.xlabel('dates')
     plt.ylabel('numbers')
     plt.grid()
     plt.show()

get_report_dates()


from os import listdir
from os.path import isfile, join, splitext
from tqdm import tqdm
import numpy as np
import csv
import sys
import pickle
import cv2

classes = ['articulated_truck', 'bicycle', 'bus', 'car',
           'motorcycle', 'non-motorized_vehicle', 'pedestrian',
           'pickup_truck', 'single_unit_truck', 'work_van', 'background']

# Load SVM model
#svmClassifier = None

with open('./model.pk1', 'rb') as f:
    svmClassifier = pickle.load(f)

def get_hog_features(images, cell, block, nbins):
    """
    Returns HoG features for all images.

    Adapted from tutorial 4.
    """
    cell_size = (cell, cell)  # h x w in pixels
    block_size = (block, block)  # h x w in cells

    img = images[0]

    # create HoG Object
    # winSize is the size of the image cropped to an multiple of the cell size
    hog = cv2.HOGDescriptor(_winSize=(img.shape[1] // cell_size[1] * cell_size[1],
                                  img.shape[0] // cell_size[0] * cell_size[0]),
                        _blockSize=(block_size[1] * cell_size[1],
                                    block_size[0] * cell_size[0]),
                        _blockStride=(cell_size[1], cell_size[0]),
                        _cellSize=(cell_size[1], cell_size[0]),
                        _nbins=nbins)

    n_cells = (img.shape[0] // cell_size[0], img.shape[1] // cell_size[1])

    hog_feats_list = []
    for img in images:
        # Compute HoG features
        hog_feats = hog.compute(img)\
                       .reshape(n_cells[1] - block_size[1] + 1,
                                n_cells[0] - block_size[0] + 1,
                                block_size[0], block_size[1], nbins) \
                       .transpose((1, 0, 2, 3, 4))  # index blocks by rows first

        # hog_feats now contains the gradient amplitudes for each direction,for each cell of its group for each group.
        # Indexing is by rows then columns.

        # computation for BlockNorm
        gradients = np.full((n_cells[0], n_cells[1], 8), 0, dtype=float)
        cell_count = np.full((n_cells[0], n_cells[1], 1), 0, dtype=int)

        for off_y in range(block_size[0]):
            for off_x in range(block_size[1]):
                gradients[off_y:n_cells[0] - block_size[0] + off_y + 1,
                          off_x:n_cells[1] - block_size[1] + off_x + 1] += \
                    hog_feats[:, :, off_y, off_x, :]
                cell_count[off_y:n_cells[0] - block_size[0] + off_y + 1,
                           off_x:n_cells[1] - block_size[1] + off_x + 1] += 1

        # Average gradients
        gradients /= cell_count

        hog_feats_list.append(gradients.ravel())

    return hog_feats_list

def classify_image(path_to_image):
    '''
    Classify the image contained in 'path_to_image'.

    You may replace this line with a call to your classification method
    '''
    image = cv2.imread(path_to_image, cv2.IMREAD_GRAYSCALE)
    images = np.array([cv2.resize(image, (128, 128))])

    test_img_features = get_hog_features(images, 4, 4, 8)
    test_img_features = np.array([x.flatten() for x in test_img_features])
    # testing
    label = svmClassifier.predict(test_img_features)[0]

    return label

def parse_dataset(path_to_dataset,nb_images):
    '''
    Parse every image contained in 'path_to_dataset' (a path to the training
    or testing set), classify each image and save in a csv file the resulting
    assignment

    dataset_result: dict structure returned by the function.  It contains the
            label of each image
    '''
    llist = listdir(path_to_dataset)
    dataset_result = {}

    do_break = False

    for count, name in enumerate(tqdm(llist)):
        dn = join(path_to_dataset, name)
        if isfile(dn):
            label = classify_image(dn)
            file_nb, file_ext = splitext(name)
            dataset_result[file_nb] = label
            if count == nb_images-1 or do_break:
                do_break = True
                break

        else:
            file_list = listdir(dn)
            for count1, file_name in enumerate(reversed(file_list)):
                file_name_with_path = join(dn, file_name)
                label = classify_image(file_name_with_path)
                file_nb, file_ext = splitext(file_name)
                if file_nb in dataset_result.keys():
                    print('error! ', file_nb, dataset_result[file_nb], ' vs ', file_name_with_path)
                dataset_result[file_nb] = label
                if count1 == nb_images-1 or do_break:
                    do_break = True
                    break

    return dataset_result

def save_classification_result(dataset_result, output_csv_file_name):
    '''
    save the dataset_result (a dict structure containing the class of every image)
    into a valid csv file.
    '''

    csvfile = open(output_csv_file_name, 'w')
    fieldnames = ['file_name', 'class_label']
    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

    for file_number in dataset_result.keys():
        writer.writerow({'file_name': str(file_number), 'class_label': dataset_result[file_number]})

    csvfile.close()

if __name__ == '__main__':

    if len(sys.argv) < 3:
        print("\nUsage : \n\t python parse_classification_dataset.py PATH OUTPUT_CSV_FILE_NAME NUMBER_OF_IMAGES\n")
        print("\t PATH : path to the training or the testing dataset")
        print("\t OUTPUT_CSV_FILE_NAME : name of the resulting csv file\n")
        print("\t NUMBER_OF_IMAGES : number of images to be testing on\n")
    else:
        print('\nProcessing: ', sys.argv[1], '\n')
        dataset_result = parse_dataset(sys.argv[1], int(sys.argv[3]))
        save_classification_result(dataset_result, sys.argv[2])


import sys
from GUI.main import GUI
from database.database_manager import DatabaseManager
from face_detection import FaceDetector
from face_recognition import FaceRecognition
from voice_authentication import VoiceAuthentication

def create_system_admin(database, _email):
    if not database.user_exists_by_email(_email):
        print("User not found, please create a user before executing this action")
        return

    if database.give_admin_privileges_by_email(_email):
        print("Admin privileges were given to", _email)
        return

    print("Something went wrong, action aborted.")

def remove_system_admin(database, _email):
    if not database.user_exists_by_email(_email):
        print("User not found, please create a user before executing this action")
        return

    if database.remove_admin_privileges_by_email(_email):
        print("Admin privileges were removed from", _email)
        return

    print("Something went wrong, action aborted.")

def main():
    database = DatabaseManager()
    if len(sys.argv) == 3:
        _email = sys.argv[2]
        if sys.argv[1] == "createadmin":
            create_system_admin(database, _email)
        elif sys.argv[1] == "removeadmin":
            remove_system_admin(database, _email)
        else:
            print("Invalid command")

    else:
        face_detector = FaceDetector(0.3, 2)
        face_recognition = FaceRecognition(database, confidence=65.0)
        voice_auth = VoiceAuthentication(database, confidence=90)
        GUI(face_detector, face_recognition, voice_auth)
    database.close()

if __name__ == "__main__":
    main()


# ! /usr/bin/env python
# - * - coding:utf-8 - * -
# __author__ : KingWolf
# createtime : 2019/1/19 4:06

from selenium import webdriver
import time
import os

"""

    处理prompt对话框

"""
#案例1

options = webdriver.ChromeOptions()
options.add_argument('user-data-dir=F:\profile')
chromedriver_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),'chromedriver.exe')
driver = webdriver.Chrome(chrome_options=options,executable_path=chromedriver_path)

driver.get(r'F:\GitExtensions_python\project_spider\exercise_learn\selenium\selenium_study_aotumation\alert_confirm_prompt\alert_confirm_prompt_test1.html')
driver.implicitly_wait(10)
# 获取prompt对话框的按钮，点击按钮，弹出prompt对话框
driver.find_element_by_id('prompt').click()
time.sleep(1)

# 获取prompt对话框
prompt = driver.switch_to.alert

# 获取对话框内容
text = prompt.text
print(text)

# 在弹框内输入信息
prompt.send_keys('狼胸最牛逼！')
# 点击“确认”按钮，提交输入的内容
prompt.accept()

time.sleep(2)
driver.quit()


import requests
import sys

from bs4 import BeautifulSoup
import pandas as pd

import config

def get_lyrics(song_title, artist_name):
    print(song_title)
    song_title = song_title.replace('?', '')
    artist_name = artist_name.replace('?', '')
    # https://github.com/willamesoares/lyrics-crawler

    base_url = 'https://api.genius.com'
    headers = {'Authorization': 'Bearer ' + config.genius_api_token}
    search_url = base_url + '/search'
    data = {'q': song_title + ' ' + artist_name}
    response = requests.get(search_url, data=data, headers=headers)
    json = response.json()
    remote_song_info = None

    for hit in json['response']['hits']:
        if artist_name.lower() in hit['result']['primary_artist']['name'].lower():
            remote_song_info = hit
            break
    if remote_song_info is None:
        # retry with only first word
        if len(song_title.split()) > 1:
            return get_lyrics(song_title.split()[0], artist_name)
        else:
            return None
    else:
        song_url = remote_song_info['result']['url']
        return scrap_song_url(song_url)

def scrap_song_url(url):
    # https://github.com/willamesoares/lyrics-crawler
    page = requests.get(url)
    html = BeautifulSoup(page.text, 'html.parser')
    [h.extract() for h in html('script')]
    lyrics = html.find('div', class_='lyrics').get_text()

    return lyrics

if __name__ == "__main__":
    if len(sys.argv) < 2:
        print("No filename provided")
        exit()
    fname = sys.argv[1].rsplit(".", 1)[0]
    songs = pd.read_csv(fname + ".csv")
    songs["lyrics"] = songs.apply(lambda row: get_lyrics(row["name"], row["artists"]), axis=1)
    songs.to_csv(fname + "_lyrics.csv")


from talon import actions
from typing import Callable

class Handler:
    def __init__(self):
        self.actions = []
        self.regions = {}
        self.screen_width = 1920
        self.screen_height = 1080

        # some convenience definitions and functions
        self.button = lambda key: lambda: actions.key(key)
        self.half_height = self.screen_height / 2
        self.third_width = self.screen_width / 3
        self.quarter_height = self.screen_height / 4

    def handle_action(self, command: str, region: int):
        """for each region and command pair there should be a unique action, returns a new state type if the action causes a transition"""
        for action in self.actions:
            if action.region == region and action.command == command:
                return action.handle_action()

    def cleanup(self):
        raise NotImplementedError(f"Handler {type(self)} didn't implement a cleanup explicitly!")

class Bounds:
    def __init__(self, x, y, width, height):
        self.x = x
        self.y = y
        self.width = width
        self.height = height

class Action:
    def __init__(self, command: str, region: int, action: Callable, bounds: Bounds):
        self.command = command
        self.region = region
        self.handle_action = action
        self.bounds = bounds

class MovementHandler(Handler):
    forward_held = False
    backward_held = False
    safe_walk = False
    held_keys = []
    held_command_and_region = None
    suspended_keys = []

    def __init__(self):
        super().__init__()
        self.regions = {
            0: Bounds(x=0, y=0, width=self.third_width, height=self.screen_height),
            1: Bounds(x=self.third_width, y=0, width=self.third_width, height=(self.quarter_height) * 3),
            2: Bounds(x=self.third_width, y=(self.quarter_height) * 3, width=self.third_width, height=self.quarter_height),
            3: Bounds(x=(self.third_width) * 2, y=0, width=self.third_width, height=self.screen_height)
        }
        self.actions = [
            Action('Hiss', 0, lambda: self.stationary_left_turn(), self.regions[0]),
            Action('Hiss', 1, lambda: self.safe_move_forward(), self.regions[1]),
            Action('Hiss', 2, lambda: self.safe_move_backward(), self.regions[2]),
            Action('Hiss', 3, lambda: self.stationary_right_turn(), self.regions[3]),
            Action('Sh', 0, lambda: self.toggle_move_left(), self.regions[0]),
            Action('Sh', 3, lambda: self.toggle_move_right(), self.regions[3]),
            Action('Cluck', 1, lambda: self.toggle_move_forward(), self.regions[1]),
            Action('Cluck', 2, lambda: self.toggle_move_backward(), self.regions[2]),
        ]
        self.actions.extend([Action('Suck', x, lambda: self.jump(), self.regions[x]) for x in range(4)])
        self.actions.extend([Action('Hiss:stop', x, lambda: self.restore_keys(), self.regions[x]) for x in range(4)])
        self.actions.extend([Action('Sh:stop', x, lambda: self.restore_keys(), self.regions[x]) for x in range(4)])

    def toggle_move_forward(self):
        """Holds forward movement key"""
        print("Walking forward")
        actions.key(f"w:{'up' if self.forward_held else 'down'}")
        self.forward_held = not self.forward_held

    def toggle_move_backward(self):
        """Holds backward movement key"""
        actions.key(f"s:{'up' if self.backward_held else 'down'}")
        self.backward_held = not self.backward_held

    def toggle_move_left(self):
        """Holds left movement key"""
        if "left" not in self.held_keys:
            print("turning left")
            actions.key("left:down")
            self.held_keys.append("left")
            return True

    def toggle_move_right(self):
        """Holds right movement key"""
        if "right" not in self.held_keys:
            print("turning right")
            actions.key("right:down")
            self.held_keys.append("right")
            return True

    def stationary_left_turn(self):
        """Turns left while suspending forward and backward movement"""
        if "a" not in self.held_keys:
            if self.forward_held:
                actions.key("w:up")
                self.suspended_keys.append("w")

            if self.backward_held:
                actions.key("s:up")
                self.suspended_keys.append("s")
            return self.toggle_move_left()

    def stationary_right_turn(self):
        """Turns right while suspending forward and backward movement"""
        if "d" not in self.held_keys:
            if self.forward_held:
                actions.key("w:up")
                self.suspended_keys.append("w")

            if self.backward_held:
                actions.key("s:up")
                self.suspended_keys.append("s")
            return self.toggle_move_right()

    def safe_move_forward(self):
        """Turns moving into a continuous command"""
        if self.safe_walk:
            actions.key("w:down")
            self.held_keys.append("w")
            return True

    def safe_move_backward(self):
        """Turns moving into a continuous command"""
        if self.safe_walk:
            actions.key("s:down")
            self.held_keys.append("s")
            return True

    def set_safe_walk(self, value: bool):
        """set safe walk"""
        self.safe_walk = value

    def jump(self):
        actions.key("space")

    def restore_keys(self):
        """Releases held keys and re-presses suspended keys"""
        [actions.key(f"{key}:up") for key in self.held_keys]
        self.held_keys = []

        [actions.key(f"{key}:down") for key in self.suspended_keys]
        self.suspended_keys = []

        self.held_command_and_region = None

    def cleanup(self):
        [actions.key(f"{key}:up") for key in self.held_keys]
        self.held_keys = []
        self.suspended_keys = []
        self.held_command_and_region = None

    def handle_action(self, command: str, region: int):
        """Calls Action Corresponding to Region but only allows one continuously held action"""
        # a held action (e.g. hissing continuously) has to end before another one can begin, so we can be sure it's always undoing last one
        # allows for different sounds to not trigger each other despite going to the same handler method
        for action in self.actions:
            if action.command == command and action.region == region:
                print(f"running command {command}")
                if action.handle_action():
                    self.held_command_and_region = (command, region)
                break

class CastingHandler(Handler):
    def __init__(self):
        super().__init__()
        self.regions = {
            0: Bounds(x=0, y=0, width=self.third_width, height=self.half_height),
            1: Bounds(x=self.third_width, y=0, width=self.third_width, height=self.half_height),
            2: Bounds(x=self.third_width*2, y=0, width=self.third_width, height=self.half_height),
            3: Bounds(x=0, y=self.half_height, width=self.third_width, height=self.half_height),
            4: Bounds(x=self.third_width, y=self.half_height, width=self.third_width, height=self.half_height),
            5: Bounds(x=self.third_width*2, y=self.half_height, width=self.third_width, height=self.half_height)
        }

        self.actions = [
            # player buffs
            Action('Ae', 0, self.button('ctrl-shift-1'), self.regions[0]),
            Action('Ah', 0, self.button('ctrl-shift-2'), self.regions[0]),
            Action('Iy', 0, self.button('ctrl-shift-3'), self.regions[0]),
            Action('Oh', 0, self.button('ctrl-shift-4'), self.regions[0]),
            Action('U', 0, self.button('ctrl-shift-5'), self.regions[0]),
            # most common rotation
            Action('Ae', 1, self.button('1'), self.regions[1]),
            Action('Ah', 1, self.button('2'), self.regions[1]),
            Action('Iy', 1, self.button('3'), self.regions[1]),
            Action('Oh', 1, self.button('4'), self.regions[1]),
            Action('U', 1, self.button('5'), self.regions[1]),
            # random garbage
            Action('Ae', 2, self.button('q'), self.regions[2]),
            Action('Ah', 2, self.button('e'), self.regions[2]),
            Action('Iy', 2, self.button('r'), self.regions[2]),
            Action('Oh', 2, self.button('t'), self.regions[2]),
            Action('U', 2, self.button('f'), self.regions[2]),
            # heals and defensives
            Action('Ae', 3, self.button('ctrl-1'), self.regions[3]),
            Action('Ah', 3, self.button('ctrl-2'), self.regions[3]),
            Action('Iy', 3, self.button('ctrl-3'), self.regions[3]),
            Action('Oh', 3, self.button('ctrl-4'), self.regions[3]),
            Action('U', 3, self.button('ctrl-5'), self.regions[3]),
            # second most common
            Action('Ae', 4, self.button('shift-1'), self.regions[4]),
            Action('Ah', 4, self.button('shift-2'), self.regions[4]),
            Action('Iy', 4, self.button('shift-3'), self.regions[4]),
            Action('Oh', 4, self.button('shift-4'), self.regions[4]),
            Action('U', 4, self.button('shift-5'), self.regions[4]),
            # big cooldowns
            Action('Ae', 5, self.button('shift-q'), self.regions[5]),
            Action('Ah', 5, self.button('shift-e'), self.regions[5]),
            Action('Iy', 5, self.button('shift-r'), self.regions[5]),
            Action('Oh', 5, self.button('shift-t'), self.regions[5]),
            Action('U', 5, self.button('shift-f'), self.regions[5]),
        ]

    def cleanup(self):
        """Providing default implementation for cleanup"""
        pass

class MiscellaneousHandler(Handler):
    def __init__(self):
        super().__init__()
        self.regions = {
            0: Bounds(x=0, y=0, width=self.third_width, height=self.screen_height)
        }
        self.actions = [
            Action('Cluck', 0, lambda: actions.key('tab'), self.regions[0])
        ]

    def cleanup(self):
        """Providing default implementation for cleanup"""
        pass

class MoveCastMiscTransitionHandler(Handler):
    def __init__(self):
        super().__init__()
        self.regions = {
            0: Bounds(x=0, y=0, width=self.third_width, height=self.screen_height),
            1: Bounds(x=self.third_width, y=0, width=self.third_width, height=(self.quarter_height) * 3),
            2: Bounds(x=self.third_width, y=(self.quarter_height) * 3, width=self.third_width, height=self.quarter_height),
            3: Bounds(x=(self.third_width) * 2, y=0, width=self.third_width, height=self.screen_height)
        }
        self.actions = [
            Action('Cluck', 3, lambda: 'ChatState', self.regions[0]),
            Action('Whistle', 1, lambda: 'MenusState', self.regions[1]),
            Action('Whistle', 2, lambda: actions.user.return_to_command_mode(), self.regions[2]),
        ]

    def cleanup(self):
        """Providing default implementation for cleanup"""
        pass

class MenusHandler(Handler):
    def __init__(self):
        super().__init__()
        self.regions = {
            0: Bounds(x=0, y=0, width=self.screen_width, height=self.screen_height / 2),
            1: Bounds(x=0, y=self.screen_height / 2, width=self.screen_width, height=self.screen_height / 2)
        }
        self.pressed = {}
        def action(b):
            self.pressed[b] = (not self.pressed[b]) if b in self.pressed else True
            actions.key(b)
        lambda_action = lambda b: (lambda: action(b))

        self.actions = [
            Action('Ae', 0, lambda_action('b'), self.regions[0]),
            Action('Ah', 0, lambda_action('c'), self.regions[0]),
            Action('Iy', 0, lambda_action('p'), self.regions[0]),
            Action('Oh', 0, lambda_action('n'), self.regions[0]),
            Action('U', 0, lambda_action('l'), self.regions[0]),
            Action('Ae', 1, lambda_action('o'), self.regions[1]),
            Action('Ah', 1, lambda_action('i'), self.regions[1]),
            Action('Iy', 1, lambda_action('escape'), self.regions[1]),
            Action('Oh', 1, lambda_action('m'), self.regions[1]),
            Action('U', 1, lambda_action('shift-p'), self.regions[1]),
        ]

    def cleanup(self):
        """Providing default implementation for cleanup"""
        # may want to make this conditional on whether or not any menus were opened
        if True in self.pressed.values():
            actions.key('escape')

class MenusTransitionHandler(Handler):
    def __init__(self):
        super().__init__()
        self.regions = {
            0: Bounds(x=0, y=0, width=self.screen_width, height=self.screen_height)
        }
        self.actions = [Action('Whistle', 0, lambda: 'MoveCastMiscState', self.regions[0])]

    def cleanup(self):
        """Providing default implementation for cleanup"""
        pass

class SubmenusHandler(Handler):
    def __init__(self):
        super().__init__()
        self.actions = []
        self.regions = {}

    def cleanup(self):
        """Providing default implementation for cleanup"""
        pass

class SubmenusTransitionHandler(Handler):
    def __init__(self):
        super().__init__()
        self.actions = []
        self.regions = {}

    def cleanup(self):
        """Providing default implementation for cleanup"""
        pass

class ChatHandler(Handler):
    def __init__(self):
        super().__init__()
        self.regions = {
            0: Bounds(x=0, y=0, width=self.screen_width / 2, height=self.screen_height),
            1: Bounds(x=self.screen_width / 2, y=0, width=self.screen_width / 2, height=self.screen_height),
        }

        self.dictating = False

        def open_chat(channel):
            if not self.dictating:
                actions.key('enter')
                actions.key('/')
                actions.insert(f"{channel} ")
                self.dictating = True
                actions.mode.enable("command")

        def send():
            if self.dictating:
                actions.key('enter')
                self.dictating = False
                actions.mode.disable("command")

        self.actions = [
            Action('Ae', 0, lambda: open_chat('p'), self.regions[0]),
            Action('Ah', 0, lambda: open_chat('s'), self.regions[0]),
            Action('Iy', 0, lambda: open_chat('g'), self.regions[0]),
            Action('Oh', 0, lambda: open_chat('2'), self.regions[0]),
            Action('U', 0, lambda: open_chat('r'), self.regions[0]),
            Action('Ae', 1, lambda: open_chat('1'), self.regions[1]),
            Action('Ah', 1, lambda: open_chat('w'), self.regions[1]),
            Action('Iy', 1, lambda: open_chat('e'), self.regions[1]),
            Action('Oh', 1, lambda: open_chat('4'), self.regions[1]),
            Action('U', 1, lambda: open_chat('raid'), self.regions[1]),
            Action('Cluck', 0, send, self.regions[0]),
            Action('Cluck', 1, send, self.regions[1]),
            Action('Suck', 0, lambda: self.back_out(), self.regions[0]),
            Action('Suck', 1, lambda: self.back_out(), self.regions[1]),
        ]

    def back_out(self):
        if self.dictating:
            actions.key('escape')
            actions.mode.disable("command")
            self.dictating = False

    def cleanup(self):
        """Providing default implementation for cleanup"""
        self.back_out()

class ChatTransitionHandler(Handler):
    def __init__(self):
        super().__init__()
        self.regions = {
            0: Bounds(x=0, y=0, width=self.screen_width, height=self.screen_height)
        }
        self.actions = [
            Action('Whistle', 0, lambda: 'MoveCastMiscState', self.regions[0])
        ]

    def cleanup(self):
        """Providing default implementation for cleanup"""
        pass


import os
import sys
import math
import argparse
import itertools
from time import time
from collections import defaultdict

import pickle
import numpy as np
import networkx as nx
import multiprocessing as mp
from scipy.sparse import dia_matrix, diags

import parallel_walks
import parallel_addition
import parallel_centrality
from algo import Algorithms
from load_data import LoadData
from random_walks import chunk_random_walk
from utils import load_bubble_diameter, get_bad_and_good_nodes, get_centralities, sort_edges

parser = argparse.ArgumentParser(description='Run the entire/partial pipeline of FairRandomWalks experiments.')
parser.add_argument('-proc', type=str, default='radius',
                    help='Part of pipeline to execute: diameter to compute the bubble diameter of all partitions, addition to compute new bubble diameter')
#parser.add_argument('-algo', type=str, default='baseline', help='If -proc is addition, choose the algo for addition')
parser.add_argument('-topic', type=str, help='Graph to analyze')
parser.add_argument('-t', type=int, default=15, help='Length of walks')
parser.add_argument('-b', type=int, default=2, help='Threshold to define good nodes')
parser.add_argument('-topk', type=int, default=10, help='Percentage of top-k central nodes to consider')
parser.add_argument('-maxedges', type=int, default=10, help='Maximum number of edges to add to the graph')
parser.add_argument('-iter', type=int, default=10, help='Number of iterations')
parser.add_argument('-unweighted', type=str, default='false', help='Weighted or unweighted graph')

#parser.add_argument('-topic', type=str, help='Graph to analyze')
args = parser.parse_args()

print(args.unweighted, type(args.unweighted), False == args.unweighted)
if args.unweighted == 'false':
    # Recall the graph of interest
    politics = LoadData(args.topic, uniform=False)
elif args.unweighted == 'true':
    politics = LoadData(args.topic, uniform=True)
else:
    print ('NO DATASET')
    exit()
id_color = politics.id_color
dictionary_weights = politics.dictionary_weights
id_name = politics.id_name
G = politics.G
edges_updated = set(list(G.edges()))

# Indeces for the adj matrix
node_id_matrix = {n:i for i,n in enumerate(list(G.nodes()))}
id_matrix_node = {j:i for i,j in node_id_matrix.items()}

# Define adj matrix
A = nx.adjacency_matrix(G)
d = diags(1/A.sum(axis=1).A.ravel())
A = A.T.dot(d).T

# Order nodes labels and colors
labels = np.array([j for i,j in sorted([(i,j) for i,j in id_matrix_node.items()], key=lambda x: x[0])])
color_nodes = np.array([id_color[j] for i,j in sorted([(i,j) for i,j in id_matrix_node.items()], key=lambda x: x[0])])

# Define RW parameters
delta = 0.05
eps = 1
r = int((args.t**2)/(eps**2)*np.log(1/delta))
print("Number of walks per node: ", r)

if args.proc == 'radius':
    if not os.path.exists(args.topic + '/'):
        os.makedirs(args.topic + '/')

    # Compute bubble diameter
    tt = time()
    result_chunks = parallel_walks.parallelization(A, labels, args.t, color_nodes, r)
    with open(args.topic + '/'  + args.topic + '_' + str(args.t) + '_bubble_diameters.pickle', 'wb') as f:
        pickle.dump(result_chunks, f)
    print(time()-tt)

    bubble_diameter = load_bubble_diameter(args.topic, id_matrix_node, args.t)
    red_bubble_diameter, blue_bubble_diameter, bad_red_vertices, bad_blue_vertices = get_bad_and_good_nodes(bubble_diameter, id_color, args.b, args.t)
    # Compute bad nodes centralities Blue
    nodes = np.array([node_id_matrix[n] for n in bad_blue_vertices])
    print('Number blue bad nodes: ', len(nodes))
    print("Number of walks per node: ", r)

    tt = time()
    result_chunks = parallel_centrality.parallelization(A, labels, args.t, color_nodes, r, nodes)
    with open(args.topic + '/'  + args.topic + '_' + 'blue' + '_' + str(args.t) + '_centralities.pickle', 'wb') as f:
        pickle.dump(result_chunks, f)
    print(time()-tt)

    # Compute bad nodes centralities Red
    nodes = np.array([node_id_matrix[n] for n in bad_red_vertices])
    print('Number red bad nodes: ', len(nodes))
    print("Number of walks per node: ", r)

    tt = time()
    result_chunks = parallel_centrality.parallelization(A, labels, args.t, color_nodes, r, nodes)
    with open(args.topic + '/'  + args.topic + '_' + 'red' + '_' + str(args.t) + '_centralities.pickle', 'wb') as f:
        pickle.dump(result_chunks, f)
    print(time()-tt)

elif args.proc == 'addition':
    bubble_diameter = load_bubble_diameter(args.topic, id_matrix_node, args.t)
    red_bubble_diameter, blue_bubble_diameter, bad_red_vertices, bad_blue_vertices = get_bad_and_good_nodes(bubble_diameter, id_color, args.b, args.t)

    # Get partitions' contribution
    s_red = np.sum(list(bad_red_vertices.values()))
    s_blue = np.sum(list(bad_blue_vertices.values()))

    perc_red = s_red/(s_red + s_blue)
    perc_blue = 1 - perc_red

    min_part = min(len(bad_red_vertices)*len(blue_bubble_diameter), len(bad_blue_vertices)*len(red_bubble_diameter))
    K_int = np.append([1],np.arange(2, min(min_part, args.maxedges), 2))

    K_R = [math.floor(i*perc_red) for i in K_int]
    K_B = [k - K_R[i] for i, k in enumerate(K_int)]

    for c in ['blue', 'red']:
        if c=='blue':
            bad = bad_blue_vertices
            K = K_B
        else:
            bad = bad_red_vertices
            K = K_R

        nodes = np.array([node_id_matrix[n] for n in bad])

        if c=='blue':
            alg = Algorithms(bad, red_bubble_diameter, labels, args.t,
                        color_nodes, r, nodes, node_id_matrix, edges_updated, 0)
        else:
            alg = Algorithms(bad, blue_bubble_diameter, labels, args.t,
                        color_nodes, r, nodes, node_id_matrix, edges_updated, 0)

        # ROV
        with open(args.topic + '/rov_candidate_edges.pickle','rb') as infile:
            possible_edges = pickle.load(infile)
        candidate_edges = [(i, j) for i,j,k in possible_edges]
        K = np.arange(2, min(len(candidate_edges), args.maxedges), 2)
        new_edges = candidate_edges
        alg._penalty_bulk_additions(A, K, new_edges,
                                args.topic, algorithm='high_to_high',
                                color=c, perc=args.topk)

        # node2vec
        with open(args.topic + '/n2v_candidate_edges.pickle','rb') as infile:
            possible_edges = pickle.load(infile)

        candidate_edges = [(node_id_matrix[i], node_id_matrix[j]) for i,j,k in possible_edges]
        K = np.arange(2, min(len(candidate_edges), args.maxedges), 2)
        new_edges = candidate_edges
        alg._penalty_bulk_additions(A, K, new_edges,
                                args.topic, algorithm='node2vec',
                                color=c, perc=args.topk)

        for ITER in range(int(args.iter)):
            if c == 'blue':
                alg = Algorithms(bad, red_bubble_diameter, labels, args.t,
                                color_nodes, r, nodes, node_id_matrix, edges_updated, ITER)
            else:
                alg = Algorithms(bad, blue_bubble_diameter, labels, args.t,
                                color_nodes, r, nodes, node_id_matrix, edges_updated, ITER)

            # Baseline
            candidate_edges = alg._compute_candidate_edge(algorithm='baseline',
                                                        top_k=100,
                                                        centrality=[])
            #K = np.arange(2, min(len(candidate_edges), args.maxedges), 2)
            alg._repeat_bulk_additions(A, K, candidate_edges,
                                    args.topic, algorithm='baseline',
                                    color=c, perc=100)

            # Recall centrality
            centrality = get_centralities(id_matrix_node, args.topic, c, args.t, old=False)
            perc = args.topk
            top_k = int(len(centrality)/100*perc)
            # Random pick central edges
            candidate_edges = alg._compute_candidate_edge('rand_central', top_k, centrality)
            alg._repeat_bulk_additions(A, K, candidate_edges,
                                    args.topic, algorithm='rand_central',
                                    color=c, perc=args.topk)
            # Random pick degree weighted central edges
            candidate_edges = alg._compute_candidate_edge('w_rand_central', top_k, centrality, G)
            alg._repeat_bulk_additions(A, K, candidate_edges,
                                    args.topic, algorithm='w_rand_central',
                                    color=c, perc=args.topk)

            # Pick sorted degree weighted central edges
            candidate_edges = alg._compute_candidate_edge('w_pen_central', top_k, centrality, G)
            new_edges = sort_edges(candidate_edges, K)
            alg._penalty_bulk_additions(A, K, new_edges,
                                    args.topic, algorithm='w_pen_central',
                                    color=c, perc=args.topk)


"""

This module provides various convenience functions and classes.

"""

import configparser
import json
import os
import tarfile
from io import BytesIO

import numpy as np
import pandas as pd

from . import EMAError, get_module_logger

# Created on 13 jan. 2011
#
# .. codeauthor:: jhkwakkel <j.h.kwakkel (at) tudelft (dot) nl>

__all__ = ["load_results", "save_results", "merge_results", "process_replications"]
_logger = get_module_logger(__name__)

def load_results(file_name):
    """
    load the specified bz2 file. the file is assumed to be saves
    using save_results.

    Parameters
    ----------
    file_name : str
                the path to the file

    Raises
    ------
    IOError if file not found

    """
    from ..em_framework.outcomes import AbstractOutcome, register

    file_name = os.path.abspath(file_name)

    with tarfile.open(file_name, "r:gz", encoding="UTF8") as archive:
        try:
            f = archive.extractfile("metadata.json")
        except KeyError:
            # old style data file
            results = load_results_old(archive)
            _logger.info(f"results loaded successfully from {file_name}")
            return results

        metadata = json.loads(f.read().decode())

        # load experiments
        f = archive.extractfile("experiments.csv")
        experiments = pd.read_csv(f)

        for name, dtype in metadata["experiments"].items():
            try:
                dtype = np.dtype(dtype)
            except TypeError:
                dtype = pd.api.types.pandas_dtype(dtype)

            if experiments[name].dtype is not dtype:
                experiments[name] = experiments[name].astype(dtype)
            # this check is for backward compatability with data stored with 2.4.
            if pd.api.types.is_object_dtype(dtype):
                experiments[name] = experiments[name].astype("category")

        # load outcomes
        outcomes = {}
        known_outcome_classes = {
            entry.__name__: entry for entry in AbstractOutcome.get_subclasses()
        }
        for outcome_type, name, filename in metadata["outcomes"]:
            outcome = known_outcome_classes[outcome_type](name)

            values = register.deserialize(name, filename, archive)
            outcomes[name] = values

    _logger.info(f"results loaded successfully from {file_name}")
    return experiments, outcomes

def load_results_old(archive):
    """
    load the specified bz2 file. the file is assumed to be saves
    using save_results.

    Parameters
    ----------
    file_name : TarFile

    Raises
    ------
    IOError if file not found

    """
    from ..em_framework.outcomes import ScalarOutcome, ArrayOutcome, register

    outcomes = {}

    # load x
    experiments = archive.extractfile("experiments.csv")
    if not (hasattr(experiments, "read")):
        raise EMAError(repr(experiments))

    experiments = pd.read_csv(experiments)

    # load experiment metadata
    metadata = archive.extractfile("experiments metadata.csv").readlines()

    for entry in metadata:
        entry = entry.decode("UTF-8")
        entry = entry.strip()
        entry = entry.split(",")
        name, dtype = (str(item) for item in entry)

        try:
            dtype = np.dtype(dtype)
        except TypeError:
            dtype = pd.api.types.pandas_dtype(dtype)

        if pd.api.types.is_object_dtype(dtype):
            experiments[name] = experiments[name].astype("category")

    # load outcome metadata
    metadata = archive.extractfile("outcomes metadata.csv").readlines()
    metadata = [entry.decode("UTF-8") for entry in metadata]
    metadata = [entry.strip() for entry in metadata]
    metadata = [tuple(entry.split(",")) for entry in metadata]
    metadata = {entry[0]: entry[1:] for entry in metadata}

    # load outcomes
    for outcome, shape in metadata.items():
        shape = list(shape)
        shape[0] = shape[0][1:]
        shape[-1] = shape[-1][0:-1]

        temp_shape = []
        for entry in shape:
            if entry:
                try:
                    temp_shape.append(int(entry))
                except ValueError:
                    temp_shape.append(int(entry[0:-1]))
        shape = tuple(temp_shape)

        if len(shape) > 2:
            nr_files = shape[-1]

            data = np.empty(shape)
            for i in range(nr_files):
                values = archive.extractfile(f"{outcome}_{i}.csv")
                values = pd.read_csv(values, index_col=False, header=None).values
                data[:, :, i] = values

        else:
            data = archive.extractfile(f"{outcome}.csv")
            data = pd.read_csv(data, index_col=False, header=None).values
            data = np.reshape(data, shape)

        outcomes[outcome] = data

    # reformat outcomes from generic dict to new style OutcomesDict
    outcomes_new = {}
    for k, v in outcomes.items():
        if v.ndim == 1:
            outcome = ScalarOutcome(k)
        else:
            outcome = ArrayOutcome(k)

        outcomes_new[outcome.name] = v

    return experiments, outcomes_new

def save_results(results, file_name):
    """
    save the results to the specified tar.gz file.

    The way in which results are stored depends. Experiments are saved
    as csv. Outcomes depend on the outcome type. Scalar, and <3D arrays are
    saved as csv files. Higher dimensional arrays are stored as .npy files.

    Parameters
    ----------
    results : tuple
              the return of perform_experiments
    file_name : str
                the path of the file

    Raises
    ------
    IOError if file not found

    """
    from ..em_framework.outcomes import register

    VERSION = 0.1
    file_name = os.path.abspath(file_name)

    def add_file(tararchive, stream, filename):
        stream.seek(0)
        tarinfo = tarfile.TarInfo(filename)
        tarinfo.size = len(stream.getbuffer())
        tararchive.addfile(tarinfo, stream)

    experiments, outcomes = results
    with tarfile.open(file_name, "w:gz") as z:
        # store experiments
        stream = BytesIO()
        stream.write(experiments.to_csv(header=True, encoding="UTF-8", index=False).encode())
        add_file(z, stream, "experiments.csv")

        # store outcomes
        outcomes_metadata = []
        for key, value in outcomes.items():
            klass = register.outcomes[key]
            stream, filename = register.serialize(key, value)
            add_file(z, stream, filename)
            outcomes_metadata.append((klass.__name__, key, filename))

        # store metadata
        metadata = {
            "version": VERSION,
            "outcomes": outcomes_metadata,
            "experiments": {k: v.name for k, v in experiments.dtypes.to_dict().items()},
        }

        stream = BytesIO()
        stream.write(json.dumps(metadata).encode())
        add_file(z, stream, "metadata.json")

    _logger.info(f"results saved successfully to {file_name}")

def merge_results(results1, results2):
    """
    convenience function for merging the return from
    :meth:`~modelEnsemble.ModelEnsemble.perform_experiments`.

    The function merges results2 with results1. For the experiments,
    it generates an empty array equal to the size of the sum of the
    experiments. As dtype is uses the dtype from the experiments in results1.
    The function assumes that the ordering of dtypes and names is identical in
    both results.

    A typical use case for this function is in combination with
    :func:`~util.experiments_to_cases`. Using :func:`~util.experiments_to_cases`
    one extracts the cases from a first set of experiments. One then
    performs these cases on a different model or policy, and then one wants to
    merge these new results with the old result for further analysis.

    Parameters
    ----------
    results1 : tuple
               first results to be merged
    results2 : tuple
               second results to be merged

    Returns
    -------
    the merged results

    """

    # start of merging
    exp1, res1 = results1
    exp2, res2 = results2

    # merge x
    merged_exp = pd.concat([exp1, exp2], axis=0)
    merged_exp.reset_index(drop=True, inplace=True)

    # only merge the results that are in both
    keys = set(res1.keys()).intersection(set(res2.keys()))
    _logger.info(f"intersection of keys: {keys}")

    # merging results
    merged_res = {}
    for key in keys:
        _logger.info(f"merge {key}")

        value1 = res1.get(key)
        value2 = res2.get(key)
        merged_value = np.concatenate([value1, value2])
        merged_res[key] = merged_value

    mr = (merged_exp, merged_res)
    return mr

def get_ema_project_home_dir():
    try:
        config_file_name = "expworkbench.cfg"
        directory = os.path.dirname(__file__)
        fn = os.path.join(directory, config_file_name)

        config = configparser.ConfigParser()
        parsed = config.read(fn)

        if parsed:
            _logger.info(f"config loaded from {parsed[0]}")
        else:
            _logger.info("no config file found")

        home_dir = config.get("ema_project_home", "home_dir")
        return home_dir
    except BaseException:
        return os.getcwd()

def process_replications(data, aggregation_func=np.mean):
    """
    Convenience function for processing the replications of a stochastic
    model's outcomes.

    The default behavior is to take the mean of the replications. This reduces
    the dimensionality of the outcomes from
    (experiments * replications * outcome_shape) to
    (experiments * outcome_shape), where outcome_shape is 0-d for scalars,
    1-d for time series, and 2-d for arrays.

    The function can take either the outcomes (dictionary: keys are outcomes
    of interest, values are arrays of data) or the results (tuple: experiments
    as DataFrame, outcomes as dictionary) of a set of simulation experiments.

    Parameters
    ----------
    data : dict, tuple
        outcomes or results of a set of experiments
    aggregation_func : callabale, optional
        aggregation function to be applied, defaults to np.mean.

    Returns
    -------
    dict, tuple

    """

    if isinstance(data, dict):
        # replications are the second dimension of the outcome arrays
        outcomes_processed = {key: aggregation_func(data[key], axis=1) for key in data.keys()}
        return outcomes_processed
    elif (
        isinstance(data, tuple) and isinstance(data[0], pd.DataFrame) and isinstance(data[1], dict)
    ):
        experiments, outcomes = data  # split results
        outcomes_processed = {
            key: aggregation_func(outcomes[key], axis=1) for key in outcomes.keys()
        }
        results_processed = (experiments.copy(deep=True), outcomes_processed)
        return results_processed

    else:
        raise EMAError(f"data should be a dict or tuple, but is a {type(data)}".format())


from pyspark.sql import SparkSession
from pyspark import SparkConf
from contextlib import contextmanager

@contextmanager
def SparkIO(conf: SparkConf = SparkConf()):
    app_name = conf.get("spark.app.name")
    master = conf.get("spark.master")
    print(f'Create SparkSession app {app_name} with {master} mode')
    spark = SparkSession.builder.config(conf=conf).getOrCreate()
    try:
        yield spark
    except Exception:
        raise Exception
    finally:
        print(f'Stop SparkSession app {app_name}')
        spark.stop()


import pytest
import os
from talon import talon
from talon import init_refs
from .helper_fns import get_db_cursor
@pytest.mark.dbunit

class TestDatabaseUpdates(object):

    def test_abundance(self):
        """ Try to add abundance entries to database in batches
        """
        conn, cursor = get_db_cursor()
        build = "toy_build"

        abundance = [ ( 1, "test", 5),
                      ( 2, "test", 1),
                      ( 3, "test", 2)]
        batch_size = 2
        talon.batch_add_abundance(cursor, abundance, batch_size)

        # Test if items are there
        query = "SELECT * FROM abundance"
        cursor.execute(query)
        assert len(cursor.fetchall()) == 3
        conn.close()

    def test_datasets(self):
        """ Try to add dataset metadata to database """
        conn, cursor = get_db_cursor()
        build = "toy_build"

        datasets = [ ( 1, "toy", "toy", "toy") ]
        talon.add_datasets(cursor, datasets)

        # Test if items are there
        query = "SELECT * FROM dataset"
        cursor.execute(query)
        assert len(cursor.fetchall()) == 1
        conn.close()

    def test_observed(self):
        """ Try to add observed entries to database in batches
        """
        conn, cursor = get_db_cursor()
        build = "toy_build"

        observed = [ ( 1, 1, 1, "read1", "test", 1, 2, 1, 1, 0, 0, 100, 0.5, "yes", "paternal", "yes", "no"),
                     ( 2, 1, 1, "read2", "test", 1, 2, 1, 1, 0, 0, 100, None, None, None, None, None),
                     ( 3, 1, 1, "read3", "test", 1, 2, 1, 1, 0, 0, 100, None, None, None, None, None),
                     ( 4, 1, 8, "read4", "test", 35, 36, 32, 32,  None, None, 100, None, None, None, None, None) ]

        # Write observed to file
        os.system("mkdir -p scratch/db_updates/")
        with open("scratch/db_updates/observed.tsv", 'w') as f:
            for obs in observed:
                f.write("\t".join([str(x) for x in obs]) + "\n")

        batch_size = 1
        talon.batch_add_observed(cursor, "scratch/db_updates/observed.tsv", batch_size)

        # Test if items are there
        query = "SELECT * FROM observed"
        cursor.execute(query)
        results = cursor.fetchall()
        assert len(results) == 4

        # Test that the 'None' values are properly recorded
        for transcript in results:
            if transcript['read_name'] == "read1":
                assert transcript['fraction_As'] == 0.5
                assert transcript['custom_label'] == "yes"
                assert transcript['allelic_label'] == "paternal"
                assert transcript['start_support'] == "yes"
                assert transcript['end_support'] == "no"
            if transcript['read_name'] == "read4":
                assert transcript['start_delta'] == None
                assert transcript['end_delta'] == None
                assert transcript['fraction_As'] == None
                assert transcript['custom_label'] == None
                assert transcript['allelic_label'] == None
                assert transcript['start_support'] == None
                assert transcript['end_support'] == None
        conn.close()

    def test_gene_annot(self):
        """ Try to add gene annotation entries to database in batches
        """
        conn, cursor = get_db_cursor()
        build = "toy_build"

        annot = [ ( 1, "toy", "TALON", "status", "NOVEL"),
                  ( 2, "toy", "TALON", "status", "NOVEL") ]

        # Write observed to file
        os.system("mkdir -p scratch/db_updates/")
        with open("scratch/db_updates/gene_annot.tsv", 'w') as f:
            for entry in annot:
                f.write("\t".join([str(x) for x in entry]) + "\n")

        batch_size = 1
        talon.batch_add_annotations(cursor, "scratch/db_updates/gene_annot.tsv",
                                    "gene", batch_size)

        # Test if items are there
        query = "SELECT * FROM gene_annotations WHERE value = 'NOVEL'"
        cursor.execute(query)
        assert len(cursor.fetchall()) == 2
        conn.close()

    def test_transcript_annot(self):
        """ Try to add transcript annotation entries to database in batches
        """
        conn, cursor = get_db_cursor()
        build = "toy_build"

        annot = [ ( 1, "toy", "TALON", "status", "NOVEL"),
                  ( 2, "toy", "TALON", "status", "NOVEL") ]

        # Write to file
        os.system("mkdir -p scratch/db_updates/")
        with open("scratch/db_updates/transcript_annot.tsv", 'w') as f:
            for entry in annot:
                f.write("\t".join([str(x) for x in entry]) + "\n")

        batch_size = 2
        talon.batch_add_annotations(cursor, "scratch/db_updates/transcript_annot.tsv",
                                    "transcript", batch_size)

        # Test if items are there
        query = "SELECT * FROM transcript_annotations WHERE value = 'NOVEL'"
        cursor.execute(query)
        assert len(cursor.fetchall()) == 2
        conn.close()

    def test_exon_annot(self):
        """ Try to add exon annotation entries to database in batches
        """
        conn, cursor = get_db_cursor()
        build = "toy_build"

        annot = [ ( 1, "toy", "TALON", "status", "NOVEL"),
                  ( 2, "toy", "TALON", "status", "NOVEL") ]

        # Write to file
        os.system("mkdir -p scratch/db_updates/")
        with open("scratch/db_updates/exon_annot.tsv", 'w') as f:
            for entry in annot:
                f.write("\t".join([str(x) for x in entry]) + "\n")

        batch_size = 3
        talon.batch_add_annotations(cursor, "scratch/db_updates/exon_annot.tsv",
                                    "exon", batch_size)

        # Test if items are there
        query = "SELECT * FROM exon_annotations WHERE value = 'NOVEL'"
        cursor.execute(query)
        assert len(cursor.fetchall()) == 2
        conn.close()

    def test_gene_update(self):
        """ Try to add novel gene entries to database while ignoring duplicates
        """
        conn, cursor = get_db_cursor()
        build = "toy_build"
        database = "scratch/toy.db"
        run_info = talon.init_run_info(database, build)
        talon.get_counters(database)

        init_refs.make_temp_novel_gene_table(cursor, build)
        talon.create_gene("chr4", 1, 1000, "+", cursor, "temp_gene")

        # Write to file
        os.system("mkdir -p scratch/db_updates/")
        with open("scratch/db_updates/genes.tsv", 'w') as f:
            cursor.execute("SELECT gene_ID, strand FROM temp_gene")
            for entry in cursor.fetchall():
                f.write("\t".join([str(x) for x in entry]) + "\n")

        talon.batch_add_genes(cursor, "scratch/db_updates/genes.tsv", 10)

        # Test if gene with ID 6 is there, but make sure we didn't add
        # duplicates of the other genes
        query = "SELECT * FROM genes"
        gene_IDs = [ x['gene_ID'] for x in cursor.execute(query)]
        assert 7 in gene_IDs
        assert len(gene_IDs) == 7
        conn.close()

    def test_transcript_update(self):
        """ Try to add novel transcript entries to database while ignoring
            duplicates
        """
        conn, cursor = get_db_cursor()
        build = "toy_build"
        transcript_dict = init_refs.make_transcript_dict(cursor, build)
        init_refs.make_temp_transcript_table(cursor, build)

        database = "scratch/toy.db"
        talon.get_counters(database)
        talon.create_transcript('+', "chr1", 1, 1000, 1, (1,), (1,2), transcript_dict,
            "temp_transcript", cursor)

        # Write to file
        os.system("mkdir -p scratch/db_updates/")
        with open("scratch/db_updates/transcripts.tsv", 'w') as f:
            for transcript in transcript_dict.values():
                if type(transcript) is dict:
                    entry = "\t".join([ str(x) for x in ( transcript['transcript_ID'],
                                                          transcript['gene_ID'],
                                                          transcript['start_exon'],
                                                          transcript['jn_path'],
                                                          transcript['end_exon'],
                                                          transcript['start_vertex'],
                                                          transcript['end_vertex'],
                                                          transcript['n_exons'] ) ])
                    f.write(entry + "\n")

        batch_size = 5
        talon.batch_add_transcripts(cursor, "scratch/db_updates/transcripts.tsv", batch_size)

        # Test if transcript with ID 8 is there, but make sure we didn't add
        # duplicates of the others
        query = "SELECT * FROM transcripts"
        cursor.execute(query)
        transcripts = cursor.fetchall()
        transcript_IDs = [ x['transcript_ID'] for x in transcripts]
        assert 8 in transcript_IDs
        assert len(transcript_IDs) == 8

        # Test if None value was handled correctly
        for transcript in transcripts:
            if transcript['transcript_ID'] == 8:
                assert transcript['jn_path'] == None

        conn.close()

    def test_edge_update(self):
        """ Try to add novel exons and introns. """
        conn, cursor = get_db_cursor()
        build = "toy_build"
        database = "scratch/toy.db"
        talon.get_counters(database)
        edge_dict = init_refs.make_edge_dict(cursor)
        run_info = talon.init_run_info(database, build)
        orig_n_edges = talon.edge_counter.value()

        talon.create_edge(2, 1, "exon", "-", edge_dict)

        # Write to file
        os.system("mkdir -p scratch/db_updates/")
        with open("scratch/db_updates/edges.tsv", 'w') as f:
            for edge in list(edge_dict.values()):
                if type(edge) is dict:
                    entry = "\t".join([str(x) for x in [edge['edge_ID'], edge['v1'],
                                                        edge['v2'], edge['edge_type'],
                                                        edge['strand']]] )
                    f.write(entry + "\n")

        batch_size = 10
        talon.batch_add_edges(cursor, "scratch/db_updates/edges.tsv", batch_size)

        # Test if the edge table has the correct number of edges now
        query = "SELECT * FROM edge"
        cursor.execute(query)
        edge_IDs = [ x['edge_ID'] for x in cursor.fetchall()]
        assert orig_n_edges + 1 in edge_IDs
        assert len(edge_IDs) == orig_n_edges + 1
        conn.close()

    def test_location_update(self):
        """ Update locations """
        conn, cursor = get_db_cursor()
        build = "toy_build"
        database = "scratch/toy.db"
        talon.get_counters(database)
        location_dict = init_refs.make_location_dict(build, cursor)
        run_info = talon.init_run_info(database, build)
        orig_n_pos = talon.vertex_counter.value()

        talon.create_vertex("chr4", 2000, location_dict, run_info)

        # Write to file
        os.system("mkdir -p scratch/db_updates/")
        with open("scratch/db_updates/loc.tsv", 'w') as f:
            for chrom_dict in location_dict.values():
                for loc in list(chrom_dict.values()):
                    if type(loc) is dict:
                        entry = ("\t".join([ str(x) for x in (loc['location_ID'],
                                                            loc['genome_build'],
                                                            loc['chromosome'],
                                                            loc['position'])]))
                        f.write(entry + "\n")

        batch_size = 10
        talon.batch_add_locations(cursor,"scratch/db_updates/loc.tsv" , batch_size)

        # Test if the table has the correct number of locations now
        query = "SELECT * FROM location"
        cursor.execute(query)
        loc_IDs = [ x['location_ID'] for x in cursor.fetchall()]
        assert orig_n_pos + 1 in loc_IDs
        assert len(loc_IDs) == orig_n_pos + 1
        conn.close()

    def test_vertex2gene_update(self):
        """ Update vertex to gene relationships """
        conn, cursor = get_db_cursor()
        build = "toy_build"
        vertex_2_gene = init_refs.make_vertex_2_gene_dict(cursor)

        talon.update_vertex_2_gene(2, (1,2), "-", vertex_2_gene)
        talon.update_vertex_2_gene(1, (1,2,3,4,5,6), "+", vertex_2_gene)

        # Write to file
        os.system("mkdir -p scratch/db_updates/")
        with open("scratch/db_updates/v2g.tsv", 'w') as f:
            for vertex_ID, gene_set in vertex_2_gene.items():
                for gene in gene_set:
                    entry = "\t".join([ str(x) for x in (vertex_ID, gene[0])])
                    f.write(entry + "\n")

        batch_size = 100
        talon.batch_add_vertex2gene(cursor, "scratch/db_updates/v2g.tsv", batch_size)

        # Use queries to check if the insert worked as expected
        query = "SELECT * FROM vertex WHERE vertex_ID = '1'"
        cursor.execute(query)
        gene_IDs = [ x['gene_ID'] for x in cursor.fetchall()]
        assert gene_IDs == [1, 2]

        query = "SELECT * FROM vertex WHERE gene_ID = '1'"
        cursor.execute(query)
        vertex_IDs = [ x['vertex_ID'] for x in cursor.fetchall()]
        assert vertex_IDs == [1, 2, 3, 4, 5, 6]

    def test_counter_update(self):
        """ Update counters """
        conn, cursor = get_db_cursor()
        build = "toy_build"
        database = "scratch/toy.db"

        talon.get_counters(database)

        # Change the counter values to some arbitrary numbers
        for i in range(10): talon.gene_counter.increment()
        for i in range(20): talon.transcript_counter.increment()
        for i in range(2): talon.edge_counter.increment()
        for i in range(5): talon.vertex_counter.increment()
        for i in range(30): talon.dataset_counter.increment()
        for i in range(6): talon.observed_counter.increment()

        # Now try the update
        talon.update_counter(cursor)

        # Check results with queries
        cursor.execute("""SELECT * FROM counters""")
        for category,value in cursor.fetchall():
            if category == "genes":
                assert value == 16
            elif category == "transcripts":
                assert value == 27
            elif category == "edge":
                assert value == 33
            elif category == "vertex":
                assert value == 39
            elif category == "observed":
                assert value == 6
            elif category == "dataset":
                assert value == 30
            else:
                if category != "genome_build":
                    pytest.fail("Unexpected entry in counters table")

        conn.close()


# -*- coding: utf-8 -*-
"""
최종 프로젝트
1. 코스피주식 업종에 따라 군집분석 및 업종 선택
2. 업종 주식가격 데이터 크롤링
3. RNN
4. 시각화
5. 업종별 주식 선택
6. 포트폴리오 최적화

"""
import pandas as pd
import urllib.request as req # 원격 서버 url 자료 요청
import numpy as np

# 1. 코스피주식 업종에 따라 군집분석 및 업종 선택

path='C:\\ITWILL\\sihun'
Dataset= pd.read_excel(path+'\\Sectors.xlsx')
Dataset = Dataset.iloc[:,0:4]
Dataset2 = Dataset.loc[:,['종목코드','업종명']]

# sector 이름 출력
Sector_name =Dataset.iloc[:,-1]
Sector=Sector_name.drop_duplicates()

'''
0             서비스업
1             기타금융
3              유통업
5             섬유의복
10           운수창고업
11            음식료품
18              증권
19              보험
20            전기전자
25             건설업
29              화학
30            철강금속
47              기계
50             의약품
57           비금속광물
69            운수장비
71             통신업
72           기타제조업
115          전기가스업
152           종이목재
201             은행
216             광업
284           의료정밀
309    농업, 임업 및 어업
Name: 업종명, dtype: object
'''
Secotor=list(Sector)

data=[]
for a in Sector:
    a=Dataset2[Dataset.iloc[:,-1] == a]
    data.append(a)

#DataFrame 생성 <- 첨삭 요구
service_industry=data[0]
Other_finance=data[1]
Distribution=data[2]
Textile_garments=data[3]
Transportation_warehousing_business=data[4]
F_B=data[5]
securities=data[6]
insurance=data[7]
electrical =data[8]
construction=data[9]
chemistry=data[10]
steel_metal=data[11]
machine=data[12]
medicines=data[13]
Non_metallic_minerals=data[14]
transport_equipment=data[15]
telecommunications=data[16]
Other_manufacturing=data[17]
Electricity_Gas=data[18]
paper_wood=data[19]
bank=data[20]
Mining=data[21]
Medical_Precision=data[22]
Agriculture_Forestry_Fisheries=data[23]

# 저평가된 주식을 찾는다는 가정하에 주식의 P/E와  ROE를 기준으로 찾는다.
# P/E는 12~15 이하인 주식 .
# ROE가 20% 이상인 주식 ...
#주당순이익이 꾸준히 증가 ...
#http://data.krx.co.kr/contents/MDC/MDI/mdiLoader/index.cmd?menuId=MDC020

df=pd.read_excel('C:\\ITWILL\\sihun\\df_P_E.xlsx')
df=df.iloc[:,0:7]
type(df)

#종목코드기준으로 P/E, 종가추출
service=service_industry.iloc[:,] # 서비스 업 주식 코드
type(service)

company=[]
for a in list(df.code):
    company.append(a)
len(company) #874

P_E=[]
cnt=0
cnt2=-1
for company1 in company:
    cnt2=+1
    try:
        if service[cnt] == company1:
            P_E.extend(df.iloc[cnt2])
            cnt=+1
        else:
            print('error')
            print(cnt)
    except Exception as e:
        print('error')

P_E=pd.DataFrame(P_E)
P_E.to_excel(excel_writer='C:\\ITWILL\\sihun\\service')

# Sector 별로 저평가된 주식을 찾고 주식을 선택한다.

#####################################################################
## 선택한 주식의 가격정보를 가지고 온다.
##################################################################
# 20페이지 일별 주식 가격 가지고 오기'
import pandas as pd
import urllib.request as req
import requests
#1. 한국 거래소 종목 가져오기
stock_code=pd.read_html('http://kind.krx.co.kr/corpgeneral/corpList.do?method=download',header=0)[0]

#한글 컬럼명을 영어로 변경
stock_code = stock_code.rename(columns={'회사명': 'company', '종목코드': 'code'})

#종목코드가 6자리이기 때문에 6자리르 맞춰주기 위해 설절해줌
stock_code.code=stock_code.code.map('{:06d}'.format)

#크롤링
company='LG화학' # [수정 필요] list형으로 변경해서 모든 선택된 주식의 정보를 가지고 온다.
code=stock_code[stock_code.company==company].code.values[0] # 한국 증권 거래소에서 가지고온 데이터

Date=[]
price=[]
for page in range(1,21):
    url=f'https://finance.naver.com/item/sise_day.nhn?code={code}&page={page}'
    table= pd.read_html(requests.get(url, headers={'User-agent': 'Mozilla/5.0'}).text)
    temp=table[0]
    temp=temp.dropna() # temp 형식 DataFrame
    date=temp.iloc[:,0]
    price_d=temp.iloc[:,1]
    price.extend(price_d) # append or extend
    Date.extend(date)

##################################################################################################
### RNN 으로 각주식의 수익 예측 후 수익성 좋은 주식을 로만 다시 주식 선택 <- tensorflow 사용
##################################################################################################

# data 가지고 오기
data = price
len(data) # 100
print(data)

# 2. RNN 구제에 맞게 reshape
#x_data
x = np.array([data[i+j] for i in range(len(data)-10) for j in range(10)])
# 90 * 10 =900
'''
0~9
1~10
2~11
3~12
  :
89~98
'''
len(x) #900

# train/val split for x
x_train = x[:700].reshape(70,10,1)
x_test = x[700:].reshape(-1,10,1)

x_train.shape #(70,10,1) - (batch_size, time steps, features)
x_test.shape # (20, 10, 1) - (batch_size, time steps, features)

#y_data
y = np.array([data[i+10] for i in range(len(data)-10)]) # 0 ~ 89
'''
10
11
 :
99
'''
len(y) # 90

# train/val split for y (70 vs 20)
y_train = y[:70].reshape(70,1)
y_test = y[70:].reshape(20,1)
y_train.shape # (70, 1)
y_test.shape #  (20, 1)

# 3. model 생성
input_shape = (10, 1)

model = Sequential()

#RNN layer 추가
model.add(SimpleRNN(units=8, input_shape=input_shape, activation='tanh'))

#hidden layer 추가가능 (복잡한 계산의 경우)

# DNN layer 추가
model.add(Dense(units=1)) # 출력 : 회귀모델 - MSE, MAE
#최종 출력은 예측된 숫자나 class 분류 결과이기 때문에 DNN을 이용합니다.

# model 학습환경
model.compile(optimizer='sgd', loss='mse', metrics=['mae']) # sgd, adam 비교해서 선택

# model training
model.fit(x_train,y_train, epochs=400, verbose=1)

# model prediction
y_pred = model.predict(x_test)
print(y_pred)

# real value vs predict value
plt.plot(y_test, 'y--', label='real value')
plt.plot(y_pred, 'r--',label='predict value')
plt.legend(loc='best')
plt.show()

#########################################################################################
### 포트폴리오 최적화(최소한의 리스크로 최대의 수익을 창출할수있는 포트폴리오 생성) <- use R
#########################################################################################