import time
import networkx as nx
import matplotlib.pyplot as plt
import pydot
from IPython.display import Image, display

# IMPORTANT!!! NEED TO COMMENT OUT DATABASE IF YOU CHOOSE TO RUN DATABASE MAIN FUNCTION!

import Database

# IMPORTANT!!! NEED TO COMMENT OUT DATABASE IF YOU CHOOSE TO RUN DATABASE MAIN FUNCTION!

class Node:
    _id_counter = 0

    def __init__(self, move, turn=None, depth=0, parent=None, opening=None):
        self.id = Node._id_counter
        Node._id_counter += 1
        self.turn = turn
        self.parent = parent
        self.children = {}
        self.outcomes = {'white': 0, 'black': 0, 'draw': 0}
        self.move = move
        self.previousMoves = (self.parent.previousMoves +
                              [self.move]) if self.parent is not None else [self.move]
        self.depth = depth
        self.opening = opening

    def addOutcome(self, outcome):
        if outcome == 'draw':
            self.outcomes['draw'] += 1
        else:
            self.outcomes[outcome.lower()] += 1

    def getDepth(self):
        return self.depth

    def setTurn(self, turn):
        self.turn = turn

    def getChild(self, move):
        return self.children.get(move)

    def getMove(self):
        return self.move

    def setDepth(self):
        self.depth = self.parent.getDepth() + 1

    def addChild(self, move, child):
        self.children[move] = child

    def __str__(self) -> str:
        return f"Node: {self.move} \nDepth: {self.depth} \nTurn: {self.turn} \nOutcomes: {self.outcomes} \nPrevious Moves: {self.previousMoves} \nChildren: {self.children}"

class Tree:
    _id_counter = 0

    def __init__(self, rootNode):
        self.root = rootNode
        rootNode.setTurn('white')
        #
        self.root.id = Tree._id_counter
        Tree._id_counter += 1
        self.nodes = {}  # {Node: depth}
        self.rootOutcome = None

    def addGame(self, moves, outcome):
        node = self.root
        for i, move in enumerate(moves):
            player = 'black' if i % 2 == 0 else 'white'
            parent = node

            # Check if the move already exists as a child of the current node
            existing_child = None
            for child in node.children.values():
                if child.move == move:
                    existing_child = child
                    break

            # If the move exists, update the existing child node and its outcomes
            if existing_child:
                node = existing_child
            else:
                # If the move does not exist, create a new child node
                new_child = Node(move=move, turn=player, parent=parent)
                new_child.setDepth()
                node.children[len(node.children)] = new_child
                node = new_child

            node.addOutcome(outcome)
        self.root.outcomes = self.get_depth1_outcomes()

    # Sum the outcomes from all nodes in depth 1

    def get_depth1_outcomes(self):
        depth1_outcomes = {'white': 0, 'black': 0, 'draw': 0}

        for child in self.root.children.values():
            for outcome, count in child.outcomes.items():
                depth1_outcomes[outcome] += count

        return depth1_outcomes

    def visualize(self, output_file_name="tree.png"):
        graph = pydot.Dot(graph_type='digraph', rankdir='LR')
        self._add_node_edges_to_graph(self.root, graph)
        graph.write_png(output_file_name)

    def _add_node_edges_to_graph(self, node, graph):
        if not node:
            return

        # Set the node color based on the turn
        # Set root node

        if node.turn.lower() == 'white':
            node_color = 'white'
        elif node.turn.lower() == 'black':
            node_color = 'black'

        # Include the opening name for the root node
        if node.depth == 1 and node.opening:
            node_label = f"{node.opening}\n{node.move}\n{node.depth}\nW: {node.outcomes['white']} D: {node.outcomes['draw']} B: {node.outcomes['black']}"
        else:
            node_label = f"{node.move}\n{node.depth}\nW: {node.outcomes['white']} D: {node.outcomes['draw']} B: {node.outcomes['black']}"

        graph_node = pydot.Node(str(node.id), label=node_label, shape='ellipse', style='filled',
                                fillcolor=node_color, fontcolor='black' if node_color == 'white' else 'white')
        graph.add_node(graph_node)
        for child in node.children.values():
            edge = pydot.Edge(str(node.id), str(child.id))
            graph.add_edge(edge)
            self._add_node_edges_to_graph(child, graph)

    def getRoot(self):
        return self.root

class TreeManagement:
    def __init__(self) -> None:
        self.trees = {}

    def addGame(self, moves, depth, outcome, opening):
        if moves[0] not in self.trees:
            self.trees[moves[0]] = Tree(
                Node(moves[0], opening=opening, turn='white'))
        self.trees[moves[0]].addGame(moves[1:depth], outcome)

    def addGames(self, games, depth):
        for game, moves in games.items():
            self.addGame(moves, depth, game.getWinningColor(),
                         game.getOpening())

    def getTree(self, move):
        return self.trees.get(move)

    def getAmountOfTrees(self):
        return len(self.trees)

    # Returnerer en liste for hver game med alle moves
    def importMoves(self, file):
        gameMovesList = {}

        db = Database.Database("Test")
        db.importFromPGN(file)
        # cleaner moves til riktig format for senere
        for game in db.getGames():
            gameMoves = []
            for i in range(len(game.getMoves())):
                gameMoves.append(game.getMoves()[i].getFirstMove())
                gameMoves.append(game.getMoves()[i].getSecondMove())
            gameMoves.pop()
            gameMovesList[game] = gameMoves
        return gameMovesList

    def visualize_opening_trees(self):
        for opening, tree in self.trees.items():
            tree.visualize(f"{opening}_tree.png")

def main():
    st = time.time()
    treeManagement = TreeManagement()
    db = "/Users/christian/performanceengineering/Task 2/Database/Stockfish_15_64-bit.commented.[2600] 2.pgn"
    # (treeManagement.importMoves(db))

    ####CREATES THE TREE FILES AS PGN
    treeManagement.addGames(treeManagement.importMoves(db), 130)
    treeManagement.visualize_opening_trees()
    ####

    et = time.time()
    elapsed_time = et - st
    print('Execution time:', elapsed_time, 'seconds')

main()


import os
import random
import shutil
import tempfile
from itertools import count
from random import randint
import requests
import undetected_chromedriver.v2 as uc
from time import sleep
from selenium import webdriver
import json
import string
from PyQt5.QtWidgets import *
from PyQt5.QtCore import QThread,pyqtSignal
from PyQt5 import uic

class UI(QMainWindow):
    def __init__(self):
        super(UI,self).__init__()
        uic.loadUi('Data/uiRobocat.ui',self)
        self.pushButton = self.findChild(QPushButton,'pushButton')
        self.pushButton.clicked.connect(self.start)
        self.spinBox = self.findChild(QSpinBox,'spinBox')
        self.spinBox_2 = self.findChild(QSpinBox,'spinBox_2')
        self.label_3 = self.findChild(QLabel,'label_3')
        self.label_4 = self.findChild(QLabel,'label_4')
        self.lineEdit = self.findChild(QLineEdit,'lineEdit')
        self.lineEdit_2 = self.findChild(QLineEdit,'lineEdit_2')
        self.loadApiSim()
        self.loadURL()
        self.show()
        self.callThread = StartThread()
    def showStatus(self,status):
        self.label_4.setText(status)
    def totalSuccess(self,sucess):
        self.label_3.setText(sucess)
    def loadApiSim(self):
        try:
            with open('Data/Reload/ApiSim.txt','r') as Api:self.lineEdit.setText(Api.read())
        except:pass
    def saveApiSim(self):
        try:
            with open('Data/Reload/ApiSim.txt', 'w') as Api:
                Api.write(self.lineEdit.text())
        except:pass
    def loadURL(self):
        try:
            with open('Data/Reload/URL.txt','r') as URL:self.lineEdit_2.setText(URL.read())
        except:pass
    def saveURL(self):
        try:
            with open('Data/Reload/URL.txt', 'w') as URL:
                URL.write(self.lineEdit_2.text())
        except:pass
    def start(self):
        self.saveApiSim()
        self.saveURL()
        self.callThread.spinBox = self.spinBox
        self.callThread.spinBox_2 = self.spinBox_2
        self.callThread.lineEdit = self.lineEdit
        self.callThread.lineEdit_2 = self.lineEdit_2
        self.callThread.start()
        self.callThread.lbstatus.connect(self.showStatus)
        self.callThread.totalSucessfull.connect(self.totalSuccess)

class StartThread(QThread):
    totalSucessfull = pyqtSignal(str)
    countTotal = count(0)
    lbstatus = pyqtSignal(str)
    def __init__(self):
        super(StartThread,self).__init__()
    def run(self):
        count = 0
        while count < self.spinBox_2.value():
            try:
                self.active()
                count += 1
                timesleep = random.randint(0,self.spinBox.value()*60)
                for i in range(timesleep,0,-1):
                    self.lbstatus.emit(f'Hệ Thống Sẽ Khởi Động Sau {i}s')
                    sleep(1)
            except:count += 1

    def getPhonenumber(self):
        # Get PhoneNumber
        dataPhoneNumber = requests.get(f'https://chothuesimcode.com/api?act=number&apik={self.lineEdit.text()}&appId=1249')
        jsonData = json.loads(dataPhoneNumber.text)
        phoneNumber = jsonData['Result']['Number']
        idNumber = jsonData['Result']['Id']
        return phoneNumber, idNumber

    def getCodeSMS(self,idNumber):
        count = 0
        while count <= 30:
            try:
                dataCodeSms = requests.get(f'https://chothuesimcode.com/api?act=code&apik={self.lineEdit.text()}&id={idNumber}')

                jsonData = json.loads(dataCodeSms.text)

                smsCode = jsonData['Result']['Code']
                return smsCode
                break
            except:
                sleep(5)
                count += 1

    def fullName(self):
        lastName = ['Nguyễn', 'Phạm', 'Trần', 'Lê', 'Hoàng', 'Phan', 'Vũ', 'Võ', 'Đặng', 'Bùi', 'Đỗ', 'Hồ', 'Ngô']
        midName = ['Hoài', 'Tú', 'Ngọc', 'Bảo', 'Nguyệt', 'Minh', 'Hương', 'Lan', 'Mai', 'Hạc', 'Nhật', 'Vân', 'Lam']
        firstName = ['An', 'Anh', 'Bích', 'Bình', 'Cát', 'Châu', 'Chi', 'Cúc', 'Dạ', 'Diệu', 'Du', 'Dung', 'Duyên',
                     'Đan', 'Đào', 'Giang', 'Hà', 'Hạ', 'Hằng']
        return f'{lastName[randint(0, len(lastName) - 1)]} {midName[randint(0, len(midName) - 1)]} {firstName[randint(0, len(firstName) - 1)]}'

    def createNameFriends(self):
        lastName = ['Nguyễn', 'Phạm', 'Trần', 'Lê', 'Hoàng', 'Phan', 'Vũ', 'Võ', 'Đặng', 'Bùi', 'Đỗ', 'Hồ', 'Ngô']
        firstName = ['An', 'Anh', 'Bích', 'Bình', 'Cát', 'Châu', 'Chi', 'Cúc', 'Dạ', 'Diệu', 'Du', 'Dung', 'Duyên',
                     'Đan',
                     'Đào', 'Giang', 'Hà', 'Hạ', 'Hằng']
        return f'{lastName[randint(0, len(lastName) - 1)]} {firstName[randint(0, len(firstName) - 1)]}'

    def createEmail(self):
        ascii = string.ascii_lowercase
        digits = string.digits
        theLastEmail = ['@gmail.com', '@yahoo.com.vn', '@outlook.com']

        email = random.choices(ascii, k=9)
        numbers = random.choices(digits, k=5)
        email = email + numbers
        email = ''.join(email) + theLastEmail[random.randint(0, len(theLastEmail) - 1)]
        return email

    def createCMT(self):
        firstNumber = ["001", "002", "004", "006", "010", "011", "012", "014", "015", "017", "019", "020", "022", "024",
                       "025", "026", "027",
                       "030", "031", "033", "034", "035", "036", "037", "038", "040", "042", "044", "045", "046", "048",
                       "049", "051", "052",
                       "054", "056", "058", "060", "062", "064", "066", "067", "068", "070", "072", "074", "075", "077",
                       "079", '080', '083']
        cmt = [firstNumber[random.randint(0, len(firstNumber) - 1)]]
        for i in range(0, 9):
            cmt.append(str(random.randint(0, 9)))
        numberCMT = ''.join(cmt)
        return numberCMT

    def createFBID(self):
        cmt = ['1000']
        for i in range(0, 11):
            cmt.append(str(random.randint(0, 9)))
        fbID = ''.join(cmt)
        return fbID

    def createBank(self):
        firstNumber = ['970436', '970418', '970406', '970426', '970422', '970423', '970432', '970431']
        return firstNumber[random.randint(0, len(firstNumber) - 1)] + str(random.randint(1000000, 9999999)) + str(
            random.randint(100000, 999999))

    def setBrowser(self):
        self.lbstatus.emit('Đang Khởi Tạo Hệ Thống')
        self.temp = os.path.normpath(tempfile.mkdtemp())
        opts = webdriver.ChromeOptions()
        args = ["hide_console", ]
        opts.add_argument("--window-size=920,880")
        opts.add_argument("--disable-popup-blocking")
        opts.add_argument("--incognito")
        opts.add_argument('--user-data-dir=' + self.temp)
        opts.add_argument("--use-fake-ui-for-media-stream")
        self.browser = uc.Chrome(options=opts)
        with self.browser:
            self.browser.get(self.lineEdit_2.text())
        self.lbstatus.emit('Khởi Tạo Thành Công')

    def active(self):
        try:
            self.lbstatus.emit('Đang Sử Lý Vui Lòng Đợi !')
            self.setBrowser()
            sleep(1)
            phoneNumber, idNumber = self.getPhonenumber()
            print(phoneNumber)
            self.browser.find_element_by_xpath('//*[@id="phone"]').send_keys(phoneNumber)
            sleep(1)
            # Submit Form

            self.browser.find_element_by_css_selector('.brand-submit-button.check-phone-btn.get-loan-button').click()
            sleep(1)

            self.browser.find_element_by_css_selector('.phone-check-btn.continue-btn').click()
            sleep(10)

            # Get Code SMS From PhoneNumber
            smsCode = self.getCodeSMS(idNumber)

            # Input Code SMS

            self.browser.find_element_by_xpath('//*[@id="sms_code"]').send_keys(smsCode)

            sleep(0.5)

            # Submit Form

            self.browser.find_element_by_id('btnDataRegister').click()

            sleep(2)

            fullName = self.fullName()

            self.browser.find_element_by_id('full_name').send_keys(fullName)
            sleep(0.5)

            # sendEmail

            email = self.createEmail()

            self.browser.find_element_by_id('email').send_keys(email)

            sleep(0.5)

            # Active CheckBox

            self.browser.find_element_by_id('agreementContentIndex').click()

            sleep(0.5)

            numberCMT = self.createCMT()
            self.browser.find_element_by_id('personnum').send_keys(numberCMT)

            sleep(1)
            dateCMT = f'{random.randint(0, 29)}{random.randint(0, 9)}{random.randint(2015, 2020)}'

            for item in list(dateCMT):
                self.browser.find_element_by_id('document_date').send_keys(item)
                sleep(0.1)
            # Noi Cap
            self.browser.find_element_by_xpath(f'//*[@id="document_location"]/option[{random.randint(2, 67)}]').click()

            sleep(2)

            # Ngay Sinh
            birthDay = f'{random.randint(0, 29)}{random.randint(0, 9)}{random.randint(1985, 2000)}'
            for item in list(birthDay):
                self.browser.find_element_by_id('birthday').send_keys(item)
                sleep(0.1)
            # Input Facebook ID

            fbID = self.createFBID()
            self.browser.find_element_by_id('facebook_id').send_keys(fbID)
            sleep(0.5)

            # Gioi Tinh

            check = random.randint(0, 1)
            if check == 0:
                self.browser.execute_script("""document.querySelector("#male").click();""")
            else:
                self.browser.execute_script("""document.querySelector("#female").click();""")
            sleep(1)

            #
            # Tinh Trang Hon Nhan

            self.browser.find_element_by_xpath('//*[@id="marital_status"]/option[2]').click()

            sleep(0.5)

            # Tinh Trang Con Cai

            self.browser.find_element_by_xpath('//*[@id="amount_children"]/option[2]').click()

            sleep(0.5)

            # Submit Form

            self.browser.find_element_by_id('btnDataRegister').click()

            sleep(4)

            # checkConfirm

            try:
                self.browser.find_element_by_id('full_name')
                sleep(0.5)
                self.browser.find_element_by_id('btnDataRegister').click()
                sleep(4)
            except:
                pass

            self.browser.find_element_by_xpath(f'//*[@id="address-municipality_id"]/option[{random.randint(2, 64)}]').click()
            sleep(3)

            self.browser.find_element_by_xpath(f'//*[@id="address-district_id"]/option[{random.randint(2, 6)}]').click()
            sleep(3)

            # Tower
            self.browser.find_element_by_xpath(f'//*[@id="address-ward_id"]/option[{random.randint(2, 6)}]').click()
            sleep(1.5)

            # Send PostCode

            self.browser.find_element_by_id('address-postcode').send_keys(random.randint(100000, 350000))
            sleep(1)

            # Get Info

            # Send Address

            self.browser.find_element_by_id('address-street').send_keys('Số 68')

            sleep(1)

            # Confirm Form
            self.browser.find_element_by_id('btnDataRegister').click()
            sleep(3)

            # Select Work
            self.browser.find_element_by_xpath('//*[@id="social_status_id"]/option[6]').click()
            sleep(1)

            # Send Name
            friendsName = self.createNameFriends()
            self.browser.find_element_by_id('relative_name').send_keys(friendsName)
            sleep(1)

            # Send PhoneNumber

            self.browser.find_element_by_id('relative_phone').send_keys(int(phoneNumber) + randint(10, 50))

            # Send Money

            self.browser.find_element_by_id('amount_monthly').send_keys('10000000')

            sleep(1)

            self.browser.find_element_by_xpath('//*[@id="btnDataRegister"]').click()

            sleep(5)

            # select Bank

            self.browser.find_element_by_xpath('//*[@id="payment_system_name_tmp"]/option[2]').click()

            sleep(1)
            numberBank = self.createBank()
            self.browser.find_element_by_id('BankCard-account_number').send_keys(numberBank)
            sleep(1)

            self.browser.find_element_by_xpath('//*[@id="addAccount"]/div[5]/button').click()

            sleep(5)

            # Click Start Take a Picture

            self.browser.find_element_by_xpath('/html/body/div[1]/div[3]/div/div/div[2]/div[3]/a').click()
            sleep(8)
            # take-snapshot
            # save-btn
            # btn-continue

            # Take A Picture
            self.browser.find_element_by_id('take-snapshot').click()
            sleep(3)
            self.browser.find_element_by_id('save-btn').click()
            sleep(3)
            self.browser.find_element_by_id('take-snapshot').click()
            sleep(3)
            self.browser.find_element_by_id('save-btn').click()
            sleep(5)
            self.browser.find_element_by_id('btn-continue').click()
            total = self.countdown()
            self.totalSucessfull.emit(f'Thành Công: {total + 1}')

            sleep(10)

            self.lbstatus.emit('Đang Dọn Dẹp Hệ Thống !')
            shutil.rmtree(r'{}'.format(self.temp))
            self.browser.close()
            self.browser.quit()
            sleep(3)
            self.lbstatus.emit('Hoàn Thành')
        except:
            self.lbstatus.emit('Đang Khởi Tạo Hệ Thống')
            shutil.rmtree(r'{}'.format(self.temp))
            self.browser.close()
            self.browser.quit()
            sleep(3)
            self.lbstatus.emit('Xảy Ra Lỗi ')
    def countdown(self):
        value = next(self.countTotal)
        return value
if __name__ == '__main__':
    import sys
    app = QApplication(sys.argv)
    UIWindow = UI()
    app.exec()


from sklearn.preprocessing import RobustScaler
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn import metrics
from sklearn.metrics import accuracy_score
from sklearn import linear_model
from sklearn.metrics import ConfusionMatrixDisplay
import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
from pandas import DataFrame
import warnings
import category_encoders as ce
import seaborn as sn
import matplotlib.pyplot as plt
import streamlit as st

st.title ("This App can classify income based on many features such as age, occupation, education, etc.")
st.header("Let's use GaussianNB, Supporting vector machine, and LogisticRegression to compare the accuracy.")

warnings.filterwarnings('ignore')

data = 'Data_ForStreamLit/adult.csv'

df = pd.read_csv(data, header=None, sep=',\s' , nrows=10000) #reduce number of rows because svc runs very slow

st.write('Quick overview of the data:')

st.dataframe(df.head())

col_names = ['age', 'workclass', 'fnlwgt', 'education', 'education_num', 'marital_status', 'occupation', 'relationship',
             'race', 'sex', 'capital_gain', 'capital_loss', 'hours_per_week', 'native_country', 'income']

df.columns = col_names

# Declare feature vector and target variable
X = df.drop(['income'], axis=1)

y = df['income']

selection = st.selectbox("Enter the percentage of testing size:",["0.1 or 10% for testing", "0.2 or 20% for testing", "0.3 or 30% for testing"])
Size = lambda x : 0.1 if (selection=="0.1 or 10% for testing") else  (0.2 if (selection=="0.2 or 20% for testing") else 0.3)

if st.button('Train and show accuracy score'):
	#split to training and testing
	X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = Size(selection), random_state = 0)
	st.write('Training model with test size:', Size(selection))

	#divide the dataset to category and numberic
	categorical = [var for var in X_train.columns if X_train[var].dtype=='O']
	numerical = [col for col in X_train.columns if X_train[col].dtypes != 'O']

	#print(categorical)
	#print(numerical)

	# impute missing categorical variables with most frequent value
	for df2 in [X_train, X_test]:
	    df2['workclass'].fillna(X_train['workclass'].mode()[0], inplace=True)
	    df2['occupation'].fillna(X_train['occupation'].mode()[0], inplace=True)
	    df2['native_country'].fillna(X_train['native_country'].mode()[0], inplace=True)

	#print(X_train[categorical].head())
	#print(X_train[numerical].head())

	#feature scaling: pre-processing to handle highly varying magnitudes or values or units of a dominant feature

	#Example of feature scaling:  """ MIN MAX SCALER """

	# min_max_scaler = preprocessing.MinMaxScaler(feature_range =(0, 1))

	# # Scaled feature
	# x_after_min_max_scaler = min_max_scaler.fit_transform(x)

	# print ("\nAfter min max Scaling : \n", x_after_min_max_scaler)

	 #Robust Scaling:

	df = DataFrame(X_test)

	scaler = RobustScaler()

	encoder = ce.OneHotEncoder(cols=['workclass', 'education', 'marital_status', 'occupation', 'relationship',
	                                 'race', 'sex', 'native_country'])

	X_train = encoder.fit_transform(X_train)

	X_test = encoder.transform(X_test)

	cols = X_train.columns
	X_train = scaler.fit_transform(X_train)
	X_test = scaler.transform(X_test)
	X_train = pd.DataFrame(X_train, columns=[cols])
	X_test = pd.DataFrame(X_test, columns=[cols])

	#print(X_train.head())

	gnb = GaussianNB()

	# fit the model
	gnb.fit(X_train, y_train)

	y_pred = gnb.predict(X_test)

	st.write('GNB Model accuracy score: {0:0.4f}'. format(accuracy_score(y_test, y_pred)))

	#intuitive explanation:https://www.youtube.com/watch?v=O2L2Uv9pdDA

	#confusion matrix for GNB
	cm = metrics.confusion_matrix(y_test,y_pred)

	disp = ConfusionMatrixDisplay(confusion_matrix=cm , display_labels=gnb.classes_)

	disp.plot()

	svc = SVC(C=0.001, random_state=1, kernel='linear', gamma='auto')
	svc.fit(X_train, y_train)
	y_predict = svc.predict(X_test)
	# print('SVM prediction: ', y_predict[:5, :])
	st.write('SVM Accuracy score: {0:0.4f}'. format(accuracy_score(y_test, y_predict)))

	logr = linear_model.LogisticRegression()
	logr.fit(X_train, y_train)
	predicted = logr.predict(X_test)

	df["GNB Prediction"] = y_pred
	df["SVM Prediction"] = y_predict
	df["LogisticRegression Prediction"] = predicted
	df["Test label"] = y_test

	st.write('LogisticRegression Accuracy score: {0:0.4f}'. format(accuracy_score(y_test, predicted)))

	st.write('The following table include the test label and prediction from all 3 algorithms')

	st.dataframe(df)

st.write('''
The math behind NB is based on fact that when both event A and B happens and they are dependent on each other then:
The probability of B happens + (probability of A happen given B already happen) =  the probability of A happens + (probability of B given A)
Move "The probability of B happens" to the left side and you have the NB formula.
If you have tranning data set then you can figure out the frequency of event
eg: 5 out 7 email given title include "money" is spam. the p of spam given "money" is 5/7.
If there are 50 spam emails out of 100 total emails, then p of spam is 0.5.
Note that NB uses bag of word model (no order).
	''')

st.write('''
The objective of a SVM is to find the optimal separating hyperplane. Let say we pick a hyperplane (or line in 2D), then double it.
We will have another line called margin. If the line/hyperplane is far from the closest data point, then margin will be big
If the line/hyperplane is close from the closest data point, then margin will be small.
We want to maximize the margin so instead of double we can increase the margin. There can also be soft margin where some data is inside the margin.
	''')

st.write('''
Logistic Regression is used when our dependent variable is dichotomous or binary. It just means a variable that has only 2 outputs.
(NB and SVM are similar but they can be modified for multiclass classification though)
We use the logistic function to convert the output to somewhere between 0 and 1.
If the output from logistic function is greater than the threshold say 0.5 then we predict 1, if it is smaller than the threshold then we predict 0 for y.
	''')


# -*- coding: utf-8 -*

from loguru import logger

import torch

from videoanalyst.model.common_opr.common_block import (conv_bn_relu,
                                                        xcorr_depthwise)
from videoanalyst.model.module_base import ModuleBase
from videoanalyst.model.task_model.taskmodel_base import (TRACK_TASKMODELS,
                                                          VOS_TASKMODELS)

torch.set_printoptions(precision=8)

@TRACK_TASKMODELS.register
@VOS_TASKMODELS.register
class STMTrack(ModuleBase):

    default_hyper_params = dict(pretrain_model_path="",
                                head_width=256,
                                conv_weight_std=0.01,
                                corr_fea_output=False,
                                amp=False)

    support_phases = ["train", "memorize", "track"]

    def __init__(self, backbone_m, backbone_q, neck_m, neck_q, head, loss=None):
        super(STMTrack, self).__init__()
        self.basemodel_m = backbone_m
        self.basemodel_q = backbone_q
        self.neck_m = neck_m
        self.neck_q = neck_q
        self.head = head
        self.loss = loss
        self._phase = "train"

    @property
    def phase(self):
        return self._phase

    @phase.setter
    def phase(self, p):
        assert p in self.support_phases
        self._phase = p

    def memorize(self, im_crop, fg_bg_label_map):
        fm = self.basemodel_m(im_crop, fg_bg_label_map)
        fm = self.neck_m(fm)
        fm = fm.permute(1, 0, 2, 3).unsqueeze(0).contiguous()  # B, C, T, H, W
        return fm

    def train_forward(self, training_data):
        memory_img = training_data["im_m"]
        query_img = training_data["im_q"]
        # backbone feature
        assert len(memory_img.shape) == 5
        B, T, C, H, W = memory_img.shape

        memory_img = memory_img.view(-1, C, H, W)  # no memory copy
        target_fg_bg_label_map = training_data["fg_bg_label_map"].view(-1, 1, H, W)

        fm = self.basemodel_m(memory_img, target_fg_bg_label_map)
        fm = self.neck_m(fm)  # B * T, C, H, W
        fm = fm.view(B, T, *fm.shape[-3:]).contiguous()  # B, T, C, H, W
        fm = fm.permute(0, 2, 1, 3, 4).contiguous()  # B, C, T, H, W

        fq = self.basemodel_q(query_img)
        fq = self.neck_q(fq)

        fcos_cls_score_final, fcos_ctr_score_final, fcos_bbox_final, corr_fea = self.head(fm, fq)
        predict_data = dict(
            cls_pred=fcos_cls_score_final,
            ctr_pred=fcos_ctr_score_final,
            box_pred=fcos_bbox_final,
        )
        if self._hyper_params["corr_fea_output"]:
            predict_data["corr_fea"] = corr_fea
        return predict_data

    def forward(self, *args, phase=None):
        if phase is None:
            phase = self._phase
        # used during training
        if phase == 'train':
            # resolve training data
            if self._hyper_params["amp"]:
                with torch.cuda.amp.autocast():
                    return self.train_forward(args[0])
            else:
                return self.train_forward(args[0])

        elif phase == 'memorize':
            target_img, fg_bg_label_map = args
            fm = self.memorize(target_img, fg_bg_label_map)
            out_list = fm

        elif phase == 'track':
            assert len(args) == 2
            search_img, fm = args
            fq = self.basemodel_q(search_img)
            fq = self.neck_q(fq)  # B, C, H, W

            fcos_cls_score_final, fcos_ctr_score_final, fcos_bbox_final, corr_fea = self.head(
                fm, fq, search_img.size(-1))
            # apply sigmoid
            fcos_cls_prob_final = torch.sigmoid(fcos_cls_score_final)
            fcos_ctr_prob_final = torch.sigmoid(fcos_ctr_score_final)
            # apply centerness correction
            fcos_score_final = fcos_cls_prob_final * fcos_ctr_prob_final

            extra = dict()
            # output
            out_list = fcos_score_final, fcos_bbox_final, fcos_cls_prob_final, fcos_ctr_prob_final, extra
        else:
            raise ValueError("Phase non-implemented.")

        return out_list

    def update_params(self):
        self._make_convs()
        self._initialize_conv()
        super().update_params()

    def _make_convs(self):
        head_width = self._hyper_params['head_width']

        # feature adjustment
        self.r_z_k = conv_bn_relu(head_width, head_width, 1, 3, 0, has_relu=False)
        self.c_z_k = conv_bn_relu(head_width, head_width, 1, 3, 0, has_relu=False)
        self.r_x = conv_bn_relu(head_width, head_width, 1, 3, 0, has_relu=False)
        self.c_x = conv_bn_relu(head_width, head_width, 1, 3, 0, has_relu=False)

    def _initialize_conv(self, ):
        conv_weight_std = self._hyper_params['conv_weight_std']
        conv_list = [
            self.r_z_k.conv, self.c_z_k.conv, self.r_x.conv, self.c_x.conv
        ]
        for ith in range(len(conv_list)):
            conv = conv_list[ith]
            torch.nn.init.normal_(conv.weight,
                                  std=conv_weight_std)  # conv_weight_std=0.01

    def set_device(self, dev):
        if not isinstance(dev, torch.device):
            dev = torch.device(dev)
        self.to(dev)
        if self.loss is not None:
            for loss_name in self.loss:
                self.loss[loss_name].to(dev)


import sys
sys.path.append('../')

import numpy as np
import model_zoo as mz
import loss_func as l
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import xgboost as xgb
import pickle
from sklearn import svm
from sklearn import preprocessing
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import RandomForestClassifier

from utility import dataProcess as dp
from utility import general_utility as gu
from utility import featureExtractor as f_extr

class model_dict:

    def __init__(self, model, model_config):

        self.model_config = model_config
        self.get = self.get_config(model)

    def get_config(self, model_name):

        try:
            model = getattr(self, model_name)
            return model()

        except:
            print("Can not find configuration")
            raise

    def xgb(self):

        model_config = self.model_config
        return xgb.XGBClassifier(
                                    learning_rate= model_config['model_config']['learning_rate'],
                                    n_estimators=500,
                                    max_depth = model_config['model_config']['max_depth'],
                                    min_child_weight = model_config['model_config']['min_child_weight'],
                                    objective='multi:softmax', num_class=3)
    def rf(self):
        model_config = self.model_config
        return RandomForestClassifier(n_estimators = 500, max_depth=model_config['model_config']['max_depth'])

    def svc(self):
        model_config = self.model_config
        return svm.SVC(
                        C = model_config['model_config']['C'],
                        gamma=model_config['model_config']['gamma'],
                        kernel=model_config['model_config']['kernel']
                      )

stock_list =  [
                '0050', '0051',  '0052', '0053',
                '0054', '0055', '0056', '0057',
                '0058', '0059', '006201', '006203',
                '006204', '006208','00690', '00692',
                '00701', '00713'
              ]

#stock_list = ['0050']
predict_days  = list(range(5))
dow = {0:'mon', 1:'tue', 2:'wed', 3:'thu', 4:'fri'}
def get_data_label_pair(single_stock, model_config, meta, isShift=True):

    features, label = dp.get_data_from_dow(f, single_stock, meta, predict_day, model_config['features'], isShift)

    feature_concat = []
    for i in range(model_config['days']):
         for k in  features[dow[i]]:
             feature_concat.append( features[dow[i]][k])

    data_feature = np.concatenate(feature_concat, axis=1)
    data = data_feature
    label = label

    return data, label

#srcPath = '/home/ubuntu/dataset/etf_prediction/all_feature_data_Nm_1_MinMax_94.pkl'
srcPath = '../Data/0525/all_feature_data_Nm_1_MinMax_120.pkl'
metaPath = '../Data/0525/all_meta_data_Nm_1_MinMax_120.pkl'
tv_gen = dp.train_validation_generaotr()
*_,meta = gu.read_metafile(metaPath)
f = tv_gen._load_data(srcPath)
mConfig =open('/home/dashmoment/workspace/etf_prediction/trainer/config/20180526/best_config_xgb_dow_all.pkl', 'rb')
#mConfig =  open('/home/ubuntu/shared/workspace/etf_prediction/trainer/config/best_config_xgb_dow_all.pkl', 'rb')
best_config = pickle.load(mConfig)

predict_ud = {}

for s in stock_list:
     predict_ud[s] = []
     for predict_day in predict_days:

         model_config =  best_config[s][predict_day]

         single_stock = tv_gen._selectData2array(f, [s], model_config['period'])
         single_stock, meta_v = f_extr.create_velocity(single_stock, meta)
         train_data, train_label = get_data_label_pair(single_stock, model_config, meta_v)

         single_stock_test = tv_gen._selectData2array(f, [s], ['20180401', '20180620'])
         single_stock_test, meta_v = f_extr.create_velocity(single_stock_test, meta)
         test_data, test_label = get_data_label_pair(single_stock_test, model_config, meta_v,False)

         model = model_dict('xgb', model_config).get

         model.fit(train_data, train_label)

         #********For submission***********
         test_data = np.reshape(test_data[-1,:], (1,-1))
         ud = gu.map_ud(model.predict(test_data)[0])
         predict_ud[s].append(ud)

         #********For test************
#         p = model.predict(test_data)
#         print(p)
#         print(test_label)
#         print("Validation Accuracy  {}: {} ".format(predict_day, accuracy_score( test_label, p)))

import pickle
with open('../submission/20180525/predict_ud_dow.pkl', 'wb') as handle:
    pickle.dump(predict_ud, handle, protocol=pickle.HIGHEST_PROTOCOL)


# coding=utf-8
import vlc
import logging
import asyncio
import time

from typing import Union

from .youtube import YoutubeAudio
from .exceptions import PlayerException, SoundcubeException, QueueException, YoutubeException, OutsideTimeBounds
from .utilities import resolve_time, clamp, Singleton
from .queue import PlayerQueue

from ..config import DEFAULT_VOLUME
from ..api._bp_types import PlayType

log = logging.getLogger(__name__)

class Player(metaclass=Singleton):
    def __init__(self, loop: asyncio.AbstractEventLoop = asyncio.get_event_loop()):
        self.vlc: vlc.Instance = vlc.Instance()
        self.player: vlc.MediaPlayer = self.vlc.media_player_new()

        self._queue: PlayerQueue = PlayerQueue()

        self.loop: asyncio.AbstractEventLoop = loop

        self._current_volume = DEFAULT_VOLUME
        self.set_volume(self._current_volume)

    ###################
    # PLAYER FUNCTIONS
    # Note: player_queue is both a Player and Queue function
    ###################
    async def player_queue(self, url: str, play_type: PlayType = PlayType.QUEUE, position: int = None, set_playing: bool = False):
        """
        Put a new song in the player queue.
        :param url: Youtube url to queue
        :param play_type: when to play this audio
        :param position: when using PlayType.AT_POSITION, this is the position to place the song
        :param set_playing: bool indicating if the new song should be loaded (but not yet played)

        :raise: SoundcubeException: play_type is invalid
        """
        t_init = time.time()

        try:
            audio = YoutubeAudio(url)
        except OSError:
            raise YoutubeException("invalid link")

        queue_was_empty = self._queue.current_audio is None

        log.debug(f"Got audio from '{audio.title}':{audio.best_audio}, parsing took {round(time.time() - t_init, 3)}")

        # All if clauses keep track of song_index for use later
        # Puts the song at the end of queue
        if play_type == PlayType.QUEUE:
            song_index = self._queue.append_to_queue(audio)
        # Always puts the song after the current one
        elif play_type == PlayType.NEXT:
            song_index = self._queue.current_index + 1
            self._queue.insert_into_queue(audio, song_index)
        elif play_type == PlayType.AT_POSITION:
            if position is None:
                raise RuntimeError("missing arguments")

            song_index = position
            self._queue.insert_into_queue(audio, position)
        else:
            raise SoundcubeException(f"invalid PlayType: '{play_type}'")

        # Load the song if specified or if it is the first song
        if set_playing is True or queue_was_empty:
            self._queue.set_current_song(song_index)
            await self.player_load()

    async def player_load(self) -> YoutubeAudio:
        """
        Loads the current song
        :return: YoutubeAudio that was loaded
        """
        # If no song is set, start with the first one
        if self._queue.current_index is None:
            raise QueueException("no song is loaded")

        # Get current song:
        audio: YoutubeAudio = self._queue.current_audio

        self._update_media(audio)
        return audio

    async def player_play(self) -> YoutubeAudio:
        """
        Plays the current song in queue.
        :return: YoutubeAudio object

        :raise: QueueException: no song in queue
        :raise: PlayerException: problem while trying to create a new Media instance
        """
        # If no song is set, start with the first one
        if self._queue.current_index is None:
            self._queue.set_current_song(0)

        # Get current song from queue
        audio: YoutubeAudio = self._queue.current_audio

        if audio is None:
            raise QueueException("no song to play")

        self._update_media(audio)
        self.player.play()

        # Show info and return
        self._show_play_info()

        return audio

    async def player_pause(self) -> bool:
        """
        Pauses the current song (if one is playing).

        :return: bool indicating if the song was paused
        """
        if not self.player.is_playing():
            log.info("Tried to pause, was not even playing")
            return False

        self.player.pause()
        log.info("Song paused")
        return True

    async def player_resume(self) -> bool:
        """
        Resume the current song (and load one if not already via play()).

        :return: bool indicating if the song was resumed

        :raise: QueueException: no song in queue
        """
        if self.player.is_playing():
            log.info("Tried to resume, was already playing")
            return False

        # If no tracks are loaded, try loading the current song first
        if not self.player.get_media():
            await self.player_play()
        else:
            self.player.play()
            log.info("Song resumed")

        return True

    async def player_stop(self) -> bool:
        """
        Stops the current song (if one is playing).

        :return: bool indicating if the song was stopped
        """
        # First pause, then unload
        if not self.player.get_media():
            return False

        was_playing = await self.player_pause()

        self.player.set_media(None)
        return was_playing

    async def player_next(self) -> YoutubeAudio:
        """
        Play next song in queue
        :return: YoutubeAudio object

        :raise: QueueException: no next song
        :raise: PlayerException: problem while trying to create a new Media instance
        """
        audio: YoutubeAudio = self._queue.next_audio
        if not audio:
            raise QueueException("no next song")

        if self.player_is_playing():
            await self.player_stop()

        self._update_media(audio)
        self.player.play()

        # Show info and return
        self._show_play_info()

        return audio

    async def player_previous(self) -> YoutubeAudio:
        """
        Play previous song in queue
        :return: YoutubeAudio object

        :raise: QueueException: no next song
        :raise: PlayerException: problem while trying to create a new Media instance
        """
        audio: YoutubeAudio = self._queue.previous_audio
        if not audio:
            raise QueueException("no previous song")

        if self.player_is_playing():
            await self.player_stop()

        self._update_media(audio)
        self.player.play()

        # Show info and return
        self._show_play_info()

        return audio

    async def player_set_time(self, time_: float):
        """
        Sets the time for the current song.

        :param time_: time to set in seconds
        :return: bool indicating if the time was changed

        :raise: OutsideTimeBounds: requested time is outside the audio bounds
        """
        if type(time_) not in [int, float]:
            raise RuntimeError(f"'time': expected float, got {type(time)}")

        if not self.player.get_media():
            return False

        # Find the current song length
        audio_length = self._queue.current_audio.length

        if time_ > audio_length:
            raise OutsideTimeBounds("invalid time")

        self.player.set_time(int(time_ * 1000))
        return True

    def player_is_playing(self) -> bool:
        """
        :return: a boolean indicating if a song is currently being played
        """
        if not self.player.get_media():
            return False

        return bool(self.player.is_playing())

    def player_is_song_loaded(self) -> bool:
        """
        :return: A boolean indicating if a song is loaded / loading.
        """
        return bool(self.player.get_media())

    async def player_get_time(self) -> Union[None, float]:
        """
        :return: None if no song is being played
        :return: float representing seconds of time
        """
        time_ms = self.player.get_time()

        # Returned by vlc module if there is no media
        if time_ms == -1:
            return None
        else:
            return time_ms / 1000

    ###################
    # QUEUE FUNCTIONS
    ###################
    async def queue_remove(self, position: int):
        """
        Remove a song from the queue
        :param position: song's index in the queue

        :raise: QueueException: no song at this position
        """
        # Move to a different song if playing the one that is being removed
        if self._queue.current_index == position:
            log.info("Trying to remove song, but it is playing; loading next/previous")
            try:
                # The entire queue moved up, adjust for this
                self._queue.set_current_song(self._queue.current_index - 1)

                await self.player_next()
            except QueueException:
                try:
                    await self.player_previous()
                except QueueException:
                    log.info("No song remaining, stopping")
            finally:
                await self.player_stop()

        self._queue.remove_from_queue(position)

    async def queue_move(self, current_position: int, new_index: int):
        """
        Move a song in the queue
        :param current_position: song's current index in the queue
        :param new_index: which index to move the song to

        :raise: QueueException: no song at this position
        """
        # This will raise a QueueException if the index doesn't exist,
        # so there is no need for backtracking if something goes haywire later
        audio = self._queue.get_song_at(current_position)
        audio2 = self._queue.get_song_at(new_index)

        # makes sure that (if a moved song is currently playing) the index of playing is updated as well
        if self._queue.current_audio == audio:
            update_playing_index = new_index
        elif self._queue.current_audio == audio2:
            update_playing_index = current_position
        else:
            update_playing_index = False

        # The order is reversed depending on the position of indexes
        if current_position < new_index:
            self._queue.insert_into_queue(audio, new_index + 1)
            self._queue.remove_from_queue(current_position)
        else:
            self._queue.remove_from_queue(current_position)
            self._queue.insert_into_queue(audio, new_index)

        if update_playing_index is not False:
            log.debug(f"Updating index after moving: {update_playing_index}")
            self._queue.set_current_song(update_playing_index)

    ###################
    # VOLUME FUNCTIONS
    ###################
    def get_volume(self) -> int:
        """
        Returns the player volume.
        :return: int - volume (0-100)
        """
        return int(self.player.audio_get_volume())

    def set_volume(self, amount: int):
        """
        Sets the player volume.
        :param amount: int between 0 and 100
        """
        amount = clamp(amount, 0, 100)

        self.player.audio_set_volume(amount)

    ###################
    # INTERNAL FUNCTIONS
    ###################
    def _update_media(self, audio: YoutubeAudio):
        media_obj: vlc.Media = self.vlc.media_new(audio.best_audio.url)
        if media_obj is None:
            raise PlayerException("Error while creating vlc Media object")

        self.player.set_media(media_obj)
        log.info("Updated Media on player")

    def _show_play_info(self):
        audio = self._queue.current_audio
        log.info(f"Playing: {audio.title} (length: {resolve_time(audio.length)})")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import matplotlib.pyplot as plt
import numpy as np

class Data(object):
    def __init__(self, filename, labellist):
        self.rawdata = np.loadtxt(filename)
        self.labellist = labellist
        self.make_data()

    def make_data(self):
        for i, label in enumerate(self.labellist):
            setattr(self, label, self.rawdata[:, i])

def load(filename):
    labellist = ['t',
                 'xd', 'yd', 'zd',
                 'x', 'y', 'z',
                 'vx', 'vy', 'vz',
                 'xz', 'yz', 'zz']
    data = Data(filename, labellist)
    return data

def plot_z(data):
    plt.figure()
    plt.plot(data.t, data.zd, label='zd', c='k', ls='--')
    plt.plot(data.t, data.z, label='z')
    plt.legend()
    plt.title('COM regulation along vertical direction')
    plt.xlabel('time [s]')
    plt.ylabel('z [m]')
    plt.grid(True)

def main():
    import sys
    data = load(sys.argv[1])
    plot_z(data)
    plt.show()

if __name__ == '__main__':
    main()


import numpy as np
import cv2
from collections import deque
import argparse

ap = argparse.ArgumentParser()
ap.add_argument("-v", "--video",
	help="path to the (optional) video file")
ap.add_argument("-b", "--buffer", type=int, default=64,
	help="max buffer size")
args = vars(ap.parse_args())

cap = cv2.VideoCapture(0)

greenLower = (29, 86, 6)
greenUpper = (64, 255, 255)
pts = deque(maxlen=args["buffer"])

while(True):
    # Capture frame-by-frame
    ret, frame = cap.read()

    # Our operations on the frame come here
    #gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)

    # Display the resulting frame
    hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

    green_mask = cv2.inRange(hsv, greenLower, greenUpper)
    green_mask = cv2.erode(green_mask, None, iterations=2)
    green_mask = cv2.dilate(green_mask, None, iterations=2)
#    cnts = cv2.findContours(green_mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    cnts = cv2.findContours(green_mask, cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_SIMPLE)[-2]
    center = None
    if len(cnts) > 0:
        c = max(cnts, key=cv2.contourArea)
        ((x, y), radius) = cv2.minEnclosingCircle(c)
        M = cv2.moments(c)
        center = (int(M["m10"] / M["m00"]), int(M["m01"] / M["m00"]))
        if radius > 10:
			# draw the circle and centroid on the frame,
			# then update the list of tracked points
            cv2.circle(frame, (int(x), int(y)), int(radius),
                       (0, 255, 255), 2)
            cv2.circle(frame, center, 5, (0, 0, 255), -1)

    pts.appendleft(center)
    for i in xrange(1, len(pts)):
        # if either of the tracked points are None, ignore
        if pts[i - 1] is None or pts[i] is None:
            continue

        # otherwise, compute the thickness of the line and
        # draw the connecting lines
        thickness = int(np.sqrt(args["buffer"] / float(i + 1)) * 2.5)
        cv2.line(frame, pts[i - 1], pts[i], (0, 0, 255), thickness)

# show the frame to our screen
    cv2.imshow("Frame", frame)
    if cv2.waitKey(1) & 0xFF == ord('q'):
        break

#    res_green = cv2.bitwise_and(frame,frame, mask= green_mask)
#    green_mask = cv2.erode(green_mask, None, iterations=2)
#    green_mask = cv2.dilate(green_mask, None, iterations=2)
#    cv2.imshow('frame',frame)
#    cv2.imshow('gree mark',res_green)
#    if cv2.waitKey(1) & 0xFF == ord('q'):
#        break

# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()


from config.mac_config import MacConfig
import yaml
import os
import sys
import time
from mac_manager import MacManager
from subprocess_manager import SubProcessManager

class YtRun:
    def __init__(self):
        mac_config = MacConfig()
        mac_config.verify_browser_installation()

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        pass

    def run(self):
        if sys.platform.startswith('win'):
            print("Running on Windows...")
        elif sys.platform.startswith('darwin'):
            print("Running on macOS...")
            self.run_mac()
        else:
            print("Not supported... check with Admin")

    def run_mac(self):
        with MacManager() as mm:
            mm.run()

    def run_win(self):
        print('TODO - Implemention')

if __name__ == "__main__":
    with YtRun() as yt:
        yt.run()


import boto3

# Using Amazon s3
client = boto3.client('s3')

BUCKET_NAME = ''

S3_URL = ''

# Upload a file
def upload_file(file, key):
    client.upload_file(file, BUCKET_NAME, key)
    image_url = F'{S3_URL}/{BUCKET_NAME}/{key}'
    return image_url

# Upload a file obj
def upload_file_obj(file_obj, key):
    with open(file_obj, 'rb') as data:
        client.upload_fileobj(data, BUCKET_NAME, key)
        image_url = F'{S3_URL}/{BUCKET_NAME}/{key}'
        return image_url

# Download an S3 object to a file.
def download_s3_file(file_name, key):
    client.Object(BUCKET_NAME, key).download_file(file_name)


import os
import sys

from pyspark.ml.feature import StringIndexer
from pyspark.ml.tuning import ParamGridBuilder, CrossValidator
from pyspark.rdd import RDD
from pyspark.sql import Row
from pyspark.sql import DataFrame
from pyspark.sql import SparkSession
from pyspark.sql.functions import lit, regexp_replace
from pyspark.sql.functions import desc
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.ml.recommendation import ALS
import pandas as pd
from pyspark.sql.functions import col, avg
from pyspark.sql.types import IntegerType

def init_spark():
    spark = SparkSession \
        .builder \
        .appName("Python Spark SQL basic example") \
        .config("spark.some.config.option", "some-value") \
        .getOrCreate()
    return spark

spark = init_spark()
sc = spark.sparkContext
restaurant_file_name = "../data/zomato_restaurant.csv"
reviews_file_name = "../data/zomato_reviews.csv"

restaurant_df = spark.read.option("wholeFile", True).option("multiline", True).option("header", True)\
    .option("inferSchema", "True").csv(restaurant_file_name)
restaurant_df = restaurant_df.filter(restaurant_df['business_id'] != "#NAME?")

reviews_df = spark.read.option("wholeFile", True).option("multiline", True).option("header", True)\
    .option("inferSchema","True").csv(reviews_file_name, header=True)
reviews_df = reviews_df.withColumn('user_id', regexp_replace('user_id', '=-', ''))

restaurant_df = restaurant_df.filter(restaurant_df['city'] == "Phoenix")
restaurant_df = restaurant_df.select(["business_id", "categories", "review_count", "name", "city"])

indexer = StringIndexer(inputCol="business_id", outputCol="business_index")
restaurant_df = indexer.fit(restaurant_df).transform(restaurant_df)
restaurant_df = restaurant_df.withColumn("business_index", restaurant_df["business_index"].cast(IntegerType()))

reviews_df = reviews_df.select(["business_id", "user_id", "stars"])

restaurant_reviews_df = reviews_df.join(restaurant_df, ['business_id'])
restaurant_reviews_df = restaurant_reviews_df.filter(restaurant_reviews_df.business_id.isNotNull())
restaurant_reviews_df = restaurant_reviews_df.filter(restaurant_reviews_df.user_id.isNotNull())

indexer = StringIndexer(inputCol="user_id", outputCol="user_index")
restaurant_reviews_df = indexer.fit(restaurant_reviews_df).transform(restaurant_reviews_df)
restaurant_reviews_df = restaurant_reviews_df.withColumn("user_index", restaurant_reviews_df["user_index"].cast(IntegerType()))

restaurant_reviews_city_df = restaurant_reviews_df.select(["business_index", "user_index", "business_id", "user_id", "stars"])
print(restaurant_reviews_city_df.first())

restaurant_reviews_city_df = restaurant_reviews_city_df.withColumn("stars", restaurant_reviews_city_df["stars"].cast(IntegerType()))

restaurant_reviews_city_df.toPandas().to_csv('mycsv.csv')

(training, test) = restaurant_reviews_city_df.randomSplit([0.8, 0.2], 123)

# als = ALS(maxIter=20, regParam=0.01, userCol="user_index", itemCol="business_index", ratingCol="stars",
#           coldStartStrategy="drop", rank=70).setSeed(123)
# model = als.fit(training)
# predictions = model.transform(test)
# # predictions.cache()
#
# evaluator = RegressionEvaluator(metricName="rmse", labelCol="stars", predictionCol="prediction")
# rmse = evaluator.evaluate(predictions)
# print(rmse)

als = ALS(userCol="user_index", itemCol="business_index", ratingCol="stars", coldStartStrategy="drop")
als.setSeed(123)
grid = ParamGridBuilder().addGrid(als.maxIter, [20]).addGrid(als.rank, [20,30,40,50,60,70]).addGrid(als.regParam, [0.45,0.5,0.55]).build()
evaluator = RegressionEvaluator(predictionCol=als.getPredictionCol(),labelCol=als.getRatingCol(), metricName='rmse')
cv = CrossValidator(estimator=als, estimatorParamMaps=grid, evaluator=evaluator, numFolds=5)
cvModel = cv.fit(training)

predictions = cvModel.transform(test)
predictions.cache()

print('########################## Computing RMSE ###########################')

rmse_evaluator = RegressionEvaluator(predictionCol='prediction',labelCol='stars', metricName='rmse')
rmse = rmse_evaluator.evaluate(predictions)
print(rmse)


import json

class Config:
    def __init__(self, properties, environment=None):
        self._properties = properties
        self._environment = environment
        if environment is not None:
            self._set_properties_environment(properties, environment)

    def __getitem__(self, key):
        return self._properties[key]

    def __contains__(self, key):
        return (key in self._properties)

    def __iter__(self):
        return iter(self._properties)

    def get_dict(self):
        return self._properties

    def get_environment(self):
        return self._environment

    def _set_properties_environment(self, properties, environment):
        if isinstance(properties, list):
            for i in range(len(properties)):
                if isinstance(properties[i], basestring):
                    properties[i] = self._set_str_environment(properties[i], environment)
                else:
                    self._set_properties_environment(properties[i], environment)
        elif isinstance(properties, dict):
            for key in properties.keys():
                item = properties[key]
                if isinstance(item, basestring):
                    properties[key] = self._set_str_environment(item, environment)
                else:
                    self._set_properties_environment(item, environment)

    def _set_str_environment(self, s, environment):
        for key in environment:
            s = s.replace("$!{" + key + "}", environment[key])
        return s

    def merge(self, other_config):
        environment = None
        if self._environment is not None or other_config._environment is not None:
            environment = dict()
            if self._environment is not None:
                for key in self._environment:
                    environment[key] = self._environment[key]
            if other_config._environment is not None:
                for key in other_config._environment:
                    environment[key] = other_config._environment[key]

        properties = dict()
        for key in self._properties:
            properties[key] = self._properties[key]
        for key in other_config._properties:
            properties[key] = other_config._properties[key]

        return Config(properties, environment=environment)

    @staticmethod
    def load(properties_file, environment=None):
        properties = None
        with open(properties_file, 'r') as fp:
            properties = json.load(fp)

        return Config(properties, environment=environment)

    @staticmethod
    def load_from_list(args_list, environment=None):
        properties = dict()

        if len(args_list) % 2 != 0:
            raise ValueError("Args list must be even length to parse into config")

        for i in range(len(args_list)/2):
            arg_name = args_list[i*2].replace("--", "").replace("-", "")
            arg_value = args_list[i*2+1]

            properties[arg_name] = arg_value

        return Config(properties, environment=environment)

    @staticmethod
    def load_from_dict(d, environment=None):
        properties = dict(d)
        return Config(properties, environment=environment)


# Blackjack Project
from replit import clear
import random
from art import logo

def deal_card():
  cards = [11, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10, 10, 10]
  card = random.choice(cards)
  return card

def calculate_score(cards):

  if sum(cards) == 21 in cards and len(cards)==2:
    return 0

  if 11 in cards and sum(cards) > 21:
    cards.remove(11)
    cards.append(1)

  return sum(cards)

def compare(user_score, computer_score):

  if user_score == computer_score:
    return "Draw 🙃"
  elif user_score > 21:
    return "You went over, you lose. 🙁"
  elif computer_score > 21:
    return "Opponent went over,you win. 😀"
  elif computer_score == 0:
    return "Lose, opponent has Blackjack. 🙁"
  elif user_score == 0:
    return "Win with a Blackjack. 🤩"
  elif user_score > computer_score:
    return "You win. 😀"
  else:
    return "You lose. 🙁"
def play_game():
  print(logo)
  user_cards = []
  computer_cards = []
  is_game_over = False

  for _ in range(2):
    user_cards.append(deal_card())
    computer_cards.append(deal_card())

  while not is_game_over:

    user_score = calculate_score(user_cards)
    computer_score = calculate_score(computer_cards)

    print(f"Your cards {user_cards}, current score: {user_score}")
    print(f"computer's first card: {computer_cards[0]}")

    if user_score == 0 or computer_score == 0 or user_score > 21:
      is_game_over = True

    else:
      continue_deal = input("Do you want to draw another card? 'y' or 'n': ")

      if continue_deal == 'y':
        user_cards.append(deal_card())
      else:
        is_game_over = True

  while computer_score != 0 and computer_score < 17:
    computer_cards.append(deal_card())
    computer_score = calculate_score(computer_cards)

  print(compare(user_score, computer_score))

  restart = input("Do you want to restart the game? type 'y' or 'n': ")
  if restart == 'y':
    clear()
    play_game()
  else:
    print("Goodbye!")

play_game()


import logging

from pylons import config, request, response, session, tmpl_context as c
from pylons.controllers.util import abort, redirect_to

from snakepit.lib.base import ProjectsBaseController, ProjectsMenuItem, connect, render
from snakepit.model import db, Project, WikiPage

log = logging.getLogger(__name__)

DEFAULT = """
= ${title} =
"""

class WikiController(ProjectsBaseController):

    def edit(self):
        title = request.urlvars.get('page')
        if not title:
            title = c.project.wiki.start_page

        page = db.query(WikiPage).filter_by(title=title,
                                            wiki_id=c.project.wiki.id).first()
        c.page = page
        c.title = title
        if not page:
            c.contents = DEFAULT % {'title': title}
        return render('/wiki/edit.mao')

    def view(self):
        title = request.urlvars.get('page')
        if not title:
            title = c.project.wiki.start_page

        page = db.query(WikiPage).filter_by(title=title,
                                            wiki_id=c.project.wiki.id).first()
        c.page = page
        c.title = title
        if not page:
            c.contents = DEFAULT % {'title': title}
        return render('/wiki/view.mao')

connect('/wiki/{project}', controller='wiki', action='view', page=None)
connect('/wiki/{project}/{page:.*?}/edit', controller='wiki', action='edit')
connect('/wiki/{project}/{page:.*?}', controller='wiki', action='view')
ProjectsBaseController.register_menu_item('wiki',
    ProjectsMenuItem('Wiki', 5, controller='wiki', action='view'))


import matplotlib.pyplot as plt
year = [ 1950, 1970, 1990, 2010]
pop = [2.519, 3.692, 5.263, 6.972]
# historgraph without any bin count or other parameters
plt.hist(year)
plt.show()
plt.clf()

plt.hist(pop)
plt.show()


from collections import OrderedDict
from typing import Dict, List, Tuple, Set

from bulk_update.manager import BulkUpdateManager
from django.conf import settings
from django.db import models
from django.db.models.query import QuerySet

from django_mysql.models import Model as MySqlModel, SetCharField

from util.mathutils import solve_linear_least_squares, create_matrix, create_2d_vector

class Team(MySqlModel):
    """
    Attributes:

    :attr id: Unique team id, always equal to the team_number.
    :attr active_years {@set[int]}: A set of years that the team has been active (in commission, not participated)
    :attr website {@string}: URL for the team website
    :attr team_number: FRC assigned team number
    :attr name: Long FRC Team Name (note, this can be VERY long, short_name or nickname is recommended)
    :attr short_name: Short FRC Team Name (often the more wel
    :attr nickname: Team Nickname, e.g. Shaker Robotics or Cheesy Poofs
    :attr key: Most specific team name, simply frc#### where #### is the team number
    """

    active_years = SetCharField(base_field=models.IntegerField(), max_length=255, default=set())

    website = models.URLField(null=True)
    name = models.TextField(null=True)  # longer name

    locality = models.CharField(max_length=50, null=True)  # e.g. city
    region = models.CharField(max_length=50, null=True)  # e.g. state, province
    country_name = models.CharField(max_length=50, null=True)
    location = models.CharField(max_length=150, null=True)  # full city + state + country

    team_number = models.PositiveSmallIntegerField(db_index=True)
    key = models.CharField(max_length=8)  # e.g. frc2791
    nickname = models.CharField(max_length=100, null=True)  # shorter name
    rookie_year = models.PositiveSmallIntegerField(null=True)
    motto = models.TextField(null=True)

    # TrueSkill distribution, which is a model of player skill which assumes a Gaussian distribution with
    # an initial estimate (mean mu, player is assumed to be of average skill) and confidence (standard deviation sigma).
    elo_mu = models.FloatField(default=settings.DEFAULT_MU)
    elo_sigma = models.FloatField(default=settings.DEFAULT_SIGMA)

    objects = BulkUpdateManager()

    # Stats that are too expensive to compute on the fly
    # see util.management.commands.update
    # Yes, it is trivially easy to get some of these through basic querysets. However, this allows us for a much
    # faster lookup at the expense of a very small amount of database space.
    active_event_winstreak = models.PositiveSmallIntegerField(default=0)
    longest_event_winstreak = models.PositiveSmallIntegerField(default=0)
    event_wins_count = models.PositiveSmallIntegerField(default=0)
    event_attended_count = models.PositiveSmallIntegerField(default=0)
    event_winrate = models.FloatField(default=0.0)

    match_wins_count = models.PositiveSmallIntegerField(default=0)
    match_losses_count = models.PositiveSmallIntegerField(default=0)
    match_ties_count = models.PositiveSmallIntegerField(default=0)
    match_winrate = models.FloatField(default=0.0)

    awards_count = models.PositiveSmallIntegerField(default=0)
    blue_banners_count = models.PositiveSmallIntegerField(default=0)

    def __str__(self):
        return "{0} ({1})".format(self.nickname, self.team_number)

    def get_matches(self, year=None) -> QuerySet:

        if year is None:
            return Match.objects.filter(alliances__teams__team_number=self.team_number)
        else:
            return Match.objects.filter(event__year=year).filter(alliances__teams__team_number=self.team_number)

    def get_wins(self, year=None) -> QuerySet:
        matches = Match.objects.prefetch_related('winner__teams')
        if year is None:
            return matches.filter(winner__teams__team_number=self.team_number)
        else:
            matches.select_related('event')
            return matches.filter(event__year=year).filter(winner__teams__team_number=self.team_number)

    def get_losses(self, year=None) -> QuerySet:
        matches = Match.objects.prefetch_related('alliances__teams').select_related('winner')
        if year is None:
            return matches.filter(alliances__teams__team_number=self.team_number).exclude(
                winner__teams__team_number=self.team_number).exclude(winner__isnull=True)
        else:
            matches.select_related('event')
            return matches.filter(event__year=year).filter(
                alliances__teams__team_number=self.team_number).exclude(
                winner__teams__team_number=self.team_number).exclude(winner__isnull=True)

    def get_ties(self, year=None) -> QuerySet:
        matches = Match.objects.prefetch_related('alliances__teams').select_related('winner')
        if year is None:
            return matches.filter(alliances__teams__team_number=self.team_number, winner__isnull=True)
        else:
            matches.select_related('event')
            return matches.filter(event__year=year) \
                .filter(alliances__teams__team_number=self.team_number, winner__isnull=True)

    def get_record(self) -> str:
        return "{0}-{1}-{2}".format(self.match_wins_count, self.match_losses_count, self.match_ties_count)

    def get_winrate(self) -> float:
        return self.match_winrate * 100.0

    def get_elo_standing(self) -> int:
        return Team.objects.filter(elo_mu__gte=self.elo_mu).count()

    def get_awards(self, year=None) -> QuerySet:
        if year is None:
            return Award.objects.filter(recipients=self)
        else:
            return Award.objects.filter(recipients=self, year=year)

    def count_awards_old(self) -> QuerySet:
        return Team.objects.filter(team_number=self.team_number).values_list('award__name'). \
            annotate(count=models.Count('award__award_type')).order_by('-count')

    def count_awards(self) -> Dict[str, List['Event']]:
        awards = {}
        award_types = set(self.award_set.values_list('award_type', flat=True).all())
        types_to_names = {t: Award.choice_to_display(t) for t in award_types}

        # get_events = lambda award_t: Event.objects.filter(award__award_type=award_t, award__recipients=self).values_list(flat=True)
        # This is great if the if statement in the below loop is not required...
        # awards = {types_to_names[t]: get_events(t) for t in award_types}

        for award_t in award_types:
            award = types_to_names[award_t]
            if award not in awards:
                awards[award] = Event.objects.filter(award__award_type=award_t, award__recipients=self).all()

        awards = {k: v for k, v in awards.items() if len(v) > 0}
        return OrderedDict(sorted(awards.items(), key=lambda entry: -len(entry[1])))

    def get_max_opr(self) -> float:
        return self.rankingmodel_set.order_by('-tba_opr').first().tba_opr

    def to_dict(self) -> dict:
        return dict((name, getattr(self, name)) for name in self.__dict__ if not name.startswith('_'))

class Alliance(MySqlModel):
    teams = models.ManyToManyField(Team)

    elo_mu = models.FloatField(default=settings.DEFAULT_MU)
    elo_sigma = models.FloatField(default=settings.DEFAULT_SIGMA)

    objects = BulkUpdateManager()

    def __str__(self):
        return "{0}".format(self.teams.all())

    def get_elo_standing(self) -> int:
        return Alliance.objects.filter(elo_mu__gte=self.elo_mu).count()

    def get_wins(self) -> QuerySet:
        return self.match_set.filter(winner=self)

    def get_losses(self) -> QuerySet:
        return self.match_set.exclude(winner=self).exclude(winner__isnull=True)

    def get_ties(self) -> QuerySet:
        return self.match_set.exclude(winner=self).filter(winner__isnull=True)

    def get_record(self) -> str:
        return "{0}-{1}-{2}".format(self.get_wins().count(), self.get_losses().count(), self.get_ties().count())

class Event(MySqlModel):
    """

    :attr id: Unique database id
    :attr key: Unique event key, often of the form ###w where #### is the year and w is the event code
    :attr name: Full event name (e.g. Finger Lakes Regional)
    :attr short_name: Shortened event name (e.g. Finger Lakes)
    :attr event_code: Unique event code (e.g. iri for Indiana Robotics Invitational)
    """
    key = models.CharField(max_length=10, unique=True)  # e.g. 2016cmp
    name = models.CharField(max_length=100)  # e.g. Finger Lakes Regional
    short_name = models.CharField(null=True, max_length=50)  # e.g. Finger Lakes
    event_code = models.CharField(max_length=7)  # e.g. cmp

    """https://github.com/the-blue-alliance/the-blue-alliance/blob/master/consts/event_type.py"""
    event_type_choices = (
        (0, 'Regional'),
        (1, 'District'),
        (2, 'District Championship'),
        (3, 'Championship Division'),
        (4, 'Championship Finals'),
        (99, 'Offseason'),
        (100, 'Preseason'),
        (-1, 'Unlabeled')
    )
    event_type = models.SmallIntegerField(choices=event_type_choices, null=True)

    """https://github.com/the-blue-alliance/the-blue-alliance/blob/master/consts/district_type.py#L6"""
    event_district_choices = (
        (0, 'No District'),
        (1, 'Michigan'),
        (2, 'Mid-Atlantic'),
        (3, 'New England'),
        (4, 'Pacific Northwest'),
        (5, 'Indiana'),
        (6, 'Chesapeake'),
        (7, 'North Carolina'),
        (8, 'Georgia'),
    )
    event_district = models.SmallIntegerField(choices=event_district_choices, null=True)

    year = models.PositiveSmallIntegerField()
    location = models.CharField(max_length=50, null=True)
    venue_address = models.TextField(null=True)
    timezone = models.CharField(max_length=35, null=True)
    website = models.URLField(null=True)
    official = models.BooleanField()
    teams = models.ManyToManyField(Team)
    alliances = models.ManyToManyField(Alliance)
    end_date = models.DateField(null=True)
    winning_alliance = models.ForeignKey(Alliance, related_name='winning_alliance', null=True)

    # webcast = my_webcast_model(), parse JSON data to model fields
    # district_points = my_district_points_model(), parse JSON data to model fields

    def __str__(self):
        return "{0}".format(self.key)

    def __eq__(self, other):
        if isinstance(other, Event):
            return self.key == other.key
        else:
            return False

    def has_f3_match(self) -> bool:
        """

        Returns:
            true if the event's finals set went to three games, false otherwise

        """
        return Match.objects.select_related('event') \
            .filter(comp_level__exact='f', match_number__exact=3, event__key__exact=self.key).exists()

    def get_oprs(self) -> Tuple[Tuple[Team, float]]:
        """
        Gets the OPRs for all teams participating in this event, these are returned as (team, opr) tuples.

        Done using the method of linear least squares.
        :return: Tuple of (Team, OPR) pairs
        """

        matches = Match.objects.prefetch_related('red_alliance__teams', 'blue_alliance__teams') \
            .select_related('scoring_model').filter(event=self, comp_level='qm')
        teams = set(Team.objects.filter(event=self).all())      # type: Set[Team]

        # Populate Score Vector
        s_vector = []
        for m in matches:
            s_vector.append(m.scoring_model.red_total_score)
            s_vector.append(m.scoring_model.blue_total_score)

        # Populate Match Matrix, each row consisting of 3 teams, all from either Blue or Red alliance
        m_row_array = []
        if self.year == 2016:
            from TBAW.models import RankingModel2016
            rms = RankingModel2016.objects.select_related('team') \
                .filter(event=self) \
                .annotate(total_contribution=models.F('goals_points') + models.F('auton_points')) \
                .order_by('-total_contribution')

        for match in matches:
            red_teams = set(match.red_alliance.teams.all())
            blue_teams = set(match.blue_alliance.teams.all())

            if self.year == 2016:
                max_contribution = rms.first().total_contribution
                ratios = [rms.get(team=team).c / max_contribution for team in teams]
                contribution_ratios = zip(ratios, teams)
            else:
                contribution_ratios = zip([1.0] * len(teams), teams)

            red_row = [cont_ratio if team in red_teams else 0 for cont_ratio, team in contribution_ratios]
            blue_row = [cont_ratio if team in blue_teams else 0 for cont_ratio, team in contribution_ratios]

            m_row_array.append(red_row)
            m_row_array.append(blue_row)

        # (Eq. 1): [A][OPR] = [SCORE]
        # (Eq. 2): [A]^T[A][OPR] = [A]^T[SCORE], from (Eq. 1)
        #          [P][OPR] = [S], [P] = [A]^T[A] and [S] = [A]^T[SCORE]
        # (Eq. 3): [L][L]^T[OPR] = [S], where [L] is the lower triangular matrix from Cholesky Decomposition of [P]
        # (Eq. 4): [L][y] = [S], [y] = [L]^T[OPR]
        #          Using Forward Substitution to solve for vector [y]
        #
        # [y] is then known, [y] = [L]^T[OPR], [L]^T is an upper triangular matrix,
        # backwards substitute to solve for OPR.
        oprs = solve_linear_least_squares(create_matrix(m_row_array), create_2d_vector(s_vector))
        team_oprs = zip(teams, oprs)    # type: List[Tuple[Team, float]]
        return tuple((team, opr.item(0)) for team, opr in team_oprs)

    def compare_oprs(self):
        from TBAW.models import RankingModel2016
        rms = [rm.team for rm in RankingModel2016.objects.filter(event=self).order_by('-tba_opr')]
        oth = [res[0] for res in self.get_oprs()]

        for i, (tba, frs) in enumerate(zip(rms, oth)):
            print('{0}.) {1},{0}.) {2}'.format(i + 1, tba, frs))

class Match(MySqlModel):
    key = models.CharField(max_length=20,
                           unique=True)  # yyyy{EVENT_CODE}_{COMP_LEVEL}m{MATCH_NUMBER}, e.g. 2016nyro_f1m2
    comp_level = models.CharField(max_length=6)  # e.g. qm, ef, qf, sf, f
    set_number = models.CharField(null=True, max_length=20)  # in 2016nyro_qf3m2, the set number is 3

    match_number = models.CharField(max_length=20)  # e.g. 2016nyro_qm20
    alliances = models.ManyToManyField(Alliance)
    # videos = my_videos_model, parse JSON data to model fields
    event = models.ForeignKey(Event, on_delete=models.CASCADE)
    # time_string = models.CharField(max_length=25)
    # time = models.DateTimeField(), parse UNIX timestamp to DatetimeField

    blue_alliance = models.ForeignKey(Alliance, null=True, related_name='blue_alliance')
    red_alliance = models.ForeignKey(Alliance, null=True, related_name='red_alliance')
    winner = models.ForeignKey(Alliance, null=True, related_name='winner')
    scoring_model = models.ForeignKey('TBAW.ScoringModel', null=True)

    objects = BulkUpdateManager()

    def __str__(self):
        return "{0}".format(self.key)

class Award(MySqlModel):
    name = models.CharField(max_length=150)

    # https://github.com/the-blue-alliance/the-blue-alliance/blob/master/consts/award_type.py#L15
    award_type_choices = (
        (0, "Chairman's"),
        (1, "Winner"),
        (2, "Finalist"),
        (3, "Woodie Flowers"),
        (4, "Dean's List"),
        (5, "Volunteer"),
        (6, "Founders"),
        (7, "Bart Kamen Memorial"),
        (8, "Make It Loud"),
        (9, "Engineering Inspiration"),
        (10, "Rookie All Star"),
        (11, "Gracious Professionalism"),
        (12, "Coopertition"),
        (13, "Judges"),
        (14, "Highest Rookie Seed"),
        (15, "Rookie Inspiration"),
        (16, "Industrial Design"),
        (17, "Quality"),
        (18, "Safety"),
        (19, "Sportsmanship"),
        (20, "Creativity"),
        (21, "Engineering Excellence"),
        (22, "Entrepreneurship"),
        (23, "Excellence in Design"),
        (24, "Excellence in Design (CAD)"),
        (25, "Excellence in Design (Animation)"),
        (26, "Driving Tomorrow's Technology"),
        (27, "Imagery"),
        (28, "Media and Technology"),
        (29, "Innovation in Control"),
        (30, "Spirit"),
        (31, "Website"),
        (32, "Visualization"),
        (33, "Autodesk Inventor"),
        (34, "Future Innovator"),
        (35, "Recognition of Extraordinary Service"),
        (36, "Outstanding Cart"),
        (37, "WSU Aim Higher"),
        (38, "Leadership in Control"),
        (39, "Number 1 Seed"),
        (40, "Incredible Play"),
        (41, "People's Choice Animation"),
        (42, "Rising Star Visualization"),
        (43, "Best Offensive Round"),
        (44, "Best Play of the Day"),
        (45, "Featherweight in the Finals"),
        (46, "Most Photogenic"),
        (47, "Outstanding Defense"),
        (48, "Power to Simplify"),
        (49, "Against All Odds"),
        (50, "Rising Star"),
        (51, "Chairman's Honorable Mention"),
        (52, "Content Communication Honorable Mention"),
        (53, "Technical Execution Honorable Mention"),
        (54, "Realization"),
        (55, "Realization Honorable Mention"),
        (56, "Design Your Future"),
        (57, "Design Your Future Honorable Mention"),
        (58, "Special Recognition Character Animation"),
        (59, "High Score"),
        (60, "Teacher Pioneer"),
        (61, "Best Craftsmanship"),
        (62, "Best Defensive Match"),
        (63, "Play of the Day"),
        (64, "Programming"),
        (65, "Professionalism"),
        (66, "Golden Corndog"),
        (67, "Most Improved Team"),
        (68, "Wildcard"),
    )

    blue_banner_choices = (
        (0, "Chairman's"),
        (1, "Winner"),
        (9, "Engineering Inspiration"),
    )

    non_district_point_choices = (
        (1, "Winner"),
        (2, "Finalist"),
        (3, "Woodie Flowers"),
        (4, "Dean's List"),
        (5, "Volunteer"),
        (14, "Highest Rookie Seed"),
    )

    award_type = models.PositiveSmallIntegerField(choices=award_type_choices)
    event = models.ForeignKey(Event, on_delete=models.CASCADE)
    recipients = models.ManyToManyField(Team)
    year = models.PositiveSmallIntegerField()

    def __str__(self):
        return "{0} - {1} ({2})".format(self.recipients.all(), self.name, self.event)

    @classmethod
    def choice_to_display(cls, choice):
        """

        Args:
            choice: a number inside award_type_choices

        Returns:
            The string associated with the choice. Works the same as some_award.get_award_type_display() but is a
            class method.

        """
        for c in Award.award_type_choices:
            if c[0] == choice:
                return c[1]

        return None

class Robot(MySqlModel):
    key = models.CharField(max_length=13, null=True)  # e.g. frc2791_2016
    team = models.ForeignKey(Team, null=True)
    year = models.PositiveSmallIntegerField(null=True)
    name = models.CharField(max_length=100, null=True)

class AllianceAppearance(MySqlModel):
    alliance = models.ForeignKey(Alliance, null=True)
    event = models.ForeignKey(Event, null=True)

    captain = models.ForeignKey(Team, related_name='captain', null=True)
    first_pick = models.ForeignKey(Team, related_name='first_pick', null=True)
    second_pick = models.ForeignKey(Team, related_name='second_pick', null=True)
    backup = models.ForeignKey(Team, related_name='backup', null=True)

    seed = models.PositiveSmallIntegerField(null=True)
    elo_mu_pre = models.FloatField(null=True)
    elo_mu_post = models.FloatField(null=True)
    elo_sigma_pre = models.FloatField(null=True)
    elo_sigma_post = models.FloatField(null=True)

    objects = BulkUpdateManager()


from pyspark.sql import SparkSession
from pyspark.sql.functions import *

def quiet_logs(sc):
  logger = sc._jvm.org.apache.log4j
  logger.LogManager.getLogger("org"). setLevel(logger.Level.ERROR)
  logger.LogManager.getLogger("akka").setLevel(logger.Level.ERROR)

spark = SparkSession \
    .builder \
    .appName("SSS - Kafka Wordcount") \
    .getOrCreate()

quiet_logs(spark)

comments = spark \
  .readStream \
  .format("kafka") \
  .option("kafka.bootstrap.servers", "kafka1:19092") \
  .option("subscribe", "subreddit-politics") \
  .load()

stopWordList = ["i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", "your", "yours", "yourself",\
                                                        "yourselves", "he", "him", "his", "himself", "she", "her", "hers", "herself", "it", "its", "itself",\
                                                        "they", "them", "their", "theirs", "themselves", "what", "which", "who", "whom", "this", "that", "these",\
                                                        "those", "am", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "having", "do", "does",\
                                                        "did", "doing", "a", "an", "the", "and", "but", "if", "or", "because", "as", "until", "while", "of", "at", "by",\
                                                        "for", "with", "about", "against", "between", "into", "through", "during", "before", "after", "above", "below",\
                                                        "to", "from", "up", "down", "in", "out", "on", "off", "over", "under", "again", "further", "then", "once", "here",\
                                                        "there", "when", "where", "why", "how", "all", "any", "both", "each", "few", "more", "most", "other", "some", "such",\
                                                        "no", "nor", "not", "only", "own", "same", "so", "than", "too", "very", "s", "t", "can", "will", "just", "don",\
                                                        "should", "now"]

words = comments.select(
   explode(
       split(comments.value, " ")
   ).alias("word")
).filter(~lower(col("word")).isin(stopWordList))

wordCounts = words.groupBy("word").count().orderBy(desc("count")).limit(10)

query = wordCounts \
    .writeStream \
    .outputMode("complete") \
    .format("console") \
    .start()

# query = wordCounts \
#     .writeStream \
#     .outputMode("complete") \
#     .format("console") \
#     .trigger(processingTime='2 seconds') \
#     .start()

query.awaitTermination()


import argparse
from typing import Dict, Final, List
from os import path, mkdir
import numpy as np
from json import dump

from utils.ada_boost import AdaBoost
from utils.logger import init_logger

Logger: Final = init_logger(__name__)

def get_pred_metrics(y: np.ndarray, pred: np.ndarray):
    """Get the classification metrics."""
    TP = np.sum(np.multiply(((pred) == y).astype("int"), (y == 1).astype("int")))
    TN = np.sum(np.multiply(((pred) == y).astype("int"), (y == -1).astype("int")))
    FP = np.sum(np.multiply(((pred) != y).astype("int"), (y == 1).astype("int")))
    FN = np.sum(np.multiply(((pred) != y).astype("int"), (y == -1).astype("int")))
    return(TP, FP, TN, FN)

def load_file(file_path: str) -> np.ndarray:
    return np.loadtxt(file_path, dtype="int", delimiter=",")

def test_model(dataset_path: str, clf: AdaBoost) -> Dict[int, Dict[str, float]]:
    """Tests the model on a given dataset"""
    test_res: Dict[int, Dict[str: float]] = {}
    Logger.info("Loading testing dataset")
    train_matrix = load_file(dataset_path)
    X, y = train_matrix[:, :-1], train_matrix[:, -1]

    Logger.info("Testing the model ...")
    for i in range(1, clf.num_of_est+1):
        Logger.info(f"Testing the model for round: {i} is starting")
        i_clf_prediction = clf.predict(X, num_of_est=i)
        TP, FP, TN, FN = get_pred_metrics(y, i_clf_prediction)
        i_accuracy = (TP + TN) /(TP+FP+TN+FN)
        i_precision = TP/(TP + FP)
        i_recall = TP/(TP + FN)
        try:
            i_specificity = (TN) /(TN + FP)
        except Exception as e:
            Logger.warning(f" round : {i} has not false positive or true negatives")
            i_specificity = 0
        i_f_score = (2*i_precision*i_recall)/(i_precision + i_recall)
        test_res[i] = {"accuracy": i_accuracy, "precision": i_precision,
                       "recall": i_recall, "specificity": i_specificity, "f_score": i_f_score}
        Logger.info(f"Testing the model for round: {i+1} is complete.")
    return test_res

def save_test_res(test_res: Dict[int, Dict[str, float]], test_file_path: str):
    with open(test_file_path, "w") as fp:
        dump(test_res, fp)

def test(dataset_path: str, model_path: str, output_path: str) -> None:

    if not path.exists(output_path):
        Logger.info("Creating output directory")
        mkdir(output_path)

    TEST_PATH: Final = path.join(dataset_path, "test_data.csv")

    clf: AdaBoost = AdaBoost.load(model_path)
    test_res = test_model(TEST_PATH, clf)
    TEST_RESULT_PATH: Final = path.join(output_path, f"{clf.num_of_est}_round_model_test_results.json")
    Logger.info("Saving test results")
    save_test_res(test_res, TEST_RESULT_PATH)
    Logger.info(f"Testing is complete, and results are available at {TEST_RESULT_PATH}")

def main():
    parser = argparse.ArgumentParser(
        prog = "Model Tester",
        description= "Test a given model the generated dataset"
    )
    parser.add_argument("--dataset_path", "-d", type=str)
    parser.add_argument("--model_path", "-m", type=str)
    parser.add_argument("--output_path", "-o", type=str)
    args = parser.parse_args()
    test(args.dataset_path, args.model_path, args.output_path)

if __name__ == "__main__":
    main()