import time
import os

from dtest import Tester, debug, DEFAULT_DIR
from tools import *
from assertions import *
from ccmlib.cluster import Cluster
from ccmlib import common as ccmcommon

import loadmaker

try:
    CASSANDRA_VERSION = os.environ['CASSANDRA_VERSION']
except KeyError:
    CASSANDRA_VERSION = 'git:trunk'

class TestRollingUpgrade(Tester):

    def __init__(self, *argv, **kwargs):
        super(TestRollingUpgrade, self).__init__(*argv, **kwargs)
        # When a node goes down under load it prints an error in it's log.
        # If we don't allow log errors, then the test will fail.
        self.allow_log_errors = True

    def rolling_upgrade_node(self, node, stress_node, create_ks_and_cf):
        """
        node is the node to upgrade. stress_ip is the node to run stress on.
        """
        debug("Called rolling_upgrade_node for: %s %s" % (node.name, node.address()))
        debug("Stress node log file: " + stress_node.logfilename())
        debug("Stress node address: " + stress_node.address() + ':' + stress_node.jmx_port)

        keyspace = 'rolling_ks'

        host, port = stress_node.network_interfaces['thrift']

        lm_standard = loadmaker.LoadMaker(
                host, port,
                create_ks=create_ks_and_cf, create_cf=create_ks_and_cf,
                column_family_name='rolling_cf_standard',
                consistency_level='QUORUM', keyspace_name=keyspace)
        lm_counter_standard = loadmaker.LoadMaker(
                host, port,
                create_ks=create_ks_and_cf, create_cf=create_ks_and_cf,
                column_family_name='rolling_cf_counter_standard',
                is_counter=True, consistency_level='QUORUM', num_cols=3,
                keyspace_name=keyspace)

        loader_counter = loadmaker.ContinuousLoader(
                load_makers=[lm_counter_standard],
                sleep_between=1,
                )
        loader_standard = loadmaker.ContinuousLoader(
                load_makers=[lm_standard],
                sleep_between=1,
                )
        #loader_counter.read_and_validate(step=10)

        debug("Sleeping to get some data into the cluster")
        time.sleep(5)

        debug("pausing counter-add load. This is because a "
                  "dead-but-not-detected-dead node will cause errors on "
                  "counter-add.")
        loader_counter.pause()
        time.sleep(2)

        debug("draining...")
        node.nodetool('drain')
        debug("stopping...")
        node.stop(wait_other_notice=False)
        debug("Node stopped. Sleeping to let gossip detect it as down...")
        time.sleep(30)

        debug("Resuming counter loader")
        loader_counter.unpause()

        debug("Letting the counter loader generate some load")
        time.sleep(10)

        debug("Upgrading node")
        debug("setting dir...")
        node.set_cassandra_dir(cassandra_version=CASSANDRA_VERSION)

        debug("starting...")
        node.start(wait_other_notice=True)
        debug("upgradsstables...")
        node.nodetool('upgradesstables')

        debug("validating standard data...")
        loader_standard.read_and_validate(step=10)
        loader_standard.exit()
        debug("validating counter data...")
        #loader_counter.read_and_validate(step=10)
        loader_counter.exit()

        debug("Done upgrading node %s.\n" % node.name)

    @since('1.0', max_version='1.1')
    def upgrade10_test(self):
        """ Upgrade from 1.0 """

        cluster = self.cluster
        cluster.set_cassandra_dir(cassandra_version="git:cassandra-1.0")

        cluster.populate(3, tokens=[0, 2**125, 2**126]).start()
        [node1, node2, node3] = cluster.nodelist()
        time.sleep(1)

        self.rolling_upgrade_node(node1, stress_node=node2, create_ks_and_cf=True)
        self.rolling_upgrade_node(node2, stress_node=node3, create_ks_and_cf=False)
        self.rolling_upgrade_node(node3, stress_node=node1, create_ks_and_cf=False)

    @since('1.1', max_version='1.2')
    def upgrade11_test(self):
        """ Upgrade from 1.1 """

        cluster = self.cluster
        cluster.set_cassandra_dir(cassandra_version="git:cassandra-1.1")

        cluster.populate(3, tokens=[0, 2**125, 2**126]).start()
        [node1, node2, node3] = cluster.nodelist()
        time.sleep(1)

        self.rolling_upgrade_node(node1, stress_node=node2, create_ks_and_cf=True)
        self.rolling_upgrade_node(node2, stress_node=node3, create_ks_and_cf=False)
        self.rolling_upgrade_node(node3, stress_node=node1, create_ks_and_cf=False)

    @since('1.2', max_version='1.3')
    def upgrade12_test(self):
        """ Upgrade from 1.2 """

        cluster = self.cluster
        cluster.set_cassandra_dir(cassandra_version="git:cassandra-1.2")

        cluster.populate(3, tokens=[0, 2**125, 2**126]).start()
        [node1, node2, node3] = cluster.nodelist()
        time.sleep(1)

        self.rolling_upgrade_node(node1, stress_node=node2, create_ks_and_cf=True)
        self.rolling_upgrade_node(node2, stress_node=node3, create_ks_and_cf=False)
        self.rolling_upgrade_node(node3, stress_node=node1, create_ks_and_cf=False)

    def upgrade1010_with_index(self):
        """ Test for #4262 bug """
        cluster = self.cluster
        cluster.set_cassandra_dir(cassandra_version="1.0.10")

        cluster.populate(2).start()
        [node1, node2] = cluster.nodelist()
        time.sleep(.2)

        cli = node1.cli()
        cli.do("create keyspace test with placement_strategy='org.apache.cassandra.locator.SimpleStrategy' and strategy_options = {replication_factor:2}")
        cli.do("use test")
        cli.do("create column family test with comparator=UTF8Type and key_validation_class=UTF8Type and column_metadata = [{ column_name: a, index_type: KEYS, validation_class: UTF8Type}]")
        cli.do("set test[a][a] = a")
        cli.do("set test[b][a] = a")
        cli.do("set test[c][a] = b")
        cli.do("consistencylevel as ALL")
        cli.do("get test where a = a")

        assert not cli.has_errors(), cli.errors()
        output = cli.last_output()
        assert re.search('%d Rows Returned' % 2, output) is not None, output

        node1.stop()
        time.sleep(2)
        self.set_node_to_current_version(node1)
        time.sleep(.2)
        node1.start()

        cli = node1.cli()
        cli.do("use test")
        cli.do("consistencylevel as ALL")
        cli.do("list test")
        cli.do("get test where a = a")

        assert not cli.has_errors(), cli.errors()
        output = cli.last_output()
        assert re.search('%d Rows Returned' % 2, output) is not None, output


from selenium import webdriver
import time

# link = "https://emias.info/appointment"
#
# try:
#     browser = webdriver.Chrome()
#     browser.get(link)
#
#     input1 = browser.find_element_by_css_selector("._3ba8xX._3z5tMY._3F1A0e :nth-child(1)")
#     input1.click()
#
# finally:
#     # успеваем скопировать код за 30 секунд
#     time.sleep(10)
#     # закрываем браузер после всех манипуляций
#     browser.quit()

# не забываем оставить пустую строку в конце файла

browser = webdriver.Chrome()
link = "https://SunInJuly.github.io/execute_script.html"
browser.get(link)
button = browser.find_element_by_tag_name("button")
browser.execute_script("return arguments[0].scrollIntoView(true);", button)
button.click()
assert True

time.sleep(5)
browser.quit()


class Term:
    def __init__(self,coefficient,power):
        if coefficient < 0:
            raise TypeError('Term coefficient cannot be negative number!')
        if power < 0:
            raise TypeError('Term power cannot be negative number!')
        self.coefficient = coefficient
        self.power = power

    def __eq__(self,other):
        return self.coefficient == other.coefficient and self.power == other.power

    def __str__(self):
        if self.power == 0:
            if self.coefficient == 0:
                return f'0'
            else:
                return f'{self.coefficient}'
        if self.coefficient == 1:
            if self.power == 1:
                return f'x'
            else:
                return f'x^{self.power}'
        else:
            if self.power == 1:
                return f'{self.coefficient}*x'
            else:
                return f'{self.coefficient}*x^{self.power}'
    def __repr__(self):
        return f'Term {self}'

    def __lt__(self,other):
        return self.power < other.power

class Polynomial:

    def __init__(self,terms):
        self.terms = terms

    def __str__(self):
        string = ''
        reversedList = sorted(self.terms)

        for item in reversedList[::-1]:
            if item == Term(0,0):
                continue
            else:
                string += str(item)
                string += ' + '
        if string == '':
            return '0'
        else:
            return string[:-3]

    def __repr__(self):
        return f'Polynomial {self}'

    def __eq__(self,other):
        selfHelper = self.terms
        otherHelper = other.terms

        for term in selfHelper:
            if term not in otherHelper:
                return False
            else:
                otherHelper.remove(term)
        if otherHelper != []:
            return False
        return True

    def derivative(self):
        oldListOfTerms = self.terms
        newListOfTerms = []

        for term in oldListOfTerms:
            newCoefficient = term.coefficient * term.power
            newPower = term.power - 1
            if newPower < 0:
                newListOfTerms.append(Term(0,0))
            else:
                newListOfTerms.append(Term(newCoefficient,newPower))

        newListOfTerms = [elem for elem in newListOfTerms if elem.power >= 0]

        return str(Polynomial(newListOfTerms))

p = Polynomial([Term(1,20),Term(7,8),Term(4,6),Term(3,5)])
print(p.derivative())


from skyfield.api import load, wgs84
import UnityEngine

ts = load.timescale()
t = ts.utc(2024, 1, 1)

active_url = 'http://celestrak.com/NORAD/elements/active.txt'
satellites = load.tle_file(active_url)
by_number = {sat.model.satnum: sat for sat in satellites}
satellite = by_number[31304]  #Change Sat here
positions = []
for x in range(1,32):
    for i in range(1,24):
        for y in range(1,60):
            t = ts.utc(2024, 1, x, i, y)
            geocentric = satellite.at(t)
            position = geocentric.position.km
            positions.append([str(position[0]), str(position[1]), str(position[2])])

key = "tleLoader"
UnityEngine.GameObject.FindGameObjectWithTag("external/pythonHelper").GetComponent("pythonOutput").add(key, positions)


#
# created by
# Antonio Garcia-Uceda Juarez
# PhD student
# Medical Informatics
#
# created on 09/02/2018
# Last update: 09/02/2018
########################################################################################

from Common.Constants import *
from Common.FunctionsUtil import *
from Common.WorkDirsManager import *
import argparse

def find_indexes_names_images_files(names_images_type_data, list_images_files):
    indexes_names = []
    for iname in names_images_type_data:
        ifound = False
        for i, ifile in enumerate(list_images_files):
            if iname in ifile:
                indexes_names.append(i)
                ifound = True
                break
        #endfor
        if not ifound:
            message = 'data named: \'%s\' not found' %(iname)
            CatchErrorException(message)
    return indexes_names

def find_element_repeated_two_indexes_names(names_images_type_data_1, names_images_type_data_2):
    list_names_repeated = []
    for ielem in names_images_type_data_1:
        if ielem in names_images_type_data_2:
            list_names_repeated.append(ielem)
    #endfor
    return list_names_repeated

def main(args):
    # ---------- SETTINGS ----------
    nameOrigImagesDataRelPath = 'Images_WorkData'
    nameOrigMasksDataRelPath = 'LumenDistTrans_WorkData'

    nameOriginImagesFiles = 'images*'+ getFileExtension(FORMATTRAINDATA)
    nameOriginMasksFiles = 'grndtru*'+ getFileExtension(FORMATTRAINDATA)
    # ---------- SETTINGS ----------

    workDirsManager = WorkDirsManager(args.basedir)

    OrigImagesDataPath = workDirsManager.getNameExistPath(workDirsManager.getNameBaseDataPath(), nameOrigImagesDataRelPath)
    OrigGroundTruthDataPath = workDirsManager.getNameExistPath(workDirsManager.getNameBaseDataPath(), nameOrigMasksDataRelPath)
    TrainingDataPath = workDirsManager.getNameNewPath(workDirsManager.getNameTrainingDataPath())
    ValidationDataPath = workDirsManager.getNameNewPath(workDirsManager.getNameValidationDataPath())
    TestingDataPath = workDirsManager.getNameNewPath(workDirsManager.getNameTestingDataPath())

    listImagesFiles = findFilesDir(OrigImagesDataPath, nameOriginImagesFiles)
    listGroundTruthFiles = findFilesDir(OrigGroundTruthDataPath, nameOriginMasksFiles)

    numImagesFiles = len(listImagesFiles)
    numGroundTruthFiles = len(listGroundTruthFiles)

    if (numImagesFiles != numGroundTruthFiles):
        message = "num image files \'%s\' not equal to num ground-truth files \'%s\'..." %(numImagesFiles, numGroundTruthFiles)
        CatchErrorException(message)

    if (args.distribute_fixed_names):
        print("Split dataset with Fixed Names...")
        names_repeated  = find_element_repeated_two_indexes_names(NAME_IMAGES_TRAINING, NAME_IMAGES_VALIDATION)
        names_repeated += find_element_repeated_two_indexes_names(NAME_IMAGES_TRAINING, NAME_IMAGES_TESTING)
        names_repeated += find_element_repeated_two_indexes_names(NAME_IMAGES_VALIDATION, NAME_IMAGES_TESTING)

        if names_repeated:
            message = "found names repeated in list Training / Validation / Testing names: %s" %(names_repeated)
            CatchErrorException(message)

        indexesTraining = find_indexes_names_images_files(NAME_IMAGES_TRAINING, listImagesFiles)
        indexesValidation = find_indexes_names_images_files(NAME_IMAGES_VALIDATION, listImagesFiles)
        indexesTesting = find_indexes_names_images_files(NAME_IMAGES_TESTING, listImagesFiles)
        print("Training (%s files)/ Validation (%s files)/ Testing (%s files)..." %(len(indexesTraining),
                                                                                len(indexesValidation),
                                                                                len(indexesTesting)))
    else:
        numTrainingFiles = int(args.prop_data_training * numImagesFiles)
        numValidationFiles = int(args.prop_data_validation * numImagesFiles)
        numTestingFiles = int(args.prop_data_testing * numImagesFiles)
        print("Training (%s files)/ Validation (%s files)/ Testing (%s files)..." %(numTrainingFiles,
                                                                                numValidationFiles,
                                                                                numTestingFiles))
        if (args.distribute_random):
            print("Split dataset Randomly...")
            indexesAllFiles = np.random.choice(range(numImagesFiles), size=numImagesFiles, replace=False)
        else:
            print("Split dataset In Order...")
            indexesAllFiles = range(numImagesFiles)

        indexesTraining = indexesAllFiles[0:numTrainingFiles]
        indexesValidation = indexesAllFiles[numTrainingFiles:numTrainingFiles+numValidationFiles]
        indexesTesting = indexesAllFiles[numTrainingFiles+numValidationFiles::]

    print("Files assigned to Training Data: \'%s\'" %([basename(listImagesFiles[index]) for index in indexesTraining]))
    print("Files assigned to Validation Data: \'%s\'" %([basename(listImagesFiles[index]) for index in indexesValidation]))
    print("Files assigned to Testing Data: \'%s\'" %([basename(listImagesFiles[index]) for index in indexesTesting]))

    # ******************** TRAINING DATA ********************
    for index in indexesTraining:
        makelink(listImagesFiles[index], joinpathnames(TrainingDataPath, basename(listImagesFiles[index])))
        makelink(listGroundTruthFiles[index], joinpathnames(TrainingDataPath, basename(listGroundTruthFiles[index])))
    #endfor
    # ******************** TRAINING DATA ********************

    # ******************** VALIDATION DATA ********************
    for index in indexesValidation:
        makelink(listImagesFiles[index], joinpathnames(ValidationDataPath, basename(listImagesFiles[index])))
        makelink(listGroundTruthFiles[index], joinpathnames(ValidationDataPath, basename(listGroundTruthFiles[index])))
    #endfor
    # ******************** VALIDATION DATA ********************

    # ******************** TESTING DATA ********************
    for index in indexesTesting:
        makelink(listImagesFiles[index], joinpathnames(TestingDataPath, basename(listImagesFiles[index])))
        makelink(listGroundTruthFiles[index], joinpathnames(TestingDataPath, basename(listGroundTruthFiles[index])))
    #endfor
    # ******************** TESTING DATA ********************

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('--datadir', default=DATADIR)
    parser.add_argument('--basedir', default=BASEDIR)
    parser.add_argument('--prop_data_training', type=float, default=PROP_DATA_TRAINING)
    parser.add_argument('--prop_data_validation', type=float, default=PROP_DATA_VALIDATION)
    parser.add_argument('--prop_data_testing', type=float, default=PROP_DATA_TESTING)
    parser.add_argument('--distribute_random', type=str2bool, default=DISTRIBUTE_RANDOM)
    parser.add_argument('--distribute_fixed_names', type=str2bool, default=DISTRIBUTE_FIXED_NAMES)
    args = parser.parse_args()

    print("Print input arguments...")
    for key, value in vars(args).iteritems():
        print("\'%s\' = %s" %(key, value))

    main(args)


import socket
import threading
from threading import Thread
from queue import Queue

class UDPServer:
    def __init__(self):
        self.localIP = "132.68.36.158"
        self.localPort = 20001
        self.bufferSize = 32768
        self.UDPServerSocket = None
        self._data_thread = None
        self._data = None
        self._queue = None

    def start(self):
        self.UDPServerSocket = socket.socket(family=socket.AF_INET, type=socket.SOCK_DGRAM)
        self.UDPServerSocket.bind((self.localIP, self.localPort))
        self._queue = Queue()
        self._data_thread = MyThread(self._queue, self)
        self._data_thread.start()
        print("UDP server up and listening")

    def update_data(self, data):
        print("update: ", data)
        if self._queue.empty():
            self._queue.put(data)
        else:
            self._queue.get()
            self._queue.put(data)

print_lock = threading.Lock()

class MyThread(threading.Thread):
    def __init__(self, queue, server, args=(), kwargs=None):
        threading.Thread.__init__(self, args=(), kwargs=None, target=self.run)
        self.queue = queue #for now this is data test
        self.server = server
        self.daemon = True

    def run(self):

        # Listen for incoming datagrams
        while True:
            data = self.queue.get()
            bytesAddressPair = self.server.UDPServerSocket.recvfrom(self.server.bufferSize)
            message = bytesAddressPair[0]
            address = bytesAddressPair[1]

            clientMsg = "Message from Client:{}".format(message)
            clientIP = "Client IP Address:{}".format(address)
            self.send_data(data, address)

            # Sending a reply to client
            # msgFromServer = "Hello UDP Client. My Name is Nir."
            # bytesToSend = str.encode(self._data or "")
            # self.server.UDPServerSocket.sendto(bytesToSend, address)

    def send_data(self, data, address):
        print(self.queue.qsize())
        print("data at time of sending: ", data)
        bytesToSend = str.encode(data or "")
        self.server.UDPServerSocket.sendto(bytesToSend, address)


from abc import ABCMeta, abstractmethod
import random
from hashlib import sha256
from functools import lru_cache
from typing import List

import numpy as np

import ray
from ray.serve.utils import logger

@lru_cache(maxsize=128)
def deterministic_hash(key: bytes) -> float:
    """Given an arbitrary bytes, return a deterministic value between 0 and 1.

    Note:
        This function uses stdlib random number generator because it's faster
        than numpy's. On a cache miss, the runtime of this function is about
        ~10us.
    """
    bytes_hash = sha256(key).digest()  # should return 32 bytes value
    int_seed = int.from_bytes(bytes_hash, "little", signed=False)
    random_state = random.Random(int_seed)
    return random_state.random()

class EndpointPolicy:
    """Defines the interface for a routing policy for a single endpoint.
    To add a new routing policy, a class should be defined that provides this
    interface. The class may be stateful, in which case it may also want to
    provide a non-default constructor. However, this state will be lost when
    the policy is updated (e.g., a new backend is added).
    """
    __metaclass__ = ABCMeta

    @abstractmethod
    def assign(self, query) -> List[str]:
        """Assign a query to a list of backends.

        Arguments:
            query (ray.serve.router.Query): the incoming query object.
        Returns:
            A list with length >= 1. It should contains the list of backend
            tags that had queries added to their queues. Ordered by importance.
            The first value should be the backend to assign and rest values
            correspond to shadow backends.
        """
        raise NotImplementedError()

class RandomEndpointPolicy(EndpointPolicy):
    """
    A stateless policy that makes a weighted random decision to map each query
    to a backend using the specified weights. If a shard key is provided in a
    query, the weighted random selection will be made deterministically based
    on the hash of the shard key.
    """

    def __init__(self, traffic_policy: "ray.serve.controller.TrafficPolicy"):
        self.backends = sorted(traffic_policy.traffic_dict.items())
        self.shadow_backends = list(traffic_policy.shadow_dict.items())

    def _select_backends(self, val):
        curr_sum = 0
        for name, weight in self.backends:
            curr_sum += weight
            if curr_sum > val:
                chosen_backend = name
                break
        else:
            assert False, "This should never be reached."

        shadow_backends = []
        for backend, backend_weight in self.shadow_backends:
            if val < backend_weight:
                shadow_backends.append(backend)

        return chosen_backend, shadow_backends

    def assign(self, query):
        if len(self.backends) == 0:
            raise ValueError("No backends to assign traffic to.")

        if query.metadata.shard_key is None:
            value = np.random.random()
        else:
            value = deterministic_hash(
                query.metadata.shard_key.encode("utf-8"))

        chosen_backend, shadow_backends = self._select_backends(value)
        logger.debug(f"Chosen backend {chosen_backend} for query {query}")
        return [chosen_backend] + shadow_backends


import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn import svm
from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics

change = {'google':'GOOGL','facebook':'FB','nvidia':'NVDA','intel':'INTC','microsoft':'MSFT','ibm':'IBM','verizon':'VZ','amazon':'AMZN','oracle':'ORCL','apple':'AAPL','walmart':'WMT','ups':'UPS','kroger':'KR','pepsico':'PEP','disney':'DIS','ford':'F','comcast':'CMCSA','citigroup':'C','anthem':'ANTM','nike':'NKE','3m':'MMM','abbott':'ABT','abbvie':'ABBV'}
change_dict = dict((v,k) for k,v in change.items())
def GetCompanyData():
    companyPoint = {}
    companys = ['FB','GOOGL','NVDA','INTC','AAPL','MSFT','IBM','VZ','AMZN','ORCL','WMT','UPS','KR','PEP','DIS','F','CMCSA','C','ANTM','NKE','MMM','ABT','ABBV']
    for company in companys:
        df = pd.read_csv('deltap/'+company + '_deltap.csv',converters={'deltap': clean_aa})
        df['date'] = pd.to_datetime(df['date'])
        # mask = df['date'].between('2017-12-27', '2018-12-31')
        df = df.loc[:, ~df.columns.str.contains('^Unnamed')]
        # df[mask].to_csv('deltap/'+company + '_deltap.csv')
        # df = df.drop(columns = ['close', 'open','high','low','volume','Name'])
        companyPoint[change_dict[company]] = df
    return companyPoint

def clean_aa(seq_string):
    return seq_string.replace("[", "").replace("'", "").replace(" ", "").replace("]", "").split(",")

def cal_tfidf(corpus):
    # 將corpus轉換成tf矩陣
    vectorizer = CountVectorizer()
    # 計算term出現次數
    X = vectorizer.fit_transform(corpus)
    # 取出不重複的term
    word = vectorizer.get_feature_names()
    # print(word)
    # 查看tf
    # print(X.toarray())
    # 類調用
    transformer = TfidfTransformer()
    # 將X統計成TF-IDF
    tfidf = transformer.fit_transform(X)
    return tfidf

def train_and_predict(clf, test_tfidf, bokTfidf, point_of_doc, test_df):
    clf.fit(bokTfidf, point_of_doc)
    correct = 0
    error = 0
    predicted = clf.predict(test_tfidf)
    for index in range(len(predicted)):
        if test_df.point[index] == predicted[index]:
            correct = correct + 1
        else:
            error = error + 1
        # print(tmp_df.point[index], predicted[index], clf.predict_proba(bokTfidf[index:index+1]))
    print("correct: "+str(correct)+" error: "+str(error))
    return correct/(correct+error)

def test_each_dataframe(df, company):
    company_correct = []
    for name in company:
        size = 0
        bokCorpus = []
        tmp_df = df
        if company != ["All Company"]:
            tmp_df = df[df.company == name]
        tmp_df = tmp_df[tmp_df.point != 3].reset_index(drop=True)
        tmp_df = tmp_df[tmp_df.point != 0].reset_index(drop=True)
        print(name)
        # for index in tmp_df.keywords:
        #     bokCorpus.append(' '.join(index))
        # bokTfidf = cal_tfidf(bokCorpus)
        # bokTfidf = bokTfidf.toarray()
        # temp = []
        # i = 0
        # for index in bokTfidf:
        #     index = np.append(index, tmp_df.deltap[i])
        #     # index = np.append(index, tmp_df.polarity[i])
        #     temp.append(index)
        #     i = i + 1
        temp = []
        for index in tmp_df.deltap:
            temp.append(index)
        bokTfidf = np.asarray(temp)
        size = int(len(tmp_df)/3*2)
        point_of_doc = []
        for point in tmp_df.point:
            point_of_doc.append(point)
            if len(point_of_doc) == size:
                break
        test_df = tmp_df[size:bokTfidf.shape[0]].reset_index(drop=True)
        train_df = tmp_df[0:size].reset_index(drop=True)
        train_tfidf = bokTfidf[0:size]
        test_tfidf = bokTfidf[size:bokTfidf.shape[0]]
        print('SVC\t', end='')
        value = train_and_predict(svm.SVC(gamma='auto', kernel='rbf'), test_tfidf, train_tfidf, point_of_doc, test_df)
        company_correct.append(value)
    total = 0
    for i in company_correct:
        total = total + i
    print(str(total/len(company_correct))+"\n")

def predict(companyprice):
    f = open(r'company.txt')
    company = f.readlines()[0].split(',')
    f.close()
    df = pd.read_csv('news_point_for_train_with_polarity.csv', converters={'keywords': clean_aa, 'cts': clean_aa, 'polarity': clean_aa})
    price = []
    for index,d in df.iterrows():
        mask = companyprice[d.company].date == d.date
        if(not mask.any()):
            price.append([0.0,0.0,0.0,0.0])
            continue
        tmp = list(companyprice[d.company][mask].deltap)[0]
        tmp_float = [float(t) for t in tmp]
        price.append(tmp_float)
    df['deltap'] = price
    # df['keywords'] = df['keywords'] + df['cts']
    test_each_dataframe(df, ["All Company"])
    test_each_dataframe(df, company)

company_data = GetCompanyData()
predict(company_data)


CSV_PATH = "../Data/reducedDataset001.csv"
APP_NAME = "Random Forest"
SPARK_URL = "local[*]"
RANDOM_SEED = 13579
TRAINING_DATA_RATIO = 0.7
RF_MAX_DEPTH = 10
RF_NUM_BINS = 32
NUM_TREES = 5

from pyspark import SparkContext
from pyspark.sql import SparkSession
from pyspark.ml.feature import  VectorAssembler
from pyspark.ml.classification import RandomForestClassifier
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark import SparkContext
from pyspark.ml import Pipeline

#Definimos contexto de spark
sc = SparkContext(SPARK_URL, APP_NAME)
spark = SparkSession.builder \
    .appName(APP_NAME) \
    .master(SPARK_URL) \
    .getOrCreate()
#Leemos dataset reducido para entrenar el modelo
df = spark.read \
    .options(header = "true", inferschema = "true") \
    .csv(CSV_PATH)

print("Total number of rows: %d" % df.count())
#Definimos las feature a tener en cuenta en el entrenamiento
feature_list = []
for col in df.columns:
    if col == 'label' or col == 'Sujeto':
        continue
    else:
        feature_list.append(col)
#Definimos un vector assembler para transformar las columnar de las features en una sola
assembler = VectorAssembler(inputCols=feature_list, outputCol="features")
(trainingData, testData) = df.randomSplit([0.7, 0.3])

print("Number of training set rows: %d" % trainingData.count())
print("Number of test set rows: %d" % testData.count())
#Modelo a usar
rf = RandomForestClassifier(labelCol="label", featuresCol="features", numTrees=10)

pipeline = Pipeline(stages=[assembler, rf])

# Train model.  This also runs the indexers.
model = pipeline.fit(trainingData)

# Make predictions.
predictions = model.transform(testData)

# Evaluar performance del modelo
evaluator = MulticlassClassificationEvaluator(
        labelCol="label", predictionCol="prediction", metricName="accuracy")
accuracy = evaluator.evaluate(predictions)
print("Test Error = %g" % (1.0 - accuracy))

# Guardar modelo
model.save("../Data/pySparkRFModel")
spark.stop()


from sklearn.ensemble import RandomForestClassifier

class RFC_model(RandomForestClassifier):

    def __init__(self):

        # Toggle settings here
        RandomForestClassifier.__init__(
            self,
            n_estimators=250,
            criterion='gini',
            max_features='log2',
            max_depth=None,
            min_samples_leaf=1,
            min_weight_fraction_leaf=0,
            max_leaf_nodes=None,
            min_impurity_decrease=0,
            bootstrap=True,
            random_state=None,
            verbose=0,
            warm_start=False,
            class_weight='balanced'
        )
        self.__name = 'RFC'

    def __str__(self):
        """Returns the name of the model"""
        return self.__name


"""
위장
https://programmers.co.kr/learn/courses/30/lessons/42578
"""
from collections import defaultdict

def solution(clothes):
    answer = 1
    data = defaultdict(int)
    for cname, ctype in clothes:
        data[ctype] += 1

    for v in data.values():
        answer *= v + 1

    return answer - 1

print(solution([["yellowhat", "headgear"], ["bluesunglasses", "eyewear"], ["green_turban", "headgear"]]), 5)
print(solution([["crowmask", "face"], ["bluesunglasses", "face"], ["smoky_makeup", "face"]]), 3)


import argparse
import datetime
import json
import os
import time

import httplib2
import oauth2client
from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from yr.libyr import Yr

debug = False
source =    'Webpage/Page/indexsrc.html'
destination = 'Webpage/Page/index.html'

class Data:
    def __init__(self, version=0):
        if version != 0:
            self.version_string = 'Running Weather and Agenda V%0.1f' % version
        else:
            self.version_string = 'Running Weather and Agenda'

        if debug:
            self.version_string += ' in Debug mode:'
        else:
            self.version_string += ':'

        print('\r%-100s' % self.version_string)

        print('\r[   ] Initializing.', end='', flush=True)

        self.start_time = time.time()
        if debug:
            self.pause_time = 30
        else:
            self.pause_time = 60*15

        self.check_update = True
        self.first_check = True

        self.last_checked = datetime.datetime.now()
        self.last_update = '%02i:%02i' % (self.last_checked.hour, self.last_checked.minute)

        self.forecasts = ForecastData()
        self.agenda = Agenda()
        self.webpage = WebPage()
        print('\r[ X ] Initializing.\n')

    def update(self):
        now = datetime.datetime.now()
        diff = now-self.last_checked
        diff = divmod(diff.days * 86400 + diff.seconds, 60)
        dt = diff[0]*60 + diff[1]

        if self.first_check:
            self.check_update = True
        elif dt > self.pause_time:
            self.check_update = True
        else:
            self.check_update = False

        if self.check_update:
            print('\r--------------------------------------------------------------\n', end='\n', flush=True)

            self.forecasts.get_weather()
            self.agenda.get_agenda()
            self.webpage.update(self.agenda, self.forecasts)

            if debug:
                if self.agenda.ongoing_match:
                    if self.pause_time != 10:
                        print('\r[ X ] Changed updating frequency')
                    self.pause_time = 10
                else:
                    if self.pause_time != 30:
                        print('\r[ X ] Changed updating frequency back to normal')
                    self.pause_time = 30
            elif not debug:
                if self.agenda.ongoing_match:
                    if self.pause_time != 60:
                        print('\r[ X ] Changed updating frequency')
                    self.pause_time = 60
                else:
                    if self.pause_time != 60*15:
                        print('\r[ X ] Changed updating frequency back to normal')
                    self.pause_time = 60*15

            self.make_html()
            self.last_checked = datetime.datetime.now()
            self.last_update = '%02i:%02i' % (self.last_checked.hour, self.last_checked.minute)

            self.first_check = False
            print()

        else:
            s0 = self.pause_time - dt
            s = s0 % 60
            m = int((s0 - s) / 60) % 60
            print('\r---------------- Time until next check: %02i:%02i ----------------' % (m, s), end='', flush=True)

        #time.sleep(1)

    def make_html(self):
        print('\r[   ] Writing html code. -->', end='', flush=True)
        with open(source, 'r') as src:
            with open(destination, 'w') as dst:
                for line in src:
                    if '@@city' in line:
                        newline = line.replace('@@city', self.forecasts.city)              # Title city
                    elif '@@update_time' in line:
                        newline = line.replace('@@update_time', str(self.pause_time/2))    # Refresh rate
                    elif '@@main_icon' in line:
                        newline = line.replace('@@main_icon', icons[self.webpage.icon])    # Icon
                    elif '@@degrees' in line:
                        newline = line.replace('@@degrees', str(self.webpage.weather_main.temperature))  # Temperature
                    elif '@@weather_message' in line:
                        newline = line.replace('@@weather_message', self.webpage.message.capitalize())   # Message
                    elif '@@precipitation_level' in line:
                        newline = line.replace('@@precipitation_level',
                                               '%i' % int(self.webpage.weather_main.precipitation_value))  # Precip.
                    elif '@@agenda' in line:
                        newline = line.replace('@@agenda', self.webpage.agenda_string)                     # Agenda
                    else:
                        newline = line

                    dst.write(newline)
        print('\r[ X ] Writing html code. -->  Last update at %s' % self.last_update)

    def cancel_program(self):
        with open(source, 'r') as src:
            with open(destination, 'w') as dst:
                for line in src:
                    if '@@city' in line:
                        newline = line.replace('@@city', 'city')                                    # Title city
                    elif '@@update_time' in line:
                        newline = line.replace('@@update_time', str(self.pause_time/2))             # Refresh rate
                    elif '@@main_icon' in line:
                        newline = line.replace('@@main_icon', icons[2])                             # Icon
                    elif '@@degrees' in line:
                        newline = line.replace('@@degrees', '-273.15')                              # Temperature
                    elif '@@weather_message' in line:
                        newline = line.replace('@@weather_message',
                                               'Error: Program interrupted. Restart to update.')    # Message
                    elif '@@precipitation_level' in line:
                        newline = line.replace('@@precipitation_level',
                                               '%i' % int(-999))                                    # Precipitation
                    elif '@@agenda' in line:
                        newline = line.replace('@@agenda', 'Need update')                           # Agenda
                    else:
                        newline = line

                    dst.write(newline)

    def error(self, terminal_msg, err_msg):
        self.last_checked = datetime.datetime.now()
        self.last_update = '%02i:%02i' % (self.last_checked.hour, self.last_checked.minute)
        self.first_check = False

        if debug:
            self.pause_time = 10
        else:
            self.pause_time = 60

        print("      Error: %s" % terminal_msg)
        with open(source, 'r') as src:
            with open(destination, 'w') as dst:
                for line in src:
                    if '@@city' in line:
                        newline = line.replace('@@city', self.forecasts.city)              # Title city
                    elif '@@update_time' in line:
                        newline = line.replace('@@update_time', str(self.pause_time))    # Refresh rate
                    elif '@@main_icon' in line:
                        newline = line.replace('@@main_icon', icons[8])    # Icon
                    elif '@@degrees' in line:
                        newline = line.replace('@@degrees', str(self.webpage.weather_main.temperature))  # Temperature
                    elif '@@weather_message' in line:
                        newline = line.replace('@@weather_message',
                                               '%s.' % err_msg)    # Message
                    elif '@@precipitation_level' in line:
                        newline = line.replace('@@precipitation_level',
                                               '%i' % int(self.webpage.weather_main.precipitation_value))  # Precip.
                    elif '@@agenda' in line:
                        newline = line.replace('@@agenda', self.webpage.agenda_string)                     # Agenda
                    else:
                        newline = line

                    dst.write(newline)

        print('      Trying again.')

class ForecastData:
    def __init__(self):

        self.forecast_list = [0, 0, 0, 0, 0, 0, 0, 0]
        self.country = 'Norway'
        self.state = 'Sør-Trøndelag'
        self.county = 'Trondheim'
        self.city = 'Trondheim'
        self.location = '%s/%s/%s/%s' % (self.country, self.state, self.county, self.city)

    def get_weather(self):
        print('\r[   ] Downloading forecasts.   ', end='', flush=True)
        yr = Yr(location_name=self.location)
        forecast_json = yr.forecast(as_json=True)
        print('\r[ X ] Downloading forecasts.')

        print('\r[   ] Sorting forecasts.', end='', flush=True)
        i = 0
        while i != 3:
            block = ForecastBlock(forecast_json)
            i = self.find_time_block(block)
            self.forecast_list[i] = block

        while i != 7:
            block = ForecastBlock(forecast_json)
            i = self.find_time_block(block) + 4
            self.forecast_list[i] = block

        print('\r[ X ] Sorting forecasts.')

        self.cleanup()

    def cleanup(self):
        for i in range(len(self.forecast_list)):
            if self.forecast_list[i] == 0:
                self.forecast_list[i] = ForecastBlock(self, empty=True)

    @staticmethod
    def find_time_block(block):
        hour_from = block.hour_from
        if divmod(hour_from, 6)[0] == 0:
            return 0
        elif divmod(hour_from, 6)[0] == 1:
            return 1
        elif divmod(hour_from, 6)[0] == 2:
            return 2
        elif divmod(hour_from, 6)[0] == 3:
            return 3

class ForecastBlock:
    def __init__(self, forecast, empty=False):
        if not empty:
            fc = json.loads(next(forecast))
            precipitation = fc['precipitation']

            time_from = fc['@from']
            time_to = fc['@to']

            self.year_from = int(time_from[0:4])
            self.year_to = int(time_to[0:4])

            self.month_from = int(time_from[5:7])
            self.month_to = int(time_to[5:7])

            self.date_from = int(time_from[8:10])
            self.date_to = int(time_to[8:10])

            self.hour_from = int(time_from[11:13])
            self.hour_to = int(time_to[11:13])

            self.weather = fc['symbol']['@name']
            self.precipitation_value = float(precipitation['@value'])

            try:
                self.precipitation_max = float(precipitation['@maxvalue'])
                self.precipitation_min = float(precipitation['@minvalue'])
            except KeyError:
                self.precipitation_min = 0.0
                self.precipitation_max = 0.0

            self.windDirection = fc['windDirection']['@code']

            self.windSpeed_name = fc['windSpeed']['@name']
            self.windSpeed = float(fc['windSpeed']['@mps'])

            self.temperature = int(fc['temperature']['@value'])
        else:
            self.year_from = -1
            self.year_to = -1

            self.month_from = -1
            self.month_to = -1

            self.date_from = -1
            self.date_to = -1

            self.hour_from = -1
            self.hour_to = -1

            self.weather = ''
            self.precipitation_value = -1

            self.precipitation_max = -1
            self.precipitation_min = -1

            self.windDirection = ''

            self.windSpeed_name = ''
            self.windSpeed = -1

            self.temperature = -273.15

class Agenda:
    def __init__(self):
        self.ongoing_match = False

        self.agenda_today = []
        self.agenda_allday = []
        self.events = []

    def get_agenda(self):
        self.ongoing_match = False

        print('[   ] Getting credentials for calendars.', end='', flush=True)
        credentials = self.get_credentials()
        http = credentials.authorize(httplib2.Http())
        service = discovery.build('calendar', 'v3', http=http)
        now = datetime.datetime.utcnow().isoformat() + 'Z'  # 'Z' indicates UTC time
        print('\r[ X ] Getting credentials for calendars.')

        primary_name = 'primary'
        copa_name = 'spacup_6162_%42arcelona#sports@group.v.calendar.google.com'
        laliga_name = 'spa_6162_%42arcelona#sports@group.v.calendar.google.com'
        cl_name = 'chlg_6162_%42arcelona#sports@group.v.calendar.google.com'

        primary = self.get_events(primary_name, service, now, 'Primary calendar')

        copa = self.get_events(copa_name, service, now, 'Copa del Rey')

        laliga = self.get_events(laliga_name, service, now, 'Liga BBVA')

        cl = self.get_events(cl_name, service, now, 'Champions League')

        print('\r[   ] Sorting events.', end='', flush=True)
        self.sort_events([primary, copa, laliga, cl])
        print('\r[ X ] Sorting events.')

        for i in self.agenda_today:
            j = i.summary
            match_check1 = '(0)' in j or '(1)' in j or '(2)' in j or '(3)' in j

            if match_check1:
                if i.start_hour <= time.localtime().tm_hour <= i.end_hour:
                    self.ongoing_match = True
                    break

    @staticmethod
    def get_events(name, service, now, name_string):
        print('\r[   ] Downloading events for calendar: %s' % name_string, end='', flush=True)
        events_result = service.events().list(calendarId=name,
                                              timeMin=now,
                                              maxResults=4,
                                              singleEvents=True,
                                              orderBy='startTime'
                                              ).execute()

        events = events_result.get('items', [])
        print('\r[ X ] Downloading events for calendar: %s' % name_string)

        return events

    def sort_events(self, calendars):
        unsorted_events = []
        sorted_events = []
        allday_events = []
        dates = []
        self.agenda_allday = []
        self.agenda_today = []

        for calendar in calendars:
            for event in calendar:
                if len(event['start'].get('dateTime', event['start'].get('date'))) == 25:  # Remove all-day events
                    unsorted_events.append(event)
                elif len(event['start'].get('dateTime', event['start'].get('date'))) == 10:
                    allday_events.append(event)

        for event in unsorted_events:
            dates.append(event['start'].get('dateTime'))

        dates2 = [datetime.datetime.strptime(ts, "%Y-%m-%dT%H:%M:00+%S:00") for ts in dates]
        dates2.sort()
        # noinspection PyTypeChecker
        sorted_dates = [datetime.datetime.strftime(ts, "%Y-%m-%dT%H:%M:00+01:00") for ts in dates2]

        for i in range(0, len(sorted_dates)):
            for j in range(0, len(unsorted_events)):
                if sorted_dates[i] == unsorted_events[j]['start'].get('dateTime'):
                    sorted_events.append(unsorted_events[j])

        today_date = int(time.localtime().tm_mday)
        today_month = int(time.localtime().tm_mon)

        for event_struct in sorted_events:
            event = Event(event_struct)
            start_date = event.start_date
            start_month = event.start_month
            if start_date == today_date and start_month == today_month:
                self.agenda_today.append(event)

        for event_struct in allday_events:
            event = Event(event_struct)
            start_date = event.start_date
            start_month = event.start_month
            if start_date == today_date and start_month == today_month:
                self.agenda_allday.append(event)

    @staticmethod
    def get_credentials():
        flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
        scopes = 'https://www.googleapis.com/auth/calendar.readonly'
        client_secret_file = '../Common/client_secret.json'
        application_name = 'Smart Mirror - Calendar Module'
        home_dir = os.path.expanduser('~')
        credential_dir = os.path.join(home_dir, '.credentials')
        if not os.path.exists(credential_dir):
            os.makedirs(credential_dir)
        credential_path = os.path.join(credential_dir,
                                       'calendar-python-quickstart.json')

        store = oauth2client.file.Storage(credential_path)
        credentials = store.get()
        if not credentials or credentials.invalid:
            flow = client.flow_from_clientsecrets(client_secret_file, scopes)
            flow.user_agent = application_name
            if flags:
                credentials = tools.run_flow(flow, store, flags)
                print('Storing credentials.')
            else:  # Needed only for compatibility with Python 2.6
                credentials = tools.run(flow, store)
            print('Storing credentials to ' + credential_path)
        return credentials

class Event:
    def __init__(self, event):
        start = event['start'].get('dateTime', event['start'].get('date'))
        end = event['end'].get('dateTime', event['end'].get('date'))

        if len(start) == 25:  # Not allday-event
            self.summary = event['summary']
            self.start_year = int(start[:4])
            self.start_month = int(start[5:7])
            self.start_date = int(start[8:10])
            self.start_hour = int(start[11:13])
            self.start_min = int(start[14:16])

            self.end_year = int(end[:4])
            self.end_month = int(end[5:7])
            self.end_date = int(end[8:10])
            self.end_hour = int(end[11:13])
            self.end_min = int(end[14:16])
        elif len(start) == 10:
            self.summary = event['summary']
            self.start_year = int(start[:4])
            self.start_month = int(start[5:7])
            self.start_date = int(start[8:10])
            self.start_hour = -1
            self.start_min = -1

            self.end_year = int(end[:4])
            self.end_month = int(end[5:7])
            self.end_date = int(end[8:10])
            self.end_hour = -1
            self.end_min = -1

class WebPage:
    def __init__(self):
        self.agenda = -1
        self.forecasts = -1
        self.weather_main = ForecastBlock(1, empty=True)
        self.message = ''
        self.agenda_string = ''
        self.icon = 14

    def update(self, agenda=None, forecasts=None):
        print('\rUpdating web page.', end='', flush=True)
        self.agenda = agenda
        self.forecasts = forecasts
        self.create_message()
        self.createagenda_string()
        self.find_icon()

    def create_message(self):
        hour = time.localtime().tm_hour

        # Weather dependent changes
        if 'fair' in self.forecasts.forecast_list or 'partly':
            article = 'the weather '
        else:
            article = 'there '

        # Time dependent changes
        if hour < 9:
            main_block = 2
            weather_message_time = ' today'
            weather_time = ' will be '
        elif hour < 14:
            main_block = 2
            weather_message_time = ' today'
            weather_time = ' is '
        elif hour < 19:
            main_block = 3
            weather_message_time = ' tonight'
            weather_time = ' will be '
        elif hour < 23:
            main_block = 5
            weather_message_time = ' tomorrow'
            weather_time = ' will be '
        else:
            main_block = 2
            weather_message_time = ' tomorrow'
            weather_time = ' will be '

        self.weather_main = self.forecasts.forecast_list[main_block]

        # Wind dependent changes
        if self.weather_main.windSpeed > 5.5:      # Virker ikke når klokken er etter 23.... Hva skjer med forecasts?
            wind_message = ' with ' + self.weather_main.windSpeed_name.lower()
        else:
            wind_message = ''

        self.message = '%s%s%s%s%s.' % (article,
                                        weather_message_time,
                                        weather_time,
                                        self.weather_main.weather.lower(),
                                        wind_message
                                        )

    def find_icon(self):
        w = self.weather_main.weather.lower()

        if w == 'clear sky':
            self.icon = 11
        elif w == 'fair':
            self.icon = 5
        elif w == 'partly cloudy':
            self.icon = 4
        elif 'snow' in w:
            self.icon = 8
        elif 'thunder' in w:
            self.icon = 12
        elif w == 'cloudy':
            self.icon = 1
        elif 'sleet' in w:
            self.icon = 6
        elif 'light rain' in w:
            self.icon = 6
        elif 'rain' in w and 'light' not in w:
            self.icon = 7
        elif w == 'fog':
            self.icon = 1

    def createagenda_string(self):
        normal = self.agenda.agenda_today
        allday = self.agenda.agenda_allday

        string = ''
        if len(normal) == 0 and len(allday) == 0:
            string = "<div class=agenda_empty> The agenda is empty.</div>"
        else:
            i = 0
            for event in allday:
                i += 1
                name = event.summary

                if i == 1:
                    time_string = "<div class=agenda_time> All day:</div>"
                else:
                    time_string = "<div class=agenda_time>   </div>"

                event_string = "<div class=agenda_event>- %s</div><br>" % name

                string += time_string+event_string

            for event in normal:
                start_time = '%02i:%02i' % (event.start_hour, event.start_min)
                end_time = '%02i:%02i' % (event.end_hour, event.end_min)
                name = event.summary
                time_string = "<div class=agenda_time> %s-%s</div>" % (start_time, end_time)
                event_string = "<div class=agenda_event>- %s</div><br>" % name

                string += time_string+event_string

        self.agenda_string = string

def clear():
    os.system('cls' if os.name == 'nt' else 'clear')

icons = {1: 'cloudy.svg',
         2: 'full-moon.svg',
         3: 'lightning-bolt.svg',
         4: 'mostly-cloudy.svg',
         5: 'mostly-sunny.svg',
         6: 'raining.svg',
         7: 'shower.svg',
         8: 'snowing.svg',
         9: 'sunny.svg',
         10: 'sunny-2.svg',
         11: 'sunny-3.svg',
         12: 'thunder-storm.svg',
         13: 'windy.svg',
         14: 'precipitation.svg'
         }


""" ---Refactor---"""

import unittest
from decimal import Decimal as dec

class ConversionNotPossible(Exception):
    """Impossible conversion class."""
    pass

KCDIF = dec('273.15')
KFDIF = dec('459.67')
NDIVF = dec('9') / 5
FDIVN = dec('5') / 9

F1 = {'cel': {'kel': (KCDIF, 1, 0), 'fah': (0, NDIVF, 32)},
      'kel': {'cel': (-KCDIF, 1, 0), 'fah': (0, NDIVF, -KFDIF)},
      'fah': {'cel': (-32, FDIVN, 0), 'kel': (KFDIF, FDIVN, 0)},
      'mil': {'yar': (0, dec(1760), 0), 'met': (0, dec(1609.34), 0)},
      'yar': {'mil': (0, dec(.00056818), 0), 'met': (0, dec(.9144), 0)},
      'met': {'mil': (0, dec(.00062137), 0), 'yar': (0, dec(1.09361), 0)}
     }

def convert(fromUnit, toUnit, value):

    fu = fromUnit[:3].lower()
    tu = toUnit[:3].lower()
    value = dec(value)
    if fu == tu:
        return float(value)
    elif tu in F1[fu]:
        conversion = ((value + F1[fu][tu][0]) *
                      (F1[fu][tu][1]) +
                      (F1[fu][tu][2]))

        return round(float(conversion), 3)
    else:
        raise ConversionNotPossible

class ConverstionTests(unittest.TestCase):
    """Tests for conversions_refactored.py"""
    tvals = [('Celsius', 'Kelvin', 100, 373.15),
             ('Celsius', 'fahrenheit', 300, 572.0),
             ('Kelvin', 'Celsius', 273.15, 0),
             ('Kelvin', 'Fahrenheit', 573.15, 572.0),
             ('Fahrenheit', 'Celsius', 32, 0),
             ('Fahrenheit', 'Kelvin', 32, 273.15),
             ('Miles', 'Yards', 1, 1760),
             ('Miles', 'Meters', .5, 804.67),
             ('Yards', 'Miles', 5, .003),
             ('Yards', 'Meters', 10, 9.144),
             ('Meters', 'Yards', 5, 5.468),
             ('Meters', 'Miles', 1000, .621)]

    def testConversions(self):
        """Tests that all possible unit conversions work correctly."""
        for i in self.tvals:
            self.assertEqual(convert(i[0], i[1], i[2]), i[3])

    def testSameUnit(self):
        """Tests that all possible unit conversions to the same unit type
        work correctly."""
        for c in F1:
            self.assertEqual(convert(c, c, 1.0), 1.0)

    def testIncompatibleUnits(self):
        """Tests that ConversionNotPossible exception is raised."""
        self.assertRaises(ConversionNotPossible, convert, 'Miles', 'Kelvin', 10)

if __name__ == '__main__':
    unittest.main()


from httplib import HTTPConnection, HTTPMessage
from cStringIO import StringIO
from .files import save_cassette, load_cassette
from .cassette import Cassette

class VCRHTTPResponse(object):
    def __init__(self, recorded_response):
        self.recorded_response = recorded_response
        self.reason = recorded_response['status']['message']
        self.status = recorded_response['status']['code']
        self._content = StringIO(self.recorded_response['body']['string'])

        self.msg = HTTPMessage(StringIO(''))
        for k, v in self.recorded_response['headers'].iteritems():
            self.msg.addheader(k, v)

    def read(self, chunked=False):
        return self._content.read()

class VCRHTTPConnection(HTTPConnection):

    def __init__(self, *args, **kwargs):
        self._cassette = Cassette()
        HTTPConnection.__init__(self, *args, **kwargs)

    def _save_cassette(self):
        save_cassette(self._vcr_cassette_path, self._cassette)

    def _load_old_response(self):
        old_cassette = load_cassette(self._vcr_cassette_path)
        if old_cassette:
            return old_cassette.get_response(self._vcr)

    def request(self, method, url, body=None, headers={}):
        """
        Persist the request metadata in self._vcr
        """
        self._vcr = {
            'method': method,
            'url': url,
            'body': body,
            'headers': headers,
        }
        old_cassette = load_cassette(self._vcr_cassette_path)
        if old_cassette and old_cassette.get_request(self._vcr):
            return
        self._cassette.requests.append(dict(
            method=method,
            url=url,
            body=body,
            headers=headers
        ))
        return HTTPConnection.request(self, method, url, body=body, headers=headers)

    def getresponse(self, buffering=False):
        old_response = self._load_old_response()
        if not old_response:
            response = HTTPConnection.getresponse(self)
            self._cassette.responses.append({
                'status': {'code': response.status, 'message': response.reason},
                'headers': dict(response.getheaders()),
                'body': {'string': response.read()},
            })
            self._save_cassette()
        old_response = self._load_old_response()
        return VCRHTTPResponse(old_response)

class VCRHTTPSConnection(VCRHTTPConnection):
    pass


'''
Created 2018-10-27
XGBoost Algorithm on predicting time
Author: ZHANG Yuning
'''
#!/usr/bin/python
# coding:utf8

import numpy as np
import pandas as pd
import sklearn.feature_selection
from sklearn.feature_selection import f_regression
from sklearn.preprocessing import Imputer
from sklearn import metrics
from sklearn.decomposition import PCA

import xgboost as xgb
from xgboost import XGBRegressor
from sklearn.feature_selection import VarianceThreshold
from sklearn.model_selection import train_test_split, GridSearchCV, KFold
import matplotlib.pyplot as plt
import seaborn as sns
#%matplotlib inline
#from __future__ import print_function
#计算分类的正确率
from sklearn.metrics import accuracy_score
from sklearn.preprocessing import LabelEncoder
from sklearn.preprocessing import OneHotEncoder
from sklearn import preprocessing
from sklearn.preprocessing import scale
from xgboost import plot_importance

def loadcsv():
    df = pd.read_csv('train.csv')
    dftrain = pd.read_csv('train.csv',usecols=[1,3,4,6,7,8,9,10,11,12,13])
    #print(dftrain.head())
    dflabel= pd.read_csv('train.csv',usecols=[14])
    #print(dflabel.head())
    df2 = pd.read_csv('test.csv')
    dftest=pd.read_csv('test.csv',usecols=[1,3,4,6,7,8,9,10,11,12,13])
    #print(dftest.head())
    return dftrain,dftest,dflabel,df2

def preprocess(dftrain,dftest):
    #对以下变量进行labelEncoder编码
    dftrain['penalty']=dftrain['penalty'].replace('none',0)
    dftrain['penalty']=dftrain['penalty'].replace('l2',1)
    dftrain['penalty']=dftrain['penalty'].replace('elasticnet',2)
    dftrain['penalty']=dftrain['penalty'].replace('l1',3)
    dftest['penalty']=dftest['penalty'].replace('none',0)
    dftest['penalty']=dftest['penalty'].replace('l2',1)
    dftest['penalty']=dftest['penalty'].replace('elasticnet',2)
    dftest['penalty']=dftest['penalty'].replace('l1',3)
    dftrain['new1']=dftrain['n_samples']*dftrain['n_features']/dftrain['n_jobs']
    dftest['new1']=dftest['n_samples']*dftest['n_features']/dftest['n_jobs']
    dftrain['new2']=dftrain['n_samples']*dftrain['n_features']
    dftest['new2']=dftest['n_samples']*dftest['n_features']
    dftrain=pd.DataFrame(dftrain)
    #print(dftrain.head())
    dftest=pd.DataFrame(dftest)
    #rint(dftest.head())
    return dftrain,dftest

def featureSet(df):
    df['alpha'] = -(np.log10(df['alpha']))
    df['n_jobs'] = df['n_jobs'].replace(-1,max(df['n_jobs']))
    return df

def trainXGBoost(dftrain,dftest,dflabel,df2):

    x_train,x_test,y_train,y_test=train_test_split(dftrain.values, dflabel.values, test_size=0.15,random_state=66)
    df=pd.DataFrame(x_train)
    #print(df.head())
    #print(dftest.head())
    XGB = XGBRegressor(silent=1 ,#设置成1则没有运行信息输出，最好是设置为0.是否在运行升级时打印消息。
                       #booster='gblinear',#采用线性模型进行提升计算，默认是gbtree
                       learning_rate=0.2, # 如同学习率
                        min_child_weight=1,
                        # 这个参数默认是 1，是每个叶子里面 h 的和至少是多少，对正负样本不均衡时的 0-1 分类而言
                        #，假设 h 在 0.01 附近，min_child_weight 为 1 意味着叶子节点中最少需要包含 100 个样本。
                        #这个参数非常影响结果，控制叶子节点中二阶导的和的最小值，该参数值越小，越容易 overfitting。
                        max_depth=4, # 构建树的深度，越大越容易过拟合
                        gamma=0.2,  # 树的叶子节点上作进一步分区所需的最小损失减少,越大越保守，一般0.1、0.2这样子。
                        subsample=1, # 随机采样训练样本 训练实例的子采样比
                        max_delta_step=1,#最大增量步长，我们允许每个树的权重估计。
                        colsample_bytree=1, # 生成树时进行的列采样
                        reg_lambda=6,  # 控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越不容易过拟合。
                        reg_alpha=0, # L1 正则项参数
                        scale_pos_weight=1, #如果取值大于0的话，在类别样本不平衡的情况下有助于快速收敛。平衡正负权重
                        n_estimators=360, #树的个数
                        seed=60 #随机种子
                      )
    XGB.fit(x_train,y_train,eval_metric='rmse',verbose=False)
    y_true, y_pred = y_test, XGB.predict(x_test)
    ans=XGB.predict(dftest.values)

    for t in range(len(dftest)):
        if ans[t]<=0:
            ans[t]=min(dflabel.values)

    submission_df = pd.DataFrame(data = {'Id':df2.id,'time':ans})
    submission_df.to_csv('submission.csv',index=None)

    #print('Mean Absolute Error:', metrics.mean_absolute_error(y_true, y_pred))
    print('Mean Squared Error:', metrics.mean_squared_error(y_true, y_pred))
    #print('Root Mean Squared Error:', np.sqrt(metrics.mean_squared_error(y_true, y_pred)))

    plot_importance(XGB)
    plt.show()

if __name__ == "__main__":
    dftrain,dftest,dflabel,df2 = loadcsv()
    dftrain=featureSet(dftrain)
    dftest=featureSet(dftest)
    dftrain,dftest=preprocess(dftrain,dftest)
    trainXGBoost(dftrain,dftest,dflabel,df2)


flag = True
file_name = 'quest_book.txt'

while flag:
    print('If you want exit, please enter exit')
    user_name = input('Please, enter your name: ')
    if user_name == 'exit':
        flag = False
        continue
    with open(file_name, 'a') as file_write:
        file_write.write(f'Hello {user_name.title()}\n')


# Copyright 2019 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from PIL import Image
import os

def loadImages(subdir, channels):
        images = []

        # get list of all files in subdirectory cats
        files = os.scandir(subdir)
        # read each image in and append the in-memory image to a list
        for file in files:
                   image = Image.open(file.path)
                   # convert to grayscale
                   if channels == 1:
                        image = image.convert('L')
                   # convert to RGB color
                   else:
                        image = image.convert('RGB')
                   images.append(image)
        return images

loadImages('cats', 3)


# -*- coding: utf-8 -*-
""" Print accepted talks not scheduled and not accepted talks which have been scheduled.

"""
from   django.core.management.base import BaseCommand, CommandError
from   django.core  import urlresolvers
from   conference   import models
from   conference   import utils

from   collections  import defaultdict
from   optparse     import make_option
import operator
import simplejson   as json

### Globals

TYPE_NAMES = (
    ('keynote', 'Keynotes'),
    ('s', 'Talks'),
    ('t', 'Trainings'),
    ('p', 'Poster sessions'),
    ('h', 'Help desks'),
    ('europython', 'EuroPython sessions'),
    ('i', 'Other sessions'),
    )

def _check_talk_types(type_names):
    d = dict(type_names)
    for code, entry in models.TALK_TYPE:
        assert code in d, 'Talk type code %r is missing' % code
_check_talk_types(TYPE_NAMES)

### Helpers
def talk_schedule(talk):
    event = talk.get_event()

    if not event:
        #print('ERROR: Talk ({}) {} is not scheduled.'.format(talk.type, talk))
        return ''

    #TODO: should also check if the talk has more than one event.

    timerange = event.get_time_range()
    return '{}, {}'.format(str(timerange[0]), str(timerange[1]))

def talk_type(talk):
    type_names = dict(TYPE_NAMES)

    if 'EPS' in talk.title or 'EuroPython 20' in talk.title:
        type = 'europython'
    elif talk.title.lower().startswith('keynote'):
        type = 'keynote'
    else:
        type = talk.type

    return type_names[type]

def group_talks_by_type(talks):

    # Group by types
    type_talks = defaultdict(list)
    for talk in talks:
        if 'EPS' in talk.title or 'EuroPython 20' in talk.title:
            type = 'europython'
        elif talk.title.lower().startswith('keynote'):
            type = 'keynote'
        else:
            type = talk.type

        type_talks[type].append(talk)

    return type_talks

###

class Command(BaseCommand):
    option_list = BaseCommand.option_list + (
        make_option('--all_scheduled',
             action='store_true',
             dest='all_scheduled',
             default=False,
             help='Show a list of accepted talks which have not been scheduled',
        ),
        make_option('--all_accepted',
             action='store_true',
             dest='all_accepted',
             default=False,
             help='Show a list of scheduled talks which have not been accepted',
        ),
    )

    def handle(self, *args, **options):
        try:
            conference = args[0]
        except IndexError:
            raise CommandError('conference not specified')

        if not options['all_scheduled'] and not options['all_accepted']:
            print('Please use either --all_scheduled or --all_accepted option.')

        if options['all_scheduled']:
            print('Checking that all accepted talks have been scheduled.')

            talks = (models.Talk.objects.filter(conference=conference, status='accepted'))
            type_talks = group_talks_by_type(talks)

            for type in type_talks:
                for talk in type_talks[type]:
                    talk_t = talk_type(talk)
                    if not talk_schedule(talk):
                        print('ERROR: Talk ({}) {} is not scheduled.'.format(talk_t, talk))

        if options['all_accepted']:
            print('Checking that all scheduled talks are accepted.')

            talks = (models.Talk.objects.filter(conference=conference))
            type_talks = group_talks_by_type(talks)

            for type in type_talks:
                for talk in type_talks[type]:
                    if talk.status != 'accepted' and talk_schedule(talk):
                        talk_t = talk_type(talk)
                        print('ERROR: Talk ({}) {} is scheduled but not accepted.'.format(talk_t, talk))


# This file contains the function that no longer used in the project
import Report
import numpy
import Observe
import Set
import copy
import Teach
import Learn

def Probability_Task(hypo_map, number_obs, number_hypo, number_feature, number_label, p_teacher_x_h):
      prob_list = []

      feature_list = Observe.Get_Target_Feature_Set([0, 1, 2], number_obs)
      print(feature_list)

      # Assume there is a true hypo
      # Get all posible hypothesis in the hypo map
      for hypo in range(number_hypo):
            F = []
            # Choose a feature to observe
            for feature_set in feature_list:
                  # Get the probability that L will select this feature / these features
                  # prob = Observe.Get_Probability_Map(p_teacher_x_h, hypo, feature_set)

                  # Does the learner find the true hypo ?
                  prob_find = Observe.Observe(hypo_map, hypo, feature_set)

                  F.append(prob_find)
            prob_list.append(F)

      return numpy.array(prob_list)

# Observe the target feature when there is a true hypothesis
# hypo_map: the set of all hypothesis
# true_hypo: the true hypo
# target_feature_set: a set of target features that we want to observe
def Observe(hypo_map, true_hypo, target_feature_set):
      list = []
      label_map = { }

      # Get a list of hypothesis
      for feature in target_feature_set:
            label_map[feature] = true_hypo[feature]

      for hypo in hypo_map:
            check = True
            for feature in target_feature_set:
                  if hypo[feature] != label_map[feature]:
                        check = False
            if check:
                  list.append(hypo)
      return 1 / len(list) if len(list) >= 1 else 0

# Sum over x
def Get_Prob_Table(number_obs, p_teacher_x_h, prob_list):
      # Again get the feature list
      feature_list = Observe.Get_Target_Feature_Set([0, 1, 2], number_obs)

      # The new probability map with a lenth = number of hypothesis
      new_prob_list = numpy.zeros(len(prob_list))

      for hypo in range(len(prob_list)):
            sum = 0
            for feature in range(len(feature_list)):
                  prob_select = Observe.Get_Probability(p_teacher_x_h, hypo, feature_list[feature])
                  sum += prob_list[hypo, feature] * prob_select
            new_prob_list[hypo] = sum
      return new_prob_list

def Delete_Repeated_Item(set):
      for i in set:
            new_set = copy.deepcopy(set)
            new_set.remove(i)
            for x in new_set:
                  overlap = True
                  for feature in x:
                        if feature not in i: overlap = False
                  if overlap:
                        set.remove(x)
      return

# all_feature_set: the set of all features
# num_observations: the number of observations, which is also the size of the subset
# check_repeat: delete repeated values (always true)
def Get_Target_Feature_Set(all_feature_set, num_observations, check_repeat=True):
      if num_observations == 1:
            whole_list = []
            for feature in all_feature_set:
                  list = []
                  list.append(feature)
                  whole_list.append(list)
            return whole_list
      else:
            return_list = []
            for feature in all_feature_set:
                  new_set = copy.deepcopy(all_feature_set)
                  new_set.remove(feature)
                  new_list = Get_Target_Feature_Set(new_set, num_observations - 1)
                  for i in new_list:
                        i.append(feature)
                  for x in new_list:
                        return_list.append(x)
            if check_repeat: Delete_Repeated_Item(return_list)
            return return_list

def Get_Probability(p_teacher_x_h, target_hypo, target_feature_set):
      list = []
      for feature in target_feature_set:
            list.append(p_teacher_x_h[feature, target_hypo])
      return numpy.sum(list)

# Print the table to the console or a text file
def ReportTable(self, hypo_table=True, p_teacher_xy_h=False, p_y_xh=False, p_teacher_x_h=True, p_learner_h_xy=True, format=False, textcopy=False):
      Report.Report(self.num_hypo, self.num_feature, self.num_label,
                    self.hypo_table if hypo_table == True else None,
                    self.p_teacher_xy_h if p_teacher_xy_h == True else None,
                    self.p_y_xh if p_y_xh == True else None,
                    self.p_teacher_x_h if p_teacher_x_h == True else None,
                    self.p_learner_h_xy if p_learner_h_xy == True else None, None)
      if textcopy:
            Report.Report_to_File(self.hypo_table if hypo_table == True else None,
                                  self.p_teacher_xy_h if p_teacher_xy_h == True else None,
                                  self.p_y_xh if p_y_xh == True else None,
                                  self.p_teacher_x_h if p_teacher_x_h == True else None,
                                  self.p_learner_h_xy if p_learner_h_xy == True else None)

# eq. 6b)
# PT(x|h) = Sum_y Sum_g ( Pl(g|x,y) * PT(x,y) * delta(g|h) )
# 'K' stands for the knowledgebility model
def K_PTeacher_x_h(number_hypo, number_feature, number_label, p_teacher_xh, p_learner_h_xy, delta_gh):
      p_teacher_xy = 1 / number_feature / number_label
      for h in range(number_hypo):
            for x in range(number_feature):
                  sum = 0
                  for y in range(number_label):
                        for g in range(number_hypo):
                              sum += p_learner_h_xy[g, x, y] * p_teacher_xy * delta_gh[g, h]
                  p_teacher_xh[x, h] = sum
      return

# Hypo list --> Hypo map
# Key: hypothesis
# Value: index
def Transform_to_Map(hypo_list):
      h = { }
      for hypo in range(len(hypo_list)):
            h[tuple(hypo_list[hypo])] = hypo
      return h

# Hypo map --> Hypo list
def Transform_to_List(hypo_map):
      h = []
      for hypo in hypo_map:
            h.append(hypo)
      return h

# Observe the target feature when there is a true hypothesis
# hypo_map: the set of all hypothesis
# true_hypo: the true hypo
# target_feature: the target features we want to observe
def Observe(hypo_map, true_hypo_idx, true_hypo, target_feature_idx, p_learner_h_xy):
      list = []
      # Get the true label
      true_label = true_hypo[target_feature_idx]

      for hypo in hypo_map:
            if hypo[target_feature_idx] != true_label:
                  check = False
            else:
                  list.append(hypo)
      return p_learner_h_xy[true_hypo_idx][target_feature_idx][true_label] if len(p_learner_h_xy) > 1 else 1.0, list

# hypo_map: The map of the hypothesis
# return: a map from hypothesis to observation * probability
def Probability_Task(hypo_table, number_hypo, number_feature, number_label, p_teacher_x_h, knowledgeability, iter=100):
      prob_map = { }
      feature_set = []

      # Assume there is a true hypo = hypo
      # Get all posible hypothesis in the hypo map
      for hypo_idx in range(len(hypo_table)):

            # Get the observable feature set
            for f in range(number_feature):
                  feature_set.append(f)
            obs = 0
            prob = []
            select = []
            # Make a copy of the whole hypo table, and transform to a hypo_map
            hypo_map_copy = copy.deepcopy(hypo_table)
            while True:

                  # Pass the hypo_copy to Set function
                  num_hypo, num_feature, num_label, p_teacher_x_h, p_teacher_xy_h, p_learner_h_xy, p_y_xh, delta_g_h = Set.Set(hypo_map_copy, knowledgeability=knowledgeability)
                  # Get the PT
                  p_learner_h_xy = Set.Initstep(num_hypo, num_feature, num_label, p_y_xh)
                  Knowledgeability_Task(num_hypo, num_feature, num_label, p_teacher_xy_h, p_teacher_x_h, p_learner_h_xy, p_y_xh, delta_g_h, iter)

                  # Choose a feature
                  new_hypo_idx = Observe.Get_Index(hypo_table, hypo_map_copy, hypo_idx)
                  feature = Observe.Get_Feature(feature_set, new_hypo_idx, p_teacher_x_h)
                  obs += 1
                  prob_find, hypo_map_copy = Observe.Observe(hypo_map_copy, new_hypo_idx, hypo_table[hypo_idx], feature, p_learner_h_xy)
                  prob.append(prob_find)
                  select.append(feature)

                  # remove the feature in the feature set
                  feature_set.remove(feature)
                  if len(feature_set) == 0:
                        prob_map[hypo_idx] = (prob, select)
                        break
      return prob_map

def Report(number_hypo, number_feature, number_label, h_table=None, p_teacher_xy_h=None, p_y_xh=None, p_teacher_x_h=None, p_learner_h_xy=None, knowledge_delta=None, format=False):
      # The new table PT(x|h)
      new_prob_teacher_h_x = np.zeros((number_hypo, number_feature))
      # The new table P(y|x,h)
      new_prob_h_x_y = np.zeros((number_hypo, number_feature, number_label))
      # The new table PT(x,y|h)
      new_prob_teacher_h_x_y = np.zeros((number_hypo, number_feature, number_label))

      # Rearrange all the tables in (hypothesis , features, labels) format
      for h in range(number_hypo):
            for x in range(number_feature):
                  for y in range(number_label):
                        if p_teacher_xy_h is not None:
                              new_prob_teacher_h_x_y[h, x, y] = p_teacher_xy_h[x, y, h]
                        if p_y_xh is not None:
                              new_prob_h_x_y[h, x, y] = p_y_xh[y, x, h]
                  if p_teacher_x_h is not None:
                        new_prob_teacher_h_x[h, x] = p_teacher_x_h[x, h]
      if h_table is not None:
            print("---- Hypothesis Table ----")
            print(np.array(h_table))
      if knowledge_delta is not None:
            print("---- Delta Table ----")
            print(knowledge_delta)
      if p_teacher_xy_h is not None:
            print("---- PT(x,y|h) ----")
            print(p_teacher_xy_h)
      if p_y_xh is not None:
            print("---- P(y|x,h) ----")
            print(p_y_xh)
      if p_teacher_x_h is not None:
            print("---- PT(x|h) ----")
            print(p_teacher_x_h)
      if p_learner_h_xy is not None:
            print("---- PL(h|x,y) ----")
            print(p_learner_h_xy)
      return

def Report_to_File(file_path=None, *args):
      if file_path is None:
            f = open("result.txt", mode="a")
      else:
            f = open(file_path, mode="a")
      for tables in args:
            if tables is not None:
                  f.write("----- Table -----\n")
                  f.write(str(np.array(tables)))
                  f.write("\n")


import sqlite3 as lite
from collections import Counter

from Data.models import Database
from django.db import transaction
from langdetect import detect
from googletrans import Translator

def translanguage(columns):
    """
    print("#################################################")
    translator = Translator()
    translation=translator.translate('Mon prénom est ghassen')
    print("TRANSLATING: "+translation.text)
    print("#################################################")
    """
    #set database
    with transaction.atomic():
        database_name = columns[0].database_name
        database = Database.objects.get(name=database_name)
        # set db absolute path
        db = database.sqlite_db_path_file

        # set list of tables
        all_tables_list = []
        for column in columns :
            all_tables_list.append(column.table_name)
        #print(all_tables_list)
        #delete redunded values
        all_tables_list = list(set(all_tables_list))
        #print(all_tables_list)
        # set all columns list
        all_columns_list = []
        for column in columns :
            all_columns_list.append(column.name)
        print(all_columns_list)
        # connecting to database
        con = lite.connect(db)
        cur=con.cursor()
        #print(cur)

        #fill tables with values and annotation

        for k in columns:
            print(k)
            print('select "'+k.name+ '" from "'+k.table_name+'"')
            print('##########')
            cur.execute('select "'+k.name+'", index_id from "'+k.table_name+'"')
            print('hhhhhhhhhhhhhhhhhh')
            o=cur.fetchall()
            print(o)
            lo=list()
            li=list()
            content_list = []
            content_listorigin = []
            print('~~~~~~')
            print(o)
            print('-----W')
            if isinstance(o[0][0], str):
                    for n in o:
                        print(n[0])
                        if n[0] != None:
                            print('ooo')
                            translator = Translator()
                            final_trans=translator.translate(n[0],dest='en')
                            if final_trans:
                                print("##########################################")
                                print(final_trans.text)
                                print(final_trans.origin)
                                print("##########################################")
                                lo.append(final_trans.text)
                                li.append(final_trans.origin)
                                content_list.append({'index':n[1],'name' : n[0],'TRANS':final_trans.text})
                                content_listorigin.append({'index':n[1],'name' : n[0],'TRANS':final_trans.origin})
                            else :
                                print('pass')
                    '''
                    mpmp=dict(Counter(lo))
                    print(mpmp)
                    #ordering

                    s = [(j , mpmp[j]) for j in sorted(mpmp, key=mpmp.get, reverse=True)]
                    print(content_list)

                    mpmp1=dict(Counter(li))
                    print(mpmp)
                    #ordering
                    s1 = [(j , mpmp1[j]) for j in sorted(mpmp1, key=mpmp1.get, reverse=True)]
                    print(content_listorigin)
                    '''
                    item = {"content":content_list,"column_name":k.name,
                    "table_name":k.table_name,"column":[dict(Counter(li))],"possible_translation":[dict(Counter(lo))]}
                    k.terms_translated = item
                    k.langtranslated = True
                    k.save()


import re

from collections import Counter

from . import colors

# Title, Author, and Date are tab (\t) separated
# Title (Author First, Last Middle) (Date Finished)

# Example:
# 2019
# Slaughterhouse-Five	(Vonnegut, Kurt)	(1/3)
# Anna Karenina	(Tolstoy, Leo)	(2/8)
# War and Peace	(Tolstoy, Leo)	(5/11)

# year_ex = re.compile('\d{4}')
# data_ex = re.compile('(.*[\t\n]|$)(\(.*\)\t)?(\(.*\))?')

class BookLog:
    def __init__(self, fn):
        self.log_dict = self.process_log(fn)                # year: list of books
        self.author_dict = self.build_author_dict()         # author: list of books
        self.author_counter = self.build_author_counter()   # author: number of books

    def process_log(self, fn):
        '''
        Create a dict where keys are years and values are a list of books read that year
        '''
        with open(fn, 'r') as f:
            books = f.read().splitlines()

        log_dict = dict()
        current_year = ''

        for ln in books:
            if not ln or ln.startswith('#'):
                continue

            try:
                int(ln)
                current_year = ln
                log_dict[current_year] = list()
            except ValueError:
                data = re.sub('[\(\)]', '', ln)
                data = data.split('\t')
                log_dict[current_year].append(data)
        return log_dict

    def build_author_dict(self):
        '''
        Create a dict where keys are authors and values are a list of books by author
        '''
        author_dict = dict()
        for year in self.log_dict.values():
            for book in year:
                if author_dict.get(book[1]):
                    author_dict[book[1]].append(book[0])
                else:
                    author_dict[book[1]] = [book[0]]
        return author_dict

    def build_author_counter(self):
        '''
        Create a counter where keys are authors and values are a count of books read by author
        '''
        author_counter = Counter()
        for author, books in self.author_dict.items():
            author_counter[author] = len(books)
        return author_counter

    def book_count(self, year=None):
        '''
        Return a count of books read in `year` if not none else total amount of books read
        '''
        if year:
            if not self.log_dict.get(year):
                return 0
            else:
                return len(self.log_dict[year])
        else:
            total = 0
            for year in self.log_dict:
                total += len(self.log_dict[year])
            return total

    def most_read_authors(self, n=7):
        return self.author_counter.most_common(n)

    def search_authors(self, term, partial_matches=True):
        authors = list()
        split_term = re.split(', | ', term.lower())

        for author in self.author_dict.keys():
            if partial_matches:
                if any([t in author.lower() for t in split_term]):
                    authors.append(author)
            else:
                split_author = re.split(', | |; ', author.lower())
                if all([t in split_author for t in split_term]):
                    authors.append(author)
        return authors

    def search_books(self, term, partial_matches=True):
        books = list()
        split_term = re.split(', | ', term.lower())

        for year, year_books in self.log_dict.items():
            for book in year_books:
                if partial_matches:
                    if any([t in book[0].lower() for t in split_term]):
                        books.append(book)
                else:
                    split_title = re.split(', | |; ', book[0].lower())
                    if all([t in split_title for t in split_term]):
                        books.append(book)
        return books

    def print_yearly_tally(self):
        for year in self.log_dict:
            print(f'{year}: {self.book_count(year):>3}')
        print(f'Total: {self.book_count()}')

    def print_all_and_tally(self):
        for year in self.log_dict:
            self.print_books_for_year(year)
            print(f'Total:{self.book_count(year):>3}\n')

    def print_books_for_year(self, year, sep=None):
        if not self.log_dict.get(year):
            print(f'No books found for year: {year}')
        else:
            print(year)
            for book in self.log_dict.get(year):
                if len(book) == 3:
                    title, author, date_completed = book
                else:
                    title, author = book
                    date_completed = ''

                if sep:
                    print(f'{title}{sep}{author}{sep}{date_completed}')
                else:
                    col_width = 50
                    pad_width = col_width - 5

                    if len(title) >= col_width:
                        title = f'{title[:pad_width]}...'
                    if len(author) >= col_width:
                        author= f'{author[:pad_width]}...'

                    print(f'{title:<{col_width}s}{author:<{col_width}s}{date_completed}')

    def print_author_dict(self, author=None):
        if author:
            authors = sorted(self.search_authors(author))

            for a in authors:
                print(f'{colors.GREEN}{a}:{colors.RESET}')
                for book in self.author_dict.get(a):
                    print(f'  {book}')
        else:
            for author, books in sorted(self.author_dict.items(), key=lambda x: x[0]):
                print(f'{colors.GREEN}{author}:{colors.RESET}')
                for book in books:
                    print(f'  {book}')


# coding: utf-8
from sqlalchemy import Column, Float, Index, String, Text
from sqlalchemy.dialects.mysql import TEXT, VARCHAR
from sqlalchemy.ext.declarative import declarative_base

Base = declarative_base()
metadata = Base.metadata

class FundBasic(Base):
    __tablename__ = 'fund_basic'

    ts_code = Column(TEXT, primary_key=True)
    name = Column(TEXT, nullable=False)
    management = Column(Text)
    custodian = Column(Text)
    fund_type = Column(Text)
    found_date = Column(Text)
    due_date = Column(Text)
    list_date = Column(Text)
    issue_date = Column(Text)
    delist_date = Column(Text)
    issue_amount = Column(Float(asdecimal=True))
    m_fee = Column(Float(asdecimal=True))
    c_fee = Column(Float(asdecimal=True))
    duration_year = Column(Float(asdecimal=True))
    p_value = Column(Float(asdecimal=True))
    min_amount = Column(Float(asdecimal=True))
    exp_return = Column(Text)
    benchmark = Column(Text)
    status = Column(Text)
    invest_type = Column(Text)
    type = Column(Text)
    trustee = Column(Text)
    purc_startdate = Column(Text)
    redm_startdate = Column(Text)
    market = Column(Text)

class FundNav(Base):
    __tablename__ = 'fund_nav'
    __table_args__ = (
        Index('ts_code', 'ts_code', 'end_date', unique=True),
    )

    ts_code = Column(VARCHAR(20), primary_key=True, nullable=False)
    ann_date = Column(VARCHAR(20), nullable=False)
    end_date = Column(VARCHAR(20), primary_key=True, nullable=False)
    unit_nav = Column(Float)
    accum_nav = Column(Float)
    accum_div = Column(Float)
    net_asset = Column(Float)
    total_netasset = Column(Float)

class FundManager(Base):
    __tablename__ = 'fund_manager'
    __table_args__ = (
        Index('ts_code', 'ts_code', 'ann_date', unique=True),
    )

    ts_code = Column(VARCHAR(20), primary_key=True, nullable=False)
    ann_date = Column(VARCHAR(20), primary_key=True, nullable=False)
    name = Column(VARCHAR(20))
    gender = Column(VARCHAR(20))
    birth_year = Column(VARCHAR(20))
    edu = Column(VARCHAR(20))
    nationality = Column(VARCHAR(20))
    begin_date = Column(VARCHAR(20))
    end_date = Column(VARCHAR(20))
    resume = Column(VARCHAR(20))

class FundPortfolio(Base):
    __tablename__ = 'fund_portfolio'
    __table_args__ = (
        Index('ts_code', 'ts_code', 'end_date','symbol', unique=True),
    )

    ts_code = Column(VARCHAR(20), primary_key=True, nullable=False)
    ann_date = Column(String(20))
    end_date = Column(VARCHAR(20), primary_key=True, nullable=False)
    symbol = Column(String(500), primary_key=True, nullable=False)
    mkv = Column(Float)
    amount = Column(Float)
    stk_mkv_ratio = Column(Float)
    stk_float_ratio = Column(Float)


import torch
import torch.nn.functional as F
import torchvision
from glob import glob

import numpy as np
import pandas as pd

import time
from tqdm import tqdm
import os
from PIL import Image

from models import *

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class CustomTestDataset(torch.utils.data.Dataset):
    """
    pytorch dataset that generates samples for test (data without labels)
    in format "image, filename" (tuple)
    """

    def __init__(self, root_dir, transform=None):
        """
        :param root_dir:
        :param transform:
        """
        self.root_dir = root_dir
        self.transform = transform
        self.filelist = [name for name in os.listdir(self.root_dir)
                         if os.path.isfile(os.path.join(self.root_dir, name))
                         ]

    def __len__(self):
        return len(self.filelist)

    def __getitem__(self, idx):
        """
        :param idx:
        :return: tuple. (image, filename)
        """
        img_name = os.path.join(self.root_dir,
                                self.filelist[idx])

        image = Image.open(img_name)
        sample = image

        if self.transform:
            sample = self.transform(image)

        sample = np.array(sample)
        # print(sample.shape, self.filelist[idx])
        return (sample, self.filelist[idx])

def predict(model, test_dataloader):
    """
    making prediction

    :param model:
    :param test_dataloader:
    :return: dict (filename -> label)
    """
    since = time.time()

    # define main constants
    train_root = 'data/test/'
    batch_size = 4
    num_workers = 4

    # get class names
    image_data = torchvision.datasets.ImageFolder(root=train_root)
    classes = {v: k for k, v in image_data.class_to_idx.items()}

    model.eval()  # set model to evaluate mode

    predictions = pd.DataFrame(columns=['filename', 'prediction'])

    # iterate over data
    for inputs, files in tqdm(test_dataloader, ascii=True):
        inputs = inputs.to(device)

        # forward
        # track history only if in train
        with torch.set_grad_enabled(False):
            outputs = model(inputs)
            _, preds = torch.max(outputs, 1)
            for pred, file in zip(preds.data.tolist(), files):
                predictions = predictions.append(
                    pd.Series({'filename': file, 'prediction': classes[pred]}
                              ), ignore_index=True)

    print()

    time_elapsed = time.time() - since
    print('Predict complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    return predictions

def predict_proba(model, test_dataloader):
    """
    making prediction probability (for 2 classes only!)

    :param model:
    :param test_dataloader:
    :return: dict (filename -> label)
    """
    since = time.time()

    # define main constants
    train_root = 'data/train/'
    batch_size = 16
    num_workers = 4

    model.eval()  # set model to evaluate mode

    predictions = pd.DataFrame(columns=['filename', 'prediction_proba'])

    # iterate over data
    for inputs, files in tqdm(test_dataloader, ascii=True):
        inputs = inputs.to(device)

        # forward
        # track history only if in train
        with torch.set_grad_enabled(False):
            outputs = model(inputs)
            #print(F.softmax(outputs, dim=1))
            preds = F.softmax(outputs, dim=1)
            for pred, file in zip(preds.data.tolist(), files):
                predictions = predictions.append(
                    pd.Series({'filename': file, 'prediction_proba': pred[1]}
                              ), ignore_index=True)

    print()

    time_elapsed = time.time() - since
    print('Predict complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    return predictions

def predict_proba_cycle(models_root, test_dataloader):
    paths = glob(models_root + '/*.pkl')
    print(paths)
    preds = []
    for model_path in paths:
        model_i = InceptionResnetV2(2).to(device) # todo: change models
        model_i.load_state_dict(torch.load(model_path))
        print('Load model {}'.format(model_path))
        preds.append(predict_proba(model_i, test_dataloader))

    res = preds[0]
    for result in preds[1:]:
        res['prediction_proba'] += result['prediction_proba']
    res['prediction_proba'] /= len(preds)
    return res

if __name__ == '__main__': #IDRND_FASDB_train
    test_root = 'data/test'
    preds_filename = 'results1.csv' # todo: argparse
    model_root = 'models/last_model/cycle0.pkl' # todo: argparse
    num_classes = 2 # todo:?
    batch_size = 16
    num_workers = 4

    # load trained model
    model = InceptionResnetV2(num_classes).to(device)
    model.load_state_dict(torch.load(model_root))

    test_transforms = torchvision.transforms.Compose([
        # torchvision.transforms.RandomCrop(224),
        # torchvision.transforms.CenterCrop((224, 224)),
        torchvision.transforms.Resize((299, 299)),
        # torchvision.transforms.RandomHorizontalFlip(),
        torchvision.transforms.ToTensor(),
        torchvision.transforms.Normalize(mean=[0.53912586, 0.45975342, 0.45474854],
                                         std=[0.26110232, 0.25262088, 0.24871802])
    ])
    test_dataset = CustomTestDataset(test_root, transform=test_transforms)
    test_dataloader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, num_workers=num_workers)

    res = predict_proba_cycle('models/last_model', test_dataloader)

    res.to_csv(preds_filename, header=None, index=None)
    print('Prediction complete! Results saved to {}'.format(preds_filename))