from brownie import network, config, interface
from scripts.helpful_scripts import get_account
from scripts.get_weth import get_weth
from web3 import Web3

amount = Web3.toWei(1, "ether")

def approve_erc20(spender, amount, erc20_address, account):
    print("Approving erc20...")
    # ABI for ERC20 with the address
    erc20 = interface.IERC20(erc20_address)
    print(f"Approving amount of {amount} ")
    tx = erc20.approve(spender, amount, {"from": account})
    tx.wait(1)
    print(f"Approved? {tx}")
    return tx

def get_lending_pool():
    # ABI for addresses provider
    lending_pool_addresses_provider = interface.ILendingPoolAddressesProvider(
        config["networks"][network.show_active()]["lending_pool_addresses_provider"]
    )
    # Address
    lending_pool_address = lending_pool_addresses_provider.getLendingPool()
    # ABI for lending pool
    lending_pool = interface.ILendingPool(lending_pool_address)
    return lending_pool

def get_borrowable_data(lending_pool, account):

    print("Getting data...")
    # returns 6 numbers
    # see: https://docs.aave.com/developers/v/2.0/the-core-protocol/lendingpool#getuseraccountdata
    data = lending_pool.getUserAccountData(account.address, {"from": account})
    total_collateral_eth = Web3.fromWei(data[0], "ether")
    total_debt_eth = Web3.fromWei(data[1], "ether")
    available_borrow_eth = Web3.fromWei(data[2], "ether")
    print(
        f"Collateral: {total_collateral_eth}, total_debt: {total_debt_eth}, available to borrow: {available_borrow_eth}"
    )

    return (float(available_borrow_eth), float(total_debt_eth))

def get_asset_price(asset_price_feed):
    # ABI
    # address
    dai_eth_price = interface.AggregatorV3Interface(asset_price_feed)
    # returns 5 values, but we're interested
    # only in the latest price which is at the index 1
    latest_price = dai_eth_price.latestRoundData()[1]
    converted_price = Web3.fromWei(latest_price, "ether")
    print(f"DAI/ETH price {converted_price} ")
    return float(converted_price)

def repay_all(amount, lending_pool, account):
    approve_erc20(
        lending_pool.address,
        Web3.toWei(amount, "ether"),
        config["networks"][network.show_active()]["dai_token_address"],
        account,
    )
    print("Approved")
    repay_tx = lending_pool.repay(
        config["networks"][network.show_active()]["dai_token_address"],
        amount,
        1,
        account.address,
        {"from": account},
    )
    repay_tx.wait(1)
    print("Repaid!")

def main():
    account = get_account(id="test1")
    erc20_address = config["networks"][network.show_active()]["weth_token"]
    if network.show_active() in ["mainnet-fork"]:
        get_weth()
    # Address
    # ABI
    lending_pool = get_lending_pool()
    print("Lending pool address:", lending_pool)
    # approving from erc20
    approve_erc20(lending_pool.address, amount, erc20_address, account)

    # depositing
    print("Depositing ...")
    tx = lending_pool.deposit(
        erc20_address, amount, account.address, 0, {"from": account}
    )
    tx.wait(1)
    print("Deposited!")

    available_to_borrow, total_debt = get_borrowable_data(lending_pool, account)

    # convert to DAI
    dai_eth_price_feed = config["networks"][network.show_active()]["dai_eth_price_feed"]
    dai_eth_price = get_asset_price(dai_eth_price_feed)

    # safety factor is for Health Factor
    # the lower the safety factor the higher Health Factor
    safety_factor = 0.9
    amount_dai_to_borrow = (1 / dai_eth_price) * (available_to_borrow * safety_factor)
    print(f"We are borrowing {amount_dai_to_borrow} DAI.")

    dai_address = config["networks"][network.show_active()]["dai_token_address"]
    borrow_tx = lending_pool.borrow(
        dai_address,
        Web3.toWei(amount_dai_to_borrow, "ether"),
        1,
        0,
        account.address,
        {"from": account},
    )
    borrow_tx.wait(1)
    print("Borrowed some DAI")
    print("New borrawable data:", get_borrowable_data(lending_pool, account))

    repay_all(amount, lending_pool, account)
    print(
        "We just deposited, borrowed, and repaid on AAVE all programatically! Thank you AAVE, Brownie, and Chainlink!!"
    )


import os
import librosa
import torch
import numpy as np

# Load the SERNet model
model = torch.hub.load('pyannote/pyannote-audio', 'SER_0')

# Define a list of emotion labels
emotions = ['neutral', 'calm', 'happy', 'sad', 'angry', 'fearful', 'disgust', 'surprised']

# Define the path to the directory containing the audio files
data_dir = "/Users/amitosi/PycharmProjects/chester/udi/emotion"

# Define empty lists to store the emotion labels and probabilities
emotion_labels = []
emotion_probs = []

# Set the model to evaluation mode
model.eval()

# Disable gradient calculation to speed up inference
torch.set_grad_enabled(False)

# Iterate through the audio files in the directory
for file in os.listdir(data_dir):
    try:
        # Load the audio data and resample it to 16 kHz (the same sample rate used during training)
        filepath = os.path.join(data_dir, file)
        audio_dat, sample_rate = librosa.load(filepath, sr=16000)
        audio_dat = librosa.resample(audio_dat, sample_rate, 16000)

        # Extract the features from the audio data using the SERNet model
        features = model({'audio': audio_dat})

        # Compute the probability distribution over the emotion classes
        probs = torch.nn.functional.softmax(features['log_likelihood']).numpy()[0]

        # Choose the emotion with the highest probability as the predicted emotion
        pred_emotion = emotions[np.argmax(probs)]

        # Append the predicted emotion label and probability to the respective lists
        emotion_labels.append(pred_emotion)
        emotion_probs.append(probs)
    except Exception as e:
        print(f"Error encountered while parsing {file}: {e}")


#-*- coding:utf-8 -*-
import sys
import os
import time
import random
import numpy as np
import tensorflow as tf
import cv2 as cv

SIZE = 256
WIDTH = 16
HEIGHT = 16
NUM_CLASSES = 36
iterations = 800

SAVER_DIR = "model_file/"
LOG_DIR = SAVER_DIR + "logfile/"
DATASET_DIR = "/"

def get_train():
    time_begin = time.time()
    #Get the total number of pictures in the first iteration
    input_count = 0
    for i in range(0, NUM_CLASSES):
        # dir = 'train/%s/' % i
        dir = DATASET_DIR + 'train/%s/' % i
        for root,dirs,files in os.walk(dir):
            for filename in files:
                if filename == "Thumbs.db":
                    continue
                # if filename ==
                input_count = input_count + 1

    input_images = np.array([[0]*SIZE for i in range(input_count)])
    input_labels = np.array([[0]*NUM_CLASSES for i in range(input_count)])
    #Get data content in the second iteration
    index = 0
    for i in range(0, NUM_CLASSES):
        dir = DATASET_DIR + 'train/%s/' % i
        a = 0
        for root,dirs,files in os.walk(dir):
            for filename in files:
                if filename == "Thumbs.db":
                    continue
                filename = dir + filename
                img = cv.imread(filename,0)

                height = img.shape[0]
                width = img.shape[1]
                a = a + 1
                for h in range(0,height):
                    for w in range(0,width):
                        m = img[h][w]
                        if m > 150:
                            input_images[index][w+h*width] = 1
                        else:
                            input_images[index][w+h*width] = 0
                input_labels[index][i] = 1
                index = index + 1
    print('[---------------]')
    print('train data successfully loaded!')
    return time_begin, input_count, input_images, input_labels

def get_test():
    tes_count = 0
    for i in range(0, NUM_CLASSES):
        dir = DATASET_DIR + 'test/%s/' % i
        for root,dirs,files in os.walk(dir):
            for filename in files:
                if filename == "Thumbs.db":
                    continue
                tes_count = tes_count + 1
    tes_images = np.array([[0]*SIZE for i in range(tes_count)])
    tes_labels = np.array([[0]*NUM_CLASSES for i in range(tes_count)])
    index = 0
    for i in range(0, NUM_CLASSES):
        dir = DATASET_DIR + 'test/%s/' % i
        for root,dirs,files in os.walk(dir):
            for filename in files:
                if filename == "Thumbs.db":
                    continue
                filename = dir + filename
                img = cv.imread(filename,0)
                height = img.shape[0]
                width = img.shape[1]
                for h in range(0,height):
                    for w in range(0,width):
                        m = img[h][w]
                        if m > 150:
                            tes_images[index][w+h*width] = 1
                        else:
                            tes_images[index][w+h*width] = 0
                tes_labels[index][i] = 1
                index = index + 1
    print('[---------------]')
    print('test data successfully loaded!')
    return tes_count, tes_images, tes_labels

class CNN:
    def __init__(self,iterations, time_begin, input_count, input_images,
                input_labels, tes_count, tes_images, tes_labels):
        self.W_conv1 = None
        self.b_conv1 = None
        self.W_conv2 = None
        self.b_conv2 = None
        self.W_fc1 = None
        self.b_fc1 = None
        self.W_fc2 = None
        self.b_fc2 = None
        self.keep_prob = None
        self.iterations = iterations
        self.time_begin = time_begin
        self.input_count = input_count
        self.input_images = input_images
        self.input_labels = input_labels
        self.tes_count = tes_count
        self.tes_images = tes_images
        self.tes_labels = tes_labels

    def conv_layer(self,inputs, W, b):
        L1_conv = tf.nn.conv2d(inputs,W,strides=[1,1,1,1],padding='SAME')
        L1_relu = tf.nn.relu(L1_conv + b)
        return tf.nn.max_pool(L1_relu,ksize=[1,2,2,1],strides=[1,2,2,1],padding='SAME')

    def full_connect(self,inputs,W,b):
        return tf.nn.relu(tf.matmul(inputs,W)+b)

    def average(self, seq):
        return float(sum(seq)) / len(seq)

    def fit(self):
        # with tf.name_scope('Input'):
        x = tf.placeholder(tf.float32,shape=[None,SIZE], name = "x_input")
        y_ = tf.placeholder(tf.float32,shape=[None,NUM_CLASSES])
        self.keep_prob = tf.placeholder(tf.float32, name = 'keep_prob')
        x_image = tf.reshape(x,[-1,WIDTH,HEIGHT,1])

        #The first convolution layer, 16*16*1-> 8*8*12
        self.W_conv1 = tf.Variable(tf.truncated_normal([5,5,1,12],stddev=0.1),name="W_conv1")
        self.b_conv1 = tf.Variable(tf.constant(0.1,shape=[12]),name="b_conv1")
        L1_pool = CNN.conv_layer(self, x_image, self.W_conv1, self.b_conv1)

        #The second convolution layer, 8*8*12-> 4*4*24
        self.W_conv2 = tf.Variable(tf.truncated_normal([5,5,12,24],stddev=0.1),name="W_conv2")
        self.b_conv2 = tf.Variable(tf.constant(0.1,shape=[24]),name="b_conv2")
        L2_pool = CNN.conv_layer(self, L1_pool, self.W_conv2, self.b_conv2)

        # #The third convolution layer, 4*4*24-> 2*2*36
        # self.W_conv3 = tf.Variable(tf.truncated_normal([5,5,24,36],stddev=0.1),name="W_conv3")
        # self.b_conv3 = tf.Variable(tf.constant(0.1,shape=[36]),name="b_conv3")
        # L3_pool = CNN.conv_layer(self, L2_pool, self.W_conv3, self.b_conv3)

        #The first fullconnect layer, 2*2*36-> 64
        self.W_fc1 = tf.Variable(tf.truncated_normal([4*4*24,128],stddev=0.1),name="W_fc1")
        self.b_fc1 = tf.Variable(tf.constant(0.1,shape=[128]),name="b_fc1")
        h_pool2_flat = tf.reshape(L2_pool,[-1,4*4*24])
        h_fc1 = CNN.full_connect(self, h_pool2_flat, self.W_fc1, self.b_fc1)
        h_fc1_drop = tf.nn.dropout(h_fc1,self.keep_prob)

        #readout layer, 64->NUM_CLASSES
        self.W_fc2 = tf.Variable(tf.truncated_normal([128,NUM_CLASSES],stddev=0.1),name="W_fc2")
        self.b_fc2 = tf.Variable(tf.constant(0.1,shape=[NUM_CLASSES]),name="b_fc2")
        y_conv = tf.matmul(h_fc1_drop, self.W_fc2) + self.b_fc2
        pyx = tf.nn.softmax(y_conv, name = "y_pred")

        with tf.name_scope('loss'):
            cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=y_,logits=y_conv))
            loss_summary = tf.summary.scalar('loss', cross_entropy)

        with tf.name_scope('train'):
            train_step = tf.train.AdamOptimizer((1e-5)).minimize(cross_entropy)

        with tf.name_scope('accuracy'):
            with tf.name_scope('correct_prediction'):
                correct_prediction = tf.equal(tf.argmax(y_conv,1),tf.argmax(y_,1))
            with tf.name_scope('accuracy'):
                accuracy = tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
                acc_summary = tf.summary.scalar('accuracy', accuracy)

        #Use tensorboard to record loss and accuracy charts
        merged = tf.summary.merge([loss_summary, acc_summary])
        with tf.Session() as sess:
            sess.run(tf.global_variables_initializer())

            time_elapsed = time.time() - self.time_begin
            print('[=============]')
            print("Reading data takes time %d second." % time_elapsed)
            print ("Read a total of %s training data, %s tags." % (self.input_count, self.input_count))
            print ("Read a total of %s test data %s tags." % (self.tes_count, self.tes_count))

            # saver = tf.compat.v1.train.Saver()
            saver = tf.train.Saver()
            writer = tf.summary.FileWriter(LOG_DIR, sess.graph)

            tr_start = time.time()
            batch_size = 64
            batches_count = int(self.input_count/batch_size)
            remainder = self.input_count % batch_size
            print ("Training set divided into %s batchs, Each batch before have %s datas, Last Batch have %s datas." % (batches_count+1, batch_size, remainder))

            print('[=============]')
            print('Train start!')
            for it in range(self.iterations + 1):
                sum_loss = []
                for n in range(batches_count):
                    summary, loss, out = sess.run([merged, cross_entropy, train_step], feed_dict = {x:self.input_images[n*batch_size:(n+1)*batch_size],
                                                                                y_:self.input_labels[n*batch_size:(n+1)*batch_size],self.keep_prob:0.5})
                    sum_loss.append(loss)
                if remainder > 0:
                    start_index = batches_count * batch_size
                    loss, out = sess.run([cross_entropy, train_step], feed_dict = {x:self.input_images[start_index:input_count-1],
                                                                                y_:self.input_labels[start_index:input_count-1],
                                                                                self.keep_prob:0.5})
                    sum_loss.append(loss)
                writer.add_summary(summary, it)
                avg_loss = CNN.average(self, sum_loss)
                iterate_accuracy = 0
                if it % 5 == 0:
                    loss1, iterate_accuracy = sess.run([cross_entropy,accuracy], feed_dict = {x :self.tes_images,
                                                                                y_ :self.tes_labels,self.keep_prob : 1.0})
                    print('The %0.4d iteration training accuracy is: %0.5f%% ' % (it,iterate_accuracy*100) +
                          '    Loss value is: %0.5f' % avg_loss + '    Test loss value is: %0.5f.' % loss1)
                    if iterate_accuracy >= 0.9999999:
                        break
                    acc = iterate_accuracy

            print('[=============]')
            print ('Train end!')
            tr_end = time.time() - tr_start
            print('The final accuracy of training is: %0.3f%%!' % (acc*100))
            print('The final loss of training is: %0.5f!' % (avg_loss))
            print ("This training takes a total of time: %d second." % tr_end)

            if not os.path.exists(SAVER_DIR) :
                print ('The training model save path does not exist, Automatically create a path now.')
                os.makedirs(SAVER_DIR)
            saver_path = saver.save(sess, SAVER_DIR)
            print("The model save path is: %s" % saver_path)
            print('The graph save path is: %s' % LOG_DIR)
            writer.close()

if __name__ == "__main__":

    # tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)
    time_begin, input_count, input_images, input_labels = get_train()
    tes_count, tes_images, tes_labels = get_test()

    cnn = CNN(iterations, time_begin, input_count, input_images,
              input_labels, tes_count, tes_images, tes_labels)
    cnn.fit()

#????é”Ÿæ–¤æ‹?????????????????????
    # val_count = 0
    # for i in range(NUM_CLASSES):
    #     dir = 'valid/%s/' % i
    #     for root,dirs,files in os.walk(dir):
    #         for filename in files:
    #             if filename == "Thumbs.db":
    #                 continue
    #             val_count = val_count + 1
    # val_images = np.array([[0]*SIZE for i in range(val_count)])
    # val_labels = np.array([[0]*NUM_CLASSES for i in range(val_count)])
    # index = 0
    # for i in range(NUM_CLASSES):
    #     dir = 'valid/%s/' % i
    #     for root,dirs,files in os.walk(dir):
    #         for filename in files:
    #             if filename == "Thumbs.db":
    #                 continue
    #             filename = dir + filename
    #             img = cv.imread(filename,0)
    #             height = img.shape[0]
    #             width = img.shape[1]
    #             for h in range(0,height):
    #                 for w in range(0,width):
    #                     m = img[h][w]
    #                     if m > 150:
    #                         val_images[index][w+h*width] = 1
    #                     else:
    #                         val_images[index][w+h*width] = 0
    #             val_labels[index][i] = 1
    #             index = index + 1
    #     print('[---------------]')
    #     print('%s é”Ÿæ–¤æ‹·å›¾ç‰‡é”Ÿæ–¤æ‹·é”Ÿæ–¤æ‹·é”Ÿæ–¤æ‹·é”Ÿï¿½??' % i)


import json
import mimetypes
import os
import requests

class UploadGH(object):
    def __init__(self, repo, token=None):
        self.token = os.getenv("GITHUB_TOKEN", token)
        self.endpoint = "https://api.github.com/repos/%s" % repo

    def headers(self, extra={}):
        headers = {'Authorization': "token %s" % self.token}
        headers.update(extra)
        return headers

    def create_release(self, tag):
        path = self.endpoint + "/releases"
        params = {"tag_name": tag,
                  "name": tag,
                  "draft": True,
                  "body": "Release %s" % tag}
        resp = requests.post(path, data=json.dumps(params), headers=self.headers())
        resp.raise_for_status()
        return resp

    def get_release(self, tag):
        path = self.endpoint + "/releases/tags/%s" % tag
        resp = requests.get(path)
        if resp.status_code == 404:
            resp = self.create_release(tag)
        resp.raise_for_status()
        return resp.json()

    def get_existing_asset(self, name, release_id):
        path = self.endpoint + "/releases/%s/assets" % release_id
        resp = requests.get(path)
        assets = resp.json()

        if resp.status_code < 299:
            for asset in assets:
                if asset['name'] == name:
                    return asset

        return None

    def delete_asset(self, asset):
        resp = requests.delete(asset['url'], headers=self.headers())
        resp.raise_for_status()

    def upload(self, filepath, tag, name=None,
               overwrite=True, contenttype="application/octet-stream"):
        release = self.get_release(tag)
        upload_url = release['upload_url'].split("{")[0]
        name = name or os.path.basename(filepath)
        data = open(filepath, "rb").read()

        if overwrite:
            previous_asset = self.get_existing_asset(name, release['id'])
            if previous_asset:
                self.delete_asset(previous_asset)

        if contenttype is None:
            mimetypes.init()
            contenttype = mimetypes.guess_type(filepath)[0]

        resp = requests.post(upload_url, data=data,
                             headers=self.headers({'Content-Type': contenttype}),
                             params={"name": name})

        return resp


import numpy as np
from ipywidgets import widgets
import ase
import ase.io
import ase.neighborlist
from itertools import permutations
from tqdm.notebook import tqdm, trange
import nglview
from IPython.display import display, FileLink
from zipfile import ZipFile
import os
import glob

import warnings
warnings.simplefilter('ignore')

# Class to define and operate on environments
class Environments:
    def __init__(self):
        self.myindex = -1
        self.indeces = np.array([],dtype=int)
        self.delta = np.array([])
        self.distance =  np.array([])
        # degeneracy defaults to 1 and is different from 1 for unique environments
        self.degeneracy = 1

# Class to find environments
class EnvironmentFinder:

    def __init__(self):
        # self.fractionalFlag is True if the coordinates are fractional
        self.fractionalFlag=False
        # uniqueFlags is True if unique environments are to be calculated
        self.uniqueFlag=True
        # self.verboseFlag is True if verbose mode is on
        self.verboseFlag=True
        # self.allEnvs stores all calculated environments
        self.allEnvs=np.ndarray((0,),dtype=object)
        # self.uniqueEnvs stores the unique environments
        self.uniqueEnvs = np.ndarray((0,),dtype=object)
        # self.tolerance is the tolerance to determine if two elements of the
        # distance vectors are the same
        self.tolerance=0
        # self.atom_types contains the unique atom types
        self.atom_types = np.empty(1)
        # Max number of neighbors in environment
        self.max_neighbors=1
        self.max_neighbors_flag=False
        #env=Environments()
        #env.delta.shape = (0,3)
        #uniqueEnvs = np.append(uniqueEnvs,env)
        # self.box_layout contains options to display a box
        self.box_layout = widgets.Layout(display='flex',
                    flex_flow='column',
                    align_items='stretch',
                    border='solid',
                    width='90%')

    def plotConf(self):
        """ Plot configuration using nglview """

        v = nglview.show_ase(self.conf)
        v.add_representation("unitcell")
        boxy=widgets.Box([v],layout=self.box_layout)
        display(boxy)

    def chooseConfiguration(self,filename):
        """ Choose configuration by filename

        Args:
            filename (string): the file name

        Returns:
            no value
        """

        self.conf = ase.io.read(filename)
        self.atom_types = np.unique(self.conf.get_chemical_symbols())
        self.atom_types = np.append(self.atom_types,"Any")

    def chooseAndPlotConfiguration(self,filename):
        """ Choose and plot configuration by filename

        Args:
            filename (string): the file name

        Returns:
            no value
        """

        self.chooseConfiguration(filename)
        self.plotConf()

    def setMaximumNumberOfNeighbors(self,max_neighbors):
        self.max_neighbors_flag = True
        self.max_neighbors = max_neighbors

    def calculateEnvironments(self,lista,listb,cutoff):
        """ Find environments

        This function calculates all environments centered at the atom in
        lista using atoms in listb as neighbors if they are within the cutoff.

        Args:
            lista (numpy vector): list of central atoms
            listb (numpy vector): list of neighboring atoms
            cutoff (float): cut off for the calculation of neighbors

        Returns:
            no value
        """

        self.allEnvs = np.ndarray((0,),dtype=object)
        # Find all environments
        # Loop over input particles of type atom_type_1:
        myindeces=lista
        nl = ase.neighborlist.NeighborList((cutoff/2.0)*np.ones(self.conf.get_number_of_atoms()),skin=0.1,bothways=True)
        nl.update(self.conf)
        mat = self.conf.get_cell()
        for index in myindeces:
            neighbors, offsets = nl.get_neighbors(index)
            # Iterate over the neighbors of the current particle:
            Environment = Environments()
            Environment.myindex = index
            Environment.delta.shape = (0,3)
            for neigh, offset in zip(neighbors, offsets):
                if (neigh in listb): # Only if neighbor is in listb
                    delta = self.conf.positions[neigh] + np.dot(offset, mat) - self.conf.positions[index]
                    distance=np.linalg.norm(delta)
                    if (distance>1.e-10):
                        if (not self.fractionalFlag):
                            Environment.delta = np.append(Environment.delta,np.array([[delta[0], delta[1], delta[2]]])*0.1,axis=0)
                        else:
                            Environment.delta = np.append(Environment.delta,np.array([[delta[0]/mat[0][0], delta[1]/mat[1][1], delta[2]/mat[2][2]]]),axis=0)
                        Environment.distance = np.append(Environment.distance,distance)
                        Environment.indeces = np.append(Environment.indeces,neigh)
            # Use maximum number of neighbors
            if (self.max_neighbors_flag==True):
                 # Sort
                 sortindeces=np.argsort(Environment.distance,kind='stable')
                 Environment.delta =    Environment.delta[sortindeces,:]
                 Environment.distance = Environment.distance[sortindeces]
                 Environment.indeces =  Environment.indeces[sortindeces]
                 # Choose first max_neighbors
                 Environment.delta =    Environment.delta[:self.max_neighbors,:]
                 Environment.distance = Environment.distance[:self.max_neighbors]
                 Environment.indeces =  Environment.indeces[:self.max_neighbors]
            self.allEnvs = np.append(self.allEnvs,Environment)

    def CalculateUniqueEnvironments(self):
        """ Calculate unique Environments

        This algorithm calculates unique environments by comparing all possible
        permutations of the environments. However, it discards entire sets of
        permutations if it is found that there is no possibility of match.

        The number of operations scales roughly as as N^2*(\sum_{i=1}^M M+1-i)
        where N is the total number of environments and M is the number of i
        atoms in the environments.

        Args:
            none

        Returns:
            no value
        """

        num_of_templates=self.allEnvs.shape[0]
        flag_unique=np.ones(num_of_templates)
        same_as=np.linspace(0,num_of_templates-1,num_of_templates)
        # Disregard empty environments
        for i in range(num_of_templates):
            if (self.allEnvs[i].indeces.shape[0]==0):
                flag_unique[i]=0
        for i in trange(num_of_templates,desc="Progress", leave=False):
            if ( self.allEnvs[i].indeces.shape[0]>0 and flag_unique[i]==1):
                for j in range(i+1,num_of_templates):
                    # Not empty,  unique, and same number of neighbors
                    if ( self.allEnvs[j].indeces.shape[0]>0 and flag_unique[j]==1 and self.allEnvs[i].indeces.shape[0]==self.allEnvs[j].indeces.shape[0]):
                        atom_match = np.zeros(self.allEnvs[j].indeces.shape[0])
                        for k1 in range(self.allEnvs[i].indeces.shape[0]):
                            for k2 in range(self.allEnvs[j].indeces.shape[0]):
                                if (np.count_nonzero(np.isclose(self.allEnvs[i].delta[k1,:],self.allEnvs[j].delta[k2,:],atol=self.tolerance))==3 ):
                                    #print(i,j,self.allEnvs[i].delta[k1,:],self.allEnvs[j].delta[k2,:],np.isclose(self.allEnvs[i].delta[k1,:],self.allEnvs[j].delta[k2,:],atol=self.tolerance))
                                    atom_match[k1] = 1
                                    break
                            # If no match was found for a given atom the whole self.allEnvs[j] must be discarded
                            if (atom_match[k1]==0):
                                break
                        # If a match was found for every atom in self.allEnvs[i]
                        #print(i,j,atom_match)
                        if np.all(atom_match==np.ones(atom_match.shape[0])):
                            flag_unique[j]=0
                            same_as[j] = i
                            #break
        self.uniqueEnvs = np.ndarray((0,),dtype=object)
        for i in range(num_of_templates):
            if (flag_unique[i]==1):
                self.uniqueEnvs = np.append(self.uniqueEnvs,self.allEnvs[i])
                self.uniqueEnvs[-1].degeneracy = np.sum(np.ones(num_of_templates)[same_as==i])

    def printEnvironmentSummaryInfo(self,Environments):
        if (self.verboseFlag):
            num_of_templates=Environments.shape[0]
            if (num_of_templates>0):
                avg_num_neighbors = 0.0
                for i in range(num_of_templates):
                    avg_num_neighbors += Environments[i].indeces.shape[0]
                avg_num_neighbors /= num_of_templates
            else:
                avg_num_neighbors = 0
            print("Found " + str(num_of_templates) + " environments each with " + str(int(avg_num_neighbors))  + " neighbors on average")
            if (num_of_templates>0):
                for i in range(num_of_templates):
                    print("Environment " + str(int(i+1)) + ": degeneracy = " + str(int(Environments[i].degeneracy)) + " - number of neighbors = ", str(int(Environments[i].indeces.shape[0])) )

    def printEnvironments(self,Environments):
        num_of_templates=Environments.shape[0]
        for i in range(num_of_templates):
            env_atom_types = np.asarray(self.conf.get_chemical_symbols())[Environments[i].indeces.astype(int)]
            env_positions = Environments[i].delta*10
            env = ase.Atoms(env_atom_types,env_positions)
            ase.io.write('-', env, format='proteindatabank')

    def printEnvironmentsToFile(self,Environments):
        num_of_templates=Environments.shape[0]
        for i in range(num_of_templates):
            env_atom_types = np.asarray(self.conf.get_chemical_symbols())[Environments[i].indeces.astype(int)]
            env_positions = Environments[i].delta*10
            env = ase.Atoms(env_atom_types,env_positions)
            fileName="env" + str(i+1) + ".pdb"
            ase.io.write(fileName, env, format='proteindatabank')

    def printEnvironmentsToZipFile(self,Environments):
        num_of_templates=Environments.shape[0]
        download_directory="Download"
        if not os.path.exists(download_directory):
            os.mkdir(download_directory)
        zipObj = ZipFile(download_directory + '/download.zip', 'w')
        for i in range(num_of_templates):
            env_atom_types = np.asarray(self.conf.get_chemical_symbols())[Environments[i].indeces.astype(int)]
            env_positions = Environments[i].delta*10
            env = ase.Atoms(env_atom_types,env_positions)
            fileName=download_directory + "/env" + str(i+1) + ".pdb"
            ase.io.write(fileName, env, format='proteindatabank')
            zipObj.write(fileName)
        zipObj.close()
        local_file = FileLink(download_directory + '/download.zip', result_html_prefix="Click here to download the environments in PDB format: ")
        display(local_file)
        fileList = glob.glob(download_directory + '/env*')
        for filePath in fileList:
            os.remove(filePath)

    def plotEnv(self,myEnvironment):
        env_atom_types = np.asarray(self.conf.get_chemical_symbols())[myEnvironment.indeces.astype(int)] #,np.array('C')] #Template[env_number].myindex)]
        env_atom_types = np.append(env_atom_types,np.asarray(self.conf.get_chemical_symbols())[myEnvironment.myindex])
        env_positions = np.vstack((myEnvironment.delta*10,np.array([0,0,0]) ) )
        env = ase.Atoms(env_atom_types,env_positions)
        v = nglview.show_ase(env)
        boxy=widgets.Box([v],layout=self.box_layout)
        display(boxy)

    def chooseEnvPlotAll(self,number):
        if (self.allEnvs.shape[0]>0 and ((number-1)<self.allEnvs.shape[0])):
            self.plotEnv(self.allEnvs[number-1])
        else:
            print("Error")

    def chooseEnvPlotUnique(self,number):
        if (self.uniqueEnvs.shape[0]>0):
            self.plotEnv(self.uniqueEnvs[number-1])
            if (self.verboseFlag):
                print("Degeneracy of the environment is " + str(int(self.uniqueEnvs[number-1].degeneracy)) )
                print("Number of neighbors in the environment is " + str(int(self.uniqueEnvs[number-1].indeces.shape[0])) )

    def chooseEnvPlot(self,number):
        if (self.uniqueFlag and self.uniqueEnvs.shape[0]>0):
            self.chooseEnvPlotUnique(number)
        elif (not(self.uniqueFlag) and self.allEnvs.shape[0]>0):
            self.chooseEnvPlotAll(number)
        else:
            print("Error")

    def calculateEnvironmentsType(self,atom_type_1,atom_type_2,cutoff,tolerance):
        self.tolerance=tolerance
        chemical_symbols = np.asarray(self.conf.get_chemical_symbols())
        lista=np.argwhere(chemical_symbols == atom_type_1).flatten()
        listb=np.argwhere(chemical_symbols == atom_type_2).flatten()
        if (atom_type_1=="Any"):
            lista=np.linspace(0,chemical_symbols.shape[0]-1,chemical_symbols.shape[0]).astype(int)
        if (atom_type_2=="Any"):
            listb=np.linspace(0,chemical_symbols.shape[0]-1,chemical_symbols.shape[0]).astype(int)
        self.calculateEnvironments(lista,listb,cutoff)
        if (self.uniqueFlag):
            self.CalculateUniqueEnvironments()
            self.printEnvironmentSummaryInfo(self.uniqueEnvs)
        else:
            self.printEnvironmentSummaryInfo(self.globalTemplate)

    def calculateEnvironmentsString(self,listastring,listbstring,cutoff,tolerance):
        self.tolerance=tolerance
        lista=np.fromstring(listastring, dtype=int, sep=',')-1
        listb=np.fromstring(listbstring, dtype=int, sep=',')-1
        self.calculateEnvironments(lista,listb,cutoff)
        if (self.uniqueFlag):
            self.CalculateUniqueEnvironments()
            self.printEnvironmentSummaryInfo(self.uniqueEnvs)
        else:
            self.printEnvironmentSummaryInfo(self.allEnvs)

    def calculateEnvironmentsMinMaxStride(self,mina,maxa,stridea,minb,maxb,strideb,cutoff,tolerance):
        self.tolerance=tolerance
        lista=np.arange(int(mina),int(maxa)+1,int(stridea))-1
        listb=np.arange(int(minb),int(maxb)+1,int(strideb))-1
        self.calculateEnvironments(lista,listb,cutoff)
        if (self.uniqueFlag):
            self.CalculateUniqueEnvironments()
            self.printEnvironmentSummaryInfo(self.uniqueEnvs)
        else:
            self.printEnvironmentSummaryInfo(self.allEnvs)


#!/bin/python

#------------------- Description & Notes --------------------#

'''
We can reduce the number of K-mers by:
* Clustering reverse complement counts (For relatively short K-mers, i.e. K < 5)

We can reduce the number of samples by:
* Clustering counts of related sequences
'''

#------------------- Dependencies ---------------------------#

# Standard library imports

# External imports
import pyspark.sql.functions as F
import pyspark.sql.types as T
from pyspark.ml.feature import Normalizer
from pyspark.ml.linalg import VectorUDT
from pyspark.ml.linalg import Vectors
from sklearn import preprocessing

# Internal imports
from ..common import *
from .table import insertKmerIdxCol

#------------------- Constants ------------------------------#

#------------------- Public Classes & Functions -------------#

def toProbabilities(kmerDf):
    kmerDf = kmerDf.withColumn(COUNT_COL_NAME,
        F.col(COUNT_COL_NAME).cast(T.FloatType()))
    kmerDf = (kmerDf.groupby(KMERDF_SEQINFO_COL_NAMES)
        .applyInPandas(_toProbabilities, schema=kmerDf.schema))
    return kmerDf

def toNormalised(kmerDf):
    ## Normalise counts according to Wang et al. 2005
    ## This is done by multiplying the probabilities with
    ## the total number of possible Kmers
    kmer = kmerDf.select(KMER_COL_NAME).first()[0]
    t    = getExpectedTotal(kmer)

    kmerDf = toProbabilities(kmerDf)
    kmerDf = kmerDf.withColumn(COUNT_COL_NAME, F.col(COUNT_COL_NAME) * t)
    return kmerDf

def toL2Normalised(kmerDf, withPandas=True):
    if (withPandas):
        kmerDf = kmerDf.withColumn(COUNT_COL_NAME,
            F.col(COUNT_COL_NAME).cast(T.FloatType()))
        kmerDf = (kmerDf.groupby(KMERDF_SEQINFO_COL_NAMES)
            .applyInPandas(_toL2Normalised, schema=kmerDf.schema))

    else:
        normalizer = Normalizer(inputCol=KMER_COL_NAME, outputCol='L2Norm')
        kmerDf = normalizer.transform(kmerDf)
        kmerDf = kmerDf.drop(KMER_COL_NAME)
        kmerDf = kmerDf.withColumnRenamed('L2Norm', KMER_COL_NAME)

    return kmerDf

def explode(kmerDf):
    ## (ID, [Kmer], [Count], *) => (ID, Kmer, Count, *)
    gCols = kmerDf.schema.names
    gCols.remove(KMER_COL_NAME)
    gCols.remove(COUNT_COL_NAME)

    f = F.explode(F.arrays_zip(KMER_COL_NAME, COUNT_COL_NAME))
    kmerDf = (
        kmerDf
        .withColumn('tmp', f)
        .select(
            *gCols,
            F.col('tmp.kmer').alias(KMER_COL_NAME),
            F.col('tmp.count').alias(COUNT_COL_NAME)
        )
    )
    return kmerDf

def toDict(kmerDf):
    kmerDf = kmerDf.select(KMERDF_COL_NAMES)
    kmerDf = toList(kmerDf, isSorted=False)

    f = F.map_from_arrays(KMER_COL_NAME, COUNT_COL_NAME)
    kmerDf = kmerDf.select(*KMERDF_SEQINFO_COL_NAMES, f.alias(KMER_COL_NAME))
    return kmerDf

def toList(kmerDf, isSorted=True):
    def _sortCounts(kmerPdf):
        ## Convert the counts into a sorted list based on the Kmers
        kmerPdf = kmerPdf.sort_values(by=[KMER_COL_NAME])
        counts  = kmerPdf[COUNT_COL_NAME].tolist()
        kmer    = kmerPdf[KMER_COL_NAME].tolist()

        newKmerPdf = kmerPdf.iloc[:1][KMERDF_SEQINFO_COL_NAMES]
        newKmerPdf[COUNT_COL_NAME] = [counts] * 1
        newKmerPdf[KMER_COL_NAME]  = [kmer] * 1
        return newKmerPdf

    kmerDf = kmerDf.select(KMERDF_COL_NAMES)
    if (isSorted):
        schema  = getSchema(countsAsInt=False)
        kmerDf = (
            kmerDf
            .groupby(KMERDF_SEQINFO_COL_NAMES)
            .applyInPandas(_sortCounts, schema=getSchema())
        )

    else:
        kmerDf = (
            kmerDf
            .groupby(KMERDF_SEQINFO_COL_NAMES)
            .agg(
                F.collect_list(KMER_COL_NAME).alias(KMER_COL_NAME),
                F.collect_list(COUNT_COL_NAME).alias(COUNT_COL_NAME)
            )
        )

    return kmerDf

def toVector(kmerDf, toDense=False):
    kmerDf = kmerDf.select(KMERDF_COL_NAMES)
    kmerDf = insertKmerIdxCol(kmerDf)
    kmerDf = kmerDf.withColumnRenamed(KMER_COL_NAME, 'kmerStr')

    ## Get the size of the vector
    vSize = kmerDf.select(F.max('kmerIdx')).collect()[0][0] + 1

    ## Create a dictionary containing (Kmer, Count) pairs
    kmerDf = kmerDf.withColumnRenamed('kmerIdx', KMER_COL_NAME)
    kmerDf = toDict(kmerDf)

    ## Convert the counts into SparseVectors
    f = lambda x: Vectors.sparse(vSize, x)
    f = F.udf(f, VectorUDT())
    kmerDf = kmerDf.withColumn(KMER_COL_NAME, f(KMER_COL_NAME))

    ## If applicable, convert the counts into DenseVectors
    if (toDense):
        f = lambda x: Vectors.dense(x.toArray())
        f = F.udf(f, VectorUDT())
        kmerDf = kmerDf.withColumn(KMER_COL_NAME, f(KMER_COL_NAME))

    ## Vectorising loses the Kmer string information. We probably don't need
    ## them, but if we do, then we can convert the vector indices into Kmers.
    return kmerDf

def groupBy(kmerDf, *cols, func=F.sum, sep=' '):
    nParts = kmerDf.rdd.getNumPartitions()

    ## Sum the counts for each Kmer
    x = (
        kmerDf
        .groupby(*cols, KMER_COL_NAME)
        .agg(func(COUNT_COL_NAME).alias(COUNT_COL_NAME))
    )

    ## Sum the lengths of each sequence
    y = (
        kmerDf
        .select(*cols, *KMERDF_SEQINFO_COL_NAMES).distinct()
        .groupby(*cols)
        .agg(func(SEQLEN_COL_NAME).alias(SEQLEN_COL_NAME))
    )

    ## Join the tables and format the columns
    if (len(cols) == 0):
        kmerDf = x.crossJoin(y)
        kmerDf = kmerDf.withColumn(SEQID_COL_NAME, F.lit('GroupedCounts'))
        kmerDf = kmerDf.select(*KMERDF_COL_NAMES)

    else:
        kmerDf = x.join(y, on=[*cols], how='left')
        kmerDf = kmerDf.withColumn(SEQID_COL_NAME, F.concat_ws(sep, *cols))
        kmerDf = kmerDf.select(*KMERDF_COL_NAMES)

    return kmerDf

def groupRevComp(kmerDf):
    def _f(df):
        ## Create a column containing the reverse complement of each sequence
        f = lambda x: '-'.join(sorted(x))
        df['revComp'] = df[KMER_COL_NAME].apply(seqToRevComp)
        df[KMER_COL_NAME] = df[[KMER_COL_NAME, 'revComp']].apply(f, axis=1)

        ## Sum up the counts of complementary K-mers, and double the counts
        ## for the total frequency of each K-mer.
        df = (df.groupby([*KMERDF_SEQINFO_COL_NAMES, KMER_COL_NAME])
            .sum(numeric_only=True) * 2)
        df = df.reset_index()
        return df

    kmerDf = kmerDf.select(KMERDF_COL_NAMES)
    kmerDf = (kmerDf.groupby(KMERDF_SEQINFO_COL_NAMES)
        .applyInPandas(_f, schema=kmerDf.schema))
    return kmerDf

#------------------- Private Classes & Functions ------------#

def _toProbabilities(kmerPdf):
    total = kmerPdf[COUNT_COL_NAME].sum()
    kmerPdf[COUNT_COL_NAME] = kmerPdf[COUNT_COL_NAME] / total
    kmerPdf[COUNT_COL_NAME] = kmerPdf[COUNT_COL_NAME]
    return kmerPdf

def _toL2Normalised(kmerPdf):
    counts = kmerPdf[COUNT_COL_NAME].values.reshape(1, -1)
    kmerPdf[COUNT_COL_NAME] = preprocessing.normalize(counts)[0]
    kmerPdf[COUNT_COL_NAME] = kmerPdf[COUNT_COL_NAME]
    return kmerPdf

#------------------- Private Classes & Functions ------------#

#------------------- Main -----------------------------------#

if (__name__ == "__main__"):
    main()

#------------------------------------------------------------------------------


from data import maskdata

from Xcept import xception , Xception
import time
import torch
from torchvision import datasets, models, transforms
import os
import torch.nn as nn

from sklearn.ensemble import AdaBoostClassifier

import copy
import torch.optim as optim
import torchvision
from torch.optim import lr_scheduler
from torch.utils.data import DataLoader ,Dataset , random_split

import numpy as np
import joblib
from efficientnet_pytorch import EfficientNet
from PIL import Image

augmentation = torchvision.transforms.Compose([torchvision.transforms.Resize(299),
                                                    torchvision.transforms.RandomCrop(299),
                                                    torchvision.transforms.RandomHorizontalFlip(),
                                                    torchvision.transforms.ToTensor(),
                                                    torchvision.transforms.Normalize(mean = [0.485, 0.456, 0.406],std = [0.229, 0.224, 0.225])
                                                    ])

class testdata(Dataset):
    def __init__(self ,Dir ,label ,transforms):
        self.dir = Dir
        self.label = label
        self.transforms = transforms

    def __len__(self):   #抓幾張圖片

        return 50   # idx 從 0 開始算

    def __getitem__(self  , idx ):   # 弄好 (img , label)

        if os.path.exists(self.dir+'/'+str(idx+1)+'.jpg'):
            pic = Image.open(self.dir+'/'+str(idx+1)+'.jpg').convert('RGB')
            if self.transforms is not None:
                pic = self.transforms(pic)

            sample = {'image': pic , 'label': self.label}
            return sample
        else:
            print('file not found')
            print(self.dir+'/'+str(idx+1)+'.jpg')
            return 0

def test(loader, model):
    model.eval()
    accuracy = 0.
    with torch.no_grad():
        for i,data in enumerate(loader , 0):
            inputs = data['image']
            labels = data['label']
            inputs, labels = inputs.to(device , dtype=torch.float32), labels.to(device)
            ou = model(inputs)
            _, pred = torch.max(torch.sigmoid(ou), 1)

            print()
            print(pred)
            print(labels)
            print()
            accuracy += torch.sum(pred==labels).item()

    print("Test Accuracy: {}".format(accuracy / len(face)))

    f = open('./res.txt' , 'w')
    f.write("Test Accuracy: {}".format(accuracy / len(face)))
    f.close()

def celeb_sum (path):   # 加總一個目錄下所有的
    l = []
    for i in os.listdir(path):
        l.append(i)

    for i in range(0,518):

        if i==0:
            if(len(l[i])>10):
                face = testdata(path+'/'+l[i] , 1 , transforms = augmentation)
            else:
                face = testdata(path+'/'+l[i] , 0 , transforms = augmentation)

        else:
            if(len(l[i])>10):
                face += testdata(path+'/'+l[i] , 1 , transforms = augmentation)
            else:
                face += testdata(path+'/'+l[i] , 0 , transforms = augmentation)

    return face

class MyEnsemble(nn.Module):
    def __init__(self, modelA, modelB, nb_classes=2):
        super(MyEnsemble, self).__init__()
        self.modelA = modelA
        self.modelB = modelB
        # Remove last linear layer
        self.modelA.fc = nn.Identity()
        self.modelB._fc = nn.Identity()

        # Create new classifier
        self.classifier = nn.Linear(2048+1536, nb_classes)

    def forward(self, x):
        x1 = self.modelA(x.clone())  # clone to make sure x is not changed by inplace methods
        x1 = x1.view(x1.size(0), -1)

        x2 = self.modelB(x)
        x2 = x2.view(x2.size(0), -1)

        x = torch.cat((x1, x2), dim=1)

        x = self.classifier(torch.relu(x))

        return x

def ff(fake_path , real_path):
    #fake_type = ['FaceShifter','NeuralTextures']

    fake_type = ['Deepfakes','Face2Face','FaceSwap']

    fake = []
    real = []
    for z in range(0,3):
        a = os.path.join(fake_path , fake_type[z] )
        for i in os.listdir(a):
            b = os.path.join(a , i )
            for j in range(1,41):
                c = os.path.join(b , str(j)+'.jpg' )
                fake.append(c)

    for i in os.listdir(real_path):
        b = os.path.join(real_path , i )
        for j in range(1,41):
            c = os.path.join(b , str(j)+'.jpg' )
            real.append(c)

    face = maskdata(real , fake ,transforms = augmentation)

    return face

if __name__ =="__main__":

    model = xception(pretrained=False)
    state_dict = torch.load(
        r'C:\Users\jeremy\Desktop\2021DF\model\FaceForensics-master\FaceForensics-master\classification\xception-b5690688.pth')
    for name, weights in state_dict.items():
        if 'pointwise' in name:
            state_dict[name] = weights.unsqueeze(-1).unsqueeze(-1)
    model.load_state_dict(state_dict)

    for param in model.parameters():
        param.requires_grad = False

    out_num = model.fc.in_features
    model.fc = nn.Linear(out_num,2)

    model2 = EfficientNet.from_pretrained('efficientnet-b3' , num_classes=2)

    for param in model2.parameters():
        param.requires_grad_(False)

    model3 = MyEnsemble(model, model2)

# =============================================================================
#     model = xception(pretrained=False)
#     model.fc = model.last_linear
#     del model.last_linear
#     state_dict = torch.load(
#         r'C:\Users\jeremy\Desktop\2021DF\model\FaceForensics-master\FaceForensics-master\classification\xception-b5690688.pth')
#     for name, weights in state_dict.items():
#         if 'pointwise' in name:
#             state_dict[name] = weights.unsqueeze(-1).unsqueeze(-1)
#     model.load_state_dict(state_dict)
#     model.last_linear = model.fc
#     out_num = model.fc.in_features
#     model.last_linear = nn.Linear(out_num,2)
#     model.load_state_dict(torch.load('./XceptionNet/mask2/m1.pth'))
# =============================================================================

# =============================================================================
#     model = Xception()
#     model.last_linear = model.fc
#     out_num = model.fc.in_features
#     model.last_linear = nn.Linear(out_num,2)
#     del model.fc
# =============================================================================

    #model.load_state_dict(torch.load('XceptionNet/m2.pth'))
    model3.load_state_dict(torch.load('ensemble.pth'))

# =============================================================================
#     p = r'C:\Users\jeremy\Desktop\2021DF\model\FaceForensics-master\FaceForensics-master\classification\all_c23.p'
#     q = r'C:\Users\jeremy\Desktop\2021DF\model\XceptionNet\mask\m2.pth'
#     model = torch.load(p)
#     model.load_state_dict(torch.load(q))
# =============================================================================

    #model.load_state_dict(torch.load('./EfficientNet/mask/3_5.pth'))

    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    model3 = model3.to(device)

# loaddata ===========================================================================

#----------------------------celceb-df------------------------------------------
    path = 'D:/Celeb-DF-v2/test'
    face = celeb_sum(path)

#-----------------------------FF++-----------------------------------------

    #face = ff(fake_path = r'D:\FF++\data\test',real_path = r'D:\FF++\data\test\original')
    #summary(model , (3,299,299))

    test_loader = DataLoader( face , batch_size = 16 , shuffle = True , num_workers = 0 )

    #test(test_loader , model3)


# coding=utf8
# 提取top50 数据并更新
import execjs
import pymongo
import ast
import requests
import urllib

import random
import json

def get_js():
    f = open(r"C:\env\新建文件夹\toutiao.js", 'r', encoding='UTF-8')  ##打开JS文件
    line = f.readline()
    htmlstr = ''
    while line:
        htmlstr = htmlstr + line
        line = f.readline()
    ctx = execjs.compile(htmlstr)
    ctx_dict = ast.literal_eval(ctx.call('get_as_cp_signature'))
    return ctx_dict

class Hotarticle():
    def __init__(self):
        self.clinet = pymongo.MongoClient(host='localhost', port=27017)
        self.db = self.clinet.mydb
        self.collection = self.db.article1
        self.collection_hot = self.db.hot_article

    def get_hot(self):

        sorts = self.collection.find().sort('impression_count', pymongo.DESCENDING)
        art_ids = list(sort for sort in sorts)[0:51]

        for art_id in art_ids:
            ctx_dict = get_js()
            print(art_id['article_id'])
            headers = {
                'User-Agent': 'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.121 Mobile Safari/537.36',
                'Referer': 'https://m.toutiao.com/i%s/' % art_id['article_id']
            }
            link = 'https://m.toutiao.com/' + 'i' + str(art_id['article_id']) + '/info/'  # 详情页url  获取阅读量

            data = {
                '_signature': ctx_dict['_signature'],
                'i': art_id['article_id']
            }
            url = link + '?' + urllib.parse.urlencode(data)
            print(url)
            ip = [{'http': '112.87.70.11:9999'}, {'http': '58.253.155.237:9999'}, {'http': '115.223.108.253:8010'},
                  {'http': '120.83.101.152:9999'}, {'http': '112.85.129.176:9999'}]
            ip_random = random.choice(ip)

            response = requests.get(url, proxies=ip_random, headers=headers)
            text = response.text
            bodys = json.loads(text)
            if bodys['success'] == True:
                impression = bodys['data']['impression_count']

                hot = {
                    'hot_id': art_id['article_id'],
                    'hot_review': impression
                }
                result = self.collection_hot.find({'hot_id': hot['hot_id']}).count()

                counts = self.collection_hot.find().count()
                if result:
                    hot_ids = self.collection_hot.find({'hot_id':hot['hot_id']})
                    for hot_id in hot_ids:
                        print('数据已存在，准备数据更新', result, counts,hot_id)
                        self.collection_hot.update_one(hot_id,{'$set':{'hot_review':impression}})
                        print('数据更新成功',hot_id)
                else:
                    print('未查询到文章，存入数据库',result)
                    self.collection_hot.insert(hot)
            else:
                print('此文章失效')
                self.collection_hot.delete_one({'hot_id': art_id['article_id']})
                self.collection.delete_one({'article_id':art_id['article_id']})

            # time.sleep(1)

def main():
    hot = Hotarticle()
    hot.get_hot()
    # hot.get_count()

if __name__ == '__main__':
    main()


"""
Functions to manage dataset
"""

import json
import logging
from typing import Dict, List, Tuple, Any, Callable

import numpy as np

from src.sound_actions import make_file_spectrogram, give_many_fft, wave_file_to_time_data

def load_dataset_json(json_path: str):
    """From the json annotating all the wav files, get useful info"""
    with open(json_path, "r") as jf:
        return json.load(jf)

def extract_tracks_by_family(dataset_json: Dict[str, Any], exclude_families=None,
                             exclude_source=None) -> Tuple[Dict[int, List[str]], Dict[str, int]]:
    """From the dict coming from nsynth json, group instruments by family.
    :returns: {class_index -> files} and {class_name -> class_index}
    """
    exclude_families = [] if exclude_families is None else exclude_families
    exclude_source = [] if exclude_source is None else exclude_source

    classes_n2i: Dict[str, int] = {}

    corr = {}
    for s in dataset_json.values():
        # instrument family
        family = s["instrument_family_str"]
        if family in exclude_families:
            continue

        # track name and instrument source
        name = s["note_str"]
        source = s["instrument_source_str"]

        # register track name under index of {family}_{source} key
        if source not in exclude_source:
            fs = f"{family}_{source}"
            if fs not in classes_n2i:
                classes_n2i[fs] = len(classes_n2i)
                corr[classes_n2i[fs]] = []
            corr[classes_n2i[fs]].append(name)
    return corr, classes_n2i

def make_labels_and_spectrograms(corr: Dict[int, List[str]], file_access: Callable[[str], str] = lambda x: x):
    """Make the spectrogram of each file in the data dict
    :returns: Lists for ids, labels, spectrograms
    """
    ids: List[str] = []
    labels: List[int] = []
    spectrograms: List[np.ndarray] = []

    for k, file_names in corr.items():
        logging.info(f"Making {k} data...")
        for fn in file_names:
            ids.append(fn)
            labels.append(k)
            spectrograms.append(make_file_spectrogram(file_access(fn)))

    return ids, labels, spectrograms

def make_labels_and_fft(corr: Dict[int, List[str]], file_access: Callable[[str], str] = lambda x: x, **kwargs) -> Tuple[List[int], List[np.ndarray], List[str]]:
    """Make the fft of some samples from each file in the data dict
    :returns: Lists for ids, labels, spectrograms; lookups {class_index -> class_name}, {class_name -> class_index}
    """
    labels: List[int] = []
    data: List[np.ndarray] = []
    source: List[str] = []

    for k, file_names in corr.items():
        logging.info(f"Making {k} data...")
        for fn in file_names:
            file_full_name = file_access(fn)
            for out in give_many_fft(*wave_file_to_time_data(file_full_name), **kwargs):
                labels.append(k)
                data.append(out)
                source.append(fn)
        logging.info(f"- Total: {len(labels)} entries...")

    return labels, data, source


import pandas as pd
import geopandas as gpd

pd.options.mode.chained_assignment = None

# Load different data
renacom = pd.read_csv('~/data/niger_election_data/processed/renacom_full.csv',
                      encoding="ISO-8859-1")
renaloc = pd.read_csv('~/data/niger_election_data/processed/renaloc_full.csv',
                      encoding="ISO-8859-1")
renaloc = renaloc[renaloc.level == 'Localite']
renaloc.region = renaloc.region.str.lower()
renaloc.departement = renaloc.departement.str.lower()
renaloc.commune = renaloc.commune.str.lower()
osm = pd.read_csv('~/data/osm/niger.csv', encoding="ISO-8859-1")
bureaux = pd.read_csv('~/data/niger_election_data/raw/Niger_Bureaux.csv',
                      encoding='ISO-8859-1')
health_facilities = pd.read_csv('~/data/dhis/niger/org_units_description.csv')

# Standardize names in RENALOC and RENACOM
def change_number(name, suffix, changer):
    """Change suffixes in names."""
    len_suffix = len(suffix)
    to_change = (name.str[-len_suffix:] == suffix)
    changer = name[to_change].str[0:-len_suffix] + [changer]
    name.loc[to_change] = changer
    return name

change_number(renacom.commune, ' i', ' 1')
change_number(renacom.commune, ' ii', ' 2')
change_number(renacom.commune, ' iii', ' 3')
change_number(renacom.commune, ' iv', ' 4')
change_number(renacom.commune, ' v', ' 5')
change_number(renaloc.commune, ' i', ' 1')
change_number(renaloc.commune, ' ii', ' 2')

# Add region, departement, commune in bureaux
commune = pd.read_csv('~/data/niger_election_data/raw/Niger_Communes.csv',
                      encoding='ISO-8859-1')
bureaux = bureaux.drop('Unnamed: 0', axis=1)
commune = commune.drop('Unnamed: 0', axis=1)
bureaux = bureaux.merge(commune)
departement = pd.read_csv('~/data/niger_election_data/raw/Niger_Departements.csv',
                          encoding='ISO-8859-1')
departement = departement.drop('Unnamed: 0', axis=1)
bureaux = bureaux.merge(departement)
region = pd.read_csv('~/data/niger_election_data/raw/Niger_Regions.csv',
                     encoding='ISO-8859-1')
region = region.drop('Unnamed: 0', axis=1)
bureaux = bureaux.merge(region)

bureaux = bureaux[['ID_BUREAU', 'NOM_BUREAU', 'NOM_COMMUNE', 'NOM_DEPART',
                   'NOM_REGION']]
bureaux.columns = ['locality_id', 'locality', 'commune', 'departement',
                   'region']

bureaux.commune = bureaux.commune.str.lower()
bureaux.departement = bureaux.departement.str.lower()
bureaux.region = bureaux.region.str.lower()

change_number(bureaux.commune, ' i', ' 1')
change_number(bureaux.commune, ' ii', ' 2')
change_number(bureaux.commune, ' iii', ' 3')
change_number(bureaux.commune, ' iv', ' 4')
change_number(bureaux.commune, ' v', ' 5')
renacom.commune[~renacom.commune.isin(bureaux.commune)].unique()
bureaux.commune[bureaux.commune == 'tibiri (doutchi)'] = 'tibiri'
bureaux.commune[bureaux.commune == 'gangara (aguie)'] = 'gangara'
bureaux.commune[bureaux.commune == 'gangara (tanout)'] = 'gangara'
bureaux.commune[bureaux.commune == 'tibiri (maradi)'] = 'tibiri'
bureaux.commune[bureaux.commune == 'maradi arrondissement 1'] = 'arrondissement 1'
bureaux.commune[bureaux.commune == 'maradi arrondissement 2'] = 'arrondissement 2'
bureaux.commune[bureaux.commune == 'maradi arrondissement 3'] = 'arrondissement 3'
bureaux.departement[bureaux.departement == 'tibiri (doutchi)'] = 'tibiri'

# Standardize GPS communes names

fp = '/Users/grlurton/data/niger_election_data/external/commune_shp/nigcom.shp'
commune_gps = gpd.read_file(fp)

commune_gps.GPS_NAME = commune_gps.GPS_NAME.str.lower()

renaloc_no_match = renaloc[~renaloc.commune.isin(commune_gps.GPS_NAME)]
gps_no_match = commune_gps[~commune_gps.GPS_NAME.isin(renaloc.commune)]

def standardize_name(var, pattern, replace):
    out = var.loc[:].str.replace(pattern, replace)
    return out.values

gps_no_match.loc[:, 'standard_name'] = standardize_name(gps_no_match['GPS_NAME'], "'", "")
renaloc_no_match.loc[:, 'standard_name'] = standardize_name(renaloc_no_match['commune'], "'", "")

dico = {}

for idx in gps_no_match.index:
    standard_name = gps_no_match.loc[idx, 'standard_name']
    if standard_name in renaloc_no_match.standard_name.tolist():
        print(standard_name)
        ren_name = renaloc_no_match.loc[renaloc_no_match['standard_name'] == standard_name,'commune'].unique()[0]
        dico[ren_name] = gps_no_match.loc[idx, 'GPS_NAME']
        commune_gps.loc[idx, 'GPS_NAME'] = ren_name

dico

renaloc_no_match = renaloc[~renaloc.commune.isin(commune_gps.GPS_NAME)]
gps_no_match = commune_gps[~commune_gps.GPS_NAME.isin(renaloc.commune)]

gps_no_match.loc[:, 'standard_name'] = standardize_name(gps_no_match['GPS_NAME'], "-", " ")
renaloc_no_match.loc[:, 'standard_name'] = standardize_name(renaloc_no_match['commune'], "-", " ")

for idx in gps_no_match.index:
    standard_name = gps_no_match.loc[idx, 'standard_name']
    if standard_name in renaloc_no_match.standard_name.tolist():
        print(standard_name)
        ren_name = renaloc_no_match.loc[renaloc_no_match['standard_name'] == standard_name,'commune'].unique()[0]
        dico[ren_name] = gps_no_match.loc[idx, 'GPS_NAME']
        commune_gps.loc[idx, 'GPS_NAME'] = ren_name

renaloc_no_match = renaloc[~renaloc.commune.isin(commune_gps.GPS_NAME)]
gps_no_match = commune_gps[~commune_gps.GPS_NAME.isin(renaloc.commune)]

gps_no_match.loc[:, 'standard_name'] = standardize_name(gps_no_match['GPS_NAME'], " ", "")
renaloc_no_match.loc[:, 'standard_name'] = standardize_name(renaloc_no_match['commune'], " ", "")

for idx in gps_no_match.index:
    standard_name = gps_no_match.loc[idx, 'standard_name']
    if standard_name in renaloc_no_match.standard_name.tolist():
        print(standard_name)
        ren_name = renaloc_no_match.loc[renaloc_no_match['standard_name'] == standard_name,'commune'].unique()[0]
        dico[ren_name] = gps_no_match.loc[idx, 'GPS_NAME']
        commune_gps.loc[idx, 'GPS_NAME'] = ren_name

renaloc_no_match = renaloc[~renaloc.commune.isin(commune_gps.GPS_NAME)]
gps_no_match = commune_gps[~commune_gps.GPS_NAME.isin(renaloc.commune)]

dico["attantane"] = "attatane"
dico["bankilare"] = "bankillare"
dico['birni lalle'] = 'birni nlalle'
dico['damagaram takaya'] = "damagaram-t"
dico['dan-goulbi'] = 'dan goulgi'
dico['diagourou'] = 'diagorou'
dico["dingazi"] = 'dingazi banda'
dico["djiratawa"] = 'djirataoua'
dico["dogonkiria"] = "dogon kirya"
dico['el allassane maireyrey'] = 'alhassane  mairerey'
dico['falmey/falmey haoussa'] = 'falmey'
dico['farey'] = 'farrey'
dico['galma koudawatche'] = "galma"
dico['goroubankassam'] = 'goroun bakassam'
dico['guidan amoumoune'] = 'guidan amoumane'
dico['guilladje'] = 'guiladje'
dico['kablewa'] = 'kabalewa'
dico['kourfeye centre'] = 'kourfey centre'
dico['kourteye'] = 'kourtey'
dico['makalondi'] = "parc w"
dico['matamey'] = "matameye"
dico["n'guelbely"] = 'nguel bely'
dico['ouro gueladjo'] = 'ouro gueladio'
dico['roumbou 1'] = 'roumbou i'
dico['sakoira'] = 'sakouara'
dico['sarkin haoussa'] = "serkin haoussa"
dico['sarkin yamma'] = 'sarkin yama'
dico['tagriss'] = 'tagris'
dico['tombokoirey 1'] = 'tombokoarey 1'
dico['tombokoirey 2'] = 'tombokoarey 2'
dico['tondikiwindi'] = 'tondikwindi'
dico['tsernaoua'] = 'tsernawa'

for comm in dico.keys():
    commune_gps.GPS_NAME[commune_gps.GPS_NAME == dico[comm]] = comm

renaloc_no_match = renaloc[~renaloc.commune.isin(commune_gps.GPS_NAME)]
gps_no_match = commune_gps[~commune_gps.GPS_NAME.isin(renaloc.commune)]

# [k for k, v in dico.items() if 'dogo dogo' in v]

# Geocache OSM

# OSM data from https://download.geofabrik.de/africa/niger.html

fp = '/Users/grlurton/data/osm/niger-latest-free.shp/gis_osm_places_free_1.shp'
osm_data = gpd.read_file(fp)

osm_data = osm_data.to_crs(commune_gps.crs)

osm_data = gpd.sjoin(osm_data, commune_gps, how="inner", op="within")

# TODO Add region, departement, commune in dhis

# TODO if needed geocache DHIS

# Standardize output

renacom_keep = ['MILIEU', 'region', 'departement', 'commune', 'LOCALITE',
                'TYPELOCALITE', 'MASCULIN', 'FEMININ',
                'TOTAL', 'LONGITUDE', 'LATITUDE']
renacom_out = renacom[renacom_keep]
renacom_out.columns = ['milieu', 'region', 'departement', 'commune',
                       'locality', 'locality_type', 'hommes', 'femmes',
                       'population', 'longitude', 'latitude']
renacom_out['source'] = 'renacom'
renacom_out['locality_id'] = ['RENACOM'] + pd.Series(list(range(len(renacom_out)))).astype(str)

renaloc_keep = ['locality', 'population', 'hommes', 'femmes',
                'settlement_type', 'region', 'departement', 'commune',
                'longitude', 'latitude']
renaloc_out = renaloc[pd.isnull(renaloc.milieu)]
renaloc_out = renaloc_out[renaloc_keep]
renaloc_out.columns = ['locality', 'population', 'hommes', 'femmes',
                       'locality_type', 'region', 'departement', 'commune',
                       'longitude', 'latitude']
renaloc_out['source'] = 'renaloc'
renaloc_out['locality_id'] = ['RENALOC'] + pd.Series(list(range(len(renaloc_out)))).astype(str)

osm_keep = ['osm_id', 'fclass', 'population', 'name', 'geometry', 'GPS_NAME',
            'REGION']
osm_data['longitude'] = osm_data.geometry.x
osm_data['latitude'] = osm_data.geometry.y
osm_out = osm_data[osm_keep]
osm_out.columns = ['locality_id', 'locality_type', 'population', 'locality',
                   'geometry', 'commune', 'region']
osm_out['source'] = 'osm'

bureaux_out = bureaux[~(bureaux.region == 'diaspora')]
bureaux_out['source'] = 'bureaux'

data_out = renacom_out.append(renaloc_out).append(osm_out).append(bureaux_out)

data_out.to_csv("~/data/niger_election_data/processed/pooled_data.csv")


import itertools

from bokeh import palettes
from bokeh.plotting import figure
from bokeh.charts import show

from loggraph import accumulator
from loggraph.parser import log_record_generator

def accumulators_for_definitions(graph_definitions):
    accumulator_definitions = (
        tuple(definition[:-1].split('['))
        for definition in graph_definitions
    )

    return [
        accumulator.NAME_TO_ACCUMULATOR[name](path.split('.'))
        for name, path in accumulator_definitions
    ]

def generate_graph(
    filepath,
    graph_definitions,
    time_column_name
):
    accumulators = accumulators_for_definitions(graph_definitions)
    record_generator = log_record_generator(filepath)
    by_day = accumulator.ByDayAccumulator(accumulators, time_column_name)
    for record in record_generator:
        by_day.process(record)

    colors = itertools.cycle(palettes.Set2[8])

    plot = figure(width=800, height=400, x_axis_type='datetime')
    plot.xaxis.axis_label = by_day.time_column_name.capitalize()

    for x_axis, y_axis, legend in by_day.get_rendered_axes():
        plot.line(
            x_axis,
            y_axis,
            legend=legend,
            color=next(colors)
        )

    show(plot)


import json

from code_names_bot_text_graph.text_disambiguator.consec_compound_text_disambiguator import ConsecCompoundTextDisambiguator

from config import DICTIONARY
import time

def main():
    with open(DICTIONARY, "r") as file:
        dictionary = json.loads(file.read())

    """
    token_senses = [
        ("I", []),
        ("have", []),
        ("a", []),
        ("beautiful", ["m_en_gbus0081680.006"]),
        ("dog", ["m_en_gbus0287810.007", "m_en_gbus0287810.012", "m_en_gbus0287810.013", "m_en_gbus0287810.019", "m_en_gbus0287810.020", "m_en_gbus0287810.023"])
    ]
    """

    token_senses = [
        ("The", []),
        ("Amazing", ['The_Amazing_Spider-Man_(film)']),
        ("Spider", ['The_Amazing_Spider-Man_(film)']),
        ("-", ['The_Amazing_Spider-Man_(film)']),
        ("Man", ['The_Amazing_Spider-Man_(film)']),
        ("Annual", [])
    ]

    compound_indices = [
        ("The_Amazing_Spider-Man_(film)", (2, 5)),
        ("The_Amazing_Spider-Man_(film)", (0, 5))
    ]

    disambiguator = ConsecCompoundTextDisambiguator(dictionary, True)
    start = time.time()
    senses = disambiguator.disambiguate(token_senses, compound_indices)
    print("Time", time.time() - start)
    print(senses)

if __name__ == "__main__":
    main()


import os
from typing import List

import numpy as np
import pytest

from ambrosia.preprocessing import IQRPreprocessor, RobustPreprocessor

STORAGE_PATHS = {
    "robust": "tests/configs/robust_config.json",
    "iqr": "tests/configs/iqr_config.json",
}

@pytest.mark.smoke
def test_robust_constructor():
    """
    Check RobustPreprocessor constructor works.
    """
    robust = RobustPreprocessor(verbose=False)

@pytest.mark.unit
@pytest.mark.parametrize(
    "tail, column_names, alpha, transf_name",
    [
        ("both", "feature_1", 0.05, "robust_both_0.05_feature_1"),
        ("both", ["feature_2"], 0.05, "robust_both_0.05_feature_2"),
        ("both", ["feature_2", "feature_1"], [0.05, 0.05], "robust_both_0.05_feature_2after1"),
        ("left", "feature_1", 0.1, "robust_left_0.1_feature_1"),
        ("right", "feature_1", 0.1, "robust_right_0.1_feature_1"),
    ],
)
def test_robust(tail, column_names, alpha, transf_name, data_nonlin_var, robust_moments):
    """
    Test RobustPreprocessor quantile transformations via resulted moments.
    """
    robust = RobustPreprocessor(verbose=False)
    transformed_data = robust.fit_transform(data_nonlin_var, column_names, alpha, tail, inplace=False)
    if isinstance(column_names, List):
        column_names: str = column_names[0]
    col_mean: float = transformed_data[column_names].mean()
    col_std: float = transformed_data[column_names].std()
    moments: np.ndarray = robust_moments.loc[transf_name].values
    assert np.allclose(np.array([col_mean, col_std]), moments, atol=0.000001)

@pytest.mark.unit
def test_robust_load_store(data_nonlin_var, robust_moments):
    """
    Test RobustPreprocessor save and load methods.
    """
    store_path = STORAGE_PATHS["robust"]
    robust = RobustPreprocessor(verbose=False)
    robust.fit(data_nonlin_var, ["feature_2", "feature_1"], 0.05)
    robust.store_params(store_path)
    loaded_robust = RobustPreprocessor(verbose=False)
    loaded_robust.load_params(store_path)
    os.remove(store_path)
    transformed_data = loaded_robust.transform(data_nonlin_var, inplace=False)
    col_mean: float = transformed_data["feature_2"].mean()
    col_std: float = transformed_data["feature_2"].std()
    moments: np.ndarray = robust_moments.loc["robust_both_0.05_feature_2after1"].values
    assert np.allclose(np.array([col_mean, col_std]), moments, atol=0.000001)

@pytest.mark.smoke
def test_iqr_constructor():
    """
    Check IQRPreprocessor constructor works.
    """
    iqr = IQRPreprocessor(verbose=False)

@pytest.mark.unit
@pytest.mark.parametrize(
    "column_names, transf_name",
    [
        ("feature_1", "iqr_feature_1"),
        (["feature_2"], "iqr_feature_2"),
        (["feature_2", "feature_1"], "iqr_feature_2after1"),
    ],
)
def test_iqr(column_names, transf_name, data_nonlin_var, robust_moments):
    """
    Test IQRPreprocessor transformations via resulted moments.
    """
    iqr = IQRPreprocessor(verbose=False)
    transformed_data = iqr.fit_transform(data_nonlin_var, column_names, inplace=False)
    if isinstance(column_names, List):
        column_names: str = column_names[0]
    col_mean: float = transformed_data[column_names].mean()
    col_std: float = transformed_data[column_names].std()
    moments: np.ndarray = robust_moments.loc[transf_name].values
    assert np.allclose(np.array([col_mean, col_std]), moments, atol=0.000001)

@pytest.mark.unit
def test_iqr_load_store(data_nonlin_var, robust_moments):
    """
    Test IQRPreprocessor save and load methods.
    """
    store_path = STORAGE_PATHS["iqr"]
    iqr = IQRPreprocessor(verbose=False)
    iqr.fit(data_nonlin_var, ["feature_2", "feature_1"])
    iqr.store_params(store_path)
    loaded_iqr = IQRPreprocessor(verbose=False)
    loaded_iqr.load_params(store_path)
    os.remove(store_path)
    transformed_data = loaded_iqr.transform(data_nonlin_var, inplace=False)
    col_mean, col_std = transformed_data["feature_2"].mean(), transformed_data["feature_2"].std()
    moments: np.ndarray = robust_moments.loc["iqr_feature_2after1"].values
    assert np.allclose(np.array([col_mean, col_std]), moments, atol=0.000001)


import DirectedGraph
import collections
class TopSort:
  def Kahns(self, graph):
    inDegree = dict()
    for node in graph.vertices:
      inDegree[node] = 0

    for node in graph.vertices:
      for neighbor in node.neighbors:
        inDegree[neighbor] = inDegree[neighbor] + 1

    topSort = list()
    queue = collections.deque()
    #add all elements of M whose values are 0 to Q
    for curr in list(inDegree.keys())[::-1]:
      if inDegree[curr] == 0:
        queue.append(curr)
        inDegree[curr] = inDegree[curr] - 1

    #while Q is not empty
    while queue:
      #N←first element in Q
      current = queue.popleft()
      #add N to A
      topSort.append(curr)
      for neighbor in curr.neighbors:
        #decrement the in-degrees of all dependents of N
        inDegree[neighbor] = inDegree[neighbor] -1

      for curr in list(inDegree.keys())[::-1]:
        if inDegree[curr] == 0:
          queue.append(curr)
          inDegree[curr] = inDegree[curr] - 1

    return topSort

  def mDFS(self, graph):
    stack = list()
    for vertex in graph.vertices:
      if not vertex.visited:
        self.mDFSHelper(vertex, stack)

    inOrderStack = stack[::-1]

    return inOrderStack

  def mDFSHelper(self, vertex, stack):
    vertex.visited = True
    for neighbor in vertex.neighbors:
      if not neighbor.visited:
        self.mDFSHelper(vertex, stack)

    stack.append(vertex)


import itertools
import random
import termcolor

class Minesweeper():
    """
    Minesweeper game representation
    """

    def __init__(self, height=8, width=8, mines=8):

        # Set initial width, height, and number of mines
        self.height = height
        self.width = width
        self.mines = set()

        # Initialize an empty field with no mines
        self.board = []
        for i in range(self.height):
            row = []
            for j in range(self.width):
                row.append(False)
            self.board.append(row)

        # Add mines randomly
        while len(self.mines) != mines:
            i = random.randrange(height)
            j = random.randrange(width)
            if not self.board[i][j]:
                self.mines.add((i, j))
                self.board[i][j] = True

        # At first, player has found no mines
        self.mines_found = set()

    def print(self):
        """
        Prints a text-based representation
        of where mines are located.
        """
        for i in range(self.height):
            print("--" * self.width + "-")
            for j in range(self.width):
                if self.board[i][j]:
                    print("|X", end="")
                else:
                    print("| ", end="")
            print("|")
        print("--" * self.width + "-")

    def is_mine(self, cell):
        i, j = cell
        return self.board[i][j]

    def nearby_mines(self, cell):
        """
        Returns the number of mines that are
        within one row and column of a given cell,
        not including the cell itself.
        """

        # Keep count of nearby mines
        count = 0

        # Loop over all cells within one row and column
        for i in range(cell[0] - 1, cell[0] + 2):
            for j in range(cell[1] - 1, cell[1] + 2):

                # Ignore the cell itself
                if (i, j) == cell:
                    continue

                # Update count if cell in bounds and is mine
                if 0 <= i < self.height and 0 <= j < self.width:
                    if self.board[i][j]:
                        count += 1

        return count

    def won(self):
        """
        Checks if all mines have been flagged.
        """
        return self.mines_found == self.mines

class Sentence():
    """
    Logical statement about a Minesweeper game
    A sentence consists of a set of board cells,
    and a count of the number of those cells which are mines.
    """

    def __init__(self, cells, count):
        self.cells = set(cells)
        self.count = count

    def __eq__(self, other):
        return self.cells == other.cells and self.count == other.count

    def __str__(self):
        return f"{self.cells} = {self.count}"

    # Returns the set of all cells in self.cells known to be mines.
    def known_mines(self):
        if len(self.cells) == self.count:
            return self.cells
        return set()

    # Returns the set of all cells in self.cells known to be safe.
    def known_safes(self):
        if not self.count:
            return self.cells
        return set()

    # Updates internal knowledge representation given the fact that
    # a cell is known to be a mine.
    def mark_mine(self, cell):
        if cell in self.cells:
            self.cells.remove(cell)
            self.count -= 1

    # Updates internal knowledge representation given the fact that
    # a cell is known to be safe.
    def mark_safe(self, cell):
        if cell in self.cells:
            self.cells.remove(cell)

class MinesweeperAI():
    """
    Minesweeper game player
    """

    def __init__(self, height=8, width=8):

        # Set initial height and width
        self.height = height
        self.width = width

        # Keep track of which cells have been clicked on
        self.moves_made = set()

        # Keep track of cells known to be safe or mines
        self.mines = set()
        self.safes = set()

        # List of sentences about the game known to be true
        self.knowledge = []

    # Marks a cell as a mine, and updates all knowledge
    # to mark that cell as a mine as well.
    def mark_mine(self, cell):
        self.mines.add(cell)
        for sentence in self.knowledge:
            sentence.mark_mine(cell)

    # Marks a cell as safe, and updates all knowledge
    # to mark that cell as safe as well.
    def mark_safe(self, cell):
        self.safes.add(cell)
        for sentence in self.knowledge:
            sentence.mark_safe(cell)

    def add_current_sentence_move(self, cell, count):
        #print("I add the sentence for", cell);
        neighbors = set()
        if cell not in self.safes:
            neighbors.add(cell)
        for i in range(cell[0]-1, cell[0]+2):
            for j in range(cell[1]-1, cell[1]+2):
                if 0<=i<self.height and 0<=j<self.width and (i,j) != cell:
                    if (i, j) in self.mines:
                        count -=1
                        continue
                    if (i, j) not in self.safes:
                        neighbors.add((i, j))
        sentence = Sentence(neighbors, count)
        self.knowledge.append(sentence)
        #termcolor.cprint("I am adding the sentence:", 'yellow')
        #termcolor.cprint(sentence, 'yellow')

    def subset_rule(self):

        def sub(sentence1, sentence2):
            new = Sentence(sentence1.cells - sentence2.cells, sentence1.count - sentence2.count)
            self.knowledge.remove(sentence1)
            self.knowledge.append(new)
            #termcolor.cprint("I am removing the sentence:", 'yellow')
            #termcolor.cprint(sentence1, 'yellow')
            #termcolor.cprint("I am adding the sentence:", 'yellow')
            #termcolor.cprint(new, 'yellow')

        while True:
            #print("In the while")
            found_new = False
            for sentence1 in self.knowledge:
                if found_new:
                    break
                for sentence2 in self.knowledge:
                    if found_new:
                        break
                    if sentence1 != sentence2 and sentence1.cells and sentence2.cells:
                        if sentence1.cells.issubset(sentence2.cells):
                            found_new = True
                            #termcolor.cprint("---", "blue")
                            #termcolor.cprint(sentence1.cells, "blue")
                            #termcolor.cprint(sentence2.cells, "blue")
                            #termcolor.cprint("---", "blue")
                            sub(sentence2, sentence1)
                        elif sentence2.cells.issubset(sentence1.cells):
                            found_new = True
                            #termcolor.cprint("---", "blue")
                            #termcolor.cprint(sentence1.cells, "blue")
                            #termcolor.cprint(sentence2.cells, "blue")
                            #termcolor.cprint("---", "blue")
                            sub(sentence1, sentence2)
            if not found_new:
                break

    def mark_new_cells(self):
        for sentence in self.knowledge:
            if not sentence.cells:
                self.knowledge.remove(sentence)
                break
            #print("I examine sentence:", end=' ')
            #print(sentence)
            for cell in sentence.cells.copy():
                #print("I examine cell:", end=' ')
                #print(cell)
                if cell in sentence.known_mines():
                    #print("It is mine for sure")
                    self.mark_mine(cell)
                elif cell in sentence.known_safes():
                    #print("It is safe for sure")
                    self.mark_safe(cell)

    def add_knowledge(self, cell, count):
        """
        Called when the Minesweeper board tells us, for a given
        safe cell, how many neighboring cells have mines in them.

        This function should:
            1) mark the cell as a move that has been made
            2) mark the cell as safe
            3) add a new sentence to the AI's knowledge base
               based on the value of `cell` and `count`
            4) mark any additional cells as safe or as mines
               if it can be concluded based on the AI's knowledge base
            5) add any new sentences to the AI's knowledge base
               if they can be inferred from existing knowledge
        """
        #termcolor.cprint("-----------I am adding knowledge-----------------------", "blue")
        #termcolor.cprint("Step (1) and (2)", "blue")
        self.moves_made.add(cell)
        self.mark_safe(cell)
        #self.print_state()

        # ----- (3) add a new sentence -----
        #termcolor.cprint("\t\t\tStep (3)", "blue")
        self.add_current_sentence_move(cell, count)
        #self.print_state()

        # ----- (4) mark additional cells -----
        #termcolor.cprint("\t\t\tStep (4)", "blue")
        self.mark_new_cells()
        #self.print_state()

        # ----------------------------------

        # ----- (5) add new sentences using subset rule -----
        #termcolor.cprint("\t\t\tStep (5)", "blue")
        self.subset_rule()
        #self.print_state()
        # ----- (6) i mark new cells again
        self.mark_new_cells()
        # ----------------------------------
        #termcolor.cprint("I added the knowledge", "blue")
        #self.print_state()

    def make_safe_move(self):
        """
        Returns a safe cell to choose on the Minesweeper board.
        The move must be known to be safe, and not already a move
        that has been made.

        This function may use the knowledge in self.mines, self.safes
        and self.moves_made, but should not modify any of those values.
        """

        #self.print_state()

        for cell in self.safes - self.moves_made:
            #termcolor.cprint("I am making a safe move at one of the", 'green')
            #termcolor.cprint(self.safes, 'green')
            return cell
        return None

    def make_random_move(self):
        """
        Returns a move to make on the Minesweeper board.
        Should choose randomly among cells that:
            1) have not already been chosen, and
            2) are not known to be mines
        """
        for i in range(self.width):
            for j in range(self.height):
                cell = (i, j)
                if cell not in set.union(self.mines, self.moves_made):
                    #termcolor.cprint("I am making a random move at the", 'yellow')
                    #termcolor.cprint(cell, 'yellow')
                    return cell
        return None

    def print_state(self):
        termcolor.cprint("Current state:", "white")
        print("Played:", end = ' ')
        termcolor.cprint(self.moves_made, "red")
        print("Safes:", end = ' ')
        termcolor.cprint(self.safes, "red")
        print("Safes left:", end = ' ')
        termcolor.cprint(self.safes - self.moves_made, "red")
        print("Mines", end = ' ')
        termcolor.cprint(self.mines, "red")
        self.print_knowledge()

    def print_knowledge(self):
        termcolor.cprint("Knowledge:", "green")
        for sentence in self.knowledge:
            #termcolor.cprint(sentence, "green")
            pass


import numpy as np
import pandas as pd
import numba as nb
import logging
import scipy.sparse.linalg as spl

from .autoregressive import AutoRegressive
from .baseline import HarmonicBaseline
from .greedy_grid_search import greedy_grid_search

logger = logging.getLogger(__name__)

__all__ = ['Autotune_AutoRegressive', 'AutotunedBaseline']

# TODO unisci la logica, greedy grid search, per ridurre
# la quantità di codice
def Autotune_AutoRegressive(train_residuals,
                            test_residuals,
                            future_lag,
                            P=None,
                            past_lag=None):

    P_range = range(0, train_residuals.shape[1] - 1) if P is None else [P]
    past_lag_range = range(1, future_lag *
                           2) if past_lag is None else [past_lag]

    ar_model = AutoRegressive(train_residuals,
                              test_residuals,
                              past_lag=1 if past_lag is None else past_lag,
                              future_lag=future_lag,
                              P=0 if P is None else P)

    # Ps = np.arange(0, P_range)
    # past_lags = np.arange(1, past_lag_range)
    result_RMSE = pd.DataFrame(index=past_lag_range,
                               columns=P_range)
    for test_past_lag in past_lag_range:
        ar_model.past_lag = test_past_lag
        for test_P in P_range:
            ar_model.P = test_P
            # if not np.isnan(result.loc[past_lag, P]):
            #     continue
            ar_model.fit_AR()

            print()
            print()
            print('past_lag = %d, P = %d' % (
                ar_model.past_lag, ar_model.P))
            print('test RMSE:', ar_model.test_RMSE)
            print()
            print()

            result_RMSE.loc[test_past_lag, test_P] = ar_model.test_RMSE
            if (P is not None) or ((test_P > 0) and result_RMSE.loc[
                    test_past_lag, test_P] > result_RMSE.loc[test_past_lag, test_P - 1]):
                break
        if (past_lag is not None) or (test_past_lag > 1 and result_RMSE.min(
                1)[test_past_lag] > result_RMSE.min(1)[test_past_lag - 1]):
            break
    print('converged!')
    best_P = result_RMSE.min().idxmin()
    best_past_lag = result_RMSE.min(1).idxmin()
    return AutoRegressive(train_residuals,
                          test_residuals,
                          past_lag=best_past_lag,
                          future_lag=future_lag,
                          P=best_P)

def baseline_autotune(train, test,
                      daily_harmonics=None,
                      weekly_harmonics=None,
                      annual_harmonics=None,
                      trend=None,
                      pre_gaussianize=None):

    train = train.dropna()
    test = test.dropna()

    print('autotuning baseline on %d train and %d test points' %
          (len(train), len(test)))

    daily_harmonics = np.arange(
        24) if daily_harmonics is None else [daily_harmonics]
    weekly_harmonics = np.arange(
        6) if weekly_harmonics is None else [daily_harmonics]
    annual_harmonics = np.arange(
        50) if annual_harmonics is None else [annual_harmonics]
    trend = [False, True] if trend is None else [trend]
    pre_gaussianize = [
        False, True] if pre_gaussianize is None else [pre_gaussianize]

    def test_RMSE(
            daily_harmonics,
            weekly_harmonics,
            annual_harmonics,
            trend,
            pre_gaussianize):
        baseline = HarmonicBaseline(train,
                                    daily_harmonics,
                                    weekly_harmonics,
                                    annual_harmonics,
                                    trend,
                                    pre_gaussianize)

        return np.sqrt(((test - baseline.baseline(
            test.index))**2).mean())

    res = greedy_grid_search(test_RMSE,
                             [daily_harmonics,
                              weekly_harmonics,
                              annual_harmonics,
                              trend,
                              pre_gaussianize],
                             num_steps=2)

    print('optimal params: %s' % res)
    print('test std. dev.: %.2f' % test.std())

    return res

# def baseline_autotune_OLD(train, test, min_harmonics=3,
#                           trend=None,
#                           daily_harmonics=None,
#                           weekly_harmonics=None,
#                           annual_harmonics=None):

#     results_test_RMSE = {}

#     train = train.dropna()
#     test = test.dropna()

#     BOUND_WEEKLY = 6

#     max_daily = min_harmonics
#     max_weekly = min_harmonics
#     max_annual = min_harmonics

#     while True:

#         daily_harmonics_range = range(max_daily) if \
#             daily_harmonics is None else [daily_harmonics]
#         weekly_harmonics_range = range(max_weekly) if \
#             weekly_harmonics is None else [weekly_harmonics]
#         annual_harmonics_range = range(max_annual) if \
#             annual_harmonics is None else [annual_harmonics]
#         trend_range = [False, True] if \
#             trend is None else [trend]

#         for daily_harmonics_test in daily_harmonics_range:
#             for weekly_harmonics_test in weekly_harmonics_range:
#                 for annual_harmonics_test in annual_harmonics_range:
#                     for trend_test in trend_range:
#                         if (daily_harmonics_test, weekly_harmonics_test,
#                                 annual_harmonics_test, trend_test) \
#                                 in results_test_RMSE:
#                             continue
#                         baseline = HarmonicBaseline(train,
#                                                     daily_harmonics_test,
#                                                     weekly_harmonics_test,
#                                                     annual_harmonics_test,
#                                                     trend_test)

#                         test_RMSE = np.sqrt(((test - baseline._predict_baseline(
#                             test.index))**2).mean())
#                         # print('test_RMSE:, ', test_RMSE)
#                         results_test_RMSE[(daily_harmonics_test,
#                                            weekly_harmonics_test,
#                                            annual_harmonics_test,
#                                            trend_test)] = test_RMSE

#         current_best = min(results_test_RMSE, key=results_test_RMSE.get)

#         current_best_daily = current_best[0]
#         current_best_weekly = current_best[1]
#         current_best_annual = current_best[2]

#         if ((current_best_daily < max_daily - 1) or
#             daily_harmonics is not None) and \
#            ((current_best_weekly < max_weekly - 1) or
#             weekly_harmonics is not None) and \
#            ((current_best_annual < max_annual - 1) or
#                 annual_harmonics is not None):
#             print('tried %d baseline parameter combinations' %
#                   len(results_test_RMSE))
#             print('optimal baseline parameters: ', current_best)
#             print('test RMSE: ', results_test_RMSE[current_best])
#             print('test std. dev.: ', test.std())
#             print()
#             return current_best

#         max_daily = max(max_daily, current_best_daily + 2)
#         max_weekly = max(max_weekly, current_best_weekly + 2)
#         max_annual = max(max_daily, current_best_annual + 2)

def AutotunedBaseline(train, test, **kwargs):
    print('autotuning baseline for column %s' % train.name)
    if len(train.dropna().value_counts()) == 1:
        kwargs['pre_gaussianize'] = False
    params = baseline_autotune(train.dropna(), test.dropna(), **kwargs)
    return HarmonicBaseline(train.dropna(), *params)


"""
1. This test covers verifying the auto complete suggestion.
2. Second test covers clicking the first auto suggestion link.
"""

from pages.search_page import SearchPage
from pages.result_page import ResultPage
import pytest

@pytest.mark.parametrize('phrase', ['where', 'why', 'what'])
def test_auto_complete(browser, phrase):
    search_page = SearchPage(browser)

    search_page.load()
    assert search_page.get_logo() is not None
    assert browser.title == "DuckDuckGo — Privacy, simplified."
    search_page.enter_search_item(phrase)
    suggestion_values = search_page.get_suggestion_values()
    assert phrase in suggestion_values

@pytest.mark.parametrize('phrase', ['where', 'why', 'what'])
def test_click_auto_suggestion_link(browser, phrase):
    search_page = SearchPage(browser)
    result_page = ResultPage(browser)

    search_page.load()
    assert search_page.get_logo() is not None
    assert browser.title == "DuckDuckGo — Privacy, simplified."
    search_page.enter_search_item(phrase)
    suggestion_values = search_page.get_suggestion_values()
    assert phrase in suggestion_values
    search_page.click_suggestion_link()
    assert phrase in result_page.get_title()
    assert phrase in result_page.get_search_box_value()
    for title in result_page.get_result_link_titles():
        assert phrase.lower() in title.lower()