#!/usr/bin/env python
# coding: utf-8

# In[9]:

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = "all"
get_ipython().run_line_magic('matplotlib', 'inline')
from tqdm.auto import tqdm
import concurrent.futures
from multiprocessing import Pool
import copy,os,sys,psutil
from collections import Counter

# In[35]:

import tensorflow as tf
import datetime
import gensim
import numpy as np
import pandas as pd
from zac_pyutils.ExqUtils import zprint
from zac_pyutils import ExqUtils
from collections import deque
from tqdm.auto import tqdm
from collections import Counter
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.preprocessing import OneHotEncoder
# from gensim.models.wrappers import FastText
import fasttext
import json
import re
import inspect

# # Config

# In[45]:

class TrainingConfig(object):
    epoches = 5
    batchSize = 128
    evaluateEvery = 100
    checkpointEvery = 100
    learningRate = 0.001
    minWordCnt = 5

class ModelConfig(object):
    numFilters = 64

    filterSizes = [2, 3, 4, 5]
    dropoutKeepProb = 0.5
    l2RegLambda = 0.001

class Config(object):
    job = "taste"
    basePath = "/home/zhoutong/nlp/data/textcnn"
    dataSource = basePath + "/labeled_timeliness_region_taste_emotion_sample.json"
    # dataSource = dataSource + ".sample_h10k"
    summaryDir = basePath+"/summary"
    cnnmodelPath_pb = basePath + "/textcnn_model_pb"
    cnnmodelPath_ckpt = basePath+"/textcnn_model_ckpt/model.ckpt"

    weDim = 300
    ft_modelPath = basePath + '/cc.en.300.bin'

    padSize = 16
    pad = '<PAD>'
    pad_initV = np.zeros(weDim)
    unk = '<UNK>'
    unk_initV = np.random.randn(weDim)

    # numClasses = 4  # 二分类设置为1，多分类设置为类别的数目
    numClasses_dict = {"taste":4,"timeliness":9,"emotion":3}
    numClasses = numClasses_dict[job]  # 二分类设置为1，多分类设置为类别的数目

    testRatio = 0.2  # 测试集的比例

    training = TrainingConfig()

    model = ModelConfig()

class Utils():
    # 清理符号
    @staticmethod
    def clean_punctuation(inp_text):
        res = re.sub(r"[~!@#$%^&*()_+-={\}|\[\]:\";'<>?,./“”]", r' ', inp_text)
        res = re.sub(r"\\u200[Bb]", r' ', res)
        res = re.sub(r"\n+", r" ", res)
        res = re.sub(r"\s+", " ", res)
        return res.strip()
    @staticmethod
    def pad_list(inp_list,width,pad_const):
        if len(inp_list) >= width:
            return inp_list[:width]
        else:
            return inp_list+[pad_const]*(width-len(inp_list))

config = Config()

# In[49]:

data.trainReviews.shape[0]
527480
config.training.batchSize
527480//config.training.batchSize

# In[75]:

# fail
import timeout_decorator
@timeout_decorator.timeout(seconds=3, use_signals=False, exception_message="timeout")
def get_input(res_inpt):
    res_inpt=input("是否重新生成persist数据？(y/n)")

res = None
try:
    get_input(res)
except Exception as e:
    res = "N"
print(res)

# In[ ]:

# fail
class StoppableThread(Thread):
    def __init__(self,target,args,timeout=1):
        super(Thread,self).__init__()
        self.timeout = timeout
        self.stopped = False
        self.t = Thread(target=target,args=args)
        self.t.setDaemon(True)

    def start(self):
        self.t.start()
        while not self.stopped:
            self.t.join(self.timeout)
        print("thread stopped.")

    def stop(self):
        self.stopped=True

res = "n"
t1 = StoppableThread(target=get_input,args=(res,))
t1.start()
time.sleep(3)
t1.terminate()
print(res)

# In[ ]:

class StoppableThread(Thread):
    def __init__(self,target,args,timeout=1):
        super(Thread,self).__init__()
        self.timeout = timeout
        self.stopped = False
        self.t = Thread(target=target,args=args)
        self.t.setDaemon(True)

    def start(self):
        self.t.start()
        while not self.stopped:
            self.t.join(self.timeout)
        print("thread stopped.")

    def stop(self):
        self.stopped=True

def func():
    pass

t1 = StoppableThread(target=func)
t1.start()
for i in range(5):
    time.sleep(1)
    print(i)
t1.terminate()

# In[1]:

print("abc")

# In[12]:

from threading import Thread
import time
import queue

class StoppableThread(Thread):
    class TimeoutException(Exception):
        pass

    def __init__(self,target,args=None,time_limit=1,delta=0.05):
        super(Thread,self).__init__()
        self.delta = delta
        self.stopped = False
        if args is None:
            self.t = Thread(target=target)
        else:
            self.t = Thread(target=target,args=args)
        self.t.setDaemon(True)

        self.timing_thread = Thread(target=self.timing,args=(time_limit,))
        self.timing_thread.setDaemon(True)

    def timing(self,timeout):
        time.sleep(timeout)
        self.stopped=True

    def start(self):
        self.t.start()
        self.timing_thread.start()
        while not self.stopped:
            self.t.join(self.delta)
            time.sleep(0.05)
        raise TimeoutException("thread timeout")

    def stop(self):
        self.stopped=True

q = queue.Queue()
def func(inp):
    print("input: ",inp)
    for i in range(3):
        time.sleep(1)
        print(i)
    inp += 1
    print("processed: ",inp)
    q.put(inp)

to_inp = 2
t1 = StoppableThread(target=func,args=(to_inp,), time_limit=1)
t1.start()
print("to_inp: ",to_inp)
print("result in queue:",q.queue[0])

# In[19]:

len(q.queue)

# In[14]:

print("result in queue:",q.queue[0])

# In[9]:

print(q.queue[0])
dir(q)

# In[39]:

zprint("各数据路径：")
print("basePath路径：{}\n样本数据来源: {}\nsummary目录：{}\n".format(config.basePath,config.dataSource,config.summaryDir))
zprint("模型参数如下：")
for k,v in inspect.getmembers(config.model):
    if not k.startswith("_"):
        print(k,v)
zprint("训练参数如下：")
for k,v in inspect.getmembers(config.training):
    if not k.startswith("_"):
        print(k,v)

# # TextCNN

# In[3]:

class TextCNN(object):
    """
    Text CNN 用于文本分类
    """

    def __init__(self, config, wordEmbedding):

        # 定义模型的输入
        self.inputX = tf.placeholder(tf.int32, [None, config.padSize], name="inputX")
        self.inputY = tf.placeholder(tf.int32, [None], name="inputY")

        self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb")

        # 定义l2损失
        l2Loss = tf.constant(0.0)

        # 词嵌入层
        with tf.name_scope("embedding"):
            # 利用预训练的词向量初始化词嵌入矩阵
            self.W = tf.Variable(tf.cast(wordEmbedding, dtype=tf.float32, name="word2vec"), name="W")
            # 利用词嵌入矩阵将输入的数据中的词转换成词向量，维度[batch_size, sequence_length, embedding_size]
            self.embeddedWords = tf.nn.embedding_lookup(self.W, self.inputX)
            # 卷积的输入是思维[batch_size, width, height, channel]，因此需要增加维度，用tf.expand_dims来增大维度
            self.embeddedWordsExpanded = tf.expand_dims(self.embeddedWords, -1)

        # 创建卷积和池化层
        pooledOutputs = []
        # 有三种size的filter，2, 3， 4， 5，textCNN是个多通道单层卷积的模型，可以看作三个单层的卷积模型的融合
        for i, filterSize in enumerate(config.model.filterSizes):
            with tf.name_scope("conv-maxpool-%s" % filterSize):
                # 卷积层，卷积核尺寸为filterSize * embeddingSize，卷积核的个数为numFilters
                # 初始化权重矩阵和偏置
                # 4-D (height,width,in_channels,out_channels) | out_channels是多少就用多少个 Height*Width 的卷积核
                filterShape = [filterSize, config.weDim, 1, config.model.numFilters]
                W = tf.Variable(tf.truncated_normal(filterShape, stddev=0.1), name="W")
                b = tf.Variable(tf.constant(0.1, shape=[config.model.numFilters]), name="b")
                convRes = tf.nn.conv2d(
                    input=self.embeddedWordsExpanded,
                    filter=W,
                    strides=[1, 1, 1, 1],
                    padding="VALID",
                    name="conv")

                # relu函数的非线性映射
                h = tf.nn.relu(tf.nn.bias_add(convRes, b), name="relu")

                # 池化层，最大池化，池化是对卷积后的序列取一个最大值
                pooled = tf.nn.max_pool(
                    h,
                    ksize=[1, config.padSize - filterSize + 1, 1, 1],
                    # ksize shape: [batch, height, width, channels]
                    strides=[1, 1, 1, 1],
                    padding='VALID',
                    name="pool")
                pooledOutputs.append(pooled)  # 将三种size的filter的输出一起加入到列表中

        # 得到CNN网络的输出长度
        numFiltersTotal = config.model.numFilters * len(config.model.filterSizes)

        # 池化后的维度不变，按照最后的维度channel来concat
        self.hPool = tf.concat(pooledOutputs, 3)

        # 摊平成二维的数据输入到全连接层
        self.hPoolFlat = tf.reshape(self.hPool, [-1, numFiltersTotal])

        # dropout
        with tf.name_scope("dropout"):
            self.hDrop = tf.nn.dropout(self.hPoolFlat, self.dropoutKeepProb)

        # 全连接层的输出
        with tf.name_scope("output"):
            outputW = tf.get_variable(
                "outputW",
                shape=[numFiltersTotal, config.numClasses],
                initializer=tf.contrib.layers.xavier_initializer())
            outputB = tf.Variable(tf.constant(0.1, shape=[config.numClasses]), name="outputB")
            l2Loss += tf.nn.l2_loss(outputW)
            l2Loss += tf.nn.l2_loss(outputB)
            self.logits = tf.nn.xw_plus_b(self.hDrop, outputW, outputB, name="logits")
            if config.numClasses == 1:
                self.predictions = tf.cast(tf.greater_equal(self.logits, 0.0), tf.int32, name="predictions")
            elif config.numClasses > 1:
                self.predictions = tf.argmax(self.logits, axis=-1, name="predictions")

            print(self.predictions)

        # 计算二元交叉熵损失
        with tf.name_scope("loss"):

            if config.numClasses == 1:
                losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.logits,
                                                                 labels=tf.cast(tf.reshape(self.inputY, [-1, 1]),
                                                                                dtype=tf.float32))
            elif config.numClasses > 1:
                losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY)

            self.loss = tf.reduce_mean(losses) + config.model.l2RegLambda * l2Loss

# In[4]:

class MetricUtils():
    """
    定义各类性能指标
    """
    @staticmethod
    def mean(item: list) -> float:
        """
        计算列表中元素的平均值
        :param item: 列表对象
        :return:
        """
        res = sum(item) / len(item) if len(item) > 0 else 0
        return res

    @staticmethod
    def accuracy(pred_y, true_y):
        """
        计算二类和多类的准确率
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :return:
        """
        if isinstance(pred_y[0], list):
            pred_y = [item[0] for item in pred_y]
        corr = 0
        for i in range(len(pred_y)):
            if pred_y[i] == true_y[i]:
                corr += 1
        acc = corr / len(pred_y) if len(pred_y) > 0 else 0
        return acc

    @staticmethod
    def binary_precision(pred_y, true_y, positive=1):
        """
        二类的精确率计算
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param positive: 正例的索引表示
        :return:
        """
        corr = 0
        pred_corr = 0
        for i in range(len(pred_y)):
            if pred_y[i] == positive:
                pred_corr += 1
                if pred_y[i] == true_y[i]:
                    corr += 1

        prec = corr / pred_corr if pred_corr > 0 else 0
        return prec

    @staticmethod
    def binary_recall(pred_y, true_y, positive=1):
        """
        二类的召回率
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param positive: 正例的索引表示
        :return:
        """
        corr = 0
        true_corr = 0
        for i in range(len(pred_y)):
            if true_y[i] == positive:
                true_corr += 1
                if pred_y[i] == true_y[i]:
                    corr += 1

        rec = corr / true_corr if true_corr > 0 else 0
        return rec

    @staticmethod
    def binary_f_beta(pred_y, true_y, beta=1.0, positive=1):
        """
        二类的f beta值
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param beta: beta值
        :param positive: 正例的索引表示
        :return:
        """
        precision = MetricUtils.binary_precision(pred_y, true_y, positive)
        recall = MetricUtils.binary_recall(pred_y, true_y, positive)
        try:
            f_b = (1 + beta * beta) * precision * recall / (beta * beta * precision + recall)
        except:
            f_b = 0
        return f_b

    @staticmethod
    def multi_precision(pred_y, true_y, labels):
        """
        多类的精确率
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param labels: 标签列表
        :return:
        """
        if isinstance(pred_y[0], list):
            pred_y = [item[0] for item in pred_y]

        precisions = [MetricUtils.binary_precision(pred_y, true_y, label) for label in labels]
        prec = MetricUtils.mean(precisions)
        return prec

    @staticmethod
    def multi_recall(pred_y, true_y, labels):
        """
        多类的召回率
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param labels: 标签列表
        :return:
        """
        if isinstance(pred_y[0], list):
            pred_y = [item[0] for item in pred_y]

        recalls = [MetricUtils.binary_recall(pred_y, true_y, label) for label in labels]
        rec = MetricUtils.mean(recalls)
        return rec

    @staticmethod
    def multi_f_beta(pred_y, true_y, labels, beta=1.0):
        """
        多类的f beta值
        :param pred_y: 预测结果
        :param true_y: 真实结果
        :param labels: 标签列表
        :param beta: beta值
        :return:
        """
        if isinstance(pred_y[0], list):
            pred_y = [item[0] for item in pred_y]

        f_betas = [MetricUtils.binary_f_beta(pred_y, true_y, beta, label) for label in labels]
        f_beta = MetricUtils.mean(f_betas)
        return f_beta

    @staticmethod
    def get_binary_metrics(pred_y, true_y, f_beta=1.0):
        """
        得到二分类的性能指标
        :param pred_y:
        :param true_y:
        :param f_beta:
        :return:
        """
        acc = MetricUtils.accuracy(pred_y, true_y)
        recall = MetricUtils.binary_recall(pred_y, true_y)
        precision = MetricUtils.binary_precision(pred_y, true_y)
        f_beta = MetricUtils.binary_f_beta(pred_y, true_y, f_beta)
        return acc, recall, precision, f_beta

    @staticmethod
    def get_multi_metrics(pred_y, true_y, labels, f_beta=1.0):
        """
        得到多分类的性能指标
        :param pred_y:
        :param true_y:
        :param labels:
        :param f_beta:
        :return:
        """
        acc = MetricUtils.accuracy(pred_y, true_y)
        recall = MetricUtils.multi_recall(pred_y, true_y, labels)
        precision = MetricUtils.multi_precision(pred_y, true_y, labels)
        f_beta = MetricUtils.multi_f_beta(pred_y, true_y, labels, f_beta)
        return acc, recall, precision, f_beta

# # Class Dataset

# In[5]:

class Dataset(object):
    def __init__(self, config):
        self.config = config
        self._dataSource = config.dataSource

        self.testRatio = config.testRatio
        self._we_fp = config.basePath+"/wordEmbeddingInfo"  # \t分割 词,idx,embedding
        self._tokens_arr_fp = config.basePath+"/tokens_arr.npy"
        self._labels_arr_fp = config.basePath+"/labels_arr.npy"
        self._emb_arr_fp = config.basePath+"/emb_arr.npy"
        self.ft_modelPath = config.ft_modelPath
        self.ft_model = None

        self.trainReviews = np.array([])
        self.trainLabels = np.array([])

        self.evalReviews = np.array([])
        self.evalLabels = np.array([])

        self.token2idx = {}
        self.wordEmbedding = None
        self.labelSet = []
        self.totalWordCnt = 0

    def _readData(self, filePath):
        f_iter = ExqUtils.load_file_as_iter(filePath)
        tokens_list = deque()
        label_list = deque()
        zprint("loading data from: "+filePath)
        for l in tqdm(f_iter,desc="readData"):
            info = json.loads(l)
            text,label = info['title'],info[self.config.job]
            tokens = Utils.pad_list(Utils.clean_punctuation(text).split(" "),width=self.config.padSize,pad_const=self.config.pad)
            tokens_list.append(tokens)
            label_list.append(label)
        return np.array(tokens_list), np.array(label_list)

    def _initStopWord(self, stopWordPath):
        with open(stopWordPath, "r") as fr:
            self._stopWordSet = set(fr.read().splitlines())

    def _initFasttextModel(self):
        if self.ft_model is None:
            self.ft_model = fasttext.load_model(self.ft_modelPath)

    def _tokens2idx(self,tokens_arr):
        tokensSet = set(np.unique(tokens_arr))

        pass

    def dataGen_persist(self):
        """
                初始化训练集和验证集
                """
        zprint("init fasttext model")
        self._initFasttextModel()

        # 初始化数据集
        tokens_arr, label_arr = self._readData(self._dataSource)
        self.labelSet = set(np.unique(label_arr))
        tokensSet = set(np.unique(tokens_arr))

        self.totalWordCnt = len(tokensSet)
        wordEmb = np.zeros(shape=[self.totalWordCnt, self.ft_model.get_dimension()])
        # (idx,token,emb)保存到文本文件
        zprint("预测词向量总计: {} , 词向量存入文件: {}".format(self.totalWordCnt, self._we_fp))
        with open(self._we_fp, "w") as fw:
            # 加上 <PAD> 和 <UNK> 及其初始化
            for idx, token in tqdm(enumerate(tokensSet), total=self.totalWordCnt,desc="tokensSet"):
                if token == self.config.pad:
                    emb = self.config.pad_initV
                elif token == self.config.unk:
                    emb = self.config.unk_initV
                else:
                    emb = self.ft_model[token]
                self.token2idx.update({token:idx})
                wordEmb[idx] = emb
                fw.writelines(str(idx) + "\t" + token + "\t" + ",".join([str(i) for i in list(emb)]) + "\n")

        # tokens变为idx保存为npy
        zprint("tokens映射为索引保存到npy")
        tokensIdx_arr = np.zeros_like(tokens_arr, dtype=np.int64)
        for i,tokens in enumerate(tokens_arr):
            for j,token in enumerate(tokens):
                tokensIdx_arr[i][j] = self.token2idx[token]
        np.save(self._tokens_arr_fp,tokensIdx_arr)

        zprint("labels保存到npy")
        np.save(self._labels_arr_fp,label_arr)

        zprint("idx对应的emb保存到npy")
        np.save(self._emb_arr_fp,wordEmb)

    def loadData(self):
        self.wordEmbedding = np.load(self._emb_arr_fp)

        tokensIdx_arr = np.load(self._tokens_arr_fp)
        label_arr = np.load(self._labels_arr_fp)
        self.labelSet = set(np.unique(label_arr))
        # 初始化训练集和测试集
        sss = StratifiedShuffleSplit(n_splits=1, test_size=self.testRatio, random_state=2019)
        train_idx, test_idx = list(sss.split(np.zeros(label_arr.shape[0]), label_arr))[0]

        self.trainReviews = tokensIdx_arr[train_idx]
        self.trainLabels = label_arr[train_idx]

        self.evalReviews = tokensIdx_arr[test_idx]
        self.evalLabels = label_arr[test_idx]

# # Produce Data

# In[11]:

data = Dataset(config)
# data.dataGen_persist()
data.loadData()
print("train data shape: {}".format(data.trainReviews.shape))
print("train label shape: {}".format(data.trainLabels.shape))
print("eval data shape: {}".format(data.evalReviews.shape))
print("eval data shape: {}".format(data.evalLabels.shape))
print("wordEmbedding info file: {}".format(data._we_fp))

# In[12]:

trainReviews = data.trainReviews
trainLabels = data.trainLabels
evalReviews = data.evalReviews
evalLabels = data.evalLabels

wordEmbedding = data.wordEmbedding
labelList = data.labelSet

# In[13]:

trainReviews
trainLabels

# # 开始构建计算图

# In[11]:

evalLabels.shape
batchY = np.array(evalLabels[10: 100], dtype="float32")
batchY.shape

# In[15]:

def nextBatch(x, y, batchSize):
    """
    生成batch数据集，用生成器的方式输出
    """
    perm = np.arange(len(x))
    np.random.shuffle(perm)
    x,y = x[perm],y[perm]

    numBatches = len(x) // batchSize

    for i in range(numBatches):
        start = i * batchSize
        end = start + batchSize
        batchX = np.array(x[start: end], dtype="int64")
        batchY = np.array(y[start: end], dtype="float32")

        yield batchX, batchY

# In[55]:

tf.reset_default_graph()

# In[56]:

self = cnn
# 定义模型的输入
self.inputX = tf.placeholder(tf.int32, [None, config.padSize], name="inputX")
self.inputY = tf.placeholder(tf.int32, [None], name="inputY")
self.dropoutKeepProb = tf.placeholder(tf.float32, name="dropoutKeepProb")
# 定义l2损失
l2Loss = tf.constant(0.0)
# 词嵌入层
with tf.name_scope("embedding"):
    # 利用预训练的词向量初始化词嵌入矩阵
    self.W = tf.Variable(tf.cast(wordEmbedding, dtype=tf.float32, name="word2vec"), name="W")
    # 利用词嵌入矩阵将输入的数据中的词转换成词向量，维度[batch_size, sequence_length, embedding_size]
    self.embeddedWords = tf.nn.embedding_lookup(self.W, self.inputX)
    # 卷积的输入是思维[batch_size, width, height, channel]，因此需要增加维度，用tf.expand_dims来增大维度
    self.embeddedWordsExpanded = tf.expand_dims(self.embeddedWords, -1)
# 创建卷积和池化层
pooledOutputs = []
# 有三种size的filter，2, 3， 4， 5，textCNN是个多通道单层卷积的模型，可以看作三个单层的卷积模型的融合
for i, filterSize in enumerate(config.model.filterSizes):
    with tf.name_scope("conv-maxpool-%s" % filterSize):
        # 卷积层，卷积核尺寸为filterSize * embeddingSize，卷积核的个数为numFilters
        # 初始化权重矩阵和偏置
        filterShape = [filterSize, config.weDim, 1, config.model.numFilters]
        W = tf.Variable(tf.truncated_normal(filterShape, stddev=0.1), name="W")
        b = tf.Variable(tf.constant(0.1, shape=[config.model.numFilters]), name="b")
        convRes = tf.nn.conv2d(
            input=self.embeddedWordsExpanded,
            filter=W,
            strides=[1, 1, 1, 1],
            padding="VALID",
            name="conv")
        # relu函数的非线性映射
        h = tf.nn.relu(tf.nn.bias_add(convRes, b), name="relu")
        # 池化层，最大池化，池化是对卷积后的序列取一个最大值
        pooled = tf.nn.max_pool(
            h,
            ksize=[1, config.padSize - filterSize + 1, 1, 1],
            # ksize shape: [batch, height, width, channels]
            strides=[1, 1, 1, 1],
            padding='VALID',
            name="pool")
        pooledOutputs.append(pooled)  # 将三种size的filter的输出一起加入到列表中
# 得到CNN网络的输出长度
numFiltersTotal = config.model.numFilters * len(config.model.filterSizes)
# 池化后的维度不变，按照最后的维度channel来concat
self.hPool = tf.concat(pooledOutputs, 3)
# 摊平成二维的数据输入到全连接层
self.hPoolFlat = tf.reshape(self.hPool, [-1, numFiltersTotal])
# dropout
with tf.name_scope("dropout"):
    self.hDrop = tf.nn.dropout(self.hPoolFlat, self.dropoutKeepProb)
# 全连接层的输出
with tf.name_scope("output"):
    outputW = tf.get_variable(
        "outputW",
        shape=[numFiltersTotal, config.numClasses],
        initializer=tf.contrib.layers.xavier_initializer())
    outputB = tf.Variable(tf.constant(0.1, shape=[config.numClasses]), name="outputB")
    l2Loss += tf.nn.l2_loss(outputW)
    l2Loss += tf.nn.l2_loss(outputB)
    self.logits = tf.nn.xw_plus_b(self.hDrop, outputW, outputB, name="logits")
    if config.numClasses == 1:
        self.predictions = tf.cast(tf.greater_equal(self.logits, 0.0), tf.int32, name="predictions")
    elif config.numClasses > 1:
        self.predictions = tf.argmax(self.logits, axis=-1, name="predictions")
    print(self.predictions)
# 计算二元交叉熵损失
with tf.name_scope("loss"):
    if config.numClasses == 1:
        losses = tf.nn.sigmoid_cross_entropy_with_logits(logits=self.logits,
                                                         labels=tf.cast(tf.reshape(self.inputY, [-1, 1]),
                                                                        dtype=tf.float32))
    elif config.numClasses > 1:
        losses = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY)
    self.loss = tf.reduce_mean(losses) + config.model.l2RegLambda * l2Loss

# In[82]:

self.logits.shape
self.inputY.shape
self.hDrop.shape,outputW.shape,outputB.shape
self.hPoolFlat.shape
inpX.shape,inpY.shape
filterShape
convRes.shape
pooled.shape
pooledOutputs
numFiltersTotal,"=",config.model.numFilters,"*",len(config.model.filterSizes)
self.hPool.shape
tf.concat(pooledOutputs, 3).shape,tf.concat(pooledOutputs, -1).shape
self.hPoolFlat.shape
self.hDrop.shape
self.logits.shape

# In[94]:

with tf.Session() as sess:
    fd={self.inputX:inpX,self.inputY:inpY,self.dropoutKeepProb:0.8}
    sess.run(tf.global_variables_initializer())
    sess.run(self.embeddedWordsExpanded,feed_dict=fd).shape
    filterShape
    sess.run(W).shape
    sess.run(convRes,feed_dict=fd).shape
    sess.run(pooled,feed_dict=fd).shape
    sess.run(outputW,fd).shape
    sess.run(outputB,fd).shape
    sess.run(self.logits,fd).shape
    sess.run(self.predictions,fd).shape
    sess.run(self.inputY,fd).shape
    sess.run(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=self.logits, labels=self.inputY),fd)

# In[16]:

tf.reset_default_graph()

# In[21]:

trainReviews.shape
config.batchSize

# In[61]:

from zac_pyutils import TFUtils

# In[73]:

import os
os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"]="-1"
tf.reset_default_graph() # 防止运行多次后session创建、运行会变慢
ckpt_fp="/home/zhoutong/nlp/data/textcnn/textcnn_model_ckpt/model.ckpt-20480"
pb_fp = "/home/zhoutong/nlp/data/textcnn/tmp_model.pb.sample"
saver = tf.train.import_meta_graph(ckpt_fp + '.meta', clear_devices=True)
with tf.Session() as sess:
    saver.restore(sess, ckpt_fp)
    output_tensor = sess.graph.get_tensor_by_name('output/predictions:0')
    output_tensor
    output_graph_def = tf.graph_util.convert_variables_to_constants(
            sess=sess,
            input_graph_def=sess.graph_def,
            output_node_names=['output/predictions'])
    with tf.gfile.GFile(pb_fp, "wb") as f:  # 保存模型
        f.write(output_graph_def.SerializeToString())  # 序列化输出

# In[ ]:

# In[ ]:

# In[ ]:

# import os
# os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
# os.environ["CUDA_VISIBLE_DEVICES"]="-1"
TFUtils.convert_ckpt2pb(ckpt_fp="/home/zhoutong/nlp/data/textcnn/textcnn_model_ckpt/model.ckpt-20480",
                        pb_fp = "/home/zhoutong/nlp/data/textcnn/tmp_model.pb.sample",
                        output_name_list=['output/predictions'])

# In[ ]:


import os
import scipy.io
import numpy as np
import scipy.signal
import config
import pandas as pd
import scipy.stats
import plotting

# Load Dataset
def LoadDataset(file_name, parameters):
    # Read Data File and Extract Desired Signals
    data_file = scipy.io.loadmat('./data/training/' + file_name)
    data = data_file['dataset']['data'][0][0]
    tags = data_file['dataset']['tags'][0][0].T[0]  # shape into 1d array
    selected_signals_data = np.vstack([data[parameters.signals[i]][0][0][:, j]
                                       for i in range(len(parameters.signals))
                                       for j in parameters.channels[i]])
    # Generate Sliding Windows
    signal_windows = np.expand_dims(GenerateWindowSlides(selected_signals_data[0],
                                                         parameters.window_size, parameters.step_size), axis=0)
    for i in range(1, len(selected_signals_data)):
        signal_windows = np.append(signal_windows,
                                   np.expand_dims(GenerateWindowSlides(selected_signals_data[i],
                                                                       parameters.window_size,
                                                                       parameters.step_size), axis=0), axis=0)
    action_tag_windows = GenerateWindowSlides(tags, parameters.window_size, parameters.step_size)

    # Generate Input/Output Dataset
    input_signal_data = signal_windows.swapaxes(0, 1)
    if parameters.window_tag_method == "MODE":
        output_class_data = scipy.stats.mode(action_tag_windows, axis=1)[0].flatten()
    elif parameters.window_tag_method == "LAST":
        output_class_data = action_tag_windows[:, -1]
    else:
        raise ValueError("Window Tag Method " + parameters.window_tag_method + " does not match either MODE or LAST.")

    return input_signal_data, output_class_data

# Sliding Window Function
def GenerateWindowSlides(data, window, step):
    shape = data.shape[:-1] + (data.shape[-1] - window + 1, window)
    strides = data.strides + (data.strides[-1],)
    slided_windows = np.lib.stride_tricks.as_strided(data, shape=shape, strides=strides)
    return slided_windows[::step]

# Generate X/Y Data
def GenerateInputOutputData(one_hot=True, sanity_plot=False):
    # Retrieve Parameters
    parameters = config.LoadDataProvisionParameters()

    # Retrieve Training Data Files
    data_files = os.listdir('./data/training')
    X, Y = LoadDataset(data_files[0], parameters)

    # Generate I/O Data
    for i in range(1, len(data_files)):
        x, y = LoadDataset(data_files[i], parameters)
        X = np.append(X, x, axis=0)
        Y = np.append(Y, y, axis=0)
    X = np.expand_dims(X, axis=0).swapaxes(0, 1).swapaxes(2, 3)     # conform to NHWC shape
    Y[Y == 0] = 2   # equalize move and idle

    if sanity_plot:
        plotting.PlotSequentialInputOutputWindowedData(X, Y, parameters.window_size,
                                                       parameters.window_size // parameters.step_size)

    if one_hot:
        return X, pd.get_dummies(Y).as_matrix()
    return X, Y

# Generate X/Y Data Split for Learning and Evaluation
def GenerateTrainTestData(holdout=0.3, sanity_plot=False):
    # Get Input/Output Data
    X, Y = GenerateInputOutputData(sanity_plot)

    # Split Data
    train_test_split = np.random.rand(len(Y)) > holdout     # true: training, false: testing
    train_x = X[train_test_split]
    train_y = Y[train_test_split]
    test_x = X[~train_test_split]
    test_y = Y[~train_test_split]

    return train_x, train_y, test_x, test_y


from django.conf.urls import url
from gevir import views

urlpatterns = [
	url(r'^$', views.index, name='index'),
	url(r'^results/$', views.results, name='results'), #ResultsView.as_view() # views.results
	url(r'^variant_search/$', views.variant_search, name='variant_search'),
	url(r'^variant_regions/$', views.variant_regions, name='variant_regions'),
	url(r'^faq/$', views.faq, name='faq'),
	url(r'^download/$', views.download, name='download'),
	url(r'^cite/$', views.cite, name='cite'),
	url(r'^terms/$', views.terms, name='terms'),
	url(r'^clear_gene_search/$', views.clear_gene_search, name='clear_gene_search'),
]


from pyspark.sql import SparkSession
from pyspark.sql.functions import col, avg

spark = SparkSession.builder.appName("AverageDurationByCategory").getOrCreate()
historique_visualisation = spark.read.csv("historique_visualisation.csv", header=True, inferSchema=True)
map_result = historique_visualisation.rdd.map(lambda row: (row['Topics'], row['Duration']))
reduce_result = map_result.groupByKey().map(lambda x: (x[0], sum(x[1]) / len(x[1])))
result_df = reduce_result.toDF(["Category", "AverageDuration"])
sorted_result_df = result_df.orderBy(col("Category"))
sorted_result_df.show(truncate=False)
spark.stop()


# -*- coding: utf-8 -*-
"""
Peter Nilson 2017 -- http://www.github.com/petenilson
"""
from setuptools import setup

packages = [
    'pexelpie',
]

with open('requirements.txt') as f:
    required = f.read().splitlines()

setup(
    name='pexelpie',
    version='0.0.1.1',
    description='python client for Pexel images',
    author='Peter Nilson',
    author_email='petenilson@gmail.com',
    url='https://github.com/petenilson/pexelpie/',
    license='MIT',
    packages=packages,
    include_package_data=True,
    install_requires=required,
)


from glob import glob
import numpy as np
from PIL import Image
import matplotlib.pyplot as plt

# Get Binary
def GetBinary(msk):
    msk_arr = np.array(msk)
    msk_arr[msk_arr > 127] = 255
    msk_arr[msk_arr <= 127] = 0
    new_msk = Image.fromarray(msk_arr)
    return new_msk

def rescale1400(image):
    width, height = image.size
    if width >= height:
        angle = 90
        out = image.rotate(angle, expand=True)
    else:
        out = image
    W = 1120
    H = 1880
    resized = out.resize((W, H), Image.NEAREST)
    return resized

def split4(im):

    # Setting the points for cropped image
    output = []
    for i in np.arange(4):
        for j in np.arange(4):
            img = im.crop((i*280, j*470, i*280+280, j*470+470)) #280, 470
            output.append(img)
    return output

if __name__ == "__main__":

    image_path = r"E:\Todo\cropped"
    #mask_path = r"H:\Aurélie\Data\ModelB\Train\Join"
    outpath_im = r"C:\Users\A60026184\Desktop\ModelA\Todo"
    #outpath_mask = r"\Masks"

    image_files = glob(image_path + '\\*.*')

    for f in image_files:

        filename = f.split('\\')[-1]#[8:] #CROP MODEL A

        #mask_file = glob(mask_path + '\\*' + filename)
        #if mask_file == []:
        #    print(filename, 'is missing.')
        #    continue

        img = Image.open(f)
        #msk = Image.open(mask_file[0])
        #msk = GetBinary(msk)

        new_img = rescale1400(img)
        #new_msk = rescale1400(msk)

        splitted_im = split4(new_img)
        #splitted_mk = split4(new_msk)

        for i, split_im in enumerate(splitted_im):
            if i < 10:
                num = '0'+str(i)
            else:
                num = str(i)
            split_im.save(outpath_im + "\\" + num + '_' + filename)
        #for i, split_mk in enumerate(splitted_mk):
        #    split_mk.save(outpath_mask + "\\" + str(i) + '_' + filename)
        print(filename, 'split and saved.')


##Get track length from spotify URI

import requests, json

def URIToLength(uri):
    url = "https://api.spotify.com/v1/tracks/"+uri
    r = requests.get(url)
    t = json.loads(r.text)
    time_ms =  int(t["duration_ms"])
    mod_ms = time_ms % 1000
    time_s = int((time_ms - mod_ms)/1000)
    mod_s = time_s % 60
    time_m = int((time_s - mod_s)/60)
    return (time_m,mod_s,mod_ms)


import pandas as pd
import re
import string
import numpy as np
import mywordtokenizer
from wordSenseByContext import SenseContextWordDict
from nltk.tokenize import sent_tokenize

def createSentimentDF(filename):
    sentiment_df = pd.read_csv(filename, index_col='Word')
    sentiment_df = sentiment_df[['V.Mean.Sum', 'A.Mean.Sum', 'D.Mean.Sum']]
    sentiment_df = sentiment_df.rename(columns={'V.Mean.Sum': 'Valence',
                                                'A.Mean.Sum': 'Arousal',
                                                'D.Mean.Sum': 'Dominance'})
    return sentiment_df

def gutenberg_iterator(filename):
    """Yields paragraphs (as defined simply by multiple
    newlines in a row).

    Parameters
    ----------
    filename : str
        Full path to the file.

    Yields
    ------
    multiline str

    """
    with open(filename) as f:
        contents = f.read()
    for para in re.split(r"[\n\s*]{2,}", contents):
        yield para

def comparison_table(paragraphstring, myDict, sentiment_df):
    """Create a table that represents arousal, valence, and dominance ratings for words and their most frequent
    alternatives across time within a paragraph.

    Parameters
    ----------
    paragraphstring : str
        The string representing the paragraph to convert into a table.

    Returns
    -------
    a dataframe with containing frequency, arousal, valence, and dominance ratings for words and their most
    frequent alternatives. The index numbers in this df represent the word's position in the sentence. Any number
    that is not available will be assigned np.nan.

    """
    paragraph = sent_tokenize(paragraphstring)

    comparison_table_df = pd.DataFrame(columns=['Word', 'WordA', 'WordV', 'WordD', 'MostFreqAlt',
                                                'AltA', 'AltV', 'AltD'])

    for sentstring in paragraph:
        sentence = mywordtokenizer.simple(sentstring)
        for word in sentence:
            newLine = {'Word': word}      # starts a line to be updated with all known information

            if sentiment_df.index.contains(word):
                newLine['WordA'] = sentiment_df.at[word, 'Arousal']
                newLine['WordV'] = sentiment_df.at[word, 'Valence']
                newLine['WordD'] = sentiment_df.at[word, 'Dominance']

            mostFreqAlt = myDict.mostFrequentAlternative(word, sentence)
            if word != mostFreqAlt and not mostFreqAlt is None:
                newLine['MostFreqAlt'] = mostFreqAlt
                if sentiment_df.index.contains(mostFreqAlt):
                    newLine['AltA'] = sentiment_df.at[mostFreqAlt, 'Arousal']
                    newLine['AltV'] = sentiment_df.at[mostFreqAlt, 'Valence']
                    newLine['AltD'] = sentiment_df.at[mostFreqAlt, 'Dominance']

            comparison_table_df = comparison_table_df.append(newLine, ignore_index=True) # add updated line to data frame

    return comparison_table_df

def comparison_table_available_values(table):
    """Update a table with arousal, valence, and dominance ratings for words and their most frequent
    alternatives across time within a paragraph to include only lines that have semantic dimension ratings for both
    the word and a known most frequent alternative.

    Parameters
    ----------
    table : a data frame
        The data frame representing arousal, valence, and dominance ratings for words and their most frequent
        alternatives across time within a paragraph

    Returns
    -------
    a dataframe without unknown ratings for semantic dimensions

    """
    table = table[table.WordA.notnull() &
                  table.WordV.notnull() &
                  table.WordD.notnull() &
                  table.AltA.notnull() &
                  table.AltV.notnull() &
                  table.AltD.notnull()]
    return table

def biggest_differences_words(prunedTable):
    """ Finds the words that are most different from their most frequent alternative across each semantic dimension

    Parameters
    ----------
    prunedTable : a data frame
        The data frame representing arousal, valence, and dominance ratings for words and their most frequent
        alternatives across time within a paragraph

    Returns
    -------
    a dictionary mapping from a semantic dimension to row indexing information about the word with the greatest
    difference for that dimension

    """
    prunedTable = prunedTable.assign(absADiff = (prunedTable['WordA'] - prunedTable['AltA']).abs(),
                                     absVDiff = (prunedTable['WordV'] - prunedTable['AltV']).abs(),
                                     absDDiff = (prunedTable['WordD'] - prunedTable['AltD']).abs())
    biggestDifferencesWords = {'Arousal': prunedTable.loc[prunedTable['absADiff'].idxmax()],
                               'Valence': prunedTable.loc[prunedTable['absVDiff'].idxmax()],
                               'Dominance': prunedTable.loc[prunedTable['absDDiff'].idxmax()]}
    return biggestDifferencesWords

def paragraphAverages(filename, myDict, sentiment_df, stop = -1):
    """Create a table that represents paragraphs' average arousal, valence, and dominance ratings for words and
    their most frequent alternatives across a given text.

    Parameters
    ----------
    filename : string
        The string representing the file name to be read

    (optional): int
        the number of paragraphs the user wants to be measured. Useful, if you don't want to look at the entire file
        but say the first 20 paragraphs. (reduces waiting time). the default is to process everything.

    Returns
    -------
    a dataframe with paragraphs' average arousal, valence, and dominance ratings for words and
    their most frequent alternatives across a given text

    """
    avgs = pd.DataFrame(columns=['WordAAvg', 'WordVAvg', 'WordDAvg', 'AltAAvg', 'AltVAvg','AltDAvg'])
    file_iterator = gutenberg_iterator(filename)
    paragraphNum = 0
    for paragraphstring in file_iterator:
        if stop != -1 and paragraphNum >= stop:
            return avgs
        table = comparison_table(paragraphstring, myDict, sentiment_df)
        availableValTable = comparison_table_available_values(table)
        newLine = {'WordAAvg': availableValTable['WordA'].mean(),
                   'WordVAvg': availableValTable['WordV'].mean(),
                   'WordDAvg': availableValTable['WordD'].mean(),
                   'AltAAvg': availableValTable['AltA'].mean(),
                   'AltVAvg': availableValTable['AltV'].mean(),
                   'AltDAvg': availableValTable['AltD'].mean()}
        avgs = avgs.append(newLine, ignore_index=True)
        paragraphNum+=1
    return avgs

def biggest_differences_paragraphs(table):
    """ Finds the paragraphs with greatest average differences between words and their most frequent alternative
        across each semantic dimension

    Parameters
    ----------
    table : a data frame
        The data frame representing average a paragraph's arousal, valence, and dominance ratings for words and
        their most frequent alternatives across time in a text

    Returns
    -------
    a dictionary mapping from a semantic dimension to a row indexing information about the paragraph with the greatest
    difference for that dimension

    """
    table = table.assign(absAAvgDiff = (table['WordAAvg'] - table['AltAAvg']).abs(),
                         absVAvgDiff = (table['WordVAvg'] - table['AltVAvg']).abs(),
                         absDAvgDiff = (table['WordDAvg'] - table['AltDAvg']).abs())
    biggestDifferencesParagraphs = {'Arousal': table.loc[table['absAAvgDiff'].idxmax()],
                                    'Valence': table.loc[table['absVAvgDiff'].idxmax()],
                                    'Dominance': table.loc[table['absDAvgDiff'].idxmax()]}
    return biggestDifferencesParagraphs

"""===================================================================
Place all function calls below the following conditional so that they
are called only if this module is called with

`python ling278_assign02.py`

No functions should execute if it is instead imported with

import ling278_assign02

in the interactive shell.
"""

if __name__ == '__main__':
    pass


#!/usr/bin/env python3.6
# -*- coding: utf-8 -*-

"""
    台湾  國家教育研究院提供
    入口 http://terms.naer.edu.tw/download/
"""

import requests
from bs4 import BeautifulSoup
import io
import zipfile
import sys
import re

import psycopg2

DSN = "dbname=sunqf user=sunqf"

def unzip_data(content):
    stream = io.BytesIO(content)
    with zipfile.ZipFile(stream) as myzip:
        for name in myzip.namelist():
            with myzip.open(name) as file:
                yield name, file.read().decode('utf-8')

def extract_term(data):
    for tr in BeautifulSoup(data, 'html.parser').find_all('tr')[1:]:
        id, en, zh, *_ = map(lambda td: td.text.strip(), tr.find_all('td'))
        yield en, zh

def get_html(url, timeout=10, try_time=1):
    if try_time > 10:
        raise RuntimeError('Getting url exceed 10 times.')

    try:
        html = requests.get(url, timeout=timeout)
        if html.status_code == 200:
            return html.content
        else:
            raise RuntimeError('Response status_code is %d' % html.status_code)
    except Exception as e:
        print(e)
        return get_html(url, try_time+1)

def get_list(output):
    list_html = get_html('http://terms.naer.edu.tw/download/')
    if list_html:
        html = BeautifulSoup(list_html, 'html.parser')
        with open(output, 'w') as output_file:
            for link in html.find(attrs={'class': 'list-tab-content'}).find_all('a'):
                prefix = 'http://terms.naer.edu.tw'
                domain = link.text
                link = prefix + link.attrs['href']
                print(domain, link)
                domain_html = get_html(link)
                domain_html = BeautifulSoup(domain_html, 'html.parser')

                for download_url in domain_html.find_all('a', text=re.compile('.*壓縮檔')):
                    print(download_url)
                    download_url = prefix + download_url.attrs['href']
                    # 文件较大，网络较慢等可能会影响下载速度，设置为10分钟超时
                    zip_data = get_html(download_url, timeout=600)

                    for name, data in unzip_data(zip_data):
                        yield (domain, name, data)
                        for en, zh in extract_term(data):
                            output_file.write('\t'.join([domain, en, zh]) + '\n')

def save_to_db(dsn: str, data_list):
    conn = psycopg2.connect(dsn)

    for domain, name, data in data_list:
        try:
            with conn:
                with conn.cursor() as cur:
                    psycopg2.extensions.register_type(psycopg2.extensions.UNICODE, cur)
                    cur.execute("INSERT INTO raw_terms (domain, name, data, source) VALUES (%s, %s, %s, %s)",
                                (domain, name, data, 'naer'))

        except Exception as e:
            print(e)

    conn.close()

'''
def update_db(dsn: str):
    conn = psycopg2.connect(dsn)
    try:
        with conn:
            with conn.cursor() as select_cur:
                select_cur.execute("SELECT id, data FROM raw_terms WHERE source='naer'")
                while True:
                    batch_size = 10
                    data = select_cur.fetchmany(batch_size)
                    with conn.cursor() as update_cur:
                        update_cur.execute("UPDATE raw_terms SET data = record.data"
                                           "FROM (VALUES %s) AS record (id, data)"
                                           "WHERE raw_terms.id = record.id",
                                           [(id, data.encode(encoding='latin1').decode('utf-8')) for id, data in data])

                    if len(data) < batch_size:
                        break
    except Exception as e:
        print(e)

    conn.close()
'''

data_list = get_list(sys.argv[1])

save_to_db(DSN, data_list)


# Imports
import numpy as np
import matplotlib.pyplot as plt

'''
1D Finite Element Analysis of a bar with N-1 elements and N nodes.

This program performs a finite element analysis of a 1D bar subjected to tensile or compressive forces.
The bar has variable cross-section defined by the function A(x) = A0 * exp(-x), where A0 is the initial cross-section
area.
The analysis calculates the displacements of the nodes and plots the results.
'''

def main():
    def stiffness(e_, i_, E_, A_, dL_):
        K_ = np.zeros((NUMBER_NODES, NUMBER_NODES))
        selection = np.ix_(e_, e_)
        K_[selection] += E_ * A_[i_] / dL_ * np.array([[1, -1], [-1, 1]])
        return K_

    NUMBER_ELEMENTS = 10
    NUMBER_NODES = NUMBER_ELEMENTS + 1
    NODE_CONNECTIONS = [[i, i + 1] for i in range(NUMBER_ELEMENTS)]

    X = np.linspace(0, 1, NUMBER_NODES)
    dL = np.abs(X[1] - X[0])

    E = 2 * 10 ** 9
    A0 = 1
    A = A0 * np.exp(-X)

    K = np.zeros((NUMBER_NODES, NUMBER_NODES))
    U = np.zeros((NUMBER_NODES, 1))
    F = np.zeros((NUMBER_NODES, 1))

    F[-1] = 1000.0

    # Calculate the global stiffness matrix
    for i, e in enumerate(NODE_CONNECTIONS):
        K += stiffness(e, i, E, A, dL)

    PRESCRIBED_NODES = [0]
    ACTIVE_NODES = np.setdiff1d(np.arange(0, NUMBER_NODES), PRESCRIBED_NODES)

    ACTIVE_FORCE = F[ACTIVE_NODES]
    ACTIVE_STIFFNESS = K[np.ix_(ACTIVE_NODES, ACTIVE_NODES)]

    # Solve for nodal displacements
    U_SOLUTION = np.linalg.solve(ACTIVE_STIFFNESS, ACTIVE_FORCE)

    U[ACTIVE_NODES] = U_SOLUTION

    REACTIONS = K.dot(U)

    print("Forces / Reactions: ", REACTIONS)

    # Plotting the results
    plt.plot(X, U)
    plt.title("1D Bar Element - Tensile/Compressive Forces")
    plt.xlabel("X [m]")
    plt.ylabel("U [m]")
    plt.grid()
    plt.show()

if __name__ == "__main__":
    main()


"""
In this file are only codes copied from sklearn.
We did that:
1st-to make sure this package is autonomous
2nd-to be able to modify marginaly some pieces of code.

example of mifications of our own:
-change numpy.random to use random
-being able to compute squared norm as well as non-squared norm (kmeans++ vs kmedians++)
-add comment lines
"""

import warnings
import numpy as np
import random as random_pck # not to be confused with random from numpy
import scipy.sparse as sp

# Pairwise distances
def euclidean_distances(X, Y=None, Y_norm_squared=None,X_norm_squared=None, squared=False):
    # copied from sklearn in 2019
    """
    Considering the rows of X (and Y=X) as vectors, compute the
    distance matrix between each pair of vectors.
    For efficiency reasons, the euclidean distance between a pair of row
    vector x and y is computed as:
        dist(x, y) = sqrt(dot(x, x) - 2 * dot(x, y) + dot(y, y))

    Parameters
    ----------
    X : {array-like}, shape (n_samples_1, n_features)
    Y : {array-like}, shape (n_samples_2, n_features)
    Y_norm_squared : array-like, shape (n_samples_2, ), optional
        Pre-computed dot-products of vectors in Y (e.g.,
        ``(Y**2).sum(axis=1)``)
        May be ignored in some cases, see the note below.
    squared : boolean, optional
        Return squared Euclidean distances.
    X_norm_squared : array-like, shape = [n_samples_1], optional
        Pre-computed dot-products of vectors in X (e.g.,
        ``(X**2).sum(axis=1)``)
        May be ignored in some cases, see the note below.
    Notes
    -----
    To achieve better accuracy, `X_norm_squared` and `Y_norm_squared` may be
    unused if they are passed as ``float32``.
    Returns
    -------
    distances : array, shape (n_samples_1, n_samples_2)
    Examples
    --------
    >>> from sklearn.metrics.pairwise import euclidean_distances
    >>> X = [[0, 1], [1, 1]]
    >>> # distance between rows of X
    >>> euclidean_distances(X, X)
    array([[0., 1.],
           [1., 0.]])
    >>> # get distance to origin
    >>> euclidean_distances(X, [[0, 0]])
    array([[1.        ],
           [1.41421356]])
    See also
    --------
    paired_distances : distances betweens pairs of elements of X and Y.
    """

    if X_norm_squared is not None:
    #    XX = check_array(X_norm_squared)
        if XX.shape == (1, X.shape[0]):
            XX = XX.T
        elif XX.shape != (X.shape[0], 1):
            raise ValueError("Incompatible dimensions for X and X_norm_squared")

        if XX.dtype == np.float32:
            XX = None
    elif X.dtype == np.float32:
        XX = None
    else:
        XX = row_norms(X, squared=True)[:, np.newaxis]

    if X is Y and XX is not None:
        # shortcut in the common case euclidean_distances(X, X)
        YY = XX.T
    elif Y_norm_squared is not None:
        YY = np.atleast_2d(Y_norm_squared)

        if YY.shape != (1, Y.shape[0]):
            raise ValueError(
                "Incompatible dimensions for Y and Y_norm_squared")
        if YY.dtype == np.float32:
            YY = None
    elif Y.dtype == np.float32:
        YY = None
    else:
        YY = row_norms(Y, squared=True)[np.newaxis, :]

    distances = - 2 * np.dot(X,Y.T)#safe_sparse_dot(X, Y.T, dense_output=True)
    distances += XX
    distances += YY
    np.maximum(distances, 0, out=distances)

    # Ensure that distances between vectors and themselves are set to 0.0.
    # This may not be the case due to floating point rounding errors.
    if X is Y:
        np.fill_diagonal(distances, 0)
    if squared:
        return(distances)
    else:
        return(distances**0.5)

def row_norms(X, squared=False):
    # copied from sklearn in 2019
    """Row-wise (squared) Euclidean norm of X.
    Equivalent to np.sqrt((X * X).sum(axis=1))

    Parameters
    ----------
    X : array_like
        The input array
    squared : bool, optional (default = False)
        If True, return squared norms.
    Returns
    -------
    array_like
        The row-wise (squared) Euclidean norm of X.
    """

    norms = np.einsum('ij,ij->i', X, X)

    if not squared:
        np.sqrt(norms, norms)
    return norms

# def check_pairwise_arrays(X, Y, precomputed=False, dtype=None):
#     """ Set X and Y appropriately and checks inputs
#     If Y is None, it is set as a pointer to X (i.e. not a copy).
#     If Y is given, this does not happen.
#     All distance metrics should use this function first to assert that the
#     given parameters are correct and safe to use.
#     Specifically, this function first ensures that both X and Y are arrays,
#     then checks that they are at least two dimensional while ensuring that
#     their elements are floats (or dtype if provided). Finally, the function
#     checks that the size of the second dimension of the two arrays is equal, or
#     the equivalent check for a precomputed distance matrix.
#     Parameters
#     ----------
#     X : {array-like}, shape (n_samples_a, n_features)
#     Y : {array-like}, shape (n_samples_b, n_features)
#     precomputed : bool
#         True if X is to be treated as precomputed distances to the samples in
#         Y.
#     dtype : string, type, list of types or None (default=None)
#         Data type required for X and Y. If None, the dtype will be an
#         appropriate float type selected by _return_float_dtype.
#         .. versionadded:: 0.18
#     Returns
#     -------
#     safe_X : {array-like}, shape (n_samples_a, n_features)
#         An array equal to X, guaranteed to be a numpy array.
#     safe_Y : {array-like}, shape (n_samples_b, n_features)
#         An array equal to Y if Y was not None, guaranteed to be a numpy array.
#         If Y was None, safe_Y will be a pointer to X.
#     """
#     X, Y, dtype_float = _return_float_dtype(X, Y)

#     estimator = 'check_pairwise_arrays'
#     if dtype is None:
#         dtype = dtype_float

#     if Y is X or Y is None:
#         X = Y = check_array(X, accept_sparse='csr', dtype=dtype,estimator=estimator)
#     else:
#         X = check_array(X, accept_sparse='csr', dtype=dtype,estimator=estimator)
#         Y = check_array(Y, accept_sparse='csr', dtype=dtype,estimator=estimator)

#     if precomputed:
#         if X.shape[1] != Y.shape[0]:
#             raise ValueError("Precomputed metric requires shape "
#                              "(n_queries, n_indexed). Got (%d, %d) "
#                              "for %d indexed." %
#                              (X.shape[0], X.shape[1], Y.shape[0]))
#     elif X.shape[1] != Y.shape[1]:
#         raise ValueError("Incompatible dimension for X and Y matrices: "
#                          "X.shape[1] == %d while Y.shape[1] == %d" % (
#                              X.shape[1], Y.shape[1]))

#     return X, Y

def stable_cumsum(arr, axis=None, rtol=1e-05, atol=1e-08):
    # copied from https://github.com/scikit-learn/scikit-learn/blob/main/sklearn/utils/extmath.py
    # in 2019
    """Use high precision for cumsum and check that final value matches sum
    Parameters
    ----------
    arr : array-like
        To be cumulatively summed as flat
    axis : int, optional
        Axis along which the cumulative sum is computed.
        The default (None) is to compute the cumsum over the flattened array.
    rtol : float
        Relative tolerance, see ``np.allclose``
    atol : float
        Absolute tolerance, see ``np.allclose``
    """
    out = np.cumsum(arr, axis=axis, dtype=np.float64)
    expected = np.sum(arr, axis=axis, dtype=np.float64)
    if not np.all(np.isclose(out.take(-1, axis=axis), expected, rtol=rtol,
                             atol=atol, equal_nan=True)):
        warnings.warn('cumsum was found to be unstable: '
                      'its last element does not correspond to sum',
                      RuntimeWarning)
    return out

def kmedianpp_init(X, n_clusters, x_norms, random_state=None, n_local_trials=None,squared=False):
    # function _k_init from sklearn, can be found here: https://atavory.github.io/ibex/_modules/sklearn/cluster/k_means_.html
    # copied in 2019 an manually modified to
    """Init n_clusters seeds according to k-means++
    Parameters
    ----------
    X : array or sparse matrix, shape (n_samples, n_features)
        The data to pick seeds for. To avoid memory copy, the input data
        should be double precision (dtype=np.float64).
    n_clusters : integer
        The number of seeds to choose
    x_norms : array, shape (n_samples,)
        use squared Euclidean norm of each data point for kmeans++ init,
        or use L1 norm of each point for kmedians++ init.
    random_state : int, RandomState instance, optional
        The generator used to initialize the centers. Use an int to make the
        randomness deterministic. Default is None.
        See :term:`Glossary <random_state>`.
    n_local_trials : integer, optional
        The number of seeding trials for each center (except the first),
        of which the one reducing inertia the most is greedily chosen.
        Set to None to make the number of trials depend logarithmically
        on the number of seeds (2+log(k)); this is the default.
    squared : if False then compute D(x) else compute D(x)**2

    Notes
    -----
    Selects initial cluster centers for k-means clustering in a smart way
    to speed up convergence. see: Arthur, D. and Vassilvitskii, S.
    "k-means++: the advantages of careful seeding". ACM-SIAM symposium
    on Discrete algorithms. 2007
    Version ported from http://www.stanford.edu/~darthur/kMeansppTest.zip,
    which is the implementation used in the aforementioned paper.
    This version has been updated such as kmedianpp can be used
    """
    n_samples, n_features = X.shape

    centers = np.empty((n_clusters, n_features), dtype=X.dtype)

    assert x_norms is not None, 'x_norms None in kmedianpp_init'

    # Set the number of local seeding trials if none is given
    if n_local_trials is None:
        # This is what Arthur/Vassilvitskii tried, but did not report
        # specific results for other than mentioning in the conclusion
        # that it helped.
        n_local_trials = 2 + int(np.log(n_clusters))

    # instantiate random generator
    if random_state is not None:
        random_pck.seed(random_state)

    # Pick first center randomly
    # version numpy
    # center_id = np.random.randint(n_samples, size=1) # numpy randint take from 0 to n_samples excluded, modified on 2022 august 25 by GENETAY edouard
    # version random
    center_id = random_pck.randint(0,n_samples-1) # random randint takes from 0 to n_samples-1 included, modified on 2022 august 25 by GENETAY edouard

    if sp.issparse(X):
        centers[0] = X[center_id].toarray()
    else:
        centers[0] = X[center_id]

    # Initialize list of closest distances and calculate current potential
    closest_dist_sq = euclidean_distances(
        centers[0, np.newaxis], X, Y_norm_squared=x_norms, squared=True # could also be False. it does not impact the distribution to pick centroids.
    )
    current_pot = closest_dist_sq.sum()

    # Pick the remaining n_clusters-1 points
    for c in range(1, n_clusters):
        # Choose center candidates by sampling with probability proportional
        # to the squared distance to the closest existing center

        # -------------------------
        # original code
        # rand_vals = np.random.random_sample(n_local_trials) * current_pot
        #
        # Replaced by
        rand_vals = np.array([random_pck.random() * current_pot for _ in range(n_local_trials)])
        #
        # manual replacement by GENETAY Edouard, 2022 august 24 in order to use random instead of numpy
        # so that code relies one random generator only. random was preferred over numpy because
        # discussions found on the internet point out that numpy.random.seed() is not stable though threads
        # and since KbMOM may evolve to work in parallel, random seems more appropriate.
        #--------------------------

        # cumsum(closest_dist_sq) is CMF (cumulative masse function) of the norms, rand_vals is U ~ Unif(0,current_pot),
        # candidate_ids is the new random_centroids = CMF^(-1)(U)
        # the distribution to pick new centroids is indeed proportional to D(X) or D(X)^2 depending on euclidean_distances(....,squared=...)
        candidate_ids = np.searchsorted(stable_cumsum(closest_dist_sq),
                                        rand_vals)

        # Compute distances to center candidates
        distance_to_candidates = euclidean_distances(
            X[candidate_ids], X, Y_norm_squared=x_norms, squared=True)

        # Decide which candidate is the best
        best_candidate = None
        best_pot = None
        best_dist_sq = None
        for trial in range(n_local_trials):
            # Compute potential when including center candidate
            new_dist_sq = np.minimum(closest_dist_sq,
                                     distance_to_candidates[trial])
            new_pot = new_dist_sq.sum()

            # Store result if it is the best local trial so far
            if (best_candidate is None) or (new_pot < best_pot):
                best_candidate = candidate_ids[trial]
                best_pot = new_pot
                best_dist_sq = new_dist_sq

        # Permanently add best center candidate found in local tries
        if sp.issparse(X):
            centers[c] = X[best_candidate].toarray()
        else:
            centers[c] = X[best_candidate]
        current_pot = best_pot
        closest_dist_sq = best_dist_sq

    return centers


import marshmallow as ma
from oarepo_model_builder.datatypes import DataTypeComponent, ModelDataType, Section
from oarepo_model_builder.datatypes.components import DefaultsModelComponent
from oarepo_model_builder.datatypes.components.model.utils import set_default

def get_community_records_schema():
    from ..communities import CommunityRecordsDataType

    return CommunityRecordsDataType.validator()

class CommunityRecordsComponent(DataTypeComponent):
    eligible_datatypes = [ModelDataType]
    affects = [DefaultsModelComponent]

    class ModelSchema(ma.Schema):
        community_records = ma.fields.Nested(
            get_community_records_schema,
            data_key="community-records",
            attribute="community-records",
        )

    def process_links(self, datatype, section: Section, **kwargs):
        if datatype.root.profile == "community_records":
            section.config = {}

    def before_model_prepare(self, datatype, *, context, **kwargs):
        if datatype.root.profile == "community_records":
            set_default(datatype, "permissions", {}).setdefault("skip", True)


import numpy as np
import pylab as pl
from scipy.optimize import curve_fit
from tqdm import tqdm

def FO_fit(x, y, dy, j_min, j_max, n_term1, n_term2):
    def poly(z, *c):
        poly = 0
        for i in range(n_term):
            poly = poly + c[i] * z**i
        return poly

    # TAGLIO LE MISURE ENTRO L'INTERVALLO DESIDERATO
    mask = ((x>=j_min) & (x<=j_max))
    x = x[mask]
    y = y[mask]
    dy = dy[mask]
    cc = []
    red_chisq = np.array([])
    term = np.array([])

    initParam = np.zeros(n_term1)
    for n_term in tqdm(range(n_term1, n_term2)):
        popt, pcov = curve_fit(poly, x, y, p0=initParam, sigma = dy, absolute_sigma=True)
        initParam =  np.zeros(n_term+1)

        chiq = np.sum(( (y - poly(x, *popt)) / dy )**2)
        ndof = len(y) - len(initParam)
        red_chiq = chiq/ndof
        red_chisq = np.append(red_chisq, red_chiq)
        term = np.append(term, n_term)
        cc.append(popt[0:])

    delta = np.abs(1-red_chisq)
    p = cc[np.argmin(delta)]
    red_chisq_opt = red_chisq[np.argmin(delta)]
    term_opt = term[np.argmin(delta)]

    return p, red_chisq_opt, term_opt

def FO_intersection(path_filenames, j_min, j_max, l_min, l_max, n_term_min, n_term_max, inter_min, inter_max, ctrl_plot):
    coeffs = []

    # TROVO I VARI COEFFICIENTI PER OGNI TAGLIA
    # POI CERCO LA RADICE A DUE A DUE
    n_term1 = n_term_min
    n_term2 = n_term_max
    for fname in path_filenames:
        with open(fname) as infile:
            aux_L, aux_J, aux_K, aux_ene_sp, aux_err_ene_sp, aux_ene_g, aux_err_ene_g, aux_ene_dens, aux_err_ene_dens, aux_susc, aux_err_susc, aux_G_pm, aux_err_G_pm, aux_C, aux_err_C, aux_U, aux_err_U, aux_corr_len, aux_err_corr_len, aux_K2_g, aux_err_K2_g, aux_K2_sp, aux_err_K2_sp, aux_K3_g, aux_err_K3_g, aux_K3_sp, aux_err_K3_sp = np.genfromtxt(infile, delimiter ="\t", unpack = True)

            if((aux_L[0]>=l_min) & (aux_L[0]<=l_max)):
                sorted_J = aux_J[aux_J.argsort()]
                sorted_U = aux_U[aux_J.argsort()]
                sorted_err_U = aux_err_U[aux_J.argsort()]
                sorted_corr_len = aux_corr_len[aux_J.argsort()]
                sorted_err_corr_len = aux_err_corr_len[aux_J.argsort()]
                sorted_L = aux_L[aux_J.argsort()]

                # p, red_chisq_opt_FO, n_term_opt = FO_fit(sorted_J, sorted_U, sorted_err_U, j_min, j_max, n_term1, n_term2)
                # coeffs.append(p)

                p, red_chisq_opt_FO, n_term_opt = FO_fit(sorted_J, sorted_corr_len/sorted_L, sorted_err_corr_len/sorted_L, j_min, j_max, n_term1, n_term2)
                coeffs.append(p)

                n_term1 = int(n_term_opt)
                n_term2 = int(n_term_opt+1)

                # FARE PLOT DI CONTROLLO
                if(ctrl_plot == True):
                    curve = np.polynomial.Polynomial(p)
                    x = np.linspace(sorted_J.min(), sorted_J.max(), 10000)
                    # pl.errorbar(sorted_J, sorted_U, sorted_err_U, ls = '', fillstyle = 'none', marker = 'o', label = 'L='+str(int(aux_L[0])))
                    pl.errorbar(sorted_J, sorted_corr_len/sorted_L, sorted_err_corr_len/sorted_L, ls = '', fillstyle = 'none', marker = 'o', label = 'L='+str(int(aux_L[0])))
                    pl.plot(x, curve(x))
                    pl.xlim(j_min, j_max)
                    pl.legend()

    if(ctrl_plot == True):
        pl.show()

    intersection = np.array([])

    for i in range(len(coeffs)-1):
        roots = np.polynomial.polynomial.Polynomial.roots(np.polynomial.Polynomial(coeffs[i+1]) - np.polynomial.Polynomial(coeffs[i]))
        roots = np.real(roots[np.isreal(roots)])
        mask = ((roots>=inter_min) & (roots<=inter_max))
        masked_roots = roots[mask]
        intersection = np.append(intersection, masked_roots)

    mean_intersect = np.mean(intersection)
    err_intersect = (max(intersection) - min(intersection)) / 2.0

    return mean_intersect, err_intersect, n_term_opt, red_chisq_opt_FO


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Dec  9 20:03:26 2021

@author: lby
"""
import torch
import torch.nn as nn
from torchvision import datasets, transforms
from torch.utils.data import DataLoader
import torch.nn.functional as F
import pandas as pd

ARCHITECTURES = ['SQU','ASC','DES','SYM','ENC']
ACTIVATION_FS = ['ReLu','Selu']
DROPOUT = [0, 0.5]
HIDDEN_LAYERS = [3, 5, 7, 1]
NEURONS = [4, 8, 16, 32]
def get_units(idx,neurons,architecture,layers=None):
    assert architecture in ARCHITECTURES
    if architecture == 'SQU':
        return neurons
    elif architecture == 'ASC':
        return (2**idx)*neurons
    elif architecture == 'DES':
        return (2**(layers-1-idx))*neurons
    elif architecture=='SYM':
        assert (layers is not None and layers > 2)
        if layers%2==1:
            return neurons*2**(int(layers/2) - abs(int(layers/2)-idx))
        else:
            x = int(layers/2)
            idx  = idx if idx < x else 2*x-idx -1
            return neurons*2**(int(layers/2) - abs(int(layers/2)-idx))

    elif architecture=='ENC':
        assert (layers is not None and layers > 2)
        if layers%2==0:
            x = int(layers/2)
            idx  = idx if idx < x else 2*x-idx -1
            return neurons*2**(int(layers/2)-1 -idx)
        else:
            x = int(layers/2)
            idx  = idx if idx < x else 2*x-idx
            return neurons*2**(int(layers/2) -idx)

def Data_Loader(batchsize):
    transform = transforms.ToTensor()

    train_data = datasets.CIFAR10(root='../Data', train=True, download=True, transform=transform)
    test_data = datasets.CIFAR10(root='../Data', train=False, download=True, transform=transform)

    torch.manual_seed(101)  # for consistent results

    train_loader = DataLoader(train_data, batch_size=batchsize, shuffle=True)

    test_loader = DataLoader(test_data, batch_size=batchsize, shuffle=False)

    return train_loader, test_loader

class nn_config():
    def __init__(self, architecture, activation_f, dropout, hidden_layers, neurons):
        self.architecture = architecture
        self.activation_f = activation_f
        self.dropout = dropout
        self.hidden_layers = hidden_layers
        self.neurons = neurons

        self.get_neurons()

    def get_neurons(self):
        neurons_units = []
        for i in range(self.hidden_layers):
            neurons = self.neurons
            units = get_units((i), neurons, self.architecture, self.hidden_layers)
            neurons_units.append(units)

        self.hidden_neurons_list = neurons_units

class NetworkModel(nn.Module,nn_config):
    def __init__(self, architecture, activation_f, dropout, hidden_layers, neurons):
        nn.Module.__init__(self)
        nn_config.__init__(self, architecture, activation_f, dropout, hidden_layers, neurons)

        # mnist dataset dimension
        n_in = 32*32*3
        n_out = 10
        # create a layer list

        layerlist = []
        for i in self.hidden_neurons_list:
            layerlist.append(nn.Linear(n_in,i))
            if self.activation_f == 'ReLu':
                layerlist.append(nn.ReLU(inplace=True))
            else:
                layerlist.append(nn.SELU(inplace=True))
            layerlist.append(nn.Dropout(self.dropout))
            n_in = i
        layerlist.append(nn.Linear(self.hidden_neurons_list[-1],n_out))
        self.layers = nn.Sequential(*layerlist)

    def forward(self, x):
        return F.log_softmax(self.layers(x),dim=1)

search_space = pd.read_csv('mnist_search_table.txt', index_col=None,header=None)
for instance in range(search_space.shape[0]):

    # dropout
    if search_space.iloc[instance,0] == 0:
        dropout = DROPOUT[0]
    else:
        dropout = DROPOUT[1]

    # hidden layer
    if search_space.iloc[instance,1] == 1/3:
        hidden_layers = HIDDEN_LAYERS[0]
    else:
        hidden_layers = HIDDEN_LAYERS[1]

    # neurons
    if search_space.iloc[instance,2] == 0:
        neurons = NEURONS[0]
    elif search_space.iloc[instance,2] < 1.1/7 and search_space.iloc[instance,2] > 0.9/7:
        neurons = NEURONS[1]
    elif search_space.iloc[instance,2] < 3.1/7 and search_space.iloc[instance,2] > 2.9/7:
        neurons = NEURONS[2]
    else:
        neurons = NEURONS[3]

    # activation function
    if search_space.iloc[instance,3] == 1:
        activation_f = ACTIVATION_FS[0]
    elif search_space.iloc[instance,4] == 1:
        activation_f = ACTIVATION_FS[1]
    else:
        print('activation function error')
        break

    # architectures
    if search_space.iloc[instance,5] == 1:
        architecture = ARCHITECTURES[0]
    elif search_space.iloc[instance,6] == 1:
        architecture = ARCHITECTURES[1]
    elif search_space.iloc[instance,7] == 1:
        architecture = ARCHITECTURES[2]
    elif search_space.iloc[instance,8] == 1:
        architecture = ARCHITECTURES[3]
    elif search_space.iloc[instance,9] == 1:
        architecture = ARCHITECTURES[4]
    else:
        print('architecture error')
        break

    model = NetworkModel(architecture, activation_f, dropout, hidden_layers, neurons)

    # stay with the dataset2vec code
    epochs = 10000
    batch_size = 16
    criterion = nn.CrossEntropyLoss()
    optimizer = torch.optim.Adam(model.parameters(), lr=0.001)

    train_loader, test_loader = Data_Loader(batch_size)

    for i in range(epochs):
        for b, (X_train, y_train) in enumerate(train_loader):

            y_pred = model(X_train.view(batch_size,-1))
            loss = criterion(y_pred,y_train)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

            if b>0:
                break
    tst_corr = 0
    with torch.no_grad():
            for b, (X_test, y_test) in enumerate(test_loader):

                # Apply the model
                y_val = model(X_test.view(batch_size, -1))  # Here we flatten X_test

                # Tally the number of correct predictions
                predicted = torch.max(y_val.data, 1)[1]
                batch_corr = (predicted == y_test).sum()
                tst_corr += batch_corr
            acc = tst_corr/10000
            search_space.iloc[instance,10] = acc.item()
    print(f'[{instance}/255] dropout:{dropout} act_f:{activation_f} layers:{hidden_layers} neurons:{neurons} archi:{architecture} acc:{acc}')
search_space.to_csv('cifar10_mnist_search_table.txt' ,header=None)


from typing import List, Tuple

import os
import subprocess

def configure_and_build(source_path: str, project_path: str, options: List[Tuple[str, str]]):
    base_configure_cmd = ['cmake',
                          '-B{}'.format(project_path),
                          '-H{}'.format(source_path),
                          '-DCMAKE_BUILD_TYPE=Debug',
                          '-DCATCH_DEVELOPMENT_BUILD=ON']
    for option, value in options:
        base_configure_cmd.append('-D{}={}'.format(option, value))
    try:
        subprocess.run(base_configure_cmd,
                       stdout = subprocess.PIPE,
                       stderr = subprocess.STDOUT,
                       check = True)
    except subprocess.SubprocessError as ex:
        print("Could not configure build to '{}' from '{}'".format(project_path, source_path))
        print("Return code: {}".format(ex.returncode))
        print("output: {}".format(ex.output))
        raise
    print('Configuring {} finished'.format(project_path))

    build_cmd = ['cmake',
                 '--build', '{}'.format(project_path),
                 # For now we assume that we only need Debug config
                 '--config', 'Debug']
    try:
        subprocess.run(build_cmd,
                       stdout = subprocess.PIPE,
                       stderr = subprocess.STDOUT,
                       check = True)
    except subprocess.SubprocessError as ex:
        print("Could not build project in '{}'".format(project_path))
        print("Return code: {}".format(ex.returncode))
        print("output: {}".format(ex.output))
        raise
    print('Building {} finished'.format(project_path))

def run_and_return_output(base_path: str, binary_name: str, other_options: List[str]) -> Tuple[str, str]:
    # For now we assume that Windows builds are done using MSBuild under
    # Debug configuration. This means that we need to add "Debug" folder
    # to the path when constructing it. On Linux, we don't add anything.
    config_path = "Debug" if os.name == 'nt' else ""
    full_path = os.path.join(base_path, config_path, binary_name)

    base_cmd = [full_path]
    base_cmd.extend(other_options)

    try:
        ret = subprocess.run(base_cmd,
                             stdout = subprocess.PIPE,
                             stderr = subprocess.PIPE,
                             check = True,
                             universal_newlines = True)
    except subprocess.SubprocessError as ex:
        print('Could not run "{}"'.format(base_cmd))
        print('Args: "{}"'.format(other_options))
        print('Return code: {}'.format(ex.returncode))
        print('stdout: {}'.format(ex.stdout))
        print('stderr: {}'.format(ex.stdout))
        raise

    return (ret.stdout, ret.stderr)


"""SMART test definition.

:test department:  OI
:test region:      REGION_PLACEHOLDER
:test environment: ENVIRONMENT_PLACEHOLDER
:test name:        Search Test
:test version:     1.1.1
:test owner:       OI2.0
:test author:      Yauheni.Skavarodkin@epam.com
:test description: https://confluence.wolterskluwer.io/display/GOMT/stdef__OI_Search_Test
"""
import random

from selenium.webdriver.support.ui import WebDriverWait, Select
from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as econdition
from selenium.webdriver.common.keys import Keys

from tdef import test_step, ATF_FNAME, TestStepExecutionError, NoTimeTrace, helper

def close_popups(wk):
    try:
        ok_button = wk.driver.find_element_by_id("AcceptB-All")

        if ok_button.is_displayed():
            ok_button.click()
    except WebDriverException:
        pass

@test_step(name='1. NavigateToMainPage_and_Searh')
def step1(driver, run_id, **params):
    """Navigate to main page and Login.

    :step description: Navigate to google.com page searching Selenium part.
    """
    driver.wk = helper.WKWebDriverHelper(
        driver,
        wait_timeout=params['wdrv_cond_wait_timeout'],
        wait_poll_frequency=params['wdrv_cond_poll_frequency'],
        close_popups=close_popups,
    )
    driver.wk.navigate(params['wapp_url'])
    driver.wk.wait_page_loaded()

    driver.wk.send_keys((By.XPATH, "//input[@type='text']"), 'selenium*')
    driver.wk.send_keys((By.XPATH, "//input[@type='text']"), Keys.ENTER, clear=False)
    driver.wk.wait_page_loaded()

@test_step(name='2. OpenFirstlink')
def step1(driver, run_id, **params):
    """Navigate to main page and Login.

    :step description: Forwarding to fist page of the list / Make sure the Page loads correctly.
    """

    driver.wk.wait_and_click((By.XPATH, "//h3[contains(@class,'LC20lb MBeuO DKV0Md')]"))
    driver.wk.wait_page_loaded()

@test_step(name='3. MakeSureCorrectLink')
def step1(driver, run_id, **params):
    """Navigate to main page and Login.

    :step description: Make Sure to Enter Correct link content link text.
    """
    driver.wk.wait_element_present((By.XPATH, "//*[contains(text(),'selenium')]"))
    driver.wk.wait_page_loaded()


from __future__ import print_function
from time import sleep
import argparse
import gattlib
import pygatt

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-a', '--adapter-name', type=str, help='Adapter name', default='hci0')
    args = parser.parse_args()

    print('Scanning for devices')
    service = gattlib.DiscoveryService(args.adapter_name)
    bd_addr = ''
    device_name = ''

    while bd_addr == '':
        devices = service.discover(4)
        for address, name in devices.items():
            bd_addr = address
            device_name = name

    print('Found device: {} {}'.format(device_name, bd_addr))
    print('connecting to: {} {}'.format(device_name, bd_addr))

    previous_battery_level = 0
    previous_hr_level = 0
    adapter = pygatt.GATTToolBackend()

    try:
        adapter.start()

        device = adapter.connect(bd_addr, timeout=20, address_type=pygatt.BLEAddressType.random)
        # use this to connect to a specific device:
        # device = adapter.connect('FC:A0:D7:76:E0:53', timeout=20, address_type=pygatt.BLEAddressType.random)
        print('Connected')

        while True:
            battery_level = ord(device.char_read("00002a19-0000-1000-8000-00805f9b34fb", timeout=20))
            if battery_level != previous_battery_level:
                print('Battery level: {}'.format(battery_level))
                previous_battery_level = battery_level
            hr_level = ord(device.char_read("00002a38-0000-1000-8000-00805f9b34fb", timeout=20))
            if hr_level != previous_hr_level:
                print('Heart rate level: {}'.format(hr_level))
                previous_hr_level = hr_level
            sleep(5)
    except KeyboardInterrupt:
        pass
    finally:
        print('Disconnecting')
        adapter.stop()

if __name__ == '__main__':
    main()


import numpy as np
import matplotlib.pyplot as plt

def getData():
  arch 		= open('TWTR.csv', 'r')
  adj_close = []
  for j in arch:
    var = list(j.strip().split(","))
    adj_close.append(var[5])
  adj_close.remove('Adj Close')
  arch.close()
  map(float, adj_close)
  return adj_close

#dt:   diferencial de tiempo
#b:    incremento browniano
#W:    camino browniano
def Brownian(seed, N):
  np.random.seed(seed)
  dt 	= 1./N
  b 	= np.random.normal(0., 1., int(N)) * np.sqrt(dt)
  W 	= np.cumsum(b)
  return W, b

def daily_returns(adj_close):
	returns = []
	for i in range(0, len(adj_close) - 1):
		today 			= float(adj_close[i+1])
		yesterday 		= float(adj_close[i])
		daily_return 	= (today - yesterday)/today
		returns.append(daily_return)
	return returns

#S0:		precio inicial
#mu:		media
#sigma:		desviacion
#W:			movimiento browniano
#T:			periodo de tiempo
#N:			total de incrementos
def GBM(S0, mu, sigma, W, T, N):
	t = np.linspace(0., 1., N+1)
	S = []
	S.append(S0)
	for i in range(1, int(N+1)):
		drift 		= (mu - 0.5*sigma**2) * t[i]
		diffusion 	= sigma * W[i-1]
		S_temp 		= S0*np.exp(drift + diffusion)
		S.append(S_temp)
	return S, t

def main(_iter):
	S = []
	mu 		= np.mean(daily_returns(getData()))*252.
	sigma 	= np.std(daily_returns(getData()))*np.sqrt(252.)
	for i in range(1, int(_iter)+1):
		seed 	= i
		W 		= Brownian(seed, N)[0]
		sol 	= GBM(S0, mu, sigma, W, T, N)[0]
		S_temp 	= np.mean(sol)
		S.append(S_temp)
	return S

def EM(S0, mu, sigma, b, T, N, M):
    dt 		= M * (1/N)
    L 		= N / M
    wi 		= [S0]
    for i in range(0,int(L)):
        Winc = np.sum(b[(M*(i-1)+M):(M*i + M)])
        w_i_new = wi[i]+mu*wi[i]*dt+sigma*wi[i]*Winc
        wi.append(w_i_new)
    return wi, dt

###inicialización de variables###
seed 	= np.random.randint(1, high = 100)
S0 		= float(getData()[0])
T 		= 5.
N 		= 251.
mu 		= np.mean(daily_returns(getData()))*252.
sigma 	= np.std(daily_returns(getData()))*np.sqrt(252.)
W 		= Brownian(seed, N)[0]
M 		= 1
L 		= N/M

###se obtiene una trayectoria random###
soln1 	= GBM(S0, mu, sigma, W, T, N)[0]
_time1 	= GBM(S0, mu, sigma, W, T, N)[1]

plt.plot(_time1, soln1, label='Trayectoria random', ls='--')

###otra trayectoria random###

seed 	= np.random.randint(1, high = 100)
W 		= Brownian(seed, N)[0]

soln2 	= GBM(S0, mu, sigma, W, T, N)[0]
_time2 	= GBM(S0, mu, sigma, W, T, N)[1]
plt.plot(_time2, soln2, label='Trayectoria random', ls='--')

###inicialización de método numérico###
seed 	= np.random.randint(1, high = 100)
W 		= Brownian(seed, N)[0]
b 		= Brownian(seed, N)[1]

em 		= EM(S0, mu, sigma, b, T, N, M)[0]
time 	= np.linspace(0., 1., L+1)

#print('Cantidad de trayectorias')
kk 	= input()
S 	= main(_iter = kk)
print('Valor estimado: ' , np.mean(S))

#actual = getData()
#tt = np.linspace(1, len(actual), len(actual))
#tt = [i/252. for i in tt]

#plt.plot(tt, actual)
plt.plot(time, em, label='Modelo MonteCarlo')
plt.ylabel('Precio Acción, $')
plt.title('Movimiento Browniano')
plt.legend(loc='best')
plt.show()


import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.transforms as transforms
from torch.utils.tensorboard import SummaryWriter
import torch.nn.functional as F
from torch.nn.utils.rnn import pack_padded_sequence
# from convert_text import get_loader
from utils import *
from model import *
from dataset import get_loader

data_location =  'flickr8k'
caption_file = 'flickr8k/captions.txt'

mean = [0.485, 0.456, 0.406]

std = [0.229, 0.224, 0.225]

transform = transforms.Compose(
    [transforms.Resize((256,256)),
    transforms.ToTensor(),
    transforms.Normalize(mean, std)]
)

train_loader, dataset = get_loader(
    root_folder = data_location+"/Images",
    annotation_file = data_location+"/captions.txt",
    transform = transform,
    batch_size = 1,
    num_workers = 4,
    test = False
)
test_loader, test_dataset = get_loader(
    root_folder = data_location+"/Images",
    annotation_file = data_location+"/captions.txt",
    transform = transform,
    num_workers = 4,
    test = True
)

# Test_dataset gonna come here soon
# Think about that later. We will do some training phases. It will take time but keep your calm.

torch.backends.cudnn.benchmark = True # Get some boost probaby

# device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = "cpu"
print(device)

load_model = False

save_model = False

train_CNN = False

# Hyperparameters

embed_size = 256
encoder_dim = 1024
decoder_dim = 400
attention_dim = 400
vocab_size = len(dataset.vocab)
learning_rate = 2e-4 # Earlier 3e-4
num_epochs = 21

# Tensorboard
writer = SummaryWriter("runs/flickr")

step = 0
# init model, loss
encoder = EncoderCNN() # Default arguments already given as encoder_size 14, train_CNN = False
encoder = encoder.to(device)

decoder = Decoder(encoder_dim, decoder_dim, embed_size, vocab_size, attention_dim, device)
decoder = decoder.to(device)

alpha_c = 1.0
# regularization parameter for 'doubly stochastic attention', as in the paper
# https://pytorch.org/tutorials/beginner/torchtext_translation_tutorial.html
# criterion = nn.CrossEntropyLoss(ignore_index=dataset.vocab.stoi["<PAD>"]).to(device)

criterion = nn.CrossEntropyLoss().to(device)

# Optimizer only for the decoder
optimizer = optim.Adam(decoder.parameters(), lr = learning_rate)

decoder.train()
if load_model:
    step = load_checkpoint(torch.load("../input/resnet15/LastModelResnet50_v2_16.pth.tar"), decoder, optimizer)

# Change the epoch before changing
for epoch in range(0, num_epochs):
    if save_model:
        checkpoint = {
            "state_dict" : decoder.state_dict(),
            "optimizer" : optimizer.state_dict(),
            "step" : step
        }

        if epoch % 5 == 0 and epoch != 0 or epoch > 15:
            filename = './LastModelResnet50_v2_' + str(epoch) +  '.pth.tar'
            save_checkpoint(checkpoint, filename)
    losses = []
    mvl = []
    for idx, (imgs, captions, lengths) in enumerate(train_loader):
        # optimizer.zero_grad() Init config

        imgs = imgs.to(device)
        captions = captions.to(device)
        lengths = lengths.to(device)
        # Pass through the encoder and get the annotation vector
        encoded_images = encoder(imgs)

        scores, alphas, sorted_cap, decode_lengths = decoder(encoded_images, captions, lengths)

        # We don't want <SOS>
        sorted_cap = sorted_cap[:,1:] # shape (batch_size, max_caption)

        # Remove timesteps that we didn't decode at, or are pads
        # pack_padded_sequence is an easy trick to do this

        scores = pack_padded_sequence(scores, decode_lengths, batch_first=True).data
        targets = pack_padded_sequence(sorted_cap, decode_lengths, batch_first=True).data

        # Calculate loss
        loss = criterion(scores, targets)

        # This method also works if you use ignore_index
        # batch_size = sorted_cap.shape[0]
        # caption_length = sorted_cap.shape[1]
        # loss = criterion(predictions.view(batch_size * caption_length, -1), sorted_cap.reshape(-1))

        #   Doubly stochastic attention regularization
        loss += 1.0 * ((1. - alphas.sum(dim=1)) ** 2).mean()

        losses.append(loss.item())

        decoder.zero_grad()
        encoder.zero_grad()

        loss.backward()

        print("Step", idx, loss.item())

        writer.add_scalar("training loss", loss.item(), global_step = step)

        step += 1

        optimizer.step()

        # This part is mostly manual for checking.

        if (idx + 1 )% 100 == 0:
            print("Epoch: {} loss: {:.5f}".format(epoch,loss.item()))

            decoder.eval()
            with torch.no_grad():
                # bleu_score_checker()
                dataiter = iter(train_loader)
                imgs,captions, lengths = next(dataiter)
                imgs = imgs
                captions = captions.to(device)
                encoded_output = encoder( (imgs[0].unsqueeze(0).to(device)) )

                # Does not make a difference for the caption since we are not using it
                caption_greedy, alphas = decoder.predict_caption(encoded_output, captions)
                caps_greedy =[dataset.vocab.itos[idx] for idx in caption_greedy]
                caption = ' '.join(caps_greedy)
                print("Greedy search", caption)

                caption = decoder.beam_search(encoded_output, 4)
                caps = [dataset.vocab.itos[idx] for idx in caption]
                print("Beam search", ' '.join(caps) )

                show_image(imgs[0],title=' '.join(caps))
            decoder.train()
            encoder.train() # Messed up while copying from kaggle kernel
                            # We are not training the encoder but eval switches off
                            # Batch norm and dropout

        # Valid loss without another dataloader. I created a manual split of images and captions
        if (idx + 1 ) % 200 == 0 :

            valid_losses = []

            decoder.eval() # V. Important.
            encoder.eval()

            print("Valid section")
            with torch.no_grad():
                for index, (imgs, captions, lengths) in enumerate(test_loader):

                    imgs = imgs.to(device)
                    captions = captions.to(device)
                    lengths = lengths.to(device)

                    encoded_images = encoder(imgs)
                    scores, alphas, sorted_cap, decode_lengths = decoder(encoded_images, captions, lengths)

                    # We don't want <SOS>
                    sorted_cap = sorted_cap[:,1:] # shape (batch_size, max_caption)

                    scores = pack_padded_sequence(scores, decode_lengths, batch_first=True).data
                    targets = pack_padded_sequence(sorted_cap, decode_lengths, batch_first=True).data

                    valid_loss = criterion(scores, targets)

                    # This also works if you use ignore_index.
                    # batch_size = sorted_cap.shape[0]
                    # caption_length = sorted_cap.shape[1]
                    #  valid_loss = criterion(predictions.view(batch_size * caption_length, -1), sorted_cap.reshape(-1))

                    valid_loss += 1.0 * ((1. - alphas.sum(dim=1)) ** 2).mean()

                    valid_losses.append(valid_loss.item())

                    print("Step", index, valid_loss.item())

            decoder.train()
            encoder.train()
            print("-" * 80)

            mean_valid_loss = sum(valid_losses)/len(valid_losses)

            mvl.append(mean_valid_loss)
            print(mean_valid_loss)

            print("-" * 80)

    mean_loss = sum(losses)/len(losses)
    print("Mean loss", mean_loss)
    print(mvl)