#! -*- coding:utf-8 -*-

from __future__ import print_function
import json
import numpy as np
from random import choice
from tqdm import tqdm
import pyhanlp
from gensim.models import Word2Vec
import re, os

mode = 0
char_size = 128
maxlen = 512

word2vec = Word2Vec.load('../word2vec_baike/word2vec_baike')

id2word = {i+1:j for i,j in enumerate(word2vec.wv.index2word)}
word2id = {j:i for i,j in id2word.items()}
word2vec = word2vec.wv.syn0
word_size = word2vec.shape[1]
word2vec = np.concatenate([np.zeros((1, word_size)), word2vec])

def tokenize(s):
    return [i.word for i in pyhanlp.HanLP.segment(s)]

def sent2vec(S):
    """S格式：[[w1, w2]]
    """
    V = []
    for s in S:
        V.append([])
        for w in s:
            for _ in w:
                V[-1].append(word2id.get(w, 0))
    V = seq_padding(V)
    V = word2vec[V]
    return V

total_data = json.load(open('../datasets/train_data_vote_me.json'))
id2predicate, predicate2id = json.load(open('../datasets/all_50_schemas_me.json'))
id2predicate = {int(i):j for i,j in id2predicate.items()}
id2char, char2id = json.load(open('../datasets/all_chars_me.json'))
num_classes = len(id2predicate)

if not os.path.exists('../random_order_vote.json'):
    random_order = range(len(total_data))
    np.random.shuffle(random_order)
    json.dump(
        random_order,
        open('../random_order_vote.json', 'w'),
        indent=4
    )
else:
    random_order = json.load(open('../random_order_vote.json'))

train_data = [total_data[j] for i, j in enumerate(random_order) if i % 8 != mode]
dev_data = [total_data[j] for i, j in enumerate(random_order) if i % 8 == mode]

predicates = {} # 格式：{predicate: [(subject, predicate, object)]}

def repair(d):
    d['text'] = d['text'].lower()
    something = re.findall(u'《([^《》]*?)》', d['text'])
    something = [s.strip() for s in something]
    zhuanji = []
    gequ = []
    for sp in d['spo_list']:
        sp[0] = sp[0].strip(u'《》').strip().lower()
        sp[2] = sp[2].strip(u'《》').strip().lower()
        for some in something:
            if sp[0] in some and d['text'].count(sp[0]) == 1:
                sp[0] = some
        if sp[1] == u'所属专辑':
            zhuanji.append(sp[2])
            gequ.append(sp[0])
    spo_list = []
    for sp in d['spo_list']:
        if sp[1] in [u'歌手', u'作词', u'作曲']:
            if sp[0] in zhuanji and sp[0] not in gequ:
                continue
        spo_list.append(tuple(sp))
    d['spo_list'] = spo_list

for d in train_data:
    repair(d)
    for sp in d['spo_list']:
        if sp[1] not in predicates:
            predicates[sp[1]] = []
        predicates[sp[1]].append(sp)

for d in dev_data:
    repair(d)

def random_generate(d, spo_list_key):
    r = np.random.random()
    if r > 0.5:
        return d
    else:
        k = np.random.randint(len(d[spo_list_key]))
        spi = d[spo_list_key][k]
        k = np.random.randint(len(predicates[spi[1]]))
        spo = predicates[spi[1]][k]
        F = lambda s: s.replace(spi[0], spo[0]).replace(spi[2], spo[2])
        text = F(d['text'])
        spo_list = [(F(sp[0]), sp[1], F(sp[2])) for sp in d[spo_list_key]]
        return {'text': text, spo_list_key: spo_list}

def seq_padding(X, padding=0):
    L = [len(x) for x in X]
    ML = max(L)
    return np.array([
        np.concatenate([x, [padding] * (ML - len(x))]) if len(x) < ML else x for x in X
    ])

class data_generator:
    def __init__(self, data, batch_size=64):
        self.data = data
        self.batch_size = batch_size
        self.steps = len(self.data) // self.batch_size
        if len(self.data) % self.batch_size != 0:
            self.steps += 1
    def __len__(self):
        return self.steps
    def __iter__(self):
        while True:
            idxs = range(len(self.data))
            np.random.shuffle(idxs)
            T1, T2, S1, S2, K1, K2, O1, O2, = [], [], [], [], [], [], [], []
            for i in idxs:
                spo_list_key = 'spo_list' # if np.random.random() > 0.5 else 'spo_list_with_pred'
                d = random_generate(self.data[i], spo_list_key)
                text = d['text'][:maxlen]
                text_words = tokenize(text)
                text = ''.join(text_words)
                items = {}
                for sp in d[spo_list_key]:
                    subjectid = text.find(sp[0])
                    objectid = text.find(sp[2])
                    if subjectid != -1 and objectid != -1:
                        key = (subjectid, subjectid+len(sp[0]))
                        if key not in items:
                            items[key] = []
                        items[key].append((objectid,
                                           objectid+len(sp[2]),
                                           predicate2id[sp[1]]))
                if items:
                    T1.append([char2id.get(c, 1) for c in text]) # 1是unk，0是padding
                    T2.append(text_words)
                    s1, s2 = np.zeros(len(text)), np.zeros(len(text))
                    for j in items:
                        s1[j[0]] = 1
                        s2[j[1]-1] = 1
                    k1, k2 = np.array(items.keys()).T
                    k1 = choice(k1)
                    k2 = choice(k2[k2 >= k1])
                    o1, o2 = np.zeros((len(text), num_classes)), np.zeros((len(text), num_classes))
                    for j in items.get((k1, k2), []):
                        o1[j[0]][j[2]] = 1
                        o2[j[1]-1][j[2]] = 1
                    S1.append(s1)
                    S2.append(s2)
                    K1.append([k1])
                    K2.append([k2-1])
                    O1.append(o1)
                    O2.append(o2)
                    if len(T1) == self.batch_size or i == idxs[-1]:
                        T1 = seq_padding(T1)
                        T2 = sent2vec(T2)
                        S1 = seq_padding(S1)
                        S2 = seq_padding(S2)
                        O1 = seq_padding(O1, np.zeros(num_classes))
                        O2 = seq_padding(O2, np.zeros(num_classes))
                        K1, K2 = np.array(K1), np.array(K2)
                        yield [T1, T2, S1, S2, K1, K2, O1, O2], None
                        T1, T2, S1, S2, K1, K2, O1, O2, = [], [], [], [], [], [], [], []

from keras.layers import *
from keras.models import Model
import keras.backend as K
from keras.callbacks import Callback
from keras.optimizers import Adam

def seq_gather(x):
    """seq是[None, seq_len, s_size]的格式，
    idxs是[None, 1]的格式，在seq的第i个序列中选出第idxs[i]个向量，
    最终输出[None, s_size]的向量。
    """
    seq, idxs = x
    idxs = K.cast(idxs, 'int32')
    batch_idxs = K.arange(0, K.shape(seq)[0])
    batch_idxs = K.expand_dims(batch_idxs, 1)
    idxs = K.concatenate([batch_idxs, idxs], 1)
    return K.tf.gather_nd(seq, idxs)

def seq_maxpool(x):
    """seq是[None, seq_len, s_size]的格式，
    mask是[None, seq_len, 1]的格式，先除去mask部分，
    然后再做maxpooling。
    """
    seq, mask = x
    seq -= (1 - mask) * 1e10
    return K.max(seq, 1, keepdims=True)

def dilated_gated_conv1d(seq, mask, dilation_rate=1):
    """膨胀门卷积（残差式）
    """
    dim = K.int_shape(seq)[-1]
    h = Conv1D(dim * 2, 3, padding='same', dilation_rate=dilation_rate)(seq)
    def _gate(x):
        dropout_rate = 0.1
        s, h = x
        g, h = h[:, :, :dim], h[:, :, dim:]
        g = K.in_train_phase(K.dropout(g, dropout_rate), g)
        g = K.sigmoid(g)
        return g * s + (1 - g) * h
    seq = Lambda(_gate)([seq, h])
    seq = Lambda(lambda x: x[0] * x[1])([seq, mask])
    return seq

class Attention(Layer):
    """多头注意力机制
    """
    def __init__(self, nb_head, size_per_head, **kwargs):
        self.nb_head = nb_head
        self.size_per_head = size_per_head
        self.out_dim = nb_head * size_per_head
        super(Attention, self).__init__(**kwargs)
    def build(self, input_shape):
        super(Attention, self).build(input_shape)
        q_in_dim = input_shape[0][-1]
        k_in_dim = input_shape[1][-1]
        v_in_dim = input_shape[2][-1]
        self.q_kernel = self.add_weight(name='q_kernel',
                                        shape=(q_in_dim, self.out_dim),
                                        initializer='glorot_normal')
        self.k_kernel = self.add_weight(name='k_kernel',
                                        shape=(k_in_dim, self.out_dim),
                                        initializer='glorot_normal')
        self.v_kernel = self.add_weight(name='w_kernel',
                                        shape=(v_in_dim, self.out_dim),
                                        initializer='glorot_normal')
    def mask(self, x, mask, mode='mul'):
        if mask is None:
            return x
        else:
            for _ in range(K.ndim(x) - K.ndim(mask)):
                mask = K.expand_dims(mask, K.ndim(mask))
            if mode == 'mul':
                return x * mask
            else:
                return x - (1 - mask) * 1e10
    def call(self, inputs):
        q, k, v = inputs[:3]
        v_mask, q_mask = None, None
        if len(inputs) > 3:
            v_mask = inputs[3]
            if len(inputs) > 4:
                q_mask = inputs[4]
        # 线性变换
        qw = K.dot(q, self.q_kernel)
        kw = K.dot(k, self.k_kernel)
        vw = K.dot(v, self.v_kernel)
        # 形状变换
        qw = K.reshape(qw, (-1, K.shape(qw)[1], self.nb_head, self.size_per_head))
        kw = K.reshape(kw, (-1, K.shape(kw)[1], self.nb_head, self.size_per_head))
        vw = K.reshape(vw, (-1, K.shape(vw)[1], self.nb_head, self.size_per_head))
        # 维度置换
        qw = K.permute_dimensions(qw, (0, 2, 1, 3))
        kw = K.permute_dimensions(kw, (0, 2, 1, 3))
        vw = K.permute_dimensions(vw, (0, 2, 1, 3))
        # Attention
        a = K.batch_dot(qw, kw, [3, 3]) / self.size_per_head**0.5
        a = K.permute_dimensions(a, (0, 3, 2, 1))
        a = self.mask(a, v_mask, 'add')
        a = K.permute_dimensions(a, (0, 3, 2, 1))
        a = K.softmax(a)
        # 完成输出
        o = K.batch_dot(a, vw, [3, 2])
        o = K.permute_dimensions(o, (0, 2, 1, 3))
        o = K.reshape(o, (-1, K.shape(o)[1], self.out_dim))
        o = self.mask(o, q_mask, 'mul')
        return o
    def compute_output_shape(self, input_shape):
        return (input_shape[0][0], input_shape[0][1], self.out_dim)

t1_in = Input(shape=(None,))
t2_in = Input(shape=(None, word_size))
s1_in = Input(shape=(None,))
s2_in = Input(shape=(None,))
k1_in = Input(shape=(1,))
k2_in = Input(shape=(1,))
o1_in = Input(shape=(None, num_classes))
o2_in = Input(shape=(None, num_classes))

t1, t2, s1, s2, k1, k2, o1, o2 = t1_in, t2_in, s1_in, s2_in, k1_in, k2_in, o1_in, o2_in
mask = Lambda(lambda x: K.cast(K.greater(K.expand_dims(x, 2), 0), 'float32'))(t1)

def position_id(x):
    if isinstance(x, list) and len(x) == 2:
        x, r = x
    else:
        r = 0
    pid = K.arange(K.shape(x)[1])
    pid = K.expand_dims(pid, 0)
    pid = K.tile(pid, [K.shape(x)[0], 1])
    return K.abs(pid - K.cast(r, 'int32'))

pid = Lambda(position_id)(t1)
position_embedding = Embedding(maxlen, char_size, embeddings_initializer='zeros')
pv = position_embedding(pid)

t1 = Embedding(len(char2id)+2, char_size)(t1) # 0: padding, 1: unk
t2 = Dense(char_size, use_bias=False)(t2) # 词向量也转为同样维度
t = Add()([t1, t2, pv]) # 字向量、词向量、位置向量相加
t = Dropout(0.25)(t)
t = Lambda(lambda x: x[0] * x[1])([t, mask])
t = dilated_gated_conv1d(t, mask, 1)
t = dilated_gated_conv1d(t, mask, 2)
t = dilated_gated_conv1d(t, mask, 5)
t = dilated_gated_conv1d(t, mask, 1)
t = dilated_gated_conv1d(t, mask, 2)
t = dilated_gated_conv1d(t, mask, 5)
t = dilated_gated_conv1d(t, mask, 1)
t = dilated_gated_conv1d(t, mask, 2)
t = dilated_gated_conv1d(t, mask, 5)
t = dilated_gated_conv1d(t, mask, 1)
t = dilated_gated_conv1d(t, mask, 1)
t = dilated_gated_conv1d(t, mask, 1)
t_dim = K.int_shape(t)[-1]

pn1 = Dense(char_size, activation='relu')(t)
pn1 = Dense(1, activation='sigmoid')(pn1)
pn2 = Dense(char_size, activation='relu')(t)
pn2 = Dense(1, activation='sigmoid')(pn2)

h = Attention(8, 16)([t, t, t, mask])
h = Concatenate()([t, h])
h = Conv1D(char_size, 3, activation='relu', padding='same')(h)
ps1 = Dense(1, activation='sigmoid')(h)
ps2 = Dense(1, activation='sigmoid')(h)
ps1 = Lambda(lambda x: x[0] * x[1])([ps1, pn1])
ps2 = Lambda(lambda x: x[0] * x[1])([ps2, pn2])

subject_model = Model([t1_in, t2_in], [ps1, ps2]) # 预测subject的模型

t_max = Lambda(seq_maxpool)([t, mask])
pc = Dense(char_size, activation='relu')(t_max)
pc = Dense(num_classes, activation='sigmoid')(pc)

def get_k_inter(x, n=6):
    seq, k1, k2 = x
    k_inter = [K.round(k1 * a + k2 * (1 - a)) for a in np.arange(n) / (n - 1.)]
    k_inter = [seq_gather([seq, k]) for k in k_inter]
    k_inter = [K.expand_dims(k, 1) for k in k_inter]
    k_inter = K.concatenate(k_inter, 1)
    return k_inter

k = Lambda(get_k_inter, output_shape=(6, t_dim))([t, k1, k2])
k = Bidirectional(CuDNNGRU(t_dim))(k)
k1v = position_embedding(Lambda(position_id)([t, k1]))
k2v = position_embedding(Lambda(position_id)([t, k2]))
kv = Concatenate()([k1v, k2v])
k = Lambda(lambda x: K.expand_dims(x[0], 1) + x[1])([k, kv])

h = Attention(8, 16)([t, t, t, mask])
h = Concatenate()([t, h, k])
h = Conv1D(char_size, 3, activation='relu', padding='same')(h)
po = Dense(1, activation='sigmoid')(h)
po1 = Dense(num_classes, activation='sigmoid')(h)
po2 = Dense(num_classes, activation='sigmoid')(h)
po1 = Lambda(lambda x: x[0] * x[1] * x[2] * x[3])([po, po1, pc, pn1])
po2 = Lambda(lambda x: x[0] * x[1] * x[2] * x[3])([po, po2, pc, pn2])

object_model = Model([t1_in, t2_in, k1_in, k2_in], [po1, po2]) # 输入text和subject，预测object及其关系

train_model = Model([t1_in, t2_in, s1_in, s2_in, k1_in, k2_in, o1_in, o2_in],
                    [ps1, ps2, po1, po2])

s1 = K.expand_dims(s1, 2)
s2 = K.expand_dims(s2, 2)

s1_loss = K.binary_crossentropy(s1, ps1)
s1_loss = K.sum(s1_loss * mask) / K.sum(mask)
s2_loss = K.binary_crossentropy(s2, ps2)
s2_loss = K.sum(s2_loss * mask) / K.sum(mask)

o1_loss = K.sum(K.binary_crossentropy(o1, po1), 2, keepdims=True)
o1_loss = K.sum(o1_loss * mask) / K.sum(mask)
o2_loss = K.sum(K.binary_crossentropy(o2, po2), 2, keepdims=True)
o2_loss = K.sum(o2_loss * mask) / K.sum(mask)

loss = (s1_loss + s2_loss) + (o1_loss + o2_loss)

train_model.add_loss(loss)
train_model.compile(optimizer=Adam(1e-3))
train_model.summary()

class ExponentialMovingAverage:
    """对模型权重进行指数滑动平均。
    用法：在model.compile之后、第一次训练之前使用；
    先初始化对象，然后执行inject方法。
    """
    def __init__(self, model, momentum=0.9999):
        self.momentum = momentum
        self.model = model
        self.ema_weights = [K.zeros(K.shape(w)) for w in model.weights]
    def inject(self):
        """添加更新算子到model.metrics_updates。
        """
        self.initialize()
        for w1, w2 in zip(self.ema_weights, self.model.weights):
            op = K.moving_average_update(w1, w2, self.momentum)
            self.model.metrics_updates.append(op)
    def initialize(self):
        """ema_weights初始化跟原模型初始化一致。
        """
        self.old_weights = K.batch_get_value(self.model.weights)
        K.batch_set_value(zip(self.ema_weights, self.old_weights))
    def apply_ema_weights(self):
        """备份原模型权重，然后将平均权重应用到模型上去。
        """
        self.old_weights = K.batch_get_value(self.model.weights)
        ema_weights = K.batch_get_value(self.ema_weights)
        K.batch_set_value(zip(self.model.weights, ema_weights))
    def reset_old_weights(self):
        """恢复模型到旧权重。
        """
        K.batch_set_value(zip(self.model.weights, self.old_weights))

EMAer = ExponentialMovingAverage(train_model)
EMAer.inject()

def extract_items(text_in):
    text_words = tokenize(text_in.lower())
    text_in = ''.join(text_words)
    R = []
    _t1 = [char2id.get(c, 1) for c in text_in]
    _t1 = np.array([_t1])
    _t2 = sent2vec([text_words])
    _k1, _k2 = subject_model.predict([_t1, _t2])
    _k1, _k2 = _k1[0, :, 0], _k2[0, :, 0]
    _k1, _k2 = np.where(_k1 > 0.5)[0], np.where(_k2 > 0.4)[0]
    _subjects = []
    for i in _k1:
        j = _k2[_k2 >= i]
        if len(j) > 0:
            j = j[0]
            _subject = text_in[i: j+1]
            _subjects.append((_subject, i, j))
    if _subjects:
        _t1 = np.repeat(_t1, len(_subjects), 0)
        _t2 = np.repeat(_t2, len(_subjects), 0)
        _k1, _k2 = np.array([_s[1:] for _s in _subjects]).T.reshape((2, -1, 1))
        _o1, _o2 = object_model.predict([_t1, _t2, _k1, _k2])
        for i,_subject in enumerate(_subjects):
            _oo1, _oo2 = np.where(_o1[i] > 0.5), np.where(_o2[i] > 0.4)
            for _ooo1, _c1 in zip(*_oo1):
                for _ooo2, _c2 in zip(*_oo2):
                    if _ooo1 <= _ooo2 and _c1 == _c2:
                        _object = text_in[_ooo1: _ooo2+1]
                        _predicate = id2predicate[_c1]
                        R.append((_subject[0], _predicate, _object))
                        break
        zhuanji, gequ = [], []
        for s, p, o in R[:]:
            if p == u'妻子':
                R.append((o, u'丈夫', s))
            elif p == u'丈夫':
                R.append((o, u'妻子', s))
            if p == u'所属专辑':
                zhuanji.append(o)
                gequ.append(s)
        spo_list = set()
        for s, p, o in R:
            if p in [u'歌手', u'作词', u'作曲']:
                if s in zhuanji and s not in gequ:
                    continue
            spo_list.add((s, p, o))
        return list(spo_list)
    else:
        return []

class Evaluate(Callback):
    def __init__(self):
        self.F1 = []
        self.best = 0.
        self.passed = 0
        self.stage = 0
    def on_batch_begin(self, batch, logs=None):
        """第一个epoch用来warmup，不warmup有不收敛的可能。
        """
        if self.passed < self.params['steps']:
            lr = (self.passed + 1.) / self.params['steps'] * 1e-3
            K.set_value(self.model.optimizer.lr, lr)
            self.passed += 1
    def on_epoch_end(self, epoch, logs=None):
        EMAer.apply_ema_weights()
        f1, precision, recall = self.evaluate()
        self.F1.append(f1)
        if f1 > self.best:
            self.best = f1
            train_model.save_weights('best_model.weights')
        print('f1: %.4f, precision: %.4f, recall: %.4f, best f1: %.4f\n' % (f1, precision, recall, self.best))
        EMAer.reset_old_weights()
        if epoch + 1 == 50 or (
            self.stage == 0 and epoch > 10 and
            (f1 < 0.5 or np.argmax(self.F1) < len(self.F1) - 8)
        ):
            self.stage = 1
            train_model.load_weights('best_model.weights')
            EMAer.initialize()
            K.set_value(self.model.optimizer.lr, 1e-4)
            K.set_value(self.model.optimizer.iterations, 0)
            opt_weights = K.batch_get_value(self.model.optimizer.weights)
            opt_weights = [w * 0. for w in opt_weights]
            K.batch_set_value(zip(self.model.optimizer.weights, opt_weights))
    def evaluate(self):
        orders = ['subject', 'predicate', 'object']
        A, B, C = 1e-10, 1e-10, 1e-10
        F = open('dev_pred.json', 'w')
        for d in tqdm(iter(dev_data)):
            R = set(extract_items(d['text']))
            T = set(d['spo_list'])
            A += len(R & T)
            B += len(R)
            C += len(T)
            s = json.dumps({
                'text': d['text'],
                'spo_list': [
                    dict(zip(orders, spo)) for spo in T
                ],
                'spo_list_pred': [
                    dict(zip(orders, spo)) for spo in R
                ],
                'new': [
                    dict(zip(orders, spo)) for spo in R - T
                ],
                'lack': [
                    dict(zip(orders, spo)) for spo in T - R
                ]
            }, ensure_ascii=False, indent=4)
            F.write(s.encode('utf-8') + '\n')
        F.close()
        return 2 * A / (B + C), A / B, A / C

def test(test_data):
    """输出测试结果
    """
    orders = ['subject', 'predicate', 'object', 'object_type', 'subject_type']
    F = open('test_pred.json', 'w')
    for d in tqdm(iter(test_data)):
        R = set(extract_items(d['text']))
        s = json.dumps({
            'text': d['text'],
            'spo_list': [
                dict(zip(orders, spo + ('', ''))) for spo in R
            ]
        }, ensure_ascii=False)
        F.write(s.encode('utf-8') + '\n')
    F.close()

train_D = data_generator(train_data)
evaluator = Evaluate()

if __name__ == '__main__':
    train_model.fit_generator(train_D.__iter__(),
                              steps_per_epoch=len(train_D),
                              epochs=120,
                              callbacks=[evaluator]
                              )
else:
    train_model.load_weights('best_model.weights')


from flask import request

import util, database

def change_name(output_number):

   output_name = request.form['output_name']

   database.change_name(output_number, output_name)

def add_command(day, output_number):

   time = request.form['time']
   command = request.form['command']

   time_on = '-'
   time_off = '-'

   if command =='periodic':
      time_on = request.form['time_on']
      time_off = request.form['time_off']

      try:                 #these 2 trys are to check time input was a number
         int(time_on)
         int(time_off)
         if len(time) == 0:
            database.add_command(output_number, day, time, command, time_on, time_off)
      except ValueError:
         return

   else:

         #util.log(time_on)

      if not len(time)==0:
         if not len(time_on)==0:
            if not len(time_off)==0:
               database.add_command(output_number, day, time, command, time_on, time_off)

def clear_commands(day, output_number):

   if 'clear_all' in request.form:
      util.log('clear')
      database.clear_commands(output_number, day)


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

from django.db import models, migrations

class Migration(migrations.Migration):

    dependencies = [
        ('bookings', '0002_booking_talent'),
    ]

    operations = [
        migrations.RenameField(
            model_name='booking',
            old_name='end_date',
            new_name='end',
        ),
        migrations.RenameField(
            model_name='booking',
            old_name='start_date',
            new_name='start',
        ),
        migrations.RemoveField(
            model_name='booking',
            name='text',
        ),
        migrations.AddField(
            model_name='booking',
            name='title',
            field=models.CharField(default='title', max_length=63),
            preserve_default=False,
        ),
    ]


import re
import copy
import logging
from urllib2 import urlparse
from datetime import datetime,timedelta
from cgi import parse_qsl
from urllib import urlencode

from tgimport import tg
from baseconnector import BaseConnector
from utils.utils import stripHtml, get_hash
from utils.decorators import logit
from utils.sessioninfomanager import checkSessionInfo, updateSessionInfo

log = logging.getLogger('BroncosFreaksConnector')
class BroncosFreaksConnector(BaseConnector):
    @logit(log , 'fetch')
    def fetch(self):
        try:

            if 'forumdisplay.php' in self.currenturi:
                return self.__createTasksForThreads()
            else:
                return self.__addThreadAndPosts()
                #this will fetch the thread links and Adds Tasks
        except:
            log.exception(self.log_msg('Exception in fetch for the url %s'\
                                                            %self.currenturi))
            return False

    @logit(log, '__addThreadAndPosts')
    def __addThreadAndPosts(self):
        """
        This will add the Thread info from setParentPage method and
        Add the posts of the thread in addPosts mehtod
        """
        try:
            self.genre = "Review"
            self.__hierarchy = []
            self.__baseuri = 'http://broncosfreaks.com/forums/'
            self.__task_elements_dict = {
                            'priority':self.task.priority,
                            'level': self.task.level,
                            'last_updated_time':datetime.strftime(datetime.utcnow(),"%Y-%m-%dT%H:%M:%SZ"),
                            'pickup_date':datetime.strftime(datetime.utcnow(),"%Y-%m-%dT%H:%M:%SZ"),
                            'connector_instance_log_id': self.task.connector_instance_log_id,
                            'connector_instance_id':self.task.connector_instance_id,
                            'workspace_id':self.task.workspace_id,
                            'client_id':self.task.client_id,
                            'client_name':self.task.client_name,
                            'versioned':False,
                            'category':self.task.instance_data.get('category',''),
                            'task_log_id':self.task.id }
            self.__setSoupForCurrentUri()
            self.__setParentPage()
            question_post = self.soup.find('div', id=re.compile('^edit.*?'))
            self.__addPost(question_post, True)
            self.__goToLastPage()
            while self.__iteratePosts():
                try:
                    next_page_uri = self.soup.find('a', text='<',rel='prev').parent['href']
                    data_dict = dict(parse_qsl(next_page_uri.split('?')[-1]))
                    if 's' in data_dict.keys():
                        data_dict.pop('s')
                    self.currenturi = self.__baseuri + 'showthread.php?'+ urlencode(data_dict)
                    self.__setSoupForCurrentUri()
                except:
                    log.exception(self.log_msg('Next Page link not found for url \
                                                    %s'%self.currenturi))
                    break
            return True

            """#try:
             #   question_post = self.soup.find('div', id=re.compile ('^edit\d+'))
            #except:
             #   log.info(self.log_msg('Question post cannot be added'))
            #main_page_soup = copy.copy(self.soup)
            #   self.__addPost(question_post, True)
            except:
                log.info(self.log_msg('page not added'))
            self.__goToLastPage(main_page_soup)
            while True:
                main_page_soup = copy.copy(self.soup)
                    #log.info(self.log_msg('Question post cannot be added'))
                if not self.__iteratePosts():
                    break
                try:
                    self.currenturi = main_page_soup.find('a',text='&lt').parent['href']
                    log.info(self.log_msg('uri %s'%self.currenturi))
                except:

                    log.info(self.log_msg('No Previous URL found for url \
                                                        %s'%self.currenturi))
                    break
            return True
        """

        except:
            log.exception(self.log_msg('Exception while add the theread posts \
                                            for the url %s'%self.currenturi))
            return False

    @logit(log, '__createTasksForThreads')
    def __createTasksForThreads(self):

        """
        This will create Tasks for the threads found on the given url
        The # of Tasks are limited by Config Variable
        """
        try:

            self.__total_threads_count = 0
            self.__baseuri = 'http://broncosfreaks.com/forums/'
            self.__last_timestamp = datetime( 1980,1,1 )
            self.__max_threads_count = int(tg.config.get(path='Connector', key=\
                                                'broncosfreaks_maxthreads'))
            self.__setSoupForCurrentUri()
            while self.__processForumUrl():
                try:
                    next_page_uri =self.__baseuri + self.soup.find('a', rel='next')['href']
                    data_dict = dict(parse_qsl(next_page_uri.split('?')[-1]))
                    if 's' in data_dict.keys():
                        data_dict.pop('s')
                    self.currenturi = self.__baseuri + 'forumdisplay.php?'+ urlencode(data_dict)
                    self.__setSoupForCurrentUri()
                except:
                    log.exception(self.log_msg('Next Page link not found for url \
                                                    %s'%self.currenturi))
                    break

            #log.info(self.log_msg('LINKSOUT: ' + str(len(self.linksOut))))
            #self.linksOut = [] # To Remove
            if self.linksOut:
                updateSessionInfo('Search', self.session_info_out, \
                            self.__last_timestamp , None, 'ForumThreadsPage', \
                            self.task.instance_data.get('update'))
            return True
        except:
            log.info(log_msg('Exception while creating tasks for the url %s'\
                                                         %self.currenturi))
            return False

    @logit(log, '__processForumUrl')
    def __processForumUrl(self):
        """
        It will fetch each thread and its associate infomarmation
        and add the tasks
        """
        threads = [each.findParent('tr') for each in self.soup.find('table',\
             id='threadslist').findAll('td', id=re.compile('td_threadtitle_'))]
        for thread in threads:
            try:
                if thread.find('img', src=re.compile('sticky.gif$')):
                    log.info(self.log_msg('Its a Sticky Thread, Ignore it in the \
                                       url %s'%self.currenturi))
                    continue
                self.__total_threads_count += 1
                if  self.__total_threads_count > self.__max_threads_count:
                    log.info(self.log_msg('Reaching maximum post,Return false \
                                        from the url %s'%self.currenturi))
                    return False

                try:
                    th=thread.findAll('td',recursive=False)
                    date_tag = stripHtml(th[3].renderContents()).splitlines()\
                                                                  [0] .strip()
                    if date_tag.startswith('Today'):
                        date_tag = date_tag.replace('Today', datetime.strftime\
                                                (datetime.utcnow(),'%m-%d-%Y'))
                    elif date_tag.startswith('Yesterday'):
                        date_tag = date_tag.replace('Yesterday', datetime.\
                                    strftime((datetime.utcnow()-timedelta\
                                         (days=1)),'%m-%d-%Y'))
                    thread_time = datetime.strptime(date_tag,'%m-%d-%Y  %I:%M %p')
                except:
                    log.exception(self.log_msg('last post not found in the url\
                                                    %s'%self.currenturi))
                if checkSessionInfo('Search', self.session_info_out, thread_time, \
                                self.task.instance_data.get('update')):
                    log.info(self.log_msg('Session info Returns True for %s'%\
                                                        self.currenturi))
                    log.info(thread_time)
                    return False
                self.__last_timestamp = max(thread_time , self.__last_timestamp )
                temp_task=self.task.clone()
                try:

                    s=th[2].find('a', id=re.compile('thread_title_\d+'))['href']
                    temp_task.instance_data[ 'uri' ] = self.__baseuri + re.sub\
                                ('\?s=.*?&','?',s)
                except:
                    log.exception(self.log_msg('uri not found in the url\
                                                    %s'%self.currenturi))
                    continue
                try:
                    temp_task.pagedata['edate_last_post_date'] = datetime.strftime\
                                                (thread_time, "%Y-%m-%dT%H:%M:%SZ")
                except:
                    log.info(self.log_msg('last post date not found in the url\
                                                    %s'%self.currenturi))
                try:
                    temp_task.pagedata['et_author_name'] = stripHtml(th[2].\
                                    find('div','smallfont').renderContents())
                except:
                    log.info(self.log_msg('author name  not found in the url\
                                                    %s'%self.currenturi))
                try:
                    temp_task.pagedata['et_last_post_author'] = \
                                          stripHtml(th[3].renderContents()).\
                                      splitlines()[-1].strip().replace('by','')
                except:
                    log.info(self.log_msg('last post author not found in the url\
                                                    %s'%self.currenturi))
                try:
                    temp_task.pagedata['ei_thread_views_count'] = int(stripHtml\
                                                       (th[5].renderContents()))
                except:
                    log.info(self.log_msg('views  not found in the url\
                                                    %s'%self.currenturi))
                try:
                    temp_task.pagedata['ei_thread_replies_count'] = int(stripHtml\
                                                    (th[4].renderContents()))
                except:
                    log.info(self.log_msg('reply  not found in the url\
                                                    %s'%self.currenturi))
                self.linksOut.append(temp_task)
            except:
                log.exception(self.log_msg('links not found'))
        return True

    @logit(log, '__iteratePosts')
    def __iteratePosts(self):

        try:
            #posts = self.soup.findAll('table', id=re.compile('post\d+'))
            posts = self.soup.findAll('div', id=re.compile ('^edit\d+'))
            if not posts:
                log.info(self.log_msg('No posts found'))
                return False
            log.debug(self.log_msg('Total No of Posts found is %d'%len(posts)))
            posts.reverse()
            for post in posts:
                if not self.__addPost(post):
                    return False
            return True
        except:
            log.exception(self.log_msg('can not  find the data'))
            return False

    @logit(log,'__setParentPage')
    def __setParentPage(self):
        """
        """
        page = {}
        try:
            hierarchies = [each for each in [stripHtml(x.renderContents()) for\
                                    x in self.soup.find('table','tborder').\
                                                table.findAll('td')] if each]
            self.hierarchy = [x.strip() for x in hierarchies[0].split('>')]
            page['data'] = page['title'] = hierarchies[1]
            self.hierarchy.append(page['title'])
            page['et_thread_hierarchy'] = self.hierarchy
        except:
            log.info(self.log_msg('Thread hierarchy is not found'))
            return False
        if checkSessionInfo(self.genre, self.session_info_out, self.task.instance_data['uri'],\
                                         self.task.instance_data.get('update')):
            log.info(self.log_msg('Session info return True, Already exists'))
            return False
        page_data_keys = ['et_author_name', 'ei_thread_replies_count', \
                            'ei_thread_views_count','edate_last_post_date',\
                                    'et_last_post_author']
        [page.update({each:self.task.pagedata.get(each)}) for each in \
                                page_data_keys if self.task.pagedata.get(each)]
        try:
            date_str = stripHtml(self.soup.find('div', id='posts').find('td', \
                                                    'thead').renderContents())
            if date_str.startswith('Today'):
                date_str = date_str.replace('Today', datetime.strftime\
                                                (datetime.utcnow(),'%m-%d-%Y'))
            elif date_str.startswith('Yesterday'):
                        date_str = date_str.replace('Yesterday', datetime.\
                                    strftime((datetime.utcnow()-timedelta\
                                         (days=1)),'%m-%d-%Y'))

            page['posted_date'] = datetime.strftime(datetime.strptime(date_str,\
                                    '%m-%d-%Y, %I:%M %p'),"%Y-%m-%dT%H:%M:%SZ")
        except:
            log.exception(self.log_msg('Posted date not found'))
            page['posted_date'] = datetime.strftime(datetime.utcnow(), \
                                                        "%Y-%m-%dT%H:%M:%SZ")
        try:
            result = updateSessionInfo('review', self.session_info_out, self.\
                task.instance_data['uri'], get_hash(page), 'forum', \
                                    self.task.instance_data.get('update'))
            if result['updated']:
                page['path'] = [self.task.instance_data['uri']]
                page['parent_path'] = []
                page['uri'] = self.currenturi
                page['uri_domain'] = unicode(urlparse.urlparse(page['uri'])[1])
                page['entity'] = 'thread'
                page.update(self.__task_elements_dict)
                self.pages.append(page)
            else:
                log.info(self.log_msg('Result[updated] returned True for \
                                                        uri'%self.currenturi))
        except :
            log.exception(self.log_msg("parent post couldn't be parsed"))

    @logit(log, '__goToLastPage')
    def __goToLastPage(self):
        """This will set the soup the last page of the post
        """
        try:
            pagination_tag = self.soup.find('div', 'pagenav')
            if not pagination_tag:
                return
            uri = None
            last_page_tag = pagination_tag.find('a', title=re.compile('Last Page'))
            if last_page_tag:
                uri = last_page_tag['href']
            else:
                last_page_tag = pagination_tag.findAll('a', href=True, text=re.compile('^\d+$'))
                if last_page_tag:
                    uri = last_page_tag[-1].parent['href']
            if not uri:
                log.info(self.log_msg('Post found in only one page'))
                return
            data_dict = dict(parse_qsl(uri.split('?')[-1]))
            if 's' in data_dict.keys():
                data_dict.pop('s')
            self.currenturi = self.__baseuri + 'showthread.php?'+ urlencode(data_dict)
            self.__setSoupForCurrentUri()
        except:
            log.exception(self.log_msg('Last page cannot find from the given page no \
                                    for url %s'%self.task.instance_data['uri']))

    """
    @logit(log, '__goToLastPage')
    #def __goToLastPage(self, main_page_soup):
        last_page_tag = None
        try:
            last_page_tag = main_page_soup.find('div', 'pagenav').find(title=re.compile('Last Page'))
        except:
            log.info(self.log_msg('Last page tag not found for url %s'%self.task.instance_data['uri']))
        try:
            if not last_page_tag:
                last_page_tag = main_page_soup.find('div', 'pagenav').findAll('a',\
                                        text=re.compile ('^\d+$'))[-1].parent
        except:
            log.info(self.log_msg('Last page cannot find from the given page no'))
        try:
            if not last_page_tag:
                log.info(self.log_msg('Posts exists only in the current page'))
                self.soup = main_page_soup
                return True
            self.currenturi = last_page_tag['href']
            self.__setSoupForCurrentUri()
            return True
        except:
            log.info(self.log_msg('Exception occurred while Setting the last page'))
            return False
    """
    @logit(log,'__addPost')
    def __addPost(self, post, is_question = False):
        """
        This will take the post tag , and fetch data and meta data and add it to
        self.pages
        """
        try:
            post_id = post.find('a',id=re.compile('postcount\d+'))
            unique_key = post_id['id'].replace('postcount','')
            log.info(unique_key)
            if checkSessionInfo(self.genre, self.session_info_out, unique_key,\
                         self.task.instance_data.get('update'),parent_list\
                                            = [self.task.instance_data['uri']]):
                log.info(self.log_msg('Session info returns True'))
                return False
            page = self.__getData(post, is_question)
        except:
            log.info(post)
            log.exception(self.log_msg('Cannot add the post for the url %s'%\
                                                            self.currenturi))
            return False
        try:
            page['uri'] = self.currenturi + '#' + stripHtml(post_id.renderContents())
        except:
            log.info(self.log_msg('Cannot find the uri'))
            page['uri'] = self.currenturi
        try:
            result=updateSessionInfo(self.genre, self.session_info_out, unique_key, \
                get_hash( page ),'Review', self.task.instance_data.get('update'),\
                                parent_list=[self.task.instance_data['uri']])
            if not result['updated']:
                log.info(self.log_msg('Update session info returns False'))
                return True
            page['parent_path'] = page['path'] = [self.task.instance_data['uri']]
            page['path'].append(unique_key)
            #page['uri'] = unique_key
            page['uri_domain']  = urlparse.urlparse(page['uri'])[1]
            page.update(self.__task_elements_dict)
            self.pages.append(page)
            log.info(self.log_msg('Post Added'))
            return True
        except:
            log.exception(self.log_msg('Error while adding session info'))
            return False

    def __getData(self, post, is_question):
        page = {'entity':'question' if is_question else 'answer'}

        try:
            page['title']=self.hierarchy[-1]
            if not is_question:
                 page['title'] = 'Re: ' + page['title']
        except:
            log.exception(self.log_msg('data forum not found'))
            page['title'] = ''

        try:
            data_tag = post.find('div', id=re.compile('post_message_\d+'))
            previous_message_tag = data_tag.find('div', text='Quote:')
            if previous_message_tag:
                previous_message_tag.parent.findParent('div').extract()
            #[x.extract() for x in data_tag.findNext('div')]
            page['data'] = stripHtml(data_tag.renderContents())
        except:
            log.info(self.log_msg('Data not found for the url %s'%self.currenturi))
            page['data'] = ''

        if not page['title'] and not page['data']:
            log.info(self.log_msg("Data and title not found for %s,"\
                                    " discarding this review"%self.currenturi))
            return False
        try:
            date_str = stripHtml(post.find('td').renderContents())
            if date_str.startswith('Today'):
                date_str = date_str.replace('Today', datetime.strftime\
                (datetime.utcnow(),'%m-%d-%Y'))
            elif date_str.startswith('Yesterday'):
                date_str= date_str.replace('Yesterday', datetime.strftime((datetime.utcnow()-timedelta(days=1)),'%m-%d-%Y'))

            page['posted_date'] = datetime.strftime(datetime.strptime(date_str,'%m-%d-%Y, %I:%M %p'),'%Y-%m-%dT%H:%M:%SZ')

        except:
            log.exception(self.log_msg('Posted date not found'))
            page['posted_date'] = datetime.strftime(datetime.utcnow(), "%Y-%m-%dT%H:%M:%SZ")

        try:
            page['et_author_name'] = stripHtml(post.find('a','bigusername').\
                                               renderContents())
        except:
            log.info(self.log_msg('author name not found'))

        try:
            page['et_author_category'] = stripHtml(post.find('div',id=re.\
                    compile('postmenu_\d+')).findNext('div').renderContents())
        except:
            log.info(self.log_msg('author category not found'))

        try:
            author_info = [each.strip() for each in stripHtml(post.find('br').
                           findNext('div').renderContents()).splitlines()
                                                 if each.strip()]
            #log.info(self.log_msg('author_info not found'))
            for each in author_info:
                try:
                    if each.startswith('Join Date:'):
                        join_date = '01 ' + each.replace('Join Date:','').strip()

                        page['edate_author_joined_date'] =datetime.strftime\
                        (datetime.strptime(join_date,'%d %b %Y'),"%Y-%m-%dT%H:%M:%SZ")  # Convert Date str to Date Time Format
                    if each.startswith('Location:'):
                        page['et_author_location'] = each.replace('Location:',\
                                                        '').strip()
                    if each.startswith('Age:'):
                        page['ei_author_age'] = int(each.replace('Age:',\
                                                        '').strip())
                    if each.startswith('Posts:'):
                        page['ei_author_posts_count'] = int(each.replace\
                                        ('Posts:','').replace(',','').strip())
                    if each.startswith('Salute:'):
                        page['ei_author_salute_count'] = int(each.replace\
                                                    ('Salute:','').replace(',','').strip())
                    if each.startswith('Location:'):
                        page['et_author_location'] = each.replace('Location:',\
                                                        '').strip()
                except:
                    #import traceback
                    #print traceback.format_exc()
                    log.exception(self.log_msg('cannot find the author info'))
        except:
            log.exception(self.log_msg('author posts count not found'))

        return page
    @logit(log, '__setSoupForCurrentUri')
    def __setSoupForCurrentUri(self, data=None, headers={}):
        """It will set soup object for the Current URI
        """
        res = self._getHTML(data=data, headers=headers)
        if res:
            self.rawpage = res['result']
        else:
            log.info(self.log_msg('Page Content cannot be fetched for the url: \
                                                            %s'%self.currenturi))
            raise Exception('Page content not fetched for th url %s'%self.currenturi)
        self._setCurrentPage()


"""A module for demonstrating exceptions."""
import sys

def convert(s):
    """Try to convert to an integer."""
    x = -1
    try:
        return int(s)
    except (ValueError, TypeError) as e:
        print("Conversion error: {}"
            .format(str(e)),
            file=sys.stderr)
        raise

# convert("33")
# convert("hedgehog")
# convert([1,2,3,4])

from math import log

def string_log(s):
    c = convert(s)
    return log(v)

# string_log("ouch")

def sqrt(x):

    if x < 0:
        raise ValueError("Cannot compute square root "
            "of negative number {}".format(x))

    guess = x
    i = 0
    while guess * guess != x and i < 20:
        guess = (guess + x / guess) / 2.0
        # print(guess)
        i += 1
    return guess

def main():
    try:
        print(sqrt(9))
        print(sqrt(2))
        print(sqrt(-1))
        print("this is never printed...")
    except ValueError as e:
        print(e, file=sys.stderr)

    print("Program execution continues normally here.")
if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 13 19:44:32 2021

@author: Zikantika
"""

from sklearn.neural_network import MLPClassifier
from sklearn.neural_network import MLPRegressor

import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.datasets import make_regression
from sklearn.model_selection import train_test_split

#cereal_df = pd.read_csv("/tmp/tmp07wuam09/data/cereal.csv")
diabetic_df2 = pd.read_csv("diabetes.csv")

#print(diabetic_df2.head(5))

print(diabetic_df2.head(10))

#print(diabetic_df2['Glucose'][0])

#print(diabetic_df2['Glucose'])

X=diabetic_df2['Glucose']
Y=diabetic_df2['Outcome']

x, y = np.array(X), np.array(Y)

#x=np.array(x).reshape(1,-1)
##
#y=np.array(y).reshape(1,-1)

x=np.array(x).reshape(-1,1)
#
y=np.array(y).reshape(-1,1)

print(x.shape)
print(y.shape)
#X = [[0., 0.], [1., 1.]]
#y = [[0, 1], [1, 1]]

#NeuralNetwork = MLPRegressor(verbose = True,
#                             max_iter = 10000,
#                             tol = 0,
#                             activation = "logistic")
#NeuralNetwork.fit(x,y)
##
#NeuralNetwork.fit(X,Y)

print("x shape is ")
print(x.shape)
print("y shape is")
print(y.shape)

X_train, X_test, y_train, y_test = train_test_split(X, Y,
                                                     random_state=1)

print(X_train.shape)

X_=np.array(X_test).reshape(-1,1)
print(X_.shape)

#Q=NeuralNetwork.predict(X_)
#
#print(Q)

#

Y_=np.array(y_test).reshape(-1,1)

#NeuralNetwork.score(X_, Y_)

#Q=NeuralNetwork.predict(10)
#print("Predicted Value is ")
#print(Q)

y=Y

clf = MLPClassifier(solver='lbfgs', alpha=1e-5,
                     hidden_layer_sizes=(10,), random_state=1)

clf.fit(x, y)

#MLPClassifier(alpha=1e-05, hidden_layer_sizes=(1,), random_state=1,
#              solver='lbfgs')

print(X_)
clf.predict(X_)
#
#clf.predict([[0., 0.]])


import csv

try:
  with open('data2.csv', mode='w', encoding='utf-8') as f:
    writer = csv.writer(f, delimiter=',', lineterminator='\r', skipinitialspace=True)
    while True:
      instr = input('data:')
      if instr == '':
        break
      inlist = instr.split(' ')
      writer.writerow(inlist)
except Exception as error:
  print(str(error))
print('***end.***')


import rethinkdb as r
from rethinkdb.errors import RqlRuntimeError

def createConnection(host, port):
    connection = r.connect(host, port)
    return connection

# This is just for cross-checking database and table exists
def setupDbAndTable(connection, db, table):
    print(db, connection)
    try:
        r.db_create(db).run(connection)
        print('Database setup completed.')
    except RqlRuntimeError:
        try:
            r.db(db).table_create(table).run(connection)
            print('Table creation completed')
        except:
            print('Table already exists.Nothing to do')

def connectAndSetup(options):
    host = options['host']
    port = options['port']
    db = options['db']
    table = options['table']
    connection = createConnection(host, port)
    setupDbAndTable(connection, db, table)
    return connection


'''
@Author       : sean cheng
@Email        : aya234@163.com
@CreateTime   : 2018/7/30
@Program      : （考试练习题）统计文件出现的词语和标点符号的数量
'''
import jieba

dataDict = {}

with open('天龙八部-网络版.txt','r',encoding='utf-8') as incomeFile:
    txt = incomeFile.read()

words = jieba.cut(txt)
for i in words:
    if i == '\n':
        continue
    if i == '　':
        continue
    if i in dataDict.keys():
        dataDict[i] += 1
    else:
        dataDict[i] = 1

with open('天龙八部-词语统计.txt','w',encoding='utf-8') as outFile:
    tmpList = []
    for item in dataDict.items():
        tmpList.append("{}:{}".format(item[0],item[1]))
    outFile.write(','.join(tmpList))


"""
Elliot Greenlee
Kyle Bashour
09/27/2016
583 Applied Cryptography

This program implements an ntargeted collision attack via the
small-space birthday attack as represented in "Introduction to
Modern Cryptography" section 5.4.2
"""

import hashlib
import binascii
import time

# We are only hunting for a partial collision in the first N bits
N_PARTIAL = 10
# Starting value (x0) for hunting is our last names concatenated
STARTING_VALUE = "bashourgreenlee"

while True:
    # set x' = x0
    xPrimetmp = STARTING_VALUE
    # set x = x0
    xtmp = STARTING_VALUE

    startTime = time.clock()

    # for i = 1, 2... do
    i = 1
    while True:

        # x = H(x)
        sha256HashX = hashlib.sha256()
        sha256HashX.update(xtmp)
        xtmp = (bin(int(binascii.hexlify(sha256HashX.hexdigest()), 16)))[17:N_PARTIAL + 17]

        # x' = H(H(x'))
        sha256HashXPrime = hashlib.sha256()
        sha256HashXPrime.update(xPrimetmp)
        xPrimetmp = (bin(int(binascii.hexlify(sha256HashXPrime.hexdigest()), 16)))[17:N_PARTIAL + 17]

        sha256HashHashXPrime = hashlib.sha256()
        sha256HashHashXPrime.update(xPrimetmp)
        xPrimetmp = (bin(int(binascii.hexlify(sha256HashHashXPrime.hexdigest()), 16)))[17:N_PARTIAL + 17]

        # if x = x' break (only compare first N_PARTIAL bits)
        if xtmp == xPrimetmp:
            break
        else:
            i += 1

    # x' = x
    xPrimetmp = xtmp
    # x = x0
    xtmp = STARTING_VALUE

    # for j=1 to i
    for j in range(1, i + 1):

        # if H(x) = H(x') (only compare first N_PARTIAL bits)
        sha256HashX = hashlib.sha256()
        sha256HashX.update(xtmp)
        x = xtmp
        xtmp = (bin(int(binascii.hexlify(sha256HashX.hexdigest()), 16)))[17:N_PARTIAL + 17]

        sha256HashXPrime = hashlib.sha256()
        sha256HashXPrime.update(xPrimetmp)
        xPrime = xPrimetmp
        xPrimetmp = (bin(int(binascii.hexlify(sha256HashXPrime.hexdigest()), 16)))[17:N_PARTIAL + 17]

        if xtmp == xPrimetmp:
            # return x, x' and halt
            endTime = time.clock()
            print("n", N_PARTIAL, "time", endTime-startTime, "x", x, "x'", xPrime, "hash", xtmp)
            break

    N_PARTIAL += 10


from collections import defaultdict as ddic
from itertools import combinations, permutations, product
import copy
import bisect
import heapq
import sys

rr = lambda: input()
rrs = lambda: input().strip()
rri = lambda: int(rr())
rrm = lambda: list(map(int, rr().split()))

# DEBUG = False
DEBUG = True
if DEBUG == True:
    sys.stdin = open("test_input.txt", "r")
    sys.stdout = open("test_output.txt", "w")

"""
Given a circular linked list.
Returns the node at the beginning of the loop.
Circular LL: A (corrupt) LL in which a node's next pointer points
to an earlier node, so as to make a loop in the LL
Example:
Input: A -> B -> C -> D -> E -> C [same C as earlier]
Output: C
"""

from linked_list import *

def solve1(node: SinglyNode):
    """
    Use hash table to cache the node reference
    -> O(n) time, O(n) space
    """
    tb = {}
    temp = node
    while temp:
        if temp in tb:
            return temp
        else:
            tb[temp] = 1
        temp = temp.next_

    return None

def solve2(node: SinglyNode):
    """
    Assume there're k nodes before the loop
    2 runners p1 and p2, p2 = p1's speed x2
    When p1 jumps k steps (at the loop start),
        p2 will jumps 2k steps
        or (2k-k)=k steps AHEAD the loop start
        or (LOOP_SIZE - k) steps BEHIND the loop start
    After each step, p2 comes 1-step nearer to p1's position
    Because now p2 is (LOOP_SIZE-k) step BEHIND p1,
        so, it needs (LOOP_SIZE-k) steps more to catch p1
    Now, p1 is at position = (LOOP_SIZE-k) which means
        k-step BEHIND the loop start
    -> It means they meet k-step BEHIND the loop start
    -> This is the collision point
    Now, let p3 is at the head of LL,
        p3 is BEHIND the loop start k-step,
        p1 is also,
        we move p1 and p3 together
        they will meet at loop start!
    -> O(L+k) time, O(1) space
    """
    if not node or not node.next_:
        return None

    p1 = node
    p2 = node
    while p2 and p2.next_:
        p1 = p1.next_
        p2 = p2.next_.next_
        if p1 == p2:
            break

    # No loop
    if not p2 or not p2.next_:
        return None

    p3 = node
    while p1 and p3:
        if p1 == p3:
            return p1
        p1 = p1.next_
        p3 = p3.next_

    return None

ll1 = SinglyLinkedList.example([1, 2, 3])
ll2 = SinglyLinkedList.example([4, 5, 6, 7, 8])

ll1_tail = ll1.get_tail()
ll2_tail = ll2.get_tail()

ll1_tail.next_ = ll2.head
ll2_tail.next_ = ll1_tail

res = solve1(ll1.head)
print(res.value if res else res)
res = solve2(ll1.head)
print(res.value if res else res)

ll3 = SinglyLinkedList.example([1, 2])
res = solve1(ll3.head)
print(res.value if res else res)
res = solve2(ll3.head)
print(res.value if res else res)


from itertools import permutations
inf=10**9
n, m, r1 = map(int, input().split())
r=list(map(int, input().split()))
abc=[list(map(int, input().split())) for _ in range(m)]
d=[[inf]*n for i in range(n)]
for ai, bi, ci in abc:
    d[ai-1][bi-1]=ci
    d[bi-1][ai-1]=ci
for k in range(n):
    for i in range(n-1):
        for j in range(i, n):
            dk=d[i][k]+d[k][j]
            if d[i][j]>dk:
                d[j][i]=dk
                d[i][j]=dk
permutations(r)
ans=inf
for pi in permutations(r):
    res=0
    for i in range(r1-1):
        res+=d[pi[i]-1][pi[i+1]-1]
    if ans>res:
        ans=res
print(ans)


"""
Classes for querying the information in a test coverage report.
"""
from __future__ import unicode_literals

from collections import defaultdict

import itertools
from xml.etree import cElementTree
from diff_cover.command_runner import run_command_for_code
from diff_cover.git_path import GitPathTool
from diff_cover.violationsreporters.base import BaseViolationReporter, Violation, RegexBasedDriver, QualityDriver

class CloverXmlCoverageReporter(BaseViolationReporter):
    """
    Query information from a Clover XML coverage report.
    """

    def __init__(self, xml_roots):
        """
        Load the Clover XML coverage report represented
        by the cElementTree with root element `xml_root`.
        """
        super(CloverXmlCoverageReporter, self).__init__("XML")
        self._xml_roots = xml_roots

        # Create a dict to cache violations dict results
        # Keys are source file paths, values are output of `violations()`
        self._info_cache = defaultdict(list)

    @staticmethod
    def _get_src_path_line_nodes(xml_document, src_path):
        """
        Return a list of nodes containing line information for `src_path`
        in `xml_document`.

        If file is not present in `xml_document`, return None
        """
        files = [file_tree
                 for file_tree in xml_document.findall(".//file")
                 if GitPathTool.relative_path(file_tree.get('path')) == src_path
                 or []]
        if not files:
            return None
        lines = [file_tree.findall('./line[@type="stmt"]')
                 for file_tree in files]
        return [elem for elem in itertools.chain(*lines)]

    def _cache_file(self, src_path):
        """
        Load the data from `self._xml_roots`
        for `src_path`, if it hasn't been already.
        """
        # If we have not yet loaded this source file
        if src_path not in self._info_cache:
            # We only want to keep violations that show up in each xml source.
            # Thus, each time, we take the intersection.  However, to do this
            # we must treat the first time as a special case and just add all
            # the violations from the first xml report.
            violations = None

            # A line is measured if it is measured in any of the reports, so
            # we take set union each time and can just start with the empty set
            measured = set()

            # Loop through the files that contain the xml roots
            for xml_document in self._xml_roots:
                line_nodes = self._get_src_path_line_nodes(xml_document,
                                                           src_path)

                if line_nodes is None:
                    continue

                # First case, need to define violations initially
                if violations is None:
                    violations = set(
                        Violation(int(line.get('num')), None)
                        for line in line_nodes
                        if int(line.get('count', 0)) == 0)

                # If we already have a violations set,
                # take the intersection of the new
                # violations set and its old self
                else:
                    violations = violations & set(
                        Violation(int(line.get('num')), None)
                        for line in line_nodes
                        if int(line.get('count', 0)) == 0
                    )

                # Measured is the union of itself and the new measured
                measured = measured | set(
                    int(line.get('num')) for line in line_nodes
                )

            # If we don't have any information about the source file,
            # don't report any violations
            if violations is None:
                violations = set()
            self._info_cache[src_path] = (violations, measured)

    def violations(self, src_path):
        """
        See base class comments.
        """

        self._cache_file(src_path)

        # Yield all lines not covered
        return self._info_cache[src_path][0]

    def measured_lines(self, src_path):
        """
        See base class docstring.
        """
        self._cache_file(src_path)
        return self._info_cache[src_path][1]

"""
    Report checkstyle violations.

    http://checkstyle.sourceforge.net/apidocs/com/puppycrawl/tools/checkstyle/DefaultLogger.html
    https://github.com/checkstyle/checkstyle/blob/master/src/main/java/com/puppycrawl/tools/checkstyle/AuditEventDefaultFormatter.java
"""
checkstyle_driver = RegexBasedDriver(
    name='checkstyle',
    supported_extensions=['java'],
    command=['checkstyle'],
    expression=r'^\[\w+\]\s+([^:]+):(\d+):(?:\d+:)? (.*)$',
    command_to_check_install=['java', 'com.puppycrawl.tools.checkstyle.Main', '-version']
)

class CheckstyleXmlDriver(QualityDriver):
    def __init__(self):
        """
        See super for args
        """
        super(CheckstyleXmlDriver, self).__init__(
            'checkstyle',
            ['java'],
            ['java', 'com.puppycrawl.tools.checkstyle.Main', '-c',
             '/google_checks.xml']
        )
        self.command_to_check_install = ['java', 'com.puppycrawl.tools.checkstyle.Main', '-version']

    def parse_reports(self, reports):
        """
        Args:
            reports: list[str] - output from the report
        Return:
            A dict[Str:Violation]
            Violation is a simple named tuple Defined above
        """
        violations_dict = defaultdict(list)
        for report in reports:
            xml_document = cElementTree.fromstring("".join(report))
            files = xml_document.findall(".//file")
            for file_tree in files:
                for error in file_tree.findall('error'):
                    line_number = error.get('line')
                    error_str = u"{0}: {1}".format(error.get('severity'),
                                                   error.get('message'))
                    violation = Violation(int(line_number), error_str)
                    filename = GitPathTool.relative_path(file_tree.get('name'))
                    violations_dict[filename].append(violation)
        return violations_dict

    def installed(self):
        """
        Method checks if the provided tool is installed.
        Returns: boolean True if installed
        """
        return run_command_for_code(self.command_to_check_install) == 0

class FindbugsXmlDriver(QualityDriver):
    def __init__(self):
        """
        See super for args
        """
        super(FindbugsXmlDriver, self).__init__(
            'findbugs',
            ['java'],
            ['false']
        )

    def parse_reports(self, reports):
        """
        Args:
            reports: list[str] - output from the report
        Return:
            A dict[Str:Violation]
            Violation is a simple named tuple Defined above
        """
        violations_dict = defaultdict(list)
        for report in reports:
            xml_document = cElementTree.fromstring("".join(report))
            bugs = xml_document.findall(".//BugInstance")
            for bug in bugs:
                category = bug.get('category')
                short_message = bug.find('ShortMessage').text
                line = bug.find('SourceLine')
                if line.get('start') is None or line.get('end') is None:
                    continue
                start = int(line.get('start'))
                end = int(line.get('end'))
                for line_number in range(start, end+1):
                    error_str = u"{0}: {1}".format(category, short_message)
                    violation = Violation(line_number, error_str)
                    filename = GitPathTool.relative_path(
                        line.get('sourcepath'))
                    violations_dict[filename].append(violation)

        return violations_dict

    def installed(self):
        """
        Method checks if the provided tool is installed.
        Returns: boolean False: As findbugs analyses bytecode, it would be hard to run it from outside the build framework.
        """
        return False


import re
"""
Goal:
scrape emails from the file
../Data/email_addresses_e_and_e.txt
containing the email addresses of
all the E&E faculty.

Note the following:
Multiple dots in the domain name:
justin.borevitz(at)anu(dot)edu(dot)au
reinitz(at)galton(dot)uchicago(dot)edu

Hyphen in the local part:
j-coyne(at)uchicago(dot)edu
"""

# read the file with email addresses
f = open('../Data/email_addresses_e_and_e.txt')
my_file_email = f.read()
f.close()


#!/usr/bin/env python3
# ===========================================================
# Adapted application script for Ci40 to work with LoRa RF Click Microcontroller
# https://github.com/francois-berder/PyLetMeCreate
# Marten Vijn BSD-license http://svn.martenvijn.nl/svn/LICENSE
# http://www.microchip.com/DevelopmentTools/ProductDetails.aspx?PartNO=dm164138
# https://www.thethingsnetwork.org/docs/devices/
# create an application and add a device for ABP or OTAA: https://console.thethingsnetwork.org/applications
# ===========================================================

import time
import os
import re
import struct
# import serial

# imported these Ci40 specific libraries from https://github.com/francois-berder/PyLetMeCreate/blob/master/examples/lora_example.py
# first two are for connecting the ci40 pins with LoRa pins, latter for UART (interface), and baude rate
from letmecreate.core.common import MIKROBUS_2
from letmecreate.core import gpio
from letmecreate.core import uart
from letmecreate.core.uart import UART_BD_57600

## Thermo
from letmecreate.core import i2c
from letmecreate.core.common import MIKROBUS_1
from letmecreate.click import air_quality

# write config and keys to rn2483 or not
# writeconfig=0
writeconfig=1

## Relic
# Find ports on Ci40 via ls -l /dev/tty* -> /dev/ttySC
# Serial
# SerialDev="/dev/ttyACM0"
# SerialDev="/dev/ttyUSB0"
## =======================================

## asign activation APB or OTAA
#Activation="ABP"
Activation="OTAA"

## TTN
## ABP settings
#Nwkskey="..."
#Appskey="..."
#Devaddr="..."

## my_ci40
## OTAA settings
Appkey="..."
Appeui="..."
Deveui="..."

## Kevin's Device
## OTAA settings
#Appkey="..."
#Appeui="..."
#Deveui="..."

## =======================================

#print("Test ABP TTN")
print("Test OTAA TTN")
print("Test AQ Data Transmission")

## =======================================

## sending encoded bytes via uart to lora tx (Ci40 -> lora tx)
def send(data):
# relic original script where serial instead of uart was used
	# p = serial.Serial(SerialDev, 9600 )
	# p.write(data+"\x0d\x0a")
# for lora tx to recognise command line end \r\n
	buf = data +"\x0d\x0a"
	uart.send(buf.encode())
# strip command clean from newline "garbage" at end
	# data.rstrip()
# print command sent to lora tx
	print(data)
# read what lora rx sends (back)
	rdata = readline()
# deletes last char to cleanup bytearray from \r\n
	rdata = rdata[:-1]
	print(rdata.decode())

## Ci40 receiving lines from lora tx
def readline():
	have_cr = False
# preparing a "line": empty byte array onto which chars from lora tx are appended if \r\n was encountered
	line = bytearray()
	while True:
		buff = uart.receive(1)
		char = buff [0]
		line.append(char)
# State Machine to recognise a newline concept: https://github.com/francois-berder/LetMeCreate/blob/master/src/click/lora.c
		if char == 13:
			have_cr = True
		elif char == 10 and have_cr == True:
			return line
		else:
			have_cr = False

## Air_quality Sensor
"""This example is adapted from the Thermo3 Click wrapper of the PyLetMeCreate
library.
It reads some values from the sensor and exits.
The Air_quality sensor click was inserted in Mikrobus 1 before running this program.
"""

def read_air():
    # Initialise I2C on Mikrobus 1
    i2c.init()
    i2c.select_bus(MIKROBUS_1)

    # sending temp sensor data
    # converting into byte object (hex) using struct
    # every 30 secs
    while True:
        air = air_quality.get_measure(MIKROBUS_1)
        air_hex = struct.pack('>h',air)
    # temp_hex is now a short (2B) byte object
    # hex() returns a string object with the hex digit as literals
        b = bytearray(air_hex).hex()
        air_wrapper = "0403" +b
    # sending on channel 04, hex code 02 for analog INput/ hex 03 for analog OUTput
        send("mac tx cnf 1 "+ str(air_wrapper))
        time.sleep(2)
    # Release I2C
    i2c.release()

## =======================================

## setting the pins from lora tx on output high to switch it on
# https://github.com/CreatorDev/LetMeCreate/blob/master/src/click/lora.c
# https://github.com/CreatorDev/PyLetMeCreate/blob/master/letmecreate/core/gpio.py
# gpio_init(rst_pin) < 0
#     ||  gpio_set_direction(rst_pin, GPIO_OUTPUT) < 0
# || gpio_set_value(rst_pin, 1) < 0) {}
pin = gpio.get_pin(MIKROBUS_2, gpio.TYPE_RST)
# print("pin="+pin)
gpio.init(pin)
gpio.set_direction(pin, gpio.GPIO_OUTPUT)
gpio.set_value(pin, 1)

## UART = serial protocol for pins of Ci40 and lora tx to communicate
# https://github.com/CreatorDev/PyLetMeCreate/blob/master/examples/lora_example.py
uart.init()
uart.select_bus(MIKROBUS_2)
uart.set_baudrate(UART_BD_57600)

## =======================================

send("sys reset")
time.sleep(1)

if writeconfig is 1:
  time.sleep(1)
  if ( Activation == "OTAA" ):
    send("mac set appkey {0}" .format(Appkey))
    send("mac set appeui {0}" .format(Appeui))
    send("mac set deveui {0}" .format(Deveui))
  elif ( Activation == "ABP"):
    send("mac set nwkskey {0}" .format(Nwkskey))
    send("mac set appskey {0}" .format(Appskey))
    send("mac set devaddr {0}" .format(Devaddr))
  send("mac set adr on")
  send("mac save")
  time.sleep(1)

if ( Activation == "OTAA" ):
  send("mac join otaa")
elif ( Activation == "ABP"):
  send("mac join abp")

while True:
# sending air data in different intervals
#  time.sleep(30)
  read_air()


# -*- coding:utf-8 -*-

import datetime
import logging
import os
import random
import unittest

import matplotlib.dates as mdates
import matplotlib.pyplot as plt
from pandas.plotting import register_matplotlib_converters

from tgym.envs.simple import SimpleEnv
from tgym.market import Market

logging.root.setLevel(logging.INFO)
register_matplotlib_converters()

class TestSimple(unittest.TestCase):
    @classmethod
    def setUpClass(self):
        # NOTE: 需要在环境变量中设置 TUSHARE_TOKEN
        ts_token = os.getenv("TUSHARE_TOKEN")
        self.start = "20190101"
        self.end = "20200101"
        self.codes = ["000001.SZ"]
        self.indexs = ["000001.SH", "399001.SZ"]
        # self.indexs = []
        self.show_plot = False
        self.data_dir = "/tmp/tgym"
        self.m = Market(
            ts_token=ts_token,
            start=self.start,
            end=self.end,
            codes=self.codes,
            indexs=self.indexs,
            data_dir=self.data_dir)
        self.invesment = 100000.0
        self.look_back_days = 10
        self.env = SimpleEnv(
            self.m,
            investment=self.invesment,
            look_back_days=self.look_back_days)

    def plot_portfolio_value(self, name):
        plt.figure(figsize=(15, 7))
        MTFmt = '%Y%m%d'
        ax = plt.gca()
        ax.xaxis.set_major_formatter(mdates.DateFormatter(MTFmt))
        plt.title(self.env.code + " " + name, fontsize=10)
        dates = [datetime.datetime.strptime(d, MTFmt) for d in self.env.dates[
            self.look_back_days:]]
        plt.plot(dates,
                 self.env.portfolio_value_logs,
                 label="portfolio_value")
        plt.show()

    def test_buy_and_hold(self):
        self.env.reset()
        action = self.env.get_buy_close_action(datestr=self.env.current_date)
        obs, reward, done, info, _ = self.env.step(action, only_update=False)
        while not done:
            # buy and hold, 持仓不动
            _, _, done, _, _ = self.env.step(action, only_update=True)
        self.assertEqual(158260.44, self.env.portfolio_value)
        if self.show_plot:
            self.plot_portfolio_value("buy_and_hold")

    def test_random(self):
        random.seed(0)
        self.env.reset()
        action = self.env.get_buy_close_action(datestr=self.env.current_date)
        obs, reward, done, info, _ = self.env.step(action, only_update=False)
        while not done:
            # buy and hold, 持仓不动
            action = self.env.get_random_action()
            _, _, done, _, _ = self.env.step(action, only_update=False)
        self.assertEqual(25217.5, round(self.env.portfolio_value, 1))
        if self.show_plot:
            self.plot_portfolio_value("random_action")

    def test_get_buy_close_action(self):
        actual = self.env.get_buy_close_action(datestr='20190116')
        expect = [0, 0.23438]
        self.assertEqual(expect, actual)

if __name__ == '__main__':
    unittest.main()


#二分查找
#原始数据　－value->[1,2,3,4,5,6,7,8,9,10,11,12,13]
#待查找数据-key-->9
#每次查找范围的左侧下标值：left
#每次查找范围的右侧下标值：right
# def ff(value,key,left,right):
#     if left>right:
#         #查找失败，返回－１
#         return -1
#     #获取中间元素对应下标值
#     middle=(left+right)//2
#     #对比中间元素值与指定查找值
#     #如果两者相同
#     if value[middle]==key:
#         #查找成功
#         return middle
#     elif value[middle]>key:
#         #则在左侧继续查找
#         return ff(value,key,left,middle-1)
#     else:
#         #则在右侧继续查找
#         return ff(value,key,middle+1,right)
# if __name__=='__main__':
#     #原始数据　－
#     value=[1,2,3,4,5,6,7,8,9,10,11,12,13]
#     #待查找数据
#     key=9
#     #顺序查找
#     res=ff(value,key,0,len(value))
#     if res==-1:
#         print("查找失败")
#     else:
#         print("查找成功，对应的下标值：",res)

#循环二分查找
def aa(value,key):
    #获取左侧和右侧对应下标值
    left=0
    right=len(value)
    #若存在合法查找范围则查找数据
    while left<=right:
        #获取对应值下标
        middle=(left+right)//2
        if value[middle]==key:
            return middle
        elif value[middle]>key:
            right=middle-1
        elif value[middle]<key:
            left=middle+1
    return -1
if __name__=='__main__':
    #原始数据　－
    value=[1,2,3,4,5,6,7,8,9,10,11,12,13]
    #待查找数据
    key=9
    #顺序查找
    res=aa(value,key)
    if res==-1:
        print("查找失败")
    else:
        print("查找成功，对应的下标值：",res)


from selenium import webdriver
import pandas as pd
import numpy as np
import time,datetime
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
import os

def calender(first_day, last_day):
    week1_6, week1_0, week1_1, week1_2, week1_3, week1_4, week1_5 = 0, 0, 0, 0, 0, 0, 0
    week2_6, week2_0, week2_1, week2_2, week2_3, week2_4, week2_5 = 0, 0, 0, 0, 0, 0, 0
    week3_6, week3_0, week3_1, week3_2, week3_3, week3_4, week3_5 = 0, 0, 0, 0, 0, 0, 0
    week4_6, week4_0, week4_1, week4_2, week4_3, week4_4, week4_5 = 0, 0, 0, 0, 0, 0, 0
    week5_6, week5_0, week5_1, week5_2, week5_3, week5_4, week5_5 = 0, 0, 0, 0, 0, 0, 0
    week6_6, week6_0, week6_1, week6_2, week6_3, week6_4, week6_5 = 0, 0, 0, 0, 0, 0, 0
    weekes = [[week1_6, week1_0, week1_1, week1_2, week1_3, week1_4, week1_5],
              [week2_6, week2_0, week2_1, week2_2, week2_3, week2_4, week2_5],
              [week3_6, week3_0, week3_1, week3_2, week3_3, week3_4, week3_5],
              [week4_6, week4_0, week4_1, week4_2, week4_3, week4_4, week4_5],
              [week5_6, week5_0, week5_1, week5_2, week5_3, week5_4, week5_5],
              [week6_6, week6_0, week6_1, week6_2, week6_3, week6_4, week6_5],
              ]
    axis_x = [-70, -35, 0, 35, 70, 105, 140]
    axis_y = [120, 150, 180, 210, 240, 290]
    a = np.arange(1, last_day + 1)
    if first_day == 6:
        num = 0
        try:
            for cc in range(first_day + 1):
                weekes[0][cc] = a[cc]
            for j in range(1, 7):
                num = j * 7
                for jj in range(7):
                    weekes[j][jj] = a[num + jj]
        except IndexError:
            pass

    elif first_day == 0:
        num = 0
        try:
            for cc in range(first_day + 1, 7):
                weekes[0][cc] = a[cc - 1]
            for j in range(1, 6):
                num = j * 7
                for jj in range(7):
                    weekes[j][jj] = a[num - first_day + jj - 1]
        except IndexError:
            pass

    elif first_day == 1:
        num = 0
        try:
            for cc in range(first_day + 1, 7):
                weekes[0][cc] = a[cc - 2]
            for j in range(1, 6):
                num = j * 7
                for jj in range(7):
                    weekes[j][jj] = a[num - first_day + jj - 1]
        except IndexError:
            pass

    elif first_day == 2:
        num = 0
        try:
            for cc in range(first_day + 1, 7):
                weekes[0][cc] = a[cc - 3]
            for j in range(1, 6):
                num = j * 7
                for jj in range(7):
                    weekes[j][jj] = a[num - first_day + jj - 1]
        except IndexError:
            pass

    elif first_day == 3:
        num = 0
        try:
            for cc in range(first_day + 1, 7):
                weekes[0][cc] = a[cc - 4]
            for j in range(1, 6):
                num = j * 7
                print(num)
                for jj in range(7):
                    weekes[j][jj] = a[num - first_day + jj - 1]
        except IndexError:
            pass

    elif first_day == 4:
        num = 0
        try:
            for cc in range(first_day + 1, 7):
                weekes[0][cc] = a[cc - 5]
            for j in range(1, 6):
                num = j * 7
                for jj in range(7):
                    weekes[j][jj] = a[num - first_day + jj - 1]
        except IndexError:
            pass

    else:
        num = 0
        try:
            for cc in range(first_day + 1, 7):
                weekes[0][cc] = a[cc - 6]
            for j in range(1, 6):
                num = j * 7
                for jj in range(7):
                    weekes[j][jj] = a[num - first_day + jj - 1]
        except IndexError:
            pass
    return weekes

while True:
    name_station = ['서울특별시', '부산', '제주도']
    now = datetime.datetime.today()
    day = 1
    first_day = datetime.date(now.year, now.month, day)
    first_day = first_day.weekday()
    print(first_day)
    d = datetime.date(now.year, now.month + 1, day)
    t = datetime.timedelta(days=1)
    k = d - t
    last_day = k.day
    axis_x = [-70, -35, 0, 35, 70, 105, 140]
    axis_y = [110, 150, 180, 210, 240, 290]
    weekes = calender(first_day, last_day)
    now_day = now.day
    for nums in range(7):
        if last_day < (nums + now_day):  # (now.day+num):
            if now.month != 12:
                except_firstday = datetime.date(now.year, now.month + 1, 1)
                except_firstday = except_firstday.weekday()
                except_d = datetime.date(now.year, now.month + 1, 1)
                except_t = datetime.timedelta(days=1)
                except_k = except_d - except_t
                except_lastday = except_k.day
                except_weekes = calender(except_firstday, except_lastday)  # 달력 생성
                result_day = (now_day + nums) - last_day
                for i in range(len(except_weekes)):
                    for j in range(len(except_weekes[0])):
                        if except_weekes[i][j] == result_day:
                            except_find_idx = j
                            except_find_idy = i
                            print(except_find_idx, except_find_idy)
                            print('다음달로 넘어갈때 :', axis_x[except_find_idx], axis_y[except_find_idy])
                            break
            else:
                except_firstday = datetime.date(now.year + 1, 1, 1)
                except_firstday = except_firstday.weekday()
                except_d = datetime.date(now.year + 1, 2, 1)
                except_t = datetime.timedelta(days=1)
                except_k = except_d - except_t
                except_lastday = except_k.day
                except_weekes = calender(except_firstday, except_lastday)
                result_day = (now_day + nums) - last_day
                for i in range(len(except_weekes)):
                    for j in range(len(except_weekes[0])):
                        if except_weekes[i][j] == result_day:
                            except_find_idx = j
                            except_find_idy = i
                            print(except_find_idx, except_find_idy)
                            print('12월일때 : ', axis_x[except_find_idx], axis_y[except_find_idy])
                            break
        else:
            for i in range(len(weekes)):
                for j in range(len(weekes[0])):
                    if weekes[i][j] == (now_day + nums):
                        find_idx = j
                        find_idy = i
                        print('일반', find_idx, find_idy)
                        break

        for num in range(len(name_station)):
            for num1 in range(len(name_station)):
                if name_station[num] == name_station[num1]:
                    pass
                else:
                    options = webdriver.ChromeOptions()
                    options.add_argument('headless')
                    options.add_argument('window-size=1920x1080')
                    options.add_argument("disable-gpu")
                    driver = webdriver.Chrome('chromedriver.exe', chrome_options=options)
                    driver.get('https://kr.trip.com/flights/?allianceid=14887&sid=1621876&ouid=nsd0043451&utm_source=naver&utm_medium=cpc&utm_term=nsd0043451&utm_campaign=flight&n_media=27758&n_query=%EB%B9%84%ED%96%89%EA%B8%B0%ED%91%9C%EC%98%88%EC%95%BD&n_rank=1&n_ad_group=grp-a001-01-000000005306659&n_ad=nad-a001-01-000000053067766&n_keyword_id=nkw-a001-01-000000984968160&n_keyword=%EB%B9%84%ED%96%89%EA%B8%B0%ED%91%9C%EC%98%88%EC%95%BD&n_campaign_type=1&NaPm=ct%3Djrqlt2r4%7Cci%3D0yq0003Yr5vqssTx-1m1%7Ctr%3Dsa%7Chk%3D2dc63f741cefb0576f2b86867fca2d8d49e15524&gclid=CLvDzvnLouACFUWavAodZGgGOg&gclsrc=ds')
                    elem = driver.find_element_by_id('FlightSearchBox')
                    elem11 = elem.find_element_by_id('SearchBoxTabSwitch')
                    elem11 = elem11.find_elements_by_tag_name('a')
                    for i in elem11:
                        if i.text == '편도':
                            ac = ActionChains(driver)
                            ac.move_to_element(i)
                            ac.click()
                            ac.perform()
                            time.sleep(1)
                    elem = driver.find_element_by_id('RoundRoute')
                    elem = driver.find_element_by_class_name('clearfix')
                    elem1 = driver.find_element_by_class_name('s-item-l')
                    elem1 = driver.find_element_by_class_name('ipt-borderless')
                    elem1.send_keys(Keys.DELETE)
                    elem1.send_keys(name_station[num])
                    elem1.send_keys(Keys.RETURN)
                    time.sleep(1)
                    elem2 = driver.find_element_by_class_name('s-item-r')
                    elem2 = elem2.find_element_by_class_name('ipt-borderless')
                    elem2.send_keys(Keys.DELETE)
                    elem2.send_keys(name_station[num1])
                    elem2.send_keys(Keys.RETURN)
                    time.sleep(1)
                    # 출발하는날
                    elem3 = driver.find_element_by_class_name('s-item-cln')
                    elem3 = elem3.find_element_by_class_name('ipt-borderless')
                    try:
                        if str(except_find_idx).isdigit():
                            print('다음달로 넘어갈때 처리반')
                            for i in range(1):
                                print('예외처리1')
                                ac = ActionChains(driver)
                                ac.move_to_element(elem3)
                                ac.click()
                                ac.perform()
                                ac.reset_actions()
                                ac.move_by_offset(37 * 11, 150)
                                ac.click()
                                ac.perform()
                            time.sleep(1)
                            elem3 = driver.find_element_by_class_name('s-item-cln')
                            elem3 = elem3.find_element_by_class_name('ipt-borderless')
                            ac = ActionChains(driver)
                            ac.move_to_element(elem3)
                            ac.click()
                            ac.perform()
                            ac.move_to_element(elem3)
                            #                             print(axis_x[except_find_idx],axis_y[except_find_idy])
                            ac.move_by_offset(axis_x[except_find_idx], axis_y[except_find_idy])
                            ac.click()
                            ac.perform()
                            time.sleep(1)

                    except:
                        ac = ActionChains(driver)
                        ac.move_to_element(elem3)
                        ac.click()
                        ac.perform()
                        time.sleep(1)
                        ac.reset_actions()
                        ac.move_to_element(elem3)
                        ac.move_by_offset(axis_x[find_idx], axis_y[find_idy])
                        ac.click()
                        ac.perform()
                        time.sleep(1)
                        pass
                    elem6 = driver.find_element_by_class_name('search-btn')
                    elem6 = elem6.find_element_by_id('SearchSubmit')
                    ac = ActionChains(driver)
                    ac.move_to_element(elem6)
                    ac.click()
                    ac.perform()
                    time.sleep(10)
                    depart_time = []
                    Name = []
                    price = []
                    elem1111 = driver.find_element_by_class_name('m-result-list')
                    elem22222 = elem1111.find_elements_by_class_name('J_FlightItem')
                    if len(elem22222) != 0:
                        for i in range(len(elem22222)):
                            time.sleep(1)
                            elem = driver.find_element_by_class_name('m-result-list')
                            elem = elem.find_element_by_xpath("//div[@data-index=%d]" % i)
                            elem = elem.find_element_by_class_name('J_FlightItem')

                            elem1 = elem.find_element_by_class_name('result-item-table')
                            elem8 = elem1.find_element_by_class_name('result-airline')
                            elem8 = elem8.find_element_by_class_name("c-result-airline__name")
                            Name1 = elem8.text
                            Name.append(Name1)

                            elem2 = elem1.find_element_by_class_name('result-period')
                            elem2 = elem2.find_element_by_class_name('c-result-period__cell')
                            elem2 = elem2.find_element_by_class_name('c-result-period__depart')
                            elem2 = elem2.find_element_by_class_name('c-result-period__time')
                            depart_time.append(elem2.text[:5])
                            time.sleep(1)

                            elem = driver.find_element_by_class_name('m-result-list')
                            elem = elem.find_element_by_xpath("//div[@data-index=%d]" % i)
                            elem = elem.find_element_by_class_name('J_FlightItem')
                            elem4 = elem.find_element_by_class_name('result-item-con')
                            elem4 = elem4.find_element_by_class_name('result-item-con-status')
                            elem4 = elem4.find_element_by_class_name('result-item-price')
                            elem4 = elem4.find_element_by_class_name('o-price-flight')
                            elem4 = elem4.find_element_by_class_name('o-price-flight__num')
                            price1 = elem4.text
                            price1 = int(''.join(price1.split(',')))
                            price.append(price1)
                        driver.quit()
                    else:
                        driver.quit()
                        Name, depart_time, price = ['blank'], ['blank'], ['blank']
                    print('@@@@@@@@@@@@@@@@@@@' * 2)
                    train = pd.DataFrame({'Name': Name, 'depart_time': depart_time, 'price': price})
                    # os.mkdir('airplan')
                    train['depart_time_hour'], train['depart_time_minute'] = range(train.shape[0]), range(
                        train.shape[0])
                    for i in range(train.shape[0]):
                        train['depart_time_hour'][i], train['depart_time_minute'][i] = train['depart_time'][i].split(
                            ':')
                        train['depart_time_hour'] = train['depart_time_hour'].astype('int64')
                        train['depart_time_minute'] = train['depart_time_minute'].astype('int64')
                    train.sort_values(by=['depart_time_hour', 'depart_time_minute'], ascending=True, inplace=True)
                    train.to_csv('airplan/trip_airplan_%s_%s_%s.csv' % (nums, num, num1), index=False)
        # nums 가 오늘 0 내일 1 이틀뒤 2 3일뒤 3 4일뒤 4 5일뒤 5 6일뒤 6
        # num 출발도시 num1 도착도시
        # 서울  0  부산 1 제주도 2
        # 서울에서 부산 0 _ 1
        print('하루 정보를 다가져왓습니다')
    print('일주일치를 다 가져왓습니다')
    time.sleep(60 * 60)


'''
Problem 25

The Fibonacci sequence is defined by the recurrence relation:

Fn = Fn−1 + Fn−2, where F1 = 1 and F2 = 1.
Hence the first 12 terms will be:

F1 = 1
F2 = 1
F3 = 2
F4 = 3
F5 = 5
F6 = 8
F7 = 13
F8 = 21
F9 = 34
F10 = 55
F11 = 89
F12 = 144
The 12th term, F12, is the first term to contain three digits.

What is the index of the first term in the Fibonacci sequence to contain 1000 digits?
'''

import time

def one_thousand_digit_fibonacci_number():
	prev, curr, counter, n = 1, 1, 1, 1000
	while len(str(prev)) < n:
	    prev, curr = curr, prev + curr
	    counter += 1
	return counter

if __name__ == '__main__':

	start = time.time()
	print(one_thousand_digit_fibonacci_number())
	end = time.time()

	print("Execution time: %fs" %(end - start))


"""
Class to test the synchronization by comparing the resulting metadata presentation
formats from DHIS2 and OCL.
"""
import sys
import requests
import warnings
import difflib
import deepdiff
import pprint
from operator import itemgetter
import datimconstants
import datimbase
import datimshow

class DatimSyncTest(datimbase.DatimBase):
    """
    Class to validate synchronizations between DHIS2 and OCL by comparing the resulting metadata presentation
    formats from each system.
    """

    def __init__(self, oclenv='', oclapitoken='', formats=None, dhis2env='', dhis2uid='', dhis2pwd=''):
        datimbase.DatimBase.__init__(self)
        self.oclenv = oclenv
        self.oclapitoken = oclapitoken
        self.dhis2env = dhis2env
        self.dhis2uid = dhis2uid
        self.dhis2pwd = dhis2pwd
        if formats:
            self.formats = formats
        else:
            self.formats = datimshow.DatimShow.PRESENTATION_FORMATS
        self.oclapiheaders = {
            'Authorization': 'Token ' + self.oclapitoken,
            'Content-Type': 'application/json'
        }

    def test_all(self):
        for resource_type in datimconstants.DatimConstants.SYNC_RESOURCE_TYPES:
            self.test_resource_type(resource_type)

    def test_resource_type(self, resource_type):
        if resource_type == datimconstants.DatimConstants.IMPORT_BATCH_SIMS:
            self.test_sims()
        elif resource_type == datimconstants.DatimConstants.IMPORT_BATCH_MECHANISMS:
            self.test_mechanisms()
        elif resource_type == datimconstants.DatimConstants.IMPORT_BATCH_TIERED_SUPPORT:
            self.test_tiered_support()
        elif resource_type == datimconstants.DatimConstants.IMPORT_BATCH_MER:
            self.test_mer()
        else:
            print('ERROR: Unrecognized resource_type "%s"' % resource_type)
            sys.exit(1)

    def test_sims(self):
        self.test_default(datimconstants.DatimConstants.SIMS_OCL_EXPORT_DEFS,
                          datimconstants.DatimConstants.OPENHIM_ENDPOINT_SIMS)

    def test_mechanisms(self):
        self.test_default(datimconstants.DatimConstants.MECHANISMS_OCL_EXPORT_DEFS,
                          datimconstants.DatimConstants.OPENHIM_ENDPOINT_MECHANISMS)

    def test_tiered_support(self):
        self.test_default(datimconstants.DatimConstants.TIERED_SUPPORT_OCL_EXPORT_DEFS,
                          datimconstants.DatimConstants.OPENHIM_ENDPOINT_TIERED_SUPPORT)

    def test_default(self, export_defs, openhim_endpoint):
        for export_def_key in export_defs:
            # Iterate through the formats
            for format in self.formats:
                if 'dhis2_sqlview_id' not in export_defs[export_def_key]:
                    continue

                # Build the dhis2 presentation url
                dhis2_presentation_url = self.replace_attr(
                    datimconstants.DatimConstants.DHIS2_PRESENTATION_URL_DEFAULT,
                    {'format': format, 'sqlview': export_defs[export_def_key]['dhis2_sqlview_id']})

                # Build the OCL presentation url
                openhimenv = 'https://ocl-mediator-trial.ohie.org:5000/'
                ocl_presentation_url = '%s%s?format=%s&collection=%s' % (
                    openhimenv, openhim_endpoint, format, export_def_key)

                self.test_one(format, dhis2_presentation_url, ocl_presentation_url)

    def test_mer(self):
        for export_def_key in datimconstants.DatimConstants.MER_OCL_EXPORT_DEFS:
            # Fetch the external_id from OCL, which is the DHIS2 dataSet uid
            url_ocl_repo = self.oclenv + datimconstants.DatimConstants.MER_OCL_EXPORT_DEFS[export_def_key]['endpoint']
            r = requests.get(url_ocl_repo, headers=self.oclapiheaders)
            repo = r.json()
            print('\n**** %s (dataSet.id=%s) ****' % (datimconstants.DatimConstants.MER_OCL_EXPORT_DEFS[export_def_key]['endpoint'], repo['external_id']))
            if not repo['external_id']:
                print('Skipping because no external ID...')
                continue

            # Iterate through the formats
            for format in self.formats:

                # Build the dhis2 presentation url
                dhis2_presentation_url = self.replace_attr(
                    datimconstants.DatimConstants.DHIS2_PRESENTATION_URL_MER,
                    {'format': format, 'dataset': repo['external_id']})

                # Build the OCL presentation url
                openhimenv = 'https://ocl-mediator-trial.ohie.org:5000/'
                ocl_presentation_url = '%s%s?format=%s&collection=%s' % (
                    openhimenv, datimconstants.DatimConstants.OPENHIM_ENDPOINT_MER, format, export_def_key)

                self.test_one(format, dhis2_presentation_url, ocl_presentation_url)

    def test_one(self, format, dhis2_presentation_url, ocl_presentation_url):
        print ('Format = %s' % format)
        print('DHIS2: %s' % dhis2_presentation_url)
        print('OCL:   %s' % ocl_presentation_url)
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            request_dhis2 = requests.get(dhis2_presentation_url)
            request_ocl = requests.get(ocl_presentation_url, verify=False)
        diff = None
        if format == datimshow.DatimShow.DATIM_FORMAT_JSON:
            diff = self.test_json(request_dhis2, request_ocl)
        elif format == datimshow.DatimShow.DATIM_FORMAT_HTML:
            diff = self.test_html(request_dhis2, request_ocl)
        elif format == datimshow.DatimShow.DATIM_FORMAT_XML:
            diff = self.test_xml(request_dhis2, request_ocl)
        elif format == datimshow.DatimShow.DATIM_FORMAT_CSV:
            diff = self.test_csv(request_dhis2, request_ocl)
        if diff:
            print('Diff Results:')
            pprint.pprint(diff)
        else:
            print('No diff!')
        sys.stdout.flush()

    def test_json(self, request_dhis2, request_ocl):
        # Prepare the diff
        dhis2_json = request_dhis2.json()
        ocl_json = request_ocl.json()
        dhis2_json['rows_dict'] = {}
        ocl_json['rows_dict'] = {}
        for row in dhis2_json['rows']:
            dhis2_json['rows_dict']['%s-%s' % (row[4], row[7])] = row[1:]
        for row in ocl_json['rows']:
            ocl_json['rows_dict']['%s-%s' % (row[4], row[7])] = row[1:]
        del(dhis2_json['rows'])
        del(ocl_json['rows'])
        print('Rows: DHIS2(%s), OCL(%s)' % (len(dhis2_json['rows_dict']), len(ocl_json['rows_dict'])))

        # Do the diff
        diff = deepdiff.DeepDiff(dhis2_json, ocl_json, ignore_order=False, verbose_level=2, exclude_paths={"root['title']", "root['subtitle']"})
        return diff

    def test_html(self, request_dhis2, request_ocl):
        d = difflib.Differ()
        return d.compare(request_dhis2.text.splitlines(1), request_ocl.text.splitlines(1))

    def test_xml(self, request_dhis2, request_ocl):
        d = difflib.Differ()
        return d.compare(request_dhis2.text.splitlines(1), request_ocl.text.splitlines(1))

    def test_csv(self, request_dhis2, request_ocl):
        d = difflib.Differ()
        return d.compare(request_dhis2.text.splitlines(1), request_ocl.text.splitlines(1))