""" CSE Partial Factorization and Post-Processing

The following script will perform partial factorization on SymPy expressions,
which should occur before common subexpression elimination (CSE) to prevent the
identification of undesirable patterns, and perform post-processing on the
the resulting replaced/reduced expressions after the CSE procedure was applied.
"""
# Author: Ken Sible
# Email:  ksible@outlook.com

from SIMDExprTree import ExprTree
import sympy as sp

# Input:  expr_list = single SymPy expression or list of SymPy expressions
#         prefix    = string prefix for variable names (i.e. replacement symbols)
#         declare   = declare negative one symbol (i.e. _NegativeOne_)
#         factor    = perform partial factorization (excluding negative one)
# Output: modified SymPy expression(s) where all integers and rationals were replaced
#           with temporary placeholder variables that allow for partial factorization
def cse_preprocess(expr_list, prefix='', declare=False, factor=True, debug=False):
    if not isinstance(expr_list, list):
        expr_list = [expr_list]
    def expand(a, n):
        if  n == 2:  return sp.Mul(a, a, evaluate=False)
        elif n > 2:  return sp.Mul(expand(a, n - 1), a, evaluate=False)
        return sp.Pow(expand(a, -n), -1, evaluate=False)
    _NegativeOne_ = sp.Symbol(prefix + '_NegativeOne_')
    map_sym_to_rat, map_rat_to_sym = {}, {}
    for i, expr in enumerate(expr_list):
        tree = ExprTree(expr)
        # Expand power function, preventing replacement of exponent argument
        for subtree in tree.preorder(tree.root):
            subexpr = subtree.expr
            if subexpr.func == sp.Pow:
                exponent = subtree.children[1].expr
                if exponent.func == sp.Integer and abs(exponent) > 1:
                    subtree.expr = expand(*subexpr.args)
                    tree.build(subtree, clear=True)
        # Search through expression tree for integers/rationals
        for subtree in tree.preorder():
            subexpr = subtree.expr
            if isinstance(subexpr, sp.Rational) and subexpr != sp.S.NegativeOne:
                # If rational < 0, factor out negative and declare positive rational
                sign = 1 if subexpr >= 0 else -1
                subexpr *= sign
                # Check whether rational was already declared, otherwise declare rational
                try: repl = map_rat_to_sym[subexpr]
                except KeyError:
                    p, q = subexpr.p, subexpr.q
                    var_name = prefix + '_Rational_' + str(p) + '_' + str(q) \
                        if q != 1 else prefix + '_Integer_' + str(p)
                    repl = sp.Symbol(var_name)
                    map_sym_to_rat[repl], map_rat_to_sym[subexpr] = subexpr, repl
                subtree.expr = repl * sign
                if sign < 0: tree.build(subtree, clear=True)
            # If declare == True, then declare symbol for -1 or extracted negative
            elif declare and subexpr == sp.S.NegativeOne:
                try: subtree.expr = map_rat_to_sym[sp.S.NegativeOne]
                except KeyError:
                    repl = _NegativeOne_
                    map_sym_to_rat[repl], map_rat_to_sym[subexpr] = subexpr, repl
                    subtree.expr = repl
        # If exponent was replaced with symbol (usually -1), then back-substitute
        for subtree in tree.preorder(tree.root):
            subexpr = subtree.expr
            if subexpr.func == sp.Pow:
                exponent = subtree.children[1].expr
                if exponent.func == sp.Symbol:
                    subtree.children[1].expr = map_sym_to_rat[exponent]
        expr = tree.reconstruct()
        # If factor == True, then perform partial factoring
        if factor:
            # Handle the separate case of function arguments
            for subtree in tree.preorder():
                if isinstance(subtree.expr, sp.Function):
                    for var in map_sym_to_rat:
                        if var != _NegativeOne_:
                            child = subtree.children[0]
                            child.expr = sp.collect(child.expr, var)
                            child.children.clear()
            expr = tree.reconstruct()
            # Perform partial factoring on the expression(s)
            for var in map_sym_to_rat:
                if var != _NegativeOne_:
                    expr = sp.collect(expr, var)
        # If debug == True, then back-substitute everything and check difference
        if debug:
            def lookup_rational(arg):
                if arg.func == sp.Symbol:
                    try: arg = map_sym_to_rat[arg]
                    except KeyError: pass
                return arg
            debug_tree = ExprTree(expr)
            for subtree in debug_tree.preorder():
                subexpr = subtree.expr
                if subexpr.func == sp.Symbol:
                    subtree.expr = lookup_rational(subexpr)
            debug_expr = tree.reconstruct()
            expr_diff  = expr - debug_expr
            if sp.simplify(expr_diff) != 0:
                raise Warning('Expression Difference: ' + str(expr_diff))
        expr_list[i] = expr
    if len(expr_list) == 1:
        expr_list = expr_list[0]
    return expr_list, map_sym_to_rat

# Input:  cse_output = output from SymPy CSE with tuple format: (list of ordered pairs that
#            contain substituted symbols and their replaced expressions, reduced SymPy expression)
# Output: output from SymPy CSE where postprocessing, such as back-substitution of addition/product
#            of symbols, has been applied to the replaced/reduced expression(s)
def cse_postprocess(cse_output):
    replaced, reduced = cse_output
    i = 0
    while i < len(replaced):
        sym, expr = replaced[i]
        # Search through replaced expressions for negative symbols
        if (expr.func == sp.Mul and len(expr.args) == 2 and \
                any((arg.func == sp.Symbol) for arg in expr.args) and \
                any((arg == sp.S.NegativeOne or '_NegativeOne_' in str(arg)) for arg in expr.args)):
            for k in range(i + 1, len(replaced)):
                if sym in replaced[k][1].free_symbols:
                    replaced[k] = (replaced[k][0], replaced[k][1].subs(sym, expr))
            for k in range(len(reduced)):
                if sym in reduced[k].free_symbols:
                    reduced[k] = reduced[k].subs(sym, expr)
            # Remove the replaced expression from the list
            replaced.pop(i)
            if i != 0: i -= 1
        # Search through replaced expressions for addition/product of 2 or less symbols
        if ((expr.func == sp.Add or expr.func == sp.Mul) and 0 < len(expr.args) < 3 and \
                all((arg.func == sp.Symbol or arg.is_integer or arg.is_rational) for arg in expr.args)) or \
                (expr.func == sp.Pow and expr.args[0].func == sp.Symbol and expr.args[1] == 2):
            sym_count = 0 # Count the number of occurrences of the substituted symbol
            for k in range(len(replaced) - i):
                # Check if the substituted symbol appears in the replaced expressions
                if sym in replaced[i + k][1].free_symbols:
                    for arg in sp.preorder_traversal(replaced[i + k][1]):
                        if arg.func == sp.Symbol and str(arg) == str(sym):
                            sym_count += 1
            for k in range(len(reduced)):
                # Check if the substituted symbol appears in the reduced expression
                if sym in reduced[k].free_symbols:
                    for arg in sp.preorder_traversal(reduced[k]):
                        if arg.func == sp.Symbol and str(arg) == str(sym):
                            sym_count += 1
            # If the number of occurrences of the substituted symbol is 2 or less, back-substitute
            if 0 < sym_count < 3:
                for k in range(i + 1, len(replaced)):
                    if sym in replaced[k][1].free_symbols:
                        replaced[k] = (replaced[k][0], replaced[k][1].subs(sym, expr))
                for k in range(len(reduced)):
                    if sym in reduced[k].free_symbols:
                        reduced[k] = reduced[k].subs(sym, expr)
                # Remove the replaced expression from the list
                replaced.pop(i); i -= 1
        i += 1
    return replaced, reduced


from algorithm.li.extract.utils.dbutils import dbs
import re
from urllib import parse as pa

def counter(name_list=[]):
    from collections import Counter

    count_list = Counter(name_list).most_common()

    s = []
    for c in count_list:
        s.append(c[0] + "(" + str(c[1]) + ")")
    return "，".join(s)

def data_clean():
    """
    将javascript的链接转换为正常
    :return:
    """
    data_list = dbs.getDics("SELECT * FROM `eds_985teacher` WHERE link like '%javascript%' AND school = '中南大学';")
    print(len(data_list))
    u_list = []
    for data in data_list:
        id = data['id']
        '''javascript:window.open('/blog/content2?name='+encodeURI('周雄伟'))'''
        link = data['link']
        if link != "":
            p_tuple = re.findall(r"open\('(.+?)'\+encodeURI\('(.+?)'\)\)", link)[0]

            link = p_tuple[0] + pa.quote(p_tuple[1])
            # print(pa.urljoin(data['institution_url'], link))
            link = pa.urljoin(data['institution_url'], link)
            print(link)
            u_list.append((link, id))

    print(len(u_list))
    u_sql = "UPDATE eds_985teacher SET all_link=%s WHERE id = %s"
    print(dbs.exe_many(u_sql, u_list))

def honor_clean():
    """
    清华国家科技奖励 清洗
    :return:
    """
    in_lines = open('.\\qinghua\\清华_国家科学技术进步奖.csv', 'r', encoding='utf-8').read().split('\n')
    teacher_lines = open('.\\qinghua\\清华教师名单.csv', 'r', encoding='utf-8').read().split('\n')
    teacher_dict = dict()
    for i in range(0, len(teacher_lines)):
        info = teacher_lines[i].split(',')
        if teacher_dict.get(info[1]):
            teacher_dict[info[1]].append(i)
        else:
            teacher_dict[info[1]] = [i]

    out_lines = open('.\\qinghua\\清华_国家科学技术进步奖_clean.csv', 'a', encoding='utf8')
    for line in in_lines:
        info = line.split(',')
        name = info[2]
        index_list = teacher_dict.get(name)

        if index_list:
            for index in index_list:
                print(line + ',' + teacher_lines[index])
                out_lines.write(line + ',' + teacher_lines[index]+'\n')
        else:
            print(line)
            out_lines.write(line + '\n')
    out_lines.close()

def match_title():
    title_dict = eval(open('.\\qinghua\\title.txt', 'r', encoding='utf-8').read())
    id_list = open('.\\qinghua\\id.txt', 'r', encoding='utf-8').read().split('\n')
    out_lines = open('.\\qinghua\\id_title.csv', 'a', encoding='utf8')
    result_list = []
    for id in id_list:
        if id == "":
            print(",")
            result_list.append(",")
        else:
            title = title_dict.get(id, "")
            print(id+","+title)
            result_list.append(id+","+title)

    out_lines.write('\n'.join(result_list))
    out_lines.close()

def match_level():
    level = {}

    level_list = open('.\\qinghua\\02.csv', 'r', encoding='utf-8').read().split('\n')
    for line in level_list:
        info = line.split(',')
        level[info[1]+info[2]] = info[0]
    print(len(level))
    teacher_list = open('.\\qinghua\\01.csv', 'r', encoding='utf-8').read().split('\n')
    for line in teacher_list:
        teacher = line.split(',')
        key = teacher[1] + teacher[0]
        value = level.get(key, "")
        print(teacher[1], teacher[0], value)

def match_teacher():
    import pandas as pd
    import numpy as np

    honor_df = pd.read_csv('.\\qinghua\\qinghua_jianglihuizong.csv')
    """
    honor     189 non-null object
    level     189 non-null object
    code      189 non-null object
    p_name    176 non-null object
    name      189 non-null object
    school    117 non-null object
    year      189 non-null int64
    """
    honor_dict = dict()
    for index in honor_df.index:
        honor = str(honor_df.loc[index].honor)
        level = str(honor_df.loc[index].level)
        code = str(honor_df.loc[index].code)
        p_name = str(honor_df.loc[index].p_name)
        t_name = str(honor_df.loc[index].t_name)
        school = str(honor_df.loc[index].school)
        year = str(honor_df.loc[index].year)

        the_honor = honor + "," + level + "," + code + "," + p_name + "," + school + "," + year

        the_list = honor_dict.get(t_name, [])
        if the_list:
            the_list.append(the_honor)
            honor_dict[t_name] = the_list
        else:
            the_list = [the_honor]
            honor_dict[t_name] = the_list

    mentor_df = pd.read_csv('.\\qinghua\\yuanshihezuoxuezhe.csv')
    """
    mentor        86 non-null object
    co_author     82 non-null object
    num1          86 non-null int64
    db_teahcer    82 non-null object
    num2          86 non-null int64
    """

    # for index in mentor_df.index:
    #     mentor = str(mentor_df.loc[index].mentor)
    #
    #     the_honor = honor_dict.get(mentor, "")
    #     for h in the_honor:
    #         print(mentor+","+h)

    # for index in mentor_df.index:
    #     db_teahcer = str(mentor_df.loc[index].db_teahcer)
    #     if db_teahcer == "nan":
    #         continue
    #     teacher_list = db_teahcer.split(',')
    #     for teacher in teacher_list:
    #
    #         the_honor = honor_dict.get(teacher, "")
    #         for h in the_honor:
    #             print(teacher+","+h)

    for index in mentor_df.index:
        co_author = str(mentor_df.loc[index].co_author)
        if co_author == "nan":
            continue
        teacher_list = co_author.split(',')
        for teacher in teacher_list:

            the_honor = honor_dict.get(teacher, "")
            for h in the_honor:
                print(teacher+","+h)

def zhuanli_duplicate():
    s_sql = "SELECT * FROM `pss_zhuanli` GROUP BY TIVIEW, INVIEW, APD"
    info_list = dbs.getDics(s_sql)

    update_list = []
    for info in info_list:
        update_list.append((info['TIVIEW'], info['INVIEW'], info['APD'], info['id']))

    for i in update_list:
        print(i)
    print(len(update_list))
    d_sql = '''
            DELETE FROM `pss_zhuanli_copy`
            WHERE TIVIEW=%s
            AND INVIEW=%s
            AND APD=%s
            AND id !=%s
    '''
    print(dbs.exe_many(d_sql, update_list))

def mentor_extract():
    s_sql = "SELECT * FROM `pss_zhuanli_clean`;"
    info_list = dbs.getDics(s_sql)
    mentor_dict = {}.fromkeys(open('.\\qinghua\\mentor_list.txt', 'r', encoding='utf-8').read().split('\n'))
    print(mentor_dict)
    print("*" * 10)
    update_list = []
    for item in info_list:
        author_list = item['INVIEW'].split(';')
        mentor_list = []
        for author in author_list:
            if mentor_dict.get(author, "") != "":
                mentor_list.append(author)
        print(author_list, mentor_list)
        update_list.append((";".join(mentor_list), item['id']))

    u_sql = "UPDATE `pss_zhuanli_clean` SET MENTOR = %s WHERE id=%s"
    print(len(update_list))
    print(dbs.exe_many(u_sql, update_list))

def create_sheet():

    import xlwt

    s_sql = "SELECT * FROM `pss_zhuanli_clean`;"
    info_list = dbs.getDics(s_sql)
    teacher_dict = {}.fromkeys(open('.\\qinghua\\teacher_list.txt', 'r', encoding='utf-8').read().split('\n'))
    wbk = xlwt.Workbook(encoding='utf-8')
    sheet = wbk.add_sheet('sheet1')
    row = 0
    sum = 0
    for info in info_list:

        author_list = info['INVIEW'].split(';')
        teacher_list = []
        for author in author_list:
            if teacher_dict.get(author, "") != "":
                teacher_list.append(author)

        mentor_list = info['MENTOR'].split(';')
        sum += len(mentor_list)
        for mentor in mentor_list:
            if len(mentor_list) > 1:
                print("======")
            print(mentor, ";".join(teacher_list), info['TIVIEW'], info['PAVIEW'], info['APD'], info['PD'])

            sheet.write(row, 0, mentor)
            sheet.write(row, 1, ";".join(teacher_list))
            sheet.write(row, 2, info['TIVIEW'])
            sheet.write(row, 3, info['PAVIEW'])
            sheet.write(row, 4, info['APD'])
            sheet.write(row, 5, info['PD'])
            row += 1

    wbk.save('.\\qinghua\\清华院士_专利信息_2018.9.30.xls')
    print(sum)

def mentor_team_year_honor():
    import pandas as pd
    import xlrd
    import xlwt

    teacher_dict = {}.fromkeys(open('.\\qinghua\\teacher_list.txt', 'r', encoding='utf8').read().split('\n'))
    mentor_dict = {}.fromkeys(open('.\\qinghua\\mentor_list.txt', 'r', encoding='utf8').read().split('\n'))
    mentor_dict_clean = {}.fromkeys(open('.\\qinghua\\mentor_list_clean.txt', 'r', encoding='utf8').read().split('\n'))

    honor_dict = dict()
    yuanshi_list = []

    wb_rd_1 = xlrd.open_workbook('.\\qinghua\\processed\\清华大学_奖励汇总.xls')
    wb_rd_1_sheet1 = wb_rd_1.sheet_by_index(0)

    for row in range(1, wb_rd_1_sheet1.nrows):
        line = wb_rd_1_sheet1.row_values(row)
        honor = line[0]
        level = line[1]
        code = line[2]
        p_name = line[3]
        name = line[4]
        year = str(int(line[6]))

        # honor_dict
        if teacher_dict.get(name, "") == "":
            continue
        the_list = honor_dict.get(str(code) + str(p_name), [])
        the_list.append(name)
        honor_dict[str(code) + str(p_name)] = the_list

        # yuanshi_list
        if mentor_dict.get(name, "") != "":
            yuanshi_list.append((honor, level, code, p_name, name, year))

    print(honor_dict)
    print("=" * 10)
    print(yuanshi_list)

    yuanshi_dict = dict()
    for the in yuanshi_list:
        honor = the[0]
        level = the[1]
        code = the[2]
        p_name = the[3]
        name = the[4]
        year = the[5]

        team_list = honor_dict.get(code + p_name, [])
        team_list = [i for i in team_list if i != name]
        team_str = counter(team_list)
        p_list = yuanshi_dict.get(name, [])
        if p_list:
            p_list.append((year, team_str, honor, level, p_name))
        else:
            p_list = [(year, team_str, honor, level, p_name)]
        yuanshi_dict[name] = p_list

    wb_rd_2 = xlrd.open_workbook('.\\qinghua\\tuandui\\清华院士团队及其成果.xls')
    wb_rd_2_sheet1 = wb_rd_2.sheet_by_index(0)

    wb_wt = xlwt.Workbook(encoding='utf-8')
    sheet = wb_wt.add_sheet('sheet1')
    sum = 1
    for row in range(1, wb_rd_2_sheet1.nrows):
        line = wb_rd_2_sheet1.row_values(row)
        ins = line[0]
        name = line[1]

        if mentor_dict_clean.get(name, "") == "":
            continue

        the_yuanshi = yuanshi_dict.get(name, [])

        if the_yuanshi:
            for i in the_yuanshi:
                year = i[0]
                team = i[1]
                achi = i[2] + i[3]
                p_name = i[4]
                # (year, team_str, honor, level, p_name)

                sheet.write(sum, 0, ins)
                sheet.write(sum, 1, name)
                sheet.write(sum, 2, year)
                sheet.write(sum, 3, team)
                sheet.write(sum, 4, achi)
                sheet.write(sum, 5, p_name)

                sum += 1
        else:
            year = ""
            team = ""
            achi = ""
            p_name = ""

            sheet.write(sum, 0, ins)
            sheet.write(sum, 1, name)
            sheet.write(sum, 2, year)
            sheet.write(sum, 3, team)
            sheet.write(sum, 4, achi)
            sheet.write(sum, 5, p_name)

            sum += 1

        wb_wt.save('.\\qinghua\\tuandui\\清华院士团队及其成果_国家科技奖_2018.9.30.xls')

def mentor_team_year_zhuanli():
    import xlwt
    import xlrd
    import re

    mentor_dict = {}.fromkeys(open('.\\qinghua\\mentor_list_clean.txt', 'r', encoding='utf-8').read().split('\n'))

    zhuanli_dict = dict()

    wb_rd_zhuanli = xlrd.open_workbook('.\\qinghua\\清华院士_专利信息_2018.9.30.xls')
    sheet_zhuanli = wb_rd_zhuanli.sheet_by_index(0)
    for row in range(1, sheet_zhuanli.nrows):
        line = sheet_zhuanli.row_values(row)
        mentor = line[0]

        team_list = [i for i in line[1].split(';') if i != mentor]
        p_name = line[2]
        year = int(re.findall(r'[0-9]{4}', line[5])[0])

        m_dict = zhuanli_dict.get(mentor, {})
        if m_dict:
            y_dict = m_dict.get(year, {})
            if y_dict:
                y_dict["team"].extend(team_list)
                y_dict["p_list"].append(p_name)
            else:
                y_dict["team"] = team_list
                y_dict["p_list"] = [p_name]
            m_dict[year] = y_dict
        else:
            y_dict = dict()
            y_dict["team"] = team_list
            y_dict["p_list"] = [p_name]
            m_dict[year] = y_dict

        zhuanli_dict[mentor] = m_dict

    workbook = xlrd.open_workbook('.\\qinghua\\tuandui\\清华院士团队及其成果.xls')
    sheet1 = workbook.sheet_by_index(0)

    wbk = xlwt.Workbook(encoding='utf-8')
    sheet = wbk.add_sheet('sheet1')

    sum = 0
    for row in range(1, sheet1.nrows):
        line = sheet1.row_values(row)
        ins = line[0]
        name = line[1]

        if mentor_dict.get(name, "") == "":
            continue

        the_dict = zhuanli_dict.get(name, {})

        if the_dict:
            for k, value in the_dict.items():
                year = k
                team = counter(value.get('team', []))
                achi = ';'.join(value.get('p_list', []))
                print(sum, year, team, achi)
                sheet.write(sum, 0, ins)
                sheet.write(sum, 1, name)
                sheet.write(sum, 2, year)
                sheet.write(sum, 3, team)
                sheet.write(sum, 4, achi)
                sum += 1
            pass
        else:
            year = ""
            team = ""
            achi = ""
            print(sum, year, team, achi)
            sheet.write(sum, 0, ins)
            sheet.write(sum, 1, name)
            sheet.write(sum, 2, year)
            sheet.write(sum, 3, team)
            sheet.write(sum, 4, achi)
            sum += 1

    wbk.save('.\\qinghua\\tuandui\\清华院士团队及其成果_专利_2018.9.30.xls')

def t():
    import os
    import xlwt
    import xlrd
    dir_list = os.listdir('.\\qinghua\\process\\')
    print(dir_list)

    for file in dir_list:
        print(file)
        wb_rd = xlrd.open_workbook('.\\qinghua\\process\\' + file)
        sheet1_rd = wb_rd.sheet_by_index(0)

        wbk = xlwt.Workbook(encoding='utf-8')
        sheet = wbk.add_sheet('sheet1')

        sum = 0
        for row in range(1, sheet1_rd.nrows):
            line = sheet1_rd.row_values(row)
            level = line[0]
            code = line[1]
            p_name = line[2]
            name_list = line[3].split('、')
            for name in name_list:
                print(level, code, p_name, name)
                sheet.write(sum, 0, level)
                sheet.write(sum, 1, code)
                sheet.write(sum, 2, p_name)
                sheet.write(sum, 3, name)
                sheet.write(sum, 4, "")
                sum += 1

        wbk.save('.\\qinghua\\processed\\' + file)

def merge_table():
    import os
    import xlwt
    import xlrd
    dir_list = os.listdir('.\\qinghua\\技术发明\\')
    print(dir_list)

    wbk = xlwt.Workbook(encoding='utf-8')
    sheet = wbk.add_sheet('sheet1')

    sum = 0
    for file in dir_list:
        print(file)
        wb_rd = xlrd.open_workbook('.\\qinghua\\技术发明\\' + file)
        sheet1_rd = wb_rd.sheet_by_index(0)
        year = re.findall(r'[0-9]{4}', file)[0]

        for row in range(1, sheet1_rd.nrows):
            line = sheet1_rd.row_values(row)
            level = line[0]
            code = line[1]
            p_name = line[2]
            name_list = line[3].split('、')
            if len(line) <= 4:
                school = ""
            else:
                school = line[4]
            for name in name_list:
                print(level, code, p_name, name)
                sheet.write(sum, 0, "技术发明奖")
                sheet.write(sum, 1, level)
                sheet.write(sum, 2, code)
                sheet.write(sum, 3, p_name)
                sheet.write(sum, 4, name)
                sheet.write(sum, 5, school)
                sheet.write(sum, 6, year)
                sum += 1

    wbk.save('.\\qinghua\\processed\\' + "技术发明奖.xls")

def honor_huizong():
    import os
    import xlwt
    import xlrd
    dir_list = os.listdir('.\\qinghua\\processed\\')
    print(dir_list)

    wbk = xlwt.Workbook(encoding='utf-8')
    sheet = wbk.add_sheet('sheet1')

    sum = 0
    for file in dir_list:
        print(file)
        wb_rd = xlrd.open_workbook('.\\qinghua\\processed\\' + file)
        sheet1_rd = wb_rd.sheet_by_index(0)

        for row in range(1, sheet1_rd.nrows):
            # honor	level   code    p_name	name	school	year
            line = sheet1_rd.row_values(row)
            honor = line[0]
            level = line[1]
            code = line[2]
            p_name = line[3]
            name = line[4]
            school = line[5]
            year = line[6]

            if school == "" and not re.findall('清华大学', school):
                continue
            print(honor,level,code,p_name,name,school,year)
            sheet.write(sum, 0, honor)
            sheet.write(sum, 1, level)
            sheet.write(sum, 2, code)
            sheet.write(sum, 3, p_name)
            sheet.write(sum, 4, name)
            sheet.write(sum, 5, school)
            sheet.write(sum, 6, year)
            sum += 1

    wbk.save('.\\qinghua\\processed\\' + "清华大学_奖励汇总.xls")

def t_():
    id_list = open('.\\qinghua\\id_list.txt', 'r', encoding='utf-8').read().split('\n')
    print(len(id_list))
    u_sql = "UPDATE zhuanli_search SET status=0 where id = %s"
    print(dbs.exe_many(u_sql, id_list))
    pass

if __name__ == "__main__":
    # zhuanli_duplicate()
    # mentor_extract()
    # create_sheet()
    # mentor_team_year_zhuanli()
    # merge_table()
    # honor_huizong()
    # mentor_team_year_honor()
    t_()
    pass


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.nn.utils.weight_norm import weight_norm

import math

class SleepPredictionMLP(nn.Module):
    def __init__(self, input_size, hidden_size, num_classes=3,
                 num_layers=1, dropout_p=0.0, w_norm=False):
        super(SleepPredictionMLP, self).__init__()

        self.input_size = input_size
        self.hidden_size = hidden_size
        self.num_classes = num_classes
        self.w_norm = w_norm
        layers = []

        for i in range(num_layers):
            idim = hidden_size
            odim = hidden_size
            if i == 0:
                idim = input_size
            if i == num_layers-1:
                odim = num_classes
            fc = nn.Linear(idim, odim)
            fc.weight.data.normal_(0.0,  math.sqrt(2. / idim))
            fc.bias.data.fill_(0)

            if w_norm:
                fc = weight_norm(fc, dim=None)
            layers.append(fc)

            if i != num_layers-1:
                layers.append(nn.ReLU())
                layers.append(nn.BatchNorm1d(odim)),
                layers.append(nn.Dropout(p=dropout_p))

        self.layers = nn.Sequential(*layers)

    def forward(self, x, h0=None):
        out = self.layers(x)
        return out

class SleepPredictionCNN(nn.Module):
    def __init__(self, kernel_sizes=[3,4,5], num_filters=100, embedding_dim=125,
                 num_classes=3, dropout_p = 0.0, sentence_len = 10):
        super(SleepPredictionCNN, self).__init__()
        self.kernel_sizes = kernel_sizes
        conv_blocks = []
        for kernel_size in kernel_sizes:
            conv1d = nn.Conv1d(in_channels = embedding_dim, out_channels = num_filters,
                               kernel_size = kernel_size,
                               stride = 1)
            component = nn.Sequential(
                conv1d,
                nn.ReLU(),
                nn.MaxPool1d(kernel_size = sentence_len - kernel_size +1)
            )

            conv_blocks.append(component)
        self.conv_blocks = nn.ModuleList(conv_blocks)
        self.fc = nn.Linear(num_filters*len(kernel_sizes), num_classes)
        self.dropout = nn.Dropout(p=dropout_p)

    def forward(self, x, h0=None):
        x = x.transpose(1,2)
        x_list= [conv_block(x) for conv_block in self.conv_blocks]
        out = torch.cat(x_list, 2)
        out = out.view(out.size(0), -1)
        out = self.dropout(out)
        return F.softmax(self.fc(out), dim=1)

class SleepPredictionSeq(nn.Module):
    def __init__(self, hidden_size=256, input_dropout_p=0,
                 num_classes=3, dropout_p=0, n_layers=1,
                 embed_size=125, bidirectional=False, rnn_cell='lstm'):
        super(SleepPredictionSeq, self).__init__()

        self.hidden_size = hidden_size
        self.n_layers = n_layers
        self.input_dropout_p = input_dropout_p
        self.input_dropout = nn.Dropout(p=input_dropout_p)
        self.linear = nn.Linear(11, hidden_size)
        self.rnn_cell = getattr(nn, rnn_cell.upper())
        self.dropout_p = dropout_p
        self.rnn = self.rnn_cell(embed_size, hidden_size, n_layers,
                                 batch_first=True, bidirectional=bidirectional,
                                 dropout=dropout_p)
        self.fc = nn.Linear(hidden_size*10, num_classes)

    def forward(self, x, h0=None):
        if h0 is not None:
            h0 = self.linear(h0)
            if self.rnn_cell is nn.LSTM:
                h0 = (h0.unsqueeze(0), h0.unsqueeze(0))
        embedded = self.input_dropout(x)
        output, _ = self.rnn(embedded, h0)
        output = output.contiguous().view(output.size(0), -1)
        return F.softmax(self.fc(output), dim=1)


import boto3
import base64
import pathlib
import os
from urllib.parse import urlparse

def get_analytics_page(client, analytics_addr):
  addr = urlparse(analytics_addr)
  return (
    f""
    f"  "
    f"    <link rel=\"icon\" type=\"image/png\" href=\"{addr.scheme}://{addr.netloc}/favicon.png\">"
    f"    <meta http-equiv=\"Content-Type\" content=\"text/html;charset=ISO-8859-1\">"
    f"    <title>Tweddle Group Insights - {client}</title>"
    f"  "
    f"  "
    f"    "
    f"  "
    f""
  )

def unauthorized():
  return {
    'status': '401',
    'statusDescription': 'Unauthorized',
    'body': 'Unauthorized',
    'headers': {
      'www-authenticate': [{'key': 'WWW-Authenticate', 'value':'Basic'}]
    }
  }

def analytics_response(body):
  return {
    'status': '200',
    'statusDescription': 'OK',
    'headers': {
      'cache-control': [{
        'key': 'Cache-Control',
        'value': 'max-age=100'
      }],
      'content-type': [{
        'key': 'Content-Type',
        'value': 'text/html'
      }],
      'content-encoding': [{
        'key': 'Content-Encoding',
        'value': 'UTF-8'
      }],
    },
    'body': body
  }

def parse_lambda_alias_arn(arn):
  parts = arn.split(":")
  name_parts = parts[6].split(".")
  return {
    'region': name_parts[0],
    'name': name_parts[1],
    'account_id': parts[4]
  }

def build_lambda_arn(name, region, account_id):
  return f"arn:aws:lambda:{region}:{account_id}:function:{name}"

def get_headers(raw_headers):
    headers = {}
    for header, values in raw_headers.items():
        headers[header] = []
        for value in values:
            for v in value['value'].split(","):
                headers[header].append(v)
    return headers

def handler(event, context):
  request = event["Records"][0]["cf"]["request"]
  event_type = event["Records"][0]["cf"]["config"]["eventType"]
  if event_type == "viewer-request":
      request['headers']['x-forwarded-host'] = [{ 'key': 'X-Forwarded-Host', 'value': request['headers']['host'][0]['value'] }]
  elif event_type == "origin-request":
      if 's3' in request['origin']:
          headers = get_headers(request['headers'])
          if 'x-forwarded-host' in headers:
              host = headers['x-forwarded-host'][0]
              custom_headers = get_headers(request['origin']['s3']['customHeaders'])
              if 'analytics-auth' in custom_headers:
                  if ('analytics-host' in custom_headers and host in custom_headers['analytics-host']) or 'analytics-host' not in custom_headers:
                    if not 'authorization' in headers or not headers['authorization'][0] in custom_headers['analytics-auth']:
                        return unauthorized()
                    if 'analytics-addr' in custom_headers:
                      args = parse_lambda_alias_arn(context.invoked_function_arn)
                      function_arn = build_lambda_arn(**args)
                      aws_lambda = boto3.client("lambda", region_name=args['region'])
                      response = aws_lambda.list_tags(
                        Resource=function_arn
                      )
                      tags = response['Tags']
                      client = tags['Brand'] if tags.get('Brand') else tags['Client']
                      body = get_analytics_page(client, custom_headers['analytics-addr'][0])
                      return analytics_response(body)
          if not pathlib.Path(request['uri']).suffix:
              request['uri'] = '/index.html'
      elif request['origin'].get('custom'):
        if request['uri'].startswith("/logs"):
          if request['method'] == "POST":
            custom_headers = get_headers(request['origin']['custom']['customHeaders'])
            request['uri'] = custom_headers['ingest-path'][0]
  return request


import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '..'))
from sklearn.feature_extraction.text import TfidfVectorizer
from collections import OrderedDict

def main():
    sys.path.append(os.path.join(os.path.dirname(__file__), '..'))

    from keras_en_parser_and_analyzer.library.dl_based_parser import ResumeParser
    from keras_en_parser_and_analyzer.library.utility.io_utils import read_pdf_and_docx

    current_dir = os.path.dirname(__file__)
    current_dir = current_dir if current_dir is not '' else '.'
    data_dir_path = current_dir + '/data/resume_samples'  # directory to scan for any pdf and docx files

    def parse_resume(file_path, file_content):
        print('parsing file: ', file_path)

        parser = ResumeParser()
        parser.load_model(current_dir + '/models')
        parser.parse(file_content)

        if parser.unknown is False:
            summary, expertise, experience, knowledge, project = parser.summary()
            # print("Condition Success")
            #
            # print(summary.encode('utf-8'))
            # vocabulary = []
            # for i in experience:
            #     vocabulary.append(i.encode("utf-8"))
            # print(vocabulary)

            experience_vocabulary = []
            for i in experience:
                j = i.encode('ascii', 'ignore').decode("utf-8").split(':')
                print(j)
                if len(j) != 1:
                    for k in j[1:]:
                        l = k.split(',')
                        experience_vocabulary.extend(l)
                else:
                    for k in j:
                        l = k.split(',')
                        experience_vocabulary.extend(l)
                        print(l)
            experience_vocabulary = list(filter(None, experience_vocabulary))
            experience_vocabulary = list(OrderedDict.fromkeys(experience_vocabulary))
            print(experience_vocabulary)
            expertise_vocabulary = []
            for i in expertise:
                expertise_vocabulary.append(i.encode('ascii', 'ignore').decode("utf-8"))
            knowledge_vocabulary = []
            for i in knowledge:
                knowledge_vocabulary.append(i.encode('ascii', 'ignore').decode("utf-8"))
            project_vocabulary = []
            for i in project:
                project_vocabulary.append(i.encode('ascii', 'ignore').decode("utf-8"))
            # print(summary.encode('utf-8'))
            document1 = "".join(experience_vocabulary)
            document2 = "".join(knowledge_vocabulary)
            document3 = "".join(project_vocabulary)
            corpus = dict([('expertise',document1),('knowledge',document2),('project',document3)])
            tfidf = TfidfVectorizer(vocabulary=experience_vocabulary, stop_words='english', ngram_range=(1, 2))
            tfs = tfidf.fit_transform(corpus.values())
            feature_names = tfidf.get_feature_names()
            corpus_index = [n for n in corpus]
            # rows, cols = tfs.nonzero()
            # for row, col in zip(rows, cols):
            #     print((feature_names[col], corpus_index[row]), tfs[row, col])
            import pandas as pd
            df = pd.DataFrame(tfs.T.todense(), index=feature_names, columns=corpus_index)
            print(df)
            # print(experience_vocabulary)
            # print(expertise_vocabulary)
            # print(knowledge_vocabulary)
            # print(project_vocabulary)
        print('++++++++++++++++++++++++++++++++++++++++++')

    collected = read_pdf_and_docx(data_dir_path, command_logging=True, callback=lambda index, file_path, file_content: {
        parse_resume(file_path, file_content)
    })
    print('count: ', len(collected))

if __name__ == '__main__':
    main()


import cv2

img = cv2.imread("obj1_5.jpg")

sift = cv2.xfeatures2d.SIFT_create(contrastThreshold=0.16, edgeThreshold=9)
kp, des = sift.detectAndCompute(img, None)
img = cv2.drawKeypoints(img, kp, img, flags=cv2.DRAW_MATCHES_FLAGS_DRAW_RICH_KEYPOINTS)

cv2.imshow('sift_kp', img)
cv2.waitKey(0)
cv2.destroyAllWindows()
cv2.imwrite('sift_kp.png', img)


import boto3
import os
from requests_aws4auth import AWS4Auth
#import cottoncandy as cc

# AWS S3 setting
service = 's3'
session = boto3.Session(
  aws_access_key_id=os.environ.get('AWS_ACCESS_KEY_ID', ''),
  aws_secret_access_key=os.environ.get('AWS_SECRET_ACCESS_KEY', '')
)
credentials = session.get_credentials()

region = os.environ.get('AWS_REGION', '')
http_auth = AWS4Auth(credentials.access_key, credentials.secret_key, region, service)

s3 = boto3.resource('s3')

# cci = cc.get_interface(os.environ.get('S3_BUCKET',''),
#                         ACCESS_KEY=os.environ.get('AWS_ACCESS_KEY_ID',''),
#                         SECRET_KEY=os.environ.get('AWS_SECRET_ACCESS_KEY',''),
#                         endpoint_url=os.environ.get('S3_ENDPOINT',''))


import pymongo
import json

from flask import request, Flask, Response

MONGODB_HOST = "fonkdb-mongodb.default"
MONGODB_PORT = 27017
MONGODB_NAME = "guestbook_app"
MONGODB_COLLECTION = "entries"

def main():
    try:
        client = pymongo.MongoClient("mongodb://{}".format(MONGODB_HOST), int(MONGODB_PORT))
        collection = client[MONGODB_NAME][MONGODB_COLLECTION]
        entries = []
        for doc in collection.find({}):
            doc["_id"] = str(doc["_id"])
            entries.append(doc)
        result = {"entries": entries}
        return response(json.dumps(result), 200)
    except pymongo.errors.PyMongoError as err:
        return response({"error": "MongoDB error: " + str(err)}, 500)
    except Exception as err:
        return response({"error": str(err)}, 500)

def response(body, status):
    return Response(str(body), status, {"Content-Type": "application/json", "Access-Control-Allow-Origin": "*"})


import flappy_headless
import numpy as np
import sys
from collections import defaultdict
import pickle

rewardAlive = 1
rewardKill = -10000
alpha = 0.2
gamma = 0.9

# Q[state] = (Nicht springen, springen)
Q = defaultdict(lambda: [0, 0])

def paramsToState(params):
    playerVelY = params['playerVelY']
    playery = round(params["playery"] / 10) * 10

    if int(params["upperPipes"][0]['x']) < 56:
        index = 1
    else:
        index = 0

    upperPipeX = round(int(params["upperPipes"][index]['x']) / 10) * 10
    upperPipeY = round(int(params["upperPipes"][index]['y']) / 10) * 10

    return str(playerVelY) + "_" + str(playery) + "_" + str(upperPipeX) + "_" + str(upperPipeY)

oldState = None
oldAction = None
gameCounter = 0

def onGameover(gameInfo):
    global oldState
    global oldAction
    global gameCounter

    print(str(gameCounter) + ":" + str(gameInfo['score']))

    # Q updaten für die vorherige Aktion
    #  -> Die vorherige Aktion war nicht erfolgreich!
    prevReward = Q[oldState]
    index = None
    if oldAction == False:
        index = 0
    else:
        index = 1

    prevReward[index] = (1 - alpha) * prevReward[index] + \
        alpha * rewardKill
    Q[oldState] = prevReward

    oldState = None
    oldAction = None

    if gameCounter % 10000 == 0:
        with open("Q/" + str(gameCounter) + ".pickle", "wb") as file:
            pickle.dump(dict(Q), file)

    gameCounter+=1

def shouldEmulateKeyPress(params):
    global oldState
    global oldAction

    state = paramsToState(params)
    estReward = Q[state]

    # Q updaten für die vorherige Aktion
    #  -> Die vorherige Aktion war erfolgreich!
    prevReward = Q[oldState]
    index = None
    if oldAction == False:
        index = 0
    else:
        index = 1

    prevReward[index] = (1 - alpha) * prevReward[index] + \
        alpha * (rewardAlive + gamma * max(estReward))
    Q[oldState] = prevReward

    oldState = state
    if estReward[0] >= estReward[1]:
        oldAction = False
        return False
    else:
        oldAction = True
        return True

flappy_headless.main(shouldEmulateKeyPress, onGameover)

Kursmaterialien/550 - Q-Learning fuer Flappy Bird/qlearning7.py


import abc
import logging
import os
import threading
from typing import Tuple, List, Dict

from galileodb.db import ExperimentDatabase
from galileodb.model import Experiment, Telemetry, ServiceRequestTrace, NodeInfo, ExperimentEvent

logger = logging.getLogger(__name__)

class SqlAdapter(abc.ABC):
    placeholder = '?'

    _thread_local = threading.local()

    def __init__(self, *args, **kwargs) -> None:
        super().__init__()
        self._thread_local.connection = None
        self.connect_args = args
        self.connect_kwargs = kwargs

    @property
    def connection(self):
        if 'connection' not in self._thread_local.__dict__ or self._thread_local.connection is None:
            logger.info('%s connecting to database', threading.current_thread().name)
            self._thread_local.connection = self._connect(*self.connect_args, **self.connect_kwargs)
        return self._thread_local.connection

    def reconnect(self):
        self.close()
        self.open()

    @property
    def db(self):
        return self.connection

    def cursor(self):
        return self.db.cursor()

    def execute(self, *args, **kwargs):
        cur = self.cursor()
        try:
            logger.debug('executing SQL %s %s', args, kwargs)
            cur.execute(*args, **kwargs)
            self.db.commit()
        finally:
            cur.close()

    def executemany(self, *args, **kwargs):
        cur = self.cursor()
        try:
            cur.executemany(*args, **kwargs)
            self.db.commit()
        finally:
            cur.close()

    def executescript(self, *args, **kwargs):
        cur = self.cursor()
        try:
            cur.executescript(*args, **kwargs)
            self.db.commit()
        finally:
            cur.close()

    def fetchone(self, *args, **kwargs):
        cur = self.cursor()
        try:
            cur.execute(*args, **kwargs)
            return cur.fetchone()
        finally:
            cur.close()

    def fetchmany(self, *args, **kwargs):
        cur = self.cursor()
        try:
            cur.execute(*args, **kwargs)
            return cur.fetchmany()
        finally:
            cur.close()

    def fetchall(self, *args, **kwargs):
        cur = self.cursor()
        try:
            cur.execute(*args, **kwargs)
            return cur.fetchall()
        finally:
            cur.close()

    def open(self):
        assert self.connection is not None

    def close(self):
        if 'connection' in self._thread_local.__dict__ and self._thread_local.connection is not None:
            try:
                self._thread_local.connection.close()
            finally:
                self._thread_local.connection = None

    def insert_one(self, table: str, data: Dict[str, object]):
        columns, values = list(), list()

        for key, value in data.items():
            columns.append('`%s`' % key.upper())
            values.append(value)

        columns = ','.join(columns)
        placeholders = ','.join([self.placeholder] * len(values))

        # TODO: sanitize table and column inputs
        sql = 'INSERT INTO `%s` (%s) VALUES (%s)' % (table, columns, placeholders)

        logger.debug('running insert sql: %s' % sql)

        self.execute(sql, values)

    def insert_many(self, table: str, keys, data: List):
        columns = ','.join(['`%s`' % key.upper() for key in keys])
        placeholders = ','.join([self.placeholder] * len(keys))

        # TODO: sanitize table and column inputs
        sql = 'INSERT INTO `%s` (%s) VALUES (%s)' % (table, columns, placeholders)

        logger.debug('running insert many sql on %d items: %s', len(data), sql)

        self.executemany(sql, data)

    def update_by_id(self, table: str, identity: Tuple[str, object], data: Dict[str, object]):
        set_statements, values = list(), list()
        id_col, id_val = identity

        for key, value in data.items():
            set_statements.append('`%s` = %s' % (key.upper(), self.placeholder))
            values.append(value)

        values.append(id_val)

        # TODO: sanitize table and column inputs
        sql = 'UPDATE `{table}` SET {set_statements} WHERE `{id_col}` = ' + self.placeholder
        sql = sql.format(table=table, set_statements=','.join(set_statements), id_col=id_col)

        logger.debug('running update sql: %s', sql)
        self.execute(sql, values)

    def _connect(self, *args, **kwargs):
        raise NotImplementedError

class ExperimentSQLDatabase(ExperimentDatabase):
    SCHEMA_FILE = os.path.join(os.path.dirname(__file__), 'schema.sql')

    def __init__(self, db: SqlAdapter) -> None:
        super().__init__()
        self.db = db

    def read_schema_file(self):
        with open(self.SCHEMA_FILE, 'r') as fd:
            return fd.read()

    def open(self):
        self.db.open()
        self.db.executescript(self.read_schema_file())

    def close(self):
        self.db.close()

    def save_experiment(self, experiment: Experiment):
        data = dict(experiment.__dict__)
        data['exp_id'] = data['id']
        del data['id']
        logger.debug('saving experiment with data %s', data)
        self.db.insert_one('experiments', data)

    def update_experiment(self, experiment: Experiment):
        data = dict(experiment.__dict__)
        del data['id']

        self.db.update_by_id('experiments', ('exp_id', experiment.id), data)

    def delete_experiment(self, exp_id: str):
        experiment = self.get_experiment(exp_id)
        if experiment is None:
            raise ValueError('No such experiment %s' % exp_id)

        stmts = [
            "DELETE FROM `telemetry` WHERE EXP_ID = " + self.db.placeholder,
            "DELETE FROM `traces` WHERE EXP_ID = " + self.db.placeholder,
            "DELETE FROM `experiments` WHERE EXP_ID = " + self.db.placeholder,
        ]

        for sql in stmts:
            try:
                self.db.execute(sql, (exp_id,))
            except Exception as e:
                logger.exception('Exception while executing %s', sql, e)

    def get_experiment(self, exp_id: str) -> Experiment:
        sql = "SELECT * FROM `experiments` WHERE EXP_ID = " + self.db.placeholder

        entry = self.db.fetchone(sql, (exp_id,))

        if entry:
            row = tuple(entry)
            return Experiment(*row)
        else:
            return None

    def save_traces(self, traces: List[ServiceRequestTrace]):
        self.db.insert_many('traces', ServiceRequestTrace._fields, traces)

    def touch_traces(self, experiment: Experiment):
        sql = 'UPDATE `traces` SET `EXP_ID` = ? WHERE CREATED >= ? AND CREATED <= ?'
        sql = sql.replace('?', self.db.placeholder)
        self.db.execute(sql, (experiment.id, experiment.start, experiment.end))

    def save_telemetry(self, telemetry: List[Telemetry]):
        self.db.insert_many('telemetry', Telemetry._fields, telemetry)

    def save_event(self, event: ExperimentEvent):
        self.db.insert_one('events', event._asdict())

    def save_events(self, events: List[ExperimentEvent]):
        self.db.insert_many('events', ExperimentEvent._fields, events)

    def save_nodeinfos(self, infos: List[NodeInfo]):
        keys = ['exp_id', 'node', 'info_key', 'info_value']

        data = list()
        for info in infos:
            for k, v in info.data.items():
                data.append((info.exp_id, info.node, k, v))

        self.db.insert_many('nodeinfo', keys, data)

    def find_all(self) -> List[Experiment]:
        sql = 'SELECT * FROM `experiments`'

        entries = self.db.fetchall(sql)

        return list(map(lambda x: Experiment(*(tuple(x))), entries))


# modified from librealsense/wrappers/python/examples/opencv_viewer_example.py
# and http://dlib.net/face_landmark_detection.py.html

import pyrealsense2 as rs
import numpy as np
import time
import dlib

class average:
    def __init__(self):
        self.count = 0
        self.accum = 0

    def add(self, j):
        self.count += 1
        self.accum += j

    def get(self):
        return self.accum / self.count if self.count >= 1 else 0

class timeaverage:
    def __init__(self):
        self.avg = average()
        self.t = None

    def start(self):
        if self.t is not None:
            raise Exception("started twice")
        self.t = time.time()

    def stop(self):
        if self.t is None:
            return #raise Exception("already stop")
        self.avg.add(time.time() - self.t)
        self.t = None

    def get(self):
        return self.avg.get()

facebbtimer = timeaverage()

shapestimer = timeaverage()

shortftimer = timeaverage()

otherstimer = timeaverage()

def decompose(r):
    return r.top(), r.bottom(), r.left(), r.right()

class facedetector:
    def __init__(self):
        self.currentbb = None
        self.last = None

    def start(self):
        self.pipeline = rs.pipeline()
        self.config = rs.config()
        self.config.enable_stream(rs.stream.depth, 640, 480, rs.format.z16, 60)
        self.config.enable_stream(rs.stream.color, 640, 480, rs.format.bgr8, 60)

        self.fd = dlib.get_frontal_face_detector()
        self.sp = dlib.shape_predictor("landmarks.dat")

        self.pipeline.start(self.config)

        class pyramid:
            def __init__(self):
                pass

            def __call__(self, im):
                return im

            def rect_down(self, rect):
                return rect

            def rect_up(self, rect):
                return rect

        self.scaledown = dlib.pyramid_down()

        self.alignment = rs.align(rs.stream.color)

    def face_detector(self, im):
        scaled_image = self.scaledown(im)
        def usewholeimg():
            facebbtimer.start()
            facebb = self.fd(scaled_image)
            facebbtimer.stop()
            if len(facebb) > 0:
                return facebb[0]
            return None

        def usesmallbox(t, b, l, r, scale, scale_top):
            h, w, _ = scaled_image.shape
            fh = b - t
            fw = r - l
            t = max(0, t - int(scale_top*fh))
            b = min(h, b + int(scale*fh))
            l = max(0, l - int(scale*fw))
            r = min(w, r + int(scale*fw))
            facebb = self.fd(scaled_image[t:b, l:r, :])
            if len(facebb) >= 1:
                if len(facebb) > 1:
                    print("multiple faces detected, choosing one arbitrarily")
                rect = facebb[0]
                return dlib.translate_rect(rect, dlib.point(l, t))
            else:
                return None

        if self.currentbb is None:
            self.currentbb = usewholeimg()
        else:
            shortftimer.start()
            t, b, l, r = decompose(self.currentbb)
            SCALE_TOP = 0.1
            SCALE = 0.05
            rect = usesmallbox(t, b, l, r, SCALE_TOP, SCALE)
            if not rect:
                rect = usewholeimg()
            self.currentbb = rect
            shortftimer.stop()
        if self.currentbb is not None:
            shapestimer.start()
            out = self.sp.calldpoint(
                im,
                self.scaledown.rect_up(self.currentbb)
            )
            shapestimer.stop()
            return out
        else:
            return None

    def get(self):
        frames = self.pipeline.wait_for_frames()
        frames = self.alignment.process(frames)
        depth_frame = frames.get_depth_frame()
        color_frame = frames.get_color_frame()
        t = color_frame.get_frame_metadata(rs.frame_metadata_value.backend_timestamp)
        depth_image = np.asanyarray(depth_frame.get_data())
        color_image = np.asanyarray(color_frame.get_data())
        color_image = color_image[:, :, [2,1,0]]
        shape = self.face_detector(color_image)
        return (
            t,
            depth_image,
            color_image,
            shape
        )

    def stop(self):
        self.pipeline.stop()

class midavg:
    def __init__(self, lth):
        self.last = np.zeros(lth)
        self.idx = 0
        self.valid = 0
        assert lth & 3 == 0

    def add(self, z):
        self.last[self.idx] = z
        self.idx += 1
        l = self.last.shape[0]
        if self.valid < l:
            self.valid += 1
        if self.idx >= l:
            self.idx -= l
        return self.get()

    def get(slf):
        out = np.sort(slf.last)
        l4 = slf.last.shape[0] >> 2
        return np.mean(out[l4: 3*l4])

class dampve:
    def __init__(self, damp0, damp1):
        self.x0 = None
        self.v0 = None
        self.t0 = None
        self.damp0 = damp0
        self.damp1 = damp1

    def add(self, t, z):
        if t == self.t0:
            return self.x0
        if self.x0 is None:
            self.x0 = z
        elif self.v0 is None:
            self.v0 = (z - self.x0) / (t - self.t0)
        else:
            dt = t - self.t0
            v = (z - self.x0) / dt
            dv = v - self.v0
            self.x0 += dt * (self.v0 + v) * 0.5
            self.v0 = self.damp0 * self.v0 + self.damp1 * dv
        self.t0 = t
        return self.x0

class predicteyes:
    def __init__(self):
        self.stop()
        self.out = np.zeros(3)

    def stop(self):
        self.t0 = None

    def __call__(self, t, x, y, z):
        if self.t0 is None:
            self.t0 = t
            self.mids = [
                dampve(1, 0.1),
                dampve(1, 0.1),
                dampve(1, 0.1)
            ]
        t -= self.t0
        for (j, c) in enumerate([x, y, z]):
            self.out[j] = self.mids[j].add(t, c) #(c)
        return t, self.out

    """
    def __call__(self, t, l, le, r, re):
        if self.t0 is None:
            self.t0 = t
        t -= self.t0
        scale = ((le[0]**2 + re[0]**2) / 2) ** 0.5
        print("t = %6.2f   scale = %6.2f   l = %9.2f %9.2f"% (t, scale, l[0], l[1]))
    """

import traceback as tb
PARTS_L_EYE = range(36, 42)
PARTS_R_EYE = range(42, 48)
PARTS_NOSE_TOP = 27
def average_on_box_around_point(img, x, y, scale):
    x = int(x)
    y = int(y)
    scale = int(scale + 0.5)
    scale //= 2
    l = max(0, x - scale)
    r = min(img.shape[1], x + scale)
    t = max(0, y - scale)
    b = min(img.shape[0], y + scale)
    img = img[t:b, l:r]
    ret = np.mean(img[img != 0])
    if not np.any(img != 0):
        print("ret =", ret)
    return ret

def meanof(shape, ix):
    sp = [shape.part(i) for i in ix]
    v = np.array([[p.x, p.y] for p in sp])
    return np.mean(v, axis=0), np.std(v, axis=0)

def coordinates(t, depthimage, colorimage, shape):
    left, leftv = meanof(shape, PARTS_L_EYE)
    right, rightv = meanof(shape, PARTS_R_EYE)
    nose, _ = meanof(shape, (PARTS_NOSE_TOP,))
    lscale = leftv[0]
    rscale = rightv[0]
    ld = average_on_box_around_point(depthimage, left[0], left[1], lscale)
    rd = average_on_box_around_point(depthimage, right[0], right[1], rscale)
    return np.array([
        (left[0] + right[0] + nose[0]) / 3,
        (left[1] + right[1] + nose[1]) / 3,
        (ld + rd) / 2
    ])

class imagewindow:
    def __init__(self, act = True):
        self.wi = dlib.image_window() if act else None
        self.framecount = 0

    def show(self, out):
        if not self.wi:
            return
        wi = self.wi
        t, depthimage, colorimage, shape = out
        wi.set_image(colorimage)
        wi.clear_overlay()
        self.framecount += 1
        if shape is not None:
            def topoint(dp):
                return dlib.point(dp.x, dp.y)
            if False:
                fobp = dlib.full_object_detection(shape.rect, [topoint(shape.part(i)) for i in range(shape.num_parts)])
                for jj in range(fobp.num_parts):
                    p = fobp.part(jj)
                    if jj > self.framecount % 68:
                        break
                    self.wi.add_overlay_circle(
                        dlib.dpoint(p.x, p.y),
                        2,
                        color = dlib.rgb_pixel(255, 255, 0)
                    )
                wi.add_overlay(fobp)
            #wi.add_overlay(shape.rect)

    def overlay_circle(self, x, y):
        if not self.wi:
            return
        self.wi.add_overlay_circle(dlib.dpoint(x, y), 3)

if __name__ == "__main__":
    fd = facedetector()
    fd.start()
    framecount = -1
    predict = predicteyes()
    #iw = imagewindow()
    try:
        while True:
            otherstimer.stop()
            out = fd.get()
            otherstimer.start()
            if out is not None:
                framecount += 1
                if framecount > 0:
                    if framecount % 100 == 0:
                        print(f"FPS: {framecount / (time.time() - starttime)}")
                        #print(f"Face bounding boxes take {facebbtimer.get()} on average")
                        #print(f"Shape prediction takes {shapestimer.get()} on average")
                        #print(f"Small face bounding boxes: {shortftimer.get()} on average")
                        #print(f"Other stuff: {otherstimer.get()} on average")
                        framecount = 0
                        starttime = time.time()
                else:
                    starttime = time.time()
                t, depthimage, colorimage, shape = out
                foundeyes = False
                if shape is not None:
                    (x, y, z) = coordinates(t, depthimage, colorimage, shape)
                    if np.any(np.isnan([x, y, z])):
                        pass
                    else:
                        foundeyes = True
                        #iw.show(out)
                        #iw.overlay_circle(x, y)
                        t, [x, y, z] = predict(t, x, y, z)
                        z /= 1000
                        SCREEN_HALF_WIDTH = 0.27
                        CAMERA_HALF_WIDTH = 1.60
                        SCALE = SCREEN_HALF_WIDTH / CAMERA_HALF_WIDTH
                        x = (x - 320) * z / 100 * SCALE
                        y = (y - 240) * z / 100 * SCALE
                        print("%6.2f    %6.2f    %6.2f" % (x, y, z))
                        #iw.overlay_circle(x * 100 + 320, y * 100 + 240)
                if not foundeyes:
                    predict.stop()
                    print("no eyes found this frame")
    finally:
        fd.stop()


from PyQt5 import uic
from PyQt5.QtWidgets import QApplication, QMainWindow, QTableWidgetItem, QTableWidget, QWidget
import sys
import sqlite3

class App(QMainWindow):
    def __init__(self):
        super().__init__()
        uic.loadUi("addEditCoffeeForm.ui", self)
        self.pushButton.clicked.connect(self.add)
        self.con = sqlite3.connect("coffee.sqlite")
        self.cur = self.con.cursor()
        self.initUi()

    def initUi(self):
        result = list(self.cur.execute("select * from coffee").fetchall())
        self.c = ["ID", "Сорт", "Степень обжарки", "Молотый/в зернах", "Описание вкуса", "Цена", "Объем упаковки"]
        self.table.setColumnCount(len(self.c))
        self.table.setHorizontalHeaderLabels(self.c)
        self.table.setRowCount(0)
        for i, row in enumerate(result):
            self.table.setRowCount(self.table.rowCount() + 1)
            for j, elem in enumerate(row):
                    self.table.setItem(i, j, QTableWidgetItem(str(elem)))
        self.table.resizeColumnsToContents()
        self.table.cellChanged.connect(self.run)

    def run(self, row, column):
        st = self.c[column]
        it = self.table.item(row, column).text()
        id = self.table.item(row, 0).text()
        res = self.cur.execute(f"""update coffee set '{st}' = ? where ID = ?""", (it, int(id)))
        self.con.commit()

    def add(self):
        a = Add().__init__()

class Add(QWidget):
    def __init__(self):
        super().__init__()
        uic.loadUi('add.ui', self)
        self.pushButton.clicked.connect(self.add)
        self.con = sqlite3.connect('coffee.sqlite')
        self.cur = self.con.cursor()
        self.show()

    def add(self):
        print(0)
        s = self.lineEdit_s.text()
        o = self.lineEdit_o.text()
        m = self.lineEdit_m.text()
        t = self.lineEdit_t.text()
        p = self.lineEdit_p.text()
        v = self.lineEdit_v.text()
        res = self.cur.execute("""insert into coffee('Сорт', 'Степень обжарки', 'Молотый/в зернах', 'Описание вкуса', 'Цена',
        'Объем упаковки') values(?, ?, ?, ?, ?, ?)""", (s, o, m, t, p, v))
        self.con.commit()
        App.initUi(self)
        self.close()

app = QApplication(sys.argv)
a = App()
a.show()
sys.exit(app.exec_())


from . import spider_task
from flask import (current_app, make_response, redirect, render_template, jsonify,
                   request, url_for, flash, session)
from .forms import SpiderTask
from ..models import FictionListAll, db, SpiderTask as st
from .thread_pool_task import add_spider_task_pool
import json
from datetime import datetime

@spider_task.route('/spider_task')
def index():
    form = SpiderTask().get_all()

    return render_template('spider_task/spider_task.html', form=form)

@spider_task.route('/get_spider_task_all', methods=['post', 'get'])
def get_spider_task_all():
    data = st().query.all()
    print(json.loads(str(data)))
    return jsonify(json.loads(str(data)))

@spider_task.route('/add_spider_task', methods=['post'])
def add_spider_task():
    data = request.form
    print(data)
    task = st().query.filter_by(task_name=data.get('task_name')).first()
    if task is None:
        task = st(task_name=data.get('task_name'),
                  task_type=data.get('task_type'),
                  task_status=data.get('task_status'),
                  creation_date=str(datetime.now().date()))
        db.session.add(task)
        db.session.commit()
    return 'ok'

@spider_task.route('/run_task', methods=['post'])
def run_task():
    print(request.args, request.form, request.get_data(as_text=True))
    datas = json.loads(request.get_data(as_text=True))
    for data in datas:
        res = st().query.filter_by(task_id=data['task_id']).first()
        add_spider_task_pool(task_id=data['task_id'], task=res.task_type, para=res.task_name)
        # print(res)
        db.session.commit()
    res = {'status': 'SUCCESS'}
    return jsonify(res)


#!/usr/bin/env python
from __future__ import division

import copy
import json
import logging
import pprint
import sys
import time
from datetime import datetime

from WMCore.Database.CMSCouch import CouchServer
from WMCore.Database.CMSCouch import Document as CouchDoc
from WMCore.Lexicon import splitCouchServiceURL
from WMCore.Services.McM.McM import McM, McMNoDataError
from WMCore.Services.WMStats.DataStruct.RequestInfoCollection import RequestInfoCollection
from WMCore.Services.WMStats.WMStatsReader import WMStatsReader

maxMCMCalls = 250  # Maximum # of entries to retrieve from McM per run

def gatherWMDataMiningStats(wmstatsUrl, reqmgrUrl, wmMiningUrl,
                            mcmUrl, mcmCert, mcmKey, tmpDir,
                            archived=False, log=logging.info):
    server, database = splitCouchServiceURL(wmMiningUrl)
    analyticsServer = CouchServer(server)
    couchdb = analyticsServer.connectDatabase(database)

    WMStats = WMStatsReader(wmstatsUrl, reqdbURL=reqmgrUrl, reqdbCouchApp="ReqMgr")

    reqMgrServer, reqMgrDB = splitCouchServiceURL(reqmgrUrl)

    reqMgr = CouchServer(reqMgrServer).connectDatabase(reqMgrDB, False)

    if archived:
        funcName = "Archived Requests"
    else:
        funcName = "Active Requests"

    log.info("%s: Getting job information from %s and %s. Please wait." % (
        funcName, wmstatsUrl, reqmgrUrl))

    if archived:
        checkStates = ['normal-archived', 'rejected-archived', 'aborted-archived']
        jobInfoFlag = False
    else:
        checkStates = WMStatsReader.ACTIVE_STATUS
        jobInfoFlag = True
    requests = WMStats.getRequestByStatus(checkStates, jobInfoFlag=jobInfoFlag, legacyFormat=True)

    requestCollection = RequestInfoCollection(requests)
    result = requestCollection.getJSONData()
    requestsDict = requestCollection.getData()
    log.info("%s: Total %s requests retrieved\n" % (funcName, len(result)))

    report = {}
    nMCMCalls = 0
    with McM(cert=mcmCert, key=mcmKey, url=mcmUrl, tmpDir=tmpDir) as mcm:
        for wf in result.keys():

            # Store a copy of the CouchDB document so we can compare later before updating
            if couchdb.documentExists(wf):
                oldCouchDoc = couchdb.document(wf)
                wfExists = True
            else:
                oldCouchDoc = CouchDoc(id=wf)
                wfExists = False

            newCouchDoc = copy.deepcopy(oldCouchDoc)
            ancientCouchDoc = copy.deepcopy(oldCouchDoc)
            report[wf] = oldCouchDoc
            # FIXME: remove report, only have two instances of couchDoc

            if 'filterEfficiency' not in oldCouchDoc or 'runWhiteList' not in oldCouchDoc:
                runWhiteList = []
                filterEfficiency = None
                try:
                    # log.debug("Looking up %s in ReqMgr" % wf)
                    rmDoc = reqMgr.document(wf)
                    runWhiteList = rmDoc.get('RunWhiteList', [])
                    filterEfficiency = rmDoc.get('FilterEfficiency', None)
                except:
                    pass  # ReqMgr no longer has the workflow
                report[wf].update({'filterEfficiency': filterEfficiency, 'runWhiteList': runWhiteList})

            if oldCouchDoc.get('mcmTotalEvents', 'Unknown') == 'Unknown' or \
                oldCouchDoc.get('mcmApprovalTime', 'Unknown') == 'Unknown':

                prepID = oldCouchDoc.get('prepID', None)
                if prepID and nMCMCalls <= maxMCMCalls:
                    log.info("Trying to update McM info for %s, PREPID %s" % (wf, prepID))
                    # Get information from McM. Don't call too many times, can take a long time
                    nMCMCalls += 1
                    try:
                        mcmHistory = mcm.getHistory(prepID=prepID)
                        if 'mcmApprovalTime' not in oldCouchDoc:
                            report[wf].update({'mcmApprovalTime': 'NoMcMData'})
                        found = False
                        for entry in mcmHistory:
                            if entry['action'] == 'set status' and entry['step'] == 'announced':
                                dateString = entry['updater']['submission_date']
                                dt = datetime.strptime(dateString, '%Y-%m-%d-%H-%M')
                                report[wf].update({'mcmApprovalTime': time.mktime(dt.timetuple())})
                                found = True
                        if not found:
                            log.error("History found but no approval time for %s" % wf)
                    except McMNoDataError:
                        log.error("Setting NoMcMData for %s" % wf)
                        report[wf].update({'mcmApprovalTime': 'NoMcMData'})
                    except (RuntimeError, IOError):
                        exc_type, dummy_exc_value, dummy_exc_traceback = sys.exc_info()
                        log.error("%s getting history from McM for PREP ID %s. May be transient and/or SSO problem." %
                                  (exc_type, prepID))
                    except:
                        exc_type, exc_value, exc_traceback = sys.exc_info()
                        log.error("%s getting history from McM for PREP ID %s. Unknown error." %
                                  (exc_type, prepID))

                    try:
                        mcmRequest = mcm.getRequest(prepID=prepID)
                        report[wf].update({'mcmTotalEvents': mcmRequest.get('total_events', 'NoMcMData')})
                    except (RuntimeError, IOError):
                        exc_type, exc_value, exc_traceback = sys.exc_info()
                        log.error("%s getting request from McM for PREP ID %s. May be transient and/or SSO problem." %
                                  (exc_type, prepID))
                    except:
                        exc_type, exc_value, exc_traceback = sys.exc_info()
                        log.error("%s getting request from McM for PREP ID %s. Unknown error." %
                                  (exc_type, prepID))

            # Basic parameters of the workflow
            priority = requests[wf].get('priority', 0)
            requestType = requests[wf].get('request_type', 'Unknown')
            targetLumis = requests[wf].get('input_lumis', 0)
            targetEvents = requests[wf].get('input_events', 0)
            campaign = requests[wf].get('campaign', 'Unknown')
            prep_id = requests[wf].get('prep_id', None)
            outputdatasets = requests[wf].get('outputdatasets', [])
            statuses = requests[wf].get('request_status', [])

            if not statuses:
                log.error("Could not find any status from workflow: %s" % wf)  # Should not happen but it does.

            # Remove a single  task_ from the start of PREP ID if it exists
            if prep_id and prep_id.startswith('task_'):
                prep_id.replace('task_', '', 1)

            # Can be an empty list, full list, empty string, or non-empty string!
            inputdataset = requests[wf].get('inputdataset', "")
            if isinstance(inputdataset, list):
                if inputdataset:
                    inputdataset = inputdataset[0]
                else:
                    inputdataset = ''

            outputTier = 'Unknown'
            try:
                outputTiers = []
                for ds in outputdatasets:
                    if isinstance(ds, list):
                        outputTiers.append(ds[0].split('/')[-1])
                    else:
                        outputTiers.append(ds.split('/')[-1])
            except:
                log.error(
                    "Could not decode outputdatasets: %s" % outputdatasets)  # Sometimes is a list of lists, not just a list. Bail
            if inputdataset:
                inputTier = inputdataset.split('/')[-1]
                if inputTier in ['GEN']:
                    outputTier = 'LHE'
                elif inputTier in ['RAW', 'RECO']:
                    outputTier = 'AOD'
                elif inputTier in ['GEN-SIM']:
                    outputTier = 'AODSIM'
                elif 'AODSIM' in outputTiers:
                    outputTier = 'AODSIM'

            else:
                if len(outputTiers) == 1 and 'GEN' in outputTiers:
                    if 'STEP0ATCERN' in wf:
                        outputTier = 'STEP0'
                    else:
                        outputTier = 'FullGen'
                elif 'GEN-SIM' in outputTiers and 'AODSIM' in outputTiers and requestType == 'TaskChain':
                    outputTier = 'RelVal'
                elif 'RECO' in outputTiers and requestType == 'TaskChain':
                    outputTier = 'RelVal'
                elif 'GEN-SIM' in outputTiers:
                    outputTier = 'GEN-SIM'
                elif 'AODSIM' in outputTiers:
                    outputTier = 'AODSIM'
                elif 'RECO' in outputTiers:
                    outputTier = 'AOD'
                elif 'AOD' in outputTiers:
                    outputTier = 'AOD'
                else:
                    outputTier = 'GEN-SIM'

            # Calculate completion ratios for events and lumi sections, take minimum for all datasets
            eventPercent = 200
            lumiPercent = 200
            datasetReports = requestsDict[wf].getProgressSummaryByOutputDataset()
            for dataset in datasetReports:
                dsr = datasetReports[dataset].getReport()
                events = dsr.get('events', 0)
                lumis = dsr.get('totalLumis', 0)
                if targetLumis:
                    lumiPercent = min(lumiPercent, lumis / targetLumis * 100)
                if targetEvents:
                    eventPercent = min(eventPercent, events / targetEvents * 100)
            if eventPercent > 100:
                eventPercent = 0
            if lumiPercent > 100:
                lumiPercent = 0

            # Sum up all jobs across agents to see if we've run the first, last
            successJobs = 0
            totalJobs = 0
            for agent in result[wf]:
                jobs = result[wf][agent]
                successJobs += jobs['sucess']
                totalJobs += jobs['created']
            try:
                if totalJobs and not report[wf].get('firstJobTime', None):
                    report[wf].update({'firstJobTime': int(time.time())})
                if totalJobs and successJobs == totalJobs and not report[wf].get('lastJobTime', None):
                    report[wf].update({'lastJobTime': int(time.time())})
            except:
                pass

            # Figure out current status of workflow and transition times
            finalStatus = None
            newTime = None
            approvedTime = None
            assignedTime = None
            acquireTime = None
            completedTime = None
            closeoutTime = None
            announcedTime = None
            archivedTime = None
            requestDate = None

            for status in statuses:
                finalStatus = status['status']
                if status['status'] == 'new':
                    newTime = status['update_time']
                if status['status'] == 'assignment-approved':
                    approvedTime = status['update_time']
                if status['status'] == 'assigned':
                    assignedTime = status['update_time']
                if status['status'] == 'completed':
                    completedTime = status['update_time']
                if status['status'] == 'acquired':
                    acquireTime = status['update_time']
                if status['status'] == 'closed-out':
                    closeoutTime = status['update_time']
                if status['status'] == 'announced':
                    announcedTime = status['update_time']
                if status['status'] == 'normal-archived':
                    archivedTime = status['update_time']

            # Build or modify the report dictionary for the WF
            report.setdefault(wf, {})

            if approvedTime and not report[wf].get('approvedTime', None):
                report[wf].update({'approvedTime': approvedTime})
            if assignedTime and not report[wf].get('assignedTime', None):
                report[wf].update({'assignedTime': assignedTime})
            if acquireTime and not report[wf].get('acquireTime', None):
                report[wf].update({'acquireTime': acquireTime})
            if closeoutTime and not report[wf].get('closeoutTime', None):
                report[wf].update({'closeoutTime': closeoutTime})
            if announcedTime and not report[wf].get('announcedTime', None):
                report[wf].update({'announcedTime': announcedTime})
            if completedTime and not report[wf].get('completedTime', None):
                report[wf].update({'completedTime': completedTime})
            if newTime and not report[wf].get('newTime', None):
                report[wf].update({'newTime': newTime})
            if archivedTime and not report[wf].get('archivedTime', None):
                report[wf].update({'archivedTime': archivedTime})

            try:
                dt = requests[wf]['request_date']
                requestDate = '%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d' % tuple(dt)
                report[wf].update({'requestDate': requestDate})
            except:
                pass

            report[wf].update({'priority': priority, 'status': finalStatus, 'type': requestType})
            report[wf].update({'totalLumis': targetLumis, 'totalEvents': targetEvents,})
            report[wf].update({'campaign': campaign, 'prepID': prep_id, 'outputTier': outputTier,})
            report[wf].update({'outputDatasets': outputdatasets, 'inputDataset': inputdataset,})

            report[wf].setdefault('lumiPercents', {})
            report[wf].setdefault('eventPercents', {})
            lumiProgress = 0
            eventProgress = 0
            for percentage in [1, 10, 20, 30, 40, 50, 60, 70, 80, 90, 95, 98, 99, 100]:
                percent = str(percentage)
                percentReported = report[wf]['lumiPercents'].get(percent, None)
                if not percentReported and lumiPercent >= percentage:
                    report[wf]['lumiPercents'][percent] = int(time.time())
                if lumiPercent >= percentage:
                    lumiProgress = percentage

                percentReported = report[wf]['eventPercents'].get(percent, None)
                if not percentReported and eventPercent >= percentage:
                    report[wf]['eventPercents'][percent] = int(time.time())
                if eventPercent >= percentage:
                    eventProgress = percentage

            report[wf].update({'eventProgress': eventProgress, 'lumiProgress': lumiProgress,})

            newCouchDoc.update(report[wf])

            # Queue the updated document for addition if it's changed.
            if ancientCouchDoc != newCouchDoc:
                if wfExists:
                    # log.debug("Workflow updated: %s" % wf)
                    pass
                else:
                    # log.debug("Workflow created: %s" % wf)
                    pass

                try:
                    newCouchDoc['updateTime'] = int(time.time())
                    report[wf]['updateTime'] = int(time.time())
                    dummy = json.dumps(newCouchDoc)  # Make sure it encodes before trying to queue
                    couchdb.queue(newCouchDoc)
                except:
                    log.error("Failed to queue document:%s \n" % pprint.pprint(newCouchDoc))

    log.info("%s: Finished getting job. wait for the next Cycle" % funcName)
    # Commit all changes to CouchDB
    couchdb.commit()


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import json

from django.contrib.auth.models import User

from rest_framework import status

from dibiup_common.tests.base_tests import BaseTest

from dibiup_project.models import (
    Project,
    ProjectUser,
    ProjectUserArchive
)

class TestProjectArchive(BaseTest):

    def init_test(self):
        users = self.create_user(2)
        user01_json = json.loads(users[0])
        user01_id = user01_json.get('id', None)
        user01 = User.objects.get(id=user01_id)

        user02_json = json.loads(users[1])
        user02_id = user02_json.get('id', None)
        user02 = User.objects.get(id=user02_id)

        projects = self.create_project(user01_id, 1)
        project01_json = json.loads(projects[0])
        project01_id = project01_json.get('id', None)
        project01 = Project.objects.get(id=project01_id)

        projects = self.create_project(user02_id, 1)
        project02_json = json.loads(projects[0])
        project02_id = project02_json.get('id', None)
        project02 = Project.objects.get(id=project02_id)

        return user01, user02, project01, project02

    def test_func01_set_project_archive_by_creator(self):
        print("\n\n")
        print(">>>> test_func01_set_project_archive")

        user01, user02, project01, project02 = self.init_test()

        # set project archive by creator
        url = "/users/{0}/projects/{1}/archives/".format(user01.id, project01.id)
        data = {}
        resp = self.client.post(url, data, format="json")
        assert resp.status_code == status.HTTP_201_CREATED

        project_archive = ProjectUserArchive.objects.filter(user=user01, project=project01)
        assert project_archive.count() == 1

        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == True
        print(">> set project archive by creator PASS")

    def test_func02_release_project_archive_by_creator(self):
        print("\n\n")
        print(">>>> test_func02_release_project_archive_by_creator")

        user01, user02, project01, project02 = self.init_test()

        # set project archive by creator
        url = "/users/{0}/projects/{1}/archives/".format(user01.id, project01.id)
        data = {}
        resp = self.client.post(url, data, format="json")
        assert resp.status_code == status.HTTP_201_CREATED

        project_archive = ProjectUserArchive.objects.filter(user=user01, project=project01)
        assert project_archive.count() == 1

        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == True
        print(">> set project archive by creator PASS")

        # list users archive
        url = "/users/{0}/archives/".format(user01.id)
        resp = self.client.get(url)
        assert resp.status_code == status.HTTP_200_OK
        resp_json = json.loads(resp.content)
        assert resp_json.get('count') == 1

        # release users archive
        url = "/users/{0}/unarchive/".format(user01.id)
        data = {
            'project_ids': str(project01.id)
        }
        resp = self.client.patch(url, data, format="json")
        assert resp.status_code == status.HTTP_200_OK
        project_archive = ProjectUserArchive.objects.filter(user=user01, project=project01)
        assert project_archive.count() == 0
        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == False
        print(">> release project archive by creator")

    def test_func03_set_project_archive_by_normal_user(self):
        print("\n\n")
        print(">>>> test_func03_set_project_archive_by_normal_user")

        user01, user02, project01, project02 = self.init_test()

        # join user02 to project01
        url = "/users/{0}/projects/{1}/projectusers/".format(user02.id, project01.id)
        data = {}
        resp = self.client.post(url, data, format="json")
        assert resp.status_code == status.HTTP_201_CREATED
        project_user = ProjectUser.objects.filter(user=user02, project=project01)
        assert project_user.count() == 1
        print(">> Join project01 PASS")

        # set project archive
        url = "/users/{0}/projects/{1}/archives/".format(user02.id, project01.id)
        data = {}
        resp = self.client.post(url, data, format="json")
        assert resp.status_code == status.HTTP_201_CREATED
        project_archive = ProjectUserArchive.objects.filter(user=user02, project=project01)
        assert project_archive.count() == 1
        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == False
        print (">> set archive by normal user PASS")

        # Release project archive
        url = "/users/{0}/unarchive/".format(user02.id)
        data = {
            "project_ids": str(project01.id)
        }
        resp = self.client.patch(url, data, format="json")
        assert resp.status_code == status.HTTP_200_OK
        project_archive = ProjectUserArchive.objects.filter(user=user02, project=project01)
        assert project_archive.count() == 0
        project = Project.objects.get(id=project01.id)
        assert project.is_admin_archive == False
        print(">> release archive by normal user PASS")


# -*- coding:utf-8 -*-

import cv2
import glob

IMAGE_SIZE = 80
COLOR_NUM = 64

def resize():
    image_files = glob.glob('t/*')

    for index, image_path in enumerate(image_files):
        image = cv2.imread(image_path)
        resize_image = cv2.resize(image, (IMAGE_SIZE, IMAGE_SIZE))
        cv2.imwrite('r/t%06d.jpg' % index, resize_image)

    image_files = glob.glob('e/*')

    for index, image_path in enumerate(image_files):
        image = cv2.imread(image_path)
        resize_image = cv2.resize(image, (IMAGE_SIZE, IMAGE_SIZE))
        cv2.imwrite('r/e%06d.jpg' % index, resize_image)

if __name__ == '__main__':
    resize()


from flask import Flask, render_template, request, jsonify
from flask_uploads import UploadSet, configure_uploads, AUDIO
import speech_recognition as sr

app = Flask(__name__)

wavaudio = UploadSet('files', AUDIO)

app.config['UPLOADED_FILES_DEST'] = 'static/audio'
configure_uploads(app, wavaudio)

@app.route('/', methods=['GET', 'POST'])
def upload():
    if request.method == 'POST' and 'au' in request.files:
        filename = wavaudio.save(request.files['au'])
        r = sr.Recognizer()
        audio = 'static/audio/{}'.format(filename)
        with sr.AudioFile(audio) as source:
            audio = r.record(source)
        try:
            text = r.recognize_google(audio)
            with open("output.txt","a+") as f:
                f.write("\n"+text)
            return(jsonify(title=text))
        except Exception as e:
            return(e)

    return render_template('upload.html')

if __name__ == '__main__':
    app.run(debug=True)


"""Base Element Class"""

#  Created by Egor Kostan.
#  GitHub: https://github.com/ikostan
#  LinkedIn: https://www.linkedin.com/in/egor-kostan/

import selenium.webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, \
    NoSuchElementException, \
    NoSuchAttributeException

class BaseElement:
    """
    Base Web Element
    """

    def __init__(self,
                 driver: selenium.webdriver,
                 explicit_wait_time: int,
                 locator=None):
        self.__driver = self.__set_driver(driver)
        self.__explicit_wait_time = explicit_wait_time
        self.__locator = self.__set_locator(locator)
        self.__element = self.__find_element()

    @property
    def explicit_wait_time(self):
        """
        Returns Explicit wait time
        :return:
        """
        return self.__explicit_wait_time

    @property
    def driver(self):
        """
        Returns selenium.webdriver object
        :return:
        """
        return self.__driver

    def attribute(self, attribute: str):
        """
        Returns attribute value
        :return:
        """
        try:
            atr = self.element.get_attribute(attribute)
            # print('\nattribute: {}, object: {}'.format(attribute, atr))  # debug only
            return atr

        except TimeoutException:
            raise NoSuchAttributeException(
                '\nERROR: The element has no attribute "{}".\n'
                'LOCATOR: {}\n'.format(attribute, self.locator))

    @property
    def element(self):
        """
        Returns Element instance
        :return:
        """
        return self.__element

    @property
    def locator(self):
        """
        Returns locator object (tuple)
        :return:
        """
        return self.__locator

    @locator.setter
    def locator(self, locator) -> None:
        """
        Set Base Element locator
        :param locator:
        :return:
        """
        self.__set_locator(locator)
        return None

    def __find_element(self):
        """
        Returns web element or NoSuchElementException in
        case element does not exist on the DOM
        :return:
        """
        try:
            element = WebDriverWait(self.driver, self.explicit_wait_time).until(
                EC.presence_of_element_located(self.locator))
            return element

        except TimeoutException:
            raise NoSuchElementException(
                '\nERROR: can not find element. The element is not present on the DOM.\n'
                'LOCATOR: {}\n'.format(self.locator))

    @staticmethod
    def __set_driver(driver: selenium.webdriver):
        """
        Check if driver of type: selenium.webdriver
        :param driver:
        :return:
        """
        BaseElement.__check_driver_type(driver)
        return driver

    @staticmethod
    def __check_driver_type(driver):

        # print('\nDRIVER TYPE: {}\n'.format(type(driver)))  # Debug only
        # print('\nDRIVER NAME: {}\n'.format(driver.capabilities['browserName']))  # Debug only

        if driver.capabilities['browserName'] == 'chrome':

            if not isinstance(driver, selenium.webdriver.chrome.webdriver.WebDriver):
                raise TypeError('\nERROR: driver must be of type '
                                '"selenium.webdriver.chrome_tests.webdriver.WebDriver"\n')
            return None

        if driver.capabilities['browserName'] == 'firefox':
            if not isinstance(driver, selenium.webdriver.firefox.webdriver.WebDriver):
                raise TypeError('\nERROR: driver must be of type '
                                '"selenium.webdriver.firefox.webdriver.WebDriver"\n')
            return None

        if driver.capabilities['browserName'] == 'MicrosoftEdge':
            if not isinstance(driver, selenium.webdriver.edge.webdriver.WebDriver):
                raise TypeError('\nERROR: driver must be of type '
                                '"selenium.webdriver.edge.webdriver.WebDriver"\n')
            return None

        raise TypeError('\nERROR: unsupported webdriver type: '
                        '{}. Browser: {}\n'.format(type(driver),
                                                   driver.capabilities['browserName']))

    @staticmethod
    def __set_locator(locator: tuple):
        """
        Check if locator of type: tuple
        :param locator:
        :return:
        """
        if not isinstance(locator, tuple):
            raise TypeError('\nERROR: locator must be of type TUPLE\n')
        return locator


import json
import urllib

from movie_posters.models import Image, Request
from django.core.files import File
from annoying.decorators import render_to, ajax_request
from django.conf import settings

@render_to('index.html')
def index(request):
    return {}

@ajax_request
def poster(request):
    """
    This view searches for poster of the movie with a given name.
    In case API returns an image file, it's being saved at a local storage.
    Whenever the requested image is already at the local storage, it's retreived
    from there.
    :param request:
    :return:
    """
    if request.is_ajax() and request.method == 'POST':
        title = request.POST.get('title')
        if not title:
            return {'status': 'error',
                    'msg': 'Sorry, this field must not be empty'}

        source, title_param = getattr(settings, 'API')
        resp = urllib.urlopen('%s?%s' %
                              (source, urllib.urlencode({title_param: title})))

        content = json.loads(resp.read())
        resp.close()
        if content['Response'] != u'True':
            Request.objects.create(title=title)
            return {'status': 'error', 'msg': content[u'Error']}

        img_url = content[u'Poster']
        stored_images = Image.objects.filter(url=img_url)

        if img_url == 'N/A':
            return {'status': 'error', 'msg': 'Sorry, the poster is not available'}

        if stored_images.exists():
            to_return = stored_images[0]
        else:
            result = urllib.urlretrieve(img_url)
            file = open(result[0])
            to_return = Image.objects.create(image=File(file), url=img_url)
            file.close()

        Request.objects.create(image=to_return, title=title)
        return {'status': 'success', 'img': to_return.image.url}

@render_to('history.html')
def history(request):
    return {'requests': Request.objects.all().order_by('-date')}


from api import app
from api.db_config import con, cur
from flask import request, jsonify
from flask_jwt_extended import get_jwt_identity, jwt_required
from api.v2.validations.models import Validations

validationObject = Validations()

class OrderCreation():

	@jwt_required
	def order_request(self):
		email = get_jwt_identity()
		email_check = validationObject.email_check(email)
		pick_up = request.json['pick_up']
		drop_off = request.json['drop_off']
		recipient_name = request.json['recipient_name']
		weight = request.json['weight']

		if any(user_input == "" for user_input in (pick_up, drop_off, recipient_name,weight)):
			return "fields cannot be empty"

		if weight <= 0:
			return "weight cannot be zero"

		if email_check:
			role_check = validationObject.role_check(email)
			role = "admin"

			if role_check != role:
				result = cur.execute ("SELECT name, email FROM users WHERE email = '{}';".format(email))
				result = cur.fetchall()

				if result:
					output = {"cost" : 200,
						"status" : "Order requested",
						"pick_up": pick_up,
						"drop_off": drop_off,
						"recipient_name" : recipient_name,
						"weight" : weight,
						"current_location": request.json['pick_up']
						}

					for row in result:
						output.update({'name': row[0], 'email': row[1] })

					cur.execute("INSERT INTO orders(sender_name, sender_email, pick_up, drop_off, recipient_name,weight, cost, status, current_location) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)", (output['name'], output['email'], output['pick_up'], output['drop_off'], output['recipient_name'], output['weight'], output['cost'], output['status'], output['current_location']))
					con.commit()
					return "Order created"

			return "Orders cannot be created on admin account"

		return "email not registered"


#!/usr//bin/env/python
# _*_ coding=utf-8 _*_

#################################################################
#            POWER by Fitz or call me WEN DA                    #
#      这个软件的功能主要是通过调取卓易健康这个api，并向其post  #
#      数据，以达到自己能修改微信运动的步数的功能！             #
#                                                               #
#                                                               #
#      API:http://www.789sk.cn/wxyd/api.html                    #
#                                                               #
#                有问题+VX：qq498289134                           #
#################################################################

import requests  # 调用requests库
import sys
keep = requests.Session()  # 用Session保持住会话
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1'
}                                                                         # 利用headers伪装好防止被服务器拒绝
url = 'http://www.789sk.cn/wxyd/api.php'  # 目标网址
datas = {}
while True:                                                               #构建循环以便满足多次修改步数的需求
    try:
        datas['user'] = input('你所绑定的卓易健康账号：')
        datas['walk'] = int(input('你想要刷取的步数：'))
    except ValueError:                                                     #防止有神奇宝贝乱填
        print('非法输入，请检查所输入的步数是否符合-1至98800这个范围\n')
        print('-1表示直接刷满也即是98800，注意：这比较容易被封号请谨慎使用！！！')
    post_data = requests.post(url=url, headers=headers, data=datas)        #提交数据
    if "数据提交成功！" in post_data:                                         #以下都是根据网站返回的信息作出判断
        print('\n你已经成功修改了微信运动步数，请前往查看')
    elif "账号未绑定微信！" in post_data.text:
        print('\n你的账号没有绑定微信，请去微信绑定后再使用，谢谢！')
    elif "效验码认证失败！" in post_data.text:
        print('\n效验码认证失败！可能是网站出现问题')
    elif "请输入账号！" in post_data.text:
        print('\n你似乎忘记输入账号了')
    elif "请输入步数！" in post_data.text:
        print('\n你似乎忘记输入步数了')
    else:
        pass
    if input('请问你还要继续修改吗？如果是请按回车键，想退出按q') == 'q':           #结合开头的While True 循环做最终判断
            sys.exit()


import time
from playlist.models import Playlist, Tracks

"""
This file demonstrates writing tests using the unittest module. These will pass
when you run "manage.py test".

Replace this with more appropriate tests for your application.
"""

from django.test import TestCase
from datetime import datetime

class PlayListTest(TestCase):
    def test_saving_retrieving_playlist(self):
        playlist = Playlist()
        playlist.name = "First ever playlist"
        #playlist.user = User
        playlist.created_at = datetime.now()
        time = datetime.now()
        playlist.save()

        playlist_ = Playlist.objects.all()

        firstplaylist = playlist_[0]

        self.assertEqual(firstplaylist.name, "First ever playlist")
        #self.assertEqual(firstplaylist.created_at, time)