# -*- coding: utf-8 -*-
"""
Created on Mon Apr  1 18:12:04 2019

@author: eric
"""
import os
from os import path
from datetime import datetime
import re
from requests import get
from bs4 import BeautifulSoup
from glob import glob
import inspect
from common import is_article_english,object2list, get_featurenames
#from icu996companies import icu996companies, non996companies
import pandas as pd
from db import get_conn

from config import year, month, company_blacklist, title_key_blacklist
year_month=f'{year}{month:02}'

class Job():
    #basic info
    job_id=""
    title=""
    page_title=''

    job_summary=''

    #经验 no 1_3 3_5 5_10 10+
    experience=''

    #学历， 初中以下，高中，专科，本科，硕士，博士
    edu=''

    headcount=0

    publish_date=datetime.today()
    #如果招聘信息本身都是周末发布的，这个公司应该是周末上班的。既996
    published_on_weekend=False

    monthly_salary=0

    #职能类别 软件工程师 算法工程师 系统架构设计师
    zhinengleibie=''

    #手机开发
    phone_app=False
    phone_android=False
    phone_iso=False

    #tags
    #五险一金
    job_tags=''
    tag_five_insurance=False
    #股票期权
    tag_stock=False
    #弹性工作
    tag_flexible=False
    #做六休一
    tag_rest_one_day=False
    #做五休二 周末双休 朝九晚五
    tag_rest_two_days=False
    #不加班
    tag_no_overtime=False
    #
    #tag_9_5=False
    #母婴室
    tag_baby_care=False

    #job_description
    job_description=""

    province=''
    city=''
    #languages
    english=False
    japanese=False
    #company_info
    company_title=""

    company_description=""
    #外资(欧美)
    #外资(非欧美)
    #合资
    #国企
    #民营公司
    #外企代表处
    #政府机关
    #事业单位
    #非营利组织
    #上市公司
    #创业公司
    company_type=''

    #少于50人
    #50-150人
    #150-500人
    #500-1000人
    #1000-5000人
    #5000-10000人
    #10000人以上
    company_size=''

    #计算机/互联网/通信/电子
    #会计/金融/银行/保险
    #贸易/消费/制造/营运
    #制药/医疗
    #广告/媒体
    #房地产/建筑
    #专业服务/教育/培训
    #服务业
    #物流/运输
    #能源/原材料
    #政府/非营利组织/其他
    industry=''

    _996_no=False
    _996_yes=False

    def get_tags(self, tags):
        self.tag_five_insurance=('五险一金' in tags)
        self.tag_baby_care=('母婴室' in tags)
        self.tag_flexible=('弹性工作' in tags)
        self.tag_no_overtime=('不加班' in tags)
        self.tag_rest_one_day=('做六休一' in tags)
        self.tag_rest_two_days=('做五休二' in tags or '周末双休' in tags or '朝九晚五' in tags)
        self.tag_stock=('股票期权' in tags)
        return self

    def get_salary(self, salary_string):
        #零时工，不统计
        if salary_string.endswith('元/小时'):
            self.monthly_salary=-1
        elif salary_string.endswith('万/月'):
            salary_string=salary_string.replace("万/月","")
            salary_min_max=salary_string.split('-')
            self.monthly_salary=(float(salary_min_max[0])+float(salary_min_max[1]))/2*10000
        elif salary_string.endswith('千/月'):
            salary_string=salary_string.replace("千/月","")
            salary_min_max=salary_string.split('-')
            self.monthly_salary=(float(salary_min_max[0])+float(salary_min_max[1]))/2*1000
        elif salary_string.endswith('万/年'):
            salary_string=salary_string.replace("万/年","")
            salary_min_max=salary_string.split('-')
            self.monthly_salary=(float(salary_min_max[0])+float(salary_min_max[1]))/2/12*10000
        elif salary_string.endswith('万/年'):
            salary_string=salary_string.replace("万/年","")
            salary_min_max=salary_string.split('-')
            self.monthly_salary=(float(salary_min_max[0])+float(salary_min_max[1]))/2/12*10000
        elif salary_string.endswith('万以上/月'):
            self.monthly_salary=float(salary_string.replace("万以上/月",""))*10000
        elif salary_string.endswith('万以上/年'):
            self.monthly_salary=float(salary_string.replace("万以上/年",""))/12*10000
        return self

    def check_edu(self):
        return not self.edu==''

    def get_edu(self, tag):
        if (tag in ['初中及以下','高中','大专','本科','硕士','博士']):
            self.edu=tag
        return self

    def check_working_experience(self):
        return not self.experience==''

    def get_working_experience(self, tag):
        if (tag=='无工作经验'):
            self.experience='no'
            return self
        if (tag=='10年以上经验'):
            self.experience='10+'
            return self
        if not self.check_working_experience():
            re_result = re.match(r'(\d+)-(\d+)年经验',tag)
            if re_result:
                we=(float(re_result.group(1))+float(re_result.group(2)))/2.0
                if we<3:
                    self.experience='1_3'
                elif we<5:
                    self.experience='3_5'
                elif we<10:
                    self.experience='5_10'
                else:
                    self.experience='10+'

                return self

            re_result = re.match(r'(\d+)年经验',tag)
            if re_result:
                we=float(re_result.group(1))
                if we<3:
                    self.experience='1_3'
                elif we<5:
                    self.experience='3_5'
                elif we<10:
                    self.experience='5_10'
                else:
                    self.experience='10+'
        return self

    def check_company_size(self):
        return not self.company_size==''

    def get_company_size(self, tag):
        if (tag=='少于50人'):
            self.company_size='50-'
        elif (tag=='50-150人'):
            self.company_size='50-150'
        elif (tag=='150-500人'):
            self.company_size='150-500'
        elif (tag=='500-1000人'):
            self.company_size='500-1000'
        elif (tag=='1000-5000人'):
            self.company_size='1000-5000'
        elif (tag=='5000-10000人'):
            self.company_size='5000-10000'
        elif (tag=='10000人以上'):
            self.company_size='10000+'
        return self

    def get_company_type(self, tag):
        if tag in ['外资（欧美）','外资（非欧美）','合资','国企','民营公司','外企代表处','政府机关','事业单位','非营利组织','上市公司''创业公司']:
            self.company_type=tag
        return self

    def check_company_type(self):
        return not self.company_type==''

    def check_industry(self):
        return not self.industry==''

    def get_industry(self, industry_tag):
        if industry_tag in ['计算机软件','计算机硬件','计算机服务(系统、数据服务、维修)','通信/电信/网络设备','通信/电信运营、增值服务','互联网/电子商务','网络游戏','电子技术/半导体/集成电路','仪器仪表/工业自动化']:
            self.industry='computer'
        #会计/金融/银行/保险
        if industry_tag in ['会计/审计','金融/投资/证券','银行','保险','信托/担保/拍卖/典当']:
            self.industry='finance'
        #贸易/消费/制造/营运
        if industry_tag in ['贸易/进出口','批发/零售','快速消费品(食品、饮料、化妆品)','服装/纺织/皮革','家具/家电/玩具/礼品','奢侈品/收藏品/工艺品/珠宝','办公用品及设备','机械/设备/重工','汽车及零配件']:
            self.industry='trade'
        #制药/医疗
        if industry_tag in ['制药/生物工程','医疗/护理/卫生','医疗设备/器械']:
            self.industry='medical'
        #广告/媒体
        if industry_tag in ['广告','公关/市场推广/会展','影视/媒体/艺术/文化传播','文字媒体/出版','印刷/包装/造纸']:
            self.industry='ads'
        #房地产/建筑
        if industry_tag in ['房地产','建筑/建材/工程','家居/室内设计/装潢','物业管理/商业中心']:
            self.industry='realestate'
        #专业服务/教育/培训
        if industry_tag in ['中介服务','专业服务(咨询、人力资源、财会)','外包服务','检测，认证','法律','教育/培训/院校','学术/科研','租赁服务']:
            self.industry='edu'
        #服务业
        if industry_tag in ['餐饮业','酒店/旅游','娱乐/休闲/体育','美容/保健','生活服务']:
            self.industry='service'
        #物流/运输
        if industry_tag in ['交通/运输/物流','航天/航空']:
            self.industry='logistic'
        #能源/原材料
        if industry_tag in ['石油/化工/矿产/地质','采掘业/冶炼','电气/电力/水利','新能源','原材料和加工']:
            self.industry='energy'
        #政府/非营利组织/其他
        if industry_tag in ['政府/公共事业','非营利组织','环保','农/林/牧/渔','多元化业务集团公司']:
            self.industry='gov'
        return self

    def check_all(self, raise_exception=False):
#        if not self.check_company_size():
#            raise Exception("check_company_size")
        if raise_exception:
            if self.company_type=='':
                raise Exception("check_company_type")
            if self.industry=='':
                raise Exception("check_industry")
            if self.experience=='':
                raise Exception("check_working_experience")

def printObject(o):
    print(inspect.getmembers(o))

def get_company_tags(company_link):
    response=get(company_link)
    response.encoding='gbk'
    soup=BeautifulSoup(response.text, 'html.parser')
    ltype_tag=soup.select_one('.ltype')
    if not ltype_tag:
        return []
    info_string=ltype_tag.text
    return [info.strip() for info in info_string.split('|')]

def file2job(file, zhinengleibie, province):
    job=Job()
    job.zhinengleibie=zhinengleibie

    if province=='深圳':
        job.province='广东'
    else:
        job.province=province

    job.job_id=path.split(file)[-1].replace(".html","")

    if job.job_id in ['110455749','77612262','107681687']:
        return None

    #page title

    #print(file)
    content=""
    try:
        with open(file, mode='r',encoding='gbk') as f:
            content=f.read()
            f.close()
    except UnicodeDecodeError:
        print("UnicodeDecodeError")
        return None

    soup=BeautifulSoup(content, "html.parser")
    title_tag=soup.select_one('title')
    if not title_tag:
        return None
    job.page_title=title_tag.text
    if '异地招聘' in job.page_title:
        return None
    #职业 start
    #首先判断职业，如果职业不是程序员，直接pass
    zhineng_tag=soup.find('span',{'class':'label'},text='职能类别：')
    if not zhineng_tag:
        return None

    #职业 end

    #20-40万/年
    #'1-1.5万/月'
    #10万以上/月
    #100万以上/年
    #3-4.5千/月
    #17元/小时
    salary_tag=soup.select_one('.cn strong')
    if not salary_tag:
        return None
    salary_string=salary_tag.text
    #零时工，不统计
    job.get_salary(salary_string)
    if job.monthly_salary==-1:
        return None

    job.title=soup.find("h1").text.strip()

    if any(key in job.title for key in title_key_blacklist):
        return None

    #'深圳-福田区|5-7年经验|本科|招1人|04-01发布'
    job.job_summary=soup.select_one(".msg").text.replace('\xa0','').replace(' ','').strip()
    #print(basic_info)
    infos=job.job_summary.split('|')
    #first location
    job.city=infos[0].split('-')[0]
    #remove the first one - location
    infos=infos[1:]
    for info in infos:
        if '经验' in info and not job.check_working_experience():
            job.get_working_experience(info)

        #学历
        if not job.check_edu():
            job.get_edu(info)

        #人数
        if '招' in info and '人' in info:
            headcount_string=info.replace('招','').replace('人','')
            if headcount_string=='若干':
                headcount_string='5'
            job.headcount=int(headcount_string)

        if info.endswith('发布'):

            #date
            date_string="2020-"+info.replace("发布",'')
            job.publish_date=datetime.strptime(date_string, '%Y-%m-%d')
            weekday=job.publish_date.weekday()
            job.published_on_weekend=weekday>4

        #language
        if '英语' in info or '英文' in info:
            job.english=True
        if '日语' in info or '日文' in info:
            job.japanese=True

    #tags
    tags=[tag.text for tag in soup.select('.sp4')]
    job.job_tags=','.join(tags)
    job.get_tags(tags)

    h2_span=soup.select_one('h2 span')
    job.job_description=h2_span.parent.find_next('div').text.strip()
    job_description_lower=job.job_description.lower()
    job_description_lower=job.title+" "+job_description_lower

    job.get_programming_languages(job_description_lower) \
        .get_databases(job_description_lower) \
        .get_big_data_stats(job_description_lower) \
        .get_machine_learning_stats(job_description_lower)

    #如果招聘信息本身都是英语写的，那么肯定要求英语
    if is_article_english(job_description_lower):
        job.english=True

    #公司信息
    company_info_tag=soup.find('span',text='公司信息')
    if company_info_tag:
        job.company_description=company_info_tag.parent.find_next('div').text.replace('\xa0',' ').strip()
    company_title_tag=soup.select_one('.com_name')
    if not company_title_tag:
        company_title_tag=soup.select_one('.catn')
    job.company_title=company_title_tag.text.strip()
    #['民营公司', '150-500人', '服装/纺织/皮革']
    company_tags=[p.text.strip() for p in soup.select('.com_tag .at')]

    if len(company_tags)>0:
        job.get_company_type(company_tags[0])
    if job.company_type=='':
        company_link=company_title_tag.attrs['href']
        company_tags=get_company_tags(company_link)
        for tag in company_tags:
            if job.get_company_type(tag).check_company_type():
                break

    if job.company_type=='':
        return None

    job.get_company_size(company_tags[1])
    if not job.check_company_size():
        company_link=company_title_tag.attrs['href']
        company_tags=get_company_tags(company_link)
        for tag in company_tags:
            if job.get_company_size(tag).check_company_size():
                break

    #计算机/互联网/通信/电子
    industry_tags=[p.text.strip() for p in soup.select('.com_tag .at a') if not p.text=='']

    if len(industry_tags)==0:
        company_link=soup.select_one('.com_name').attrs['href']
        industry_tags=get_company_tags(company_link)

    for industry_tag in industry_tags:
        job.get_industry(industry_tag)

    if job.company_title in ['系统集成有限责任公司','博彦科技股份有限公司']:
        job.industry='computer'
    if job.company_title=='软件与服务中心':
        job.industry='trade'
    if job.company_title== '中核集团技术经济总院':
        job.industry='energy'

    #black named companies
    if job.company_title in company_blacklist:
        return None
#    if not job.check_industry():
#        raise Exception("no industry")

    return job

def try_rename(file):
    #return
    new_file=file.replace(f"51jobs_{year_month}",f"51jobs_{year_month}_b")
    if path.isfile(new_file):
        os.remove(file)
    else:
        os.rename(file, new_file)

def file2db(file, zhinengleibie, province):
    conn=get_conn()

    filename=path.split(file)[-1]
    job_id=filename.replace(".html","")

    exists=conn.execute("select count(1) from _{} where job_id='{}'".format(year_month, job_id)).fetchall()[0][0]
    if exists:
        try_rename(file)
        return
    print(file)
    job=file2job(file, zhinengleibie, province)
    if not job:
        try_rename(file)
        return

    data=pd.DataFrame(columns=get_featurenames(job))
    l=object2list(job)
    data.loc[job.job_id]=l
    data.to_sql("_"+year_month,conn,if_exists="append", index=False)

    conn.close()
    try_rename(file)

def city2db(data_folder,zhinengleibie, province):
    city_folder=path.join(data_folder,zhinengleibie, province)
    files = glob(path.join(city_folder,"*.html"))
    for file in files:
        file2db(file, zhinengleibie, province)

def main():
    provinces=['北京','上海','广东','深圳','天津','重庆','江苏','浙江','四川','海南','福建','山东','江西','广西','安徽','河北','河南','湖北','湖南','陕西','山西','黑龙江','辽宁','吉林','云南','贵州','甘肃','内蒙古','宁夏','西藏','新疆','青海']
    data_folder = '../../data/51jobs_{}/'.format(year_month)
    back_folder = '../../data/51jobs_{}_b/'.format(year_month)
    zhinengleibies=['高级软件工程师', '软件工程师','算法工程师','机器学习工程师','深度学习工程师','图像算法工程师','图像处理工程师','语音识别工程师','图像识别工程师','机器视觉工程师','自然语言处理（NLP）','系统架构设计师','互联网软件开发工程师','手机应用开发工程师','网站架构设计师']
    for zhinengleibie in zhinengleibies:
        category_back_folder=path.join(back_folder, zhinengleibie)
        if not path.isdir(category_back_folder):
            os.mkdir(category_back_folder)
        for province in provinces:
            city_back_folder=path.join(category_back_folder, province)
            if not path.isdir(city_back_folder):
                os.mkdir(city_back_folder)

            city2db(data_folder, zhinengleibie, province)
            #t=threading.Thread(target=city2db, args=(path.join(data_folder,job_category, city), city))
            #t.start()

if __name__=='__main__':
    #global year_month

    main()


"""
OpenStack deployments
"""

import types
import tmuxp
from monster import util

class Deployment(object):
    """Base for OpenStack deployments
    """

    def __init__(self, name, os_name, branch, provisioner, status, product,
                 clients=None):
        self.name = name
        self.os_name = os_name
        self.branch = branch
        self.features = []
        self.nodes = []
        self.status = status or "provisioning"
        self.provisioner = str(provisioner)
        self.product = product
        self.clients = clients

    def __repr__(self):
        """
        Print out current instance
        """

        outl = 'class: ' + self.__class__.__name__
        for attr in self.__dict__:
            if attr == 'features':
                features = "\tFeatures: {0}".format(
                    ", ".join((str(f) for f in self.features)))
            elif attr == 'nodes':
                nodes = "\tNodes: {0}".format(
                    "".join((str(n) for n in self.nodes)))
            elif isinstance(getattr(self, attr), types.NoneType):
                outl += '\n\t{0} : {1}'.format(attr, 'None')
            else:
                outl += '\n\t{0} : {1}'.format(attr, getattr(self, attr))

        return "\n".join([outl, features, nodes])

    def destroy(self):
        """
        Destroys an OpenStack deployment
        """

        self.status = "destroying"
        util.logger.info("Destroying deployment:{0}".format(self.name))
        for node in self.nodes:
            node.destroy()
        self.status = "destroyed"

    def update_environment(self):
        """
        Pre configures node for each feature
        """

        self.status = "loading environment"
        for feature in self.features:
            log = "Deployment feature: update environment: {0}"\
                .format(str(feature))
            util.logger.debug(log)
            feature.update_environment()
        util.logger.debug(self.environment)
        self.status = "environment ready"

    def pre_configure(self):
        """
        Pre configures node for each feature
        """

        self.status = "pre-configure"
        for feature in self.features:
            log = "Deployment feature: pre-configure: {0}"\
                .format(str(feature))
            util.logger.debug(log)
            feature.pre_configure()

    def build_nodes(self):
        """
        Builds each node
        """

        self.status = "building nodes"
        for node in self.nodes:
            node.build()
        self.status = "nodes built"

    def post_configure(self):
        """
        Post configures node for each feature
        """

        self.status = "post-configure"
        for feature in self.features:
            log = "Deployment feature: post-configure: {0}"\
                .format(str(feature))
            util.logger.debug(log)
            feature.post_configure()

    def build(self):
        """
        Runs build steps for node's features
        """

        util.logger.debug("Deployment step: update environment")
        self.update_environment()
        util.logger.debug("Deployment step: pre-configure")
        self.pre_configure()
        util.logger.debug("Deployment step: build nodes")
        self.build_nodes()
        util.logger.debug("Deployment step: post-configure")
        self.post_configure()
        self.status = "post-build"
        util.logger.info(self)

    def artifact(self):
        """
        Artifacts openstack and its dependant services for a deployment
        """

        self.log_path = "/var/log"
        self.etc_path = "/etc/"
        self.misc_path = "misc/"

        if self.deployment.os_name == 'precise':
            self.list_packages_cmd = ["dpkg -l"]
        else:
            self.list_packages_cmd = ["rpm -qa"]

        # Run each features archive
        for feature in self.features:
            feature.archive()

        # Run each nodes archive
        for node in self.nodes:
            node.archive()

    def search_role(self, feature):
        """
        Returns nodes the have the desired role
        :param feature: feature to be searched for
        :type feature: string
        :rtype: Iterator (Nodes)
        """

        return (node for node in
                self.nodes if feature in
                (str(f).lower() for f in node.features))

    def feature_in(self, feature):
        """
        Boolean function to determine if a feature exists in deployment
        :param feature: feature to be searched for
        :type feature: string
        :rtype: Boolean
        """

        if feature in (feature.__class__.__name__.lower()
                       for feature in self.features):
            return True
        return False

    def tmux(self):
        """
        Creates an new tmux session with an window for each node
        """

        server = tmuxp.Server()
        session = server.new_session(session_name=self.name)
        cmd = ("sshpass -p {1} ssh -o UserKnownHostsFile=/dev/null "
               "-o StrictHostKeyChecking=no -o LogLevel=quiet -l root {0}")
        for node in self.nodes:
            name = node.name[len(self.name) + 1:]
            window = session.new_window(window_name=name)
            pane = window.panes[0]
            pane.send_keys(cmd.format(node.ipaddress, node.password))

    def feature_names(self):
        """
        Returns list features as strings
        :rtype: list (string)
        """

        return [feature.__class__.__name__.lower() for feature in
                self.features]


from flask import request, jsonify
from .auth import isAuth
from models import Kategori
from . import db

def get_kategori():
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    return jsonify([
        {
            'id_kategori': i.id_kategori,
            'nama': i.nama,
            'deskripsi' : i.deskripsi,
        } for i in Kategori.query.all()
    ])

def get_kategori_id(id):
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    kategori = Kategori.query.filter_by(id_kategori=id).first_or_404()
    return {
        'id': kategori.id_kategori,
        'nama': kategori.nama,
        'deskripsi': kategori.deskripsi
    }

def create_kategori():
    # from db import db
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    if a.is_admin == False:
        return {
            "YOU ARE NOT ADMIN"
        }
    data = request.get_json()
    if not 'nama' in data or not 'deskripsi' in data :
        return jsonify({
            'error': 'Bad Request',
            'message': 'Nama ata Deskripisi Tidak ada'
        }), 400
    k = Kategori (
                    nama = data['nama'],
                    deskripsi = data['deskripsi']
                )
    db.session.add(k)
    db.session.commit()
    return {
        'nama': k.nama,
        'deskripsi': k.deskripsi
    }, 201

def update_kategori(id):
    # from db import db
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    if a.is_admin == False:
        return {
            "YOU ARE NOT ADMIN"
        }
    data=request.get_json()
    if 'nama' not in data:
        return {
            'error': 'Bad Request',
            'message': 'Name field needs to be present'
        }, 400
    k = Kategori.query.filter_by(id_kategori=id).first_or_404()
    k.nama=data['nama']
    if 'deskripsi' in data:
        k.deskripsi= data['deskripsi']
    db.session.commit()
    return jsonify({
        'nama': k.nama,
        'deskripsi': k.deskripsi
    })

def delete_kategori(id):
    # from db import db
    a = isAuth()
    if a == None:
        return {
            "message": "Unauthorized"
        }, 400
    if a.is_admin == False:
        return {
            "YOU ARE NOT ADMIN"
        }
    k = Kategori.query.filter_by(id_kategori=id).first_or_404()
    db.session.delete(k)
    db.session.commit()
    return {
        'success': 'Data Berhasil Dihapus'
    }


from PyQt5.QtWidgets import QDialog, QMessageBox
from UI.accout.accountModify import Ui_Dialog
import datetime
from PyQt5 import QtCore
from PyQt5.QtWidgets import QTableWidgetItem, QHeaderView
from UI.accout.modify_client import modify_clinet

class account_modify_ok(QDialog):
    "A dialog class for Ui_LoginDialog, who can show itself"

    def __init__(self, parent):
        super(account_modify_ok, self).__init__(parent)
        self.ui = Ui_Dialog()
        self.ui.setupUi(self)

#继承父类的信息
        self.parent = parent
        self.db = parent.db
        self.id_number = parent.id_number
        self.show()
        self.client_id = parent.client_id
        self.account_id = parent.account_id

#按键设置
        self.ui.ModifyBtn.clicked.connect(self.Modify)
        self.ui.modify.clicked.connect(self.modifyClient)

        self.ui.tableWidget.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch)  # 设置表格等宽
        self.ui.tableWidget.horizontalHeader().setStyleSheet("QHeaderView::section{background:skyblue;}")

        self.showMessage()
        self.showTable()

#显示账户的基本信息
    def showMessage(self):
        cursor = self.db.cursor()
        sql = """ select account_type,currency_type,rate,brance,overdraft FROM account WHERE account_number = %s"""
        cursor.execute(sql, [self.account_id])
        result = cursor.fetchone()
        # print(result)
        account_type, currency_type, rate, brance, overdraft = result
        self.ui.account_id.setText(self.account_id)
        self.ui.account_type.setText(str(account_type))
        self.ui.overdraft.setText(str(overdraft))
        self.ui.currency_type.setText(str(currency_type))
        self.ui.rate.setText(str(rate))
        self.ui.brance.setText(str(brance))
        self.account_type = account_type

#显示账户的户主
    def showTable(self):
        self.ui.tableWidget.setColumnCount(2)
        self.ui.tableWidget.setHorizontalHeaderLabels(['id', '姓名'])
        cursor = self.db.cursor()
        sql = """ select own.id_number ,client.name  from own ,client where own.id_number = client.id_number and own.account_number = %s"""
        cursor.execute(sql, [self.account_id])
        tabs = cursor.fetchall()
        currentRowCount = 0
        for tab in tabs:
            self.ui.tableWidget.insertRow(currentRowCount)

            item0 = QTableWidgetItem(str(tab[0]))
            item0.setTextAlignment(QtCore.Qt.AlignCenter)
            item1 = QTableWidgetItem(str(tab[1]))
            item1.setTextAlignment(QtCore.Qt.AlignCenter)

            self.ui.tableWidget.setItem(currentRowCount, 0, item0)  # 列1
            self.ui.tableWidget.setItem(currentRowCount, 1, item1)  # 列2
            currentRowCount += 1
            self.ui.tableWidget.setRowCount(currentRowCount)

#修改账户的基本信息
    def Modify(self):
       currency_type,rate,brance,overdraft = self.ui.currency_type.text(),self.ui.rate.text(),self.ui.brance.text(),self.ui.overdraft.text()
       try:
           cursor = self.db.cursor()

           # 获取待删除账户余额
           sql = """select brance from account
                    where account_number = %s"""
           cursor.execute(sql, [self.account_id])
           pre_brance = cursor.fetchone()
           # print("1")
           #获取支行名称
           sql = """select department.name from staff,department where staff.dep_department_id = department.department_id and staff.id_number = %s"""
           cursor.execute(sql, [self.id_number])
           sub_name = cursor.fetchone()
           # print("2")

           sql = """ UPDATE account SET currency_type = %s,rate = %s, brance = %s ,overdraft = %s  WHERE account_number = %s"""
           cursor.execute(sql, [currency_type,rate,brance,overdraft, self.account_id])
           sql = """ UPDATE own SET last_date =  %s WHERE account_number = %s and id_number = %s"""
           cursor.execute(sql, [ datetime.date.today(),self.account_id,self.id_number])

           # print(pre_brance[0], brance)
           # 修改支行余额
           sql = """update sub_branch set asset = asset - %s + %s
                    where name =  %s"""
           cursor.execute(sql, [str(pre_brance[0]), str(brance), sub_name[0]])
           self.db.commit()
           QMessageBox.information(self, '提示', '修改成功', QMessageBox.Close, QMessageBox.Close)
           self.close()

       except:
           QMessageBox.information(self, '提示', '错误', QMessageBox.Close, QMessageBox.Close)
           self.close()

#不显示户主信息
    def closeClient(self):
        self.ui.tableWidget.setColumnCount(0)

#修改账户关联的户主
    def modifyClient(self):
        w = modify_clinet(self)
        #self.showTable()


import os
import numpy as np
import time
import sys
import copy
import cv2

from data import Data
from objects import Annotations, Detections, TrackingResults
from utilities import ImageSequenceCapture, draw_box, linux_path, CustomLogger, BaseParams

class Input:
    """
     read images, annotations, detections and tracking results for a sequence

    :type params: Input.Params
    :type _logger: CustomLogger
    :type all_frames: list[np.ndarray]
    :type curr_frame: np.ndarray
    :type frame_id: int
    :type frame_size: tuple
    :type _cap: cv2.VideoCapture
    :type annotations: Annotations
    :type track_res: TrackingResults
    :type detections: Detections
    """

    class Params(BaseParams):
        """
        :type annotations: Annotations.Params
        :type track_res: TrackingResults.Params
        :type detections: Detections.Params

         :ivar path: path of the directory, video file or binary file from where
        images are to be read; if this is not specified then it is computed from
        db_root_path, sequence set/name (in Params.py), source_type and img_fmt/vid_fmt

        :ivar frame_ids: two element tuple specifying the IDs of the first and
        last frames in the sub sequence; if either is less than 0, it is computed
        from ratios and offsets

        :ivar db_root_path: path of the directory that contains all datasets

        :ivar source_type: 0: image sequence 1: video file

        :ivar img_fmt: (naming scheme, extension) of each image file in the sequence

        :ivar vid_fmt: file extension of the video file

        :ivar resize_factor: multiplicative factor by which to resize the input
        images

        :ivar convert_to_gs: convert all input images to greyscale (if they are
        RGB) before they are used

        :ivar read_from_bin: write all image data to a binary file for quick and
        lossless reading

        :ivar write_to_bin: write all image data to a binary file for quick and
        lossless reading

        :ivar batch_mode: read all frames in the input sequence at once to avoid
        repeated disk accesses

        :ivar roi: four element tuple specifying the region of interest (ROI)
        as (xmin, ymin, xmax, ymax) of the corresponding bounding box; ROI is
        only enabled if xmx > xmin and ymax > ymin

        :ivar annotations: parameters for Annotations in Objects module

        :ivar track_res: parameters for tracking results (same as Annotations
        in Objects module)

        :ivar detections: parameters for Detections in Objects module

        """

        def __init__(self):
            self.path = ''
            self.frame_ids = (-1, -1)

            self.db_root_path = '/data'
            self.source_type = 0
            self.img_fmt = ('image%06d', 'jpg')
            self.vid_fmt = 'mp4'

            self.resize_factor = 1
            self.convert_to_gs = 0
            self.read_from_bin = 0
            self.write_to_bin = 0
            self.batch_mode = 1
            self.roi = (0, 0, 0, 0)

            self.annotations = Annotations.Params()
            self.track_res = TrackingResults.Params()
            self.detections = Detections.Params()

    def __init__(self, params, logger):
        """
        :type params: Input.Params
        :type logger: CustomLogger
        :rtype: None
        """

        self.params = copy.deepcopy(params)
        self._logger = logger

        self.curr_frame = None
        self.all_frames = None
        self.frame_id = 0
        self.n_pix = 0
        self.frame_size = None
        self.bin_fid = None
        self.source_path = None
        self._cap = None

        self.roi = self.params.roi
        if self.roi[0] >= self.roi[2] or self.roi[1] >= self.roi[3]:
            self.roi = None

        self.annotations = None
        self.detections = None
        self.track_res = None

        self.read_from_bin = self.params.read_from_bin

        self.start_frame_id, self.end_frame_id = self.params.frame_ids
        self.seq_name = None
        self.seq_set = None
        self.seq_n_frames = 0
        self.n_frames = 0

        self.is_initialized = False

    def initialize(self, data, read_img=True, logger=None):
        """
        :type data: Data
        :type read_img: bool | int
        :type logger: CustomLogger | logging.RootLogger
        :rtype: bool
        """

        if logger is not None:
            self._logger = logger

        self.annotations = None
        self.detections = None
        self.track_res = None

        self.frame_id = 0
        self.start_frame_id, self.end_frame_id = self.params.frame_ids

        curr_path = self.params.path

        if curr_path:
            self.seq_name = os.path.splitext(os.path.basename(curr_path))[0]
            self.seq_set = "custom"
            self.seq_n_frames = self.get_n_frames(curr_path)
            if self.seq_n_frames <= 0:
                raise AssertionError('No. of frames could not be obtained from: {}'.format(curr_path))

            if self.start_frame_id < 0:
                self.start_frame_id = 0
            if self.end_frame_id < 0:
                self.end_frame_id = self.seq_n_frames - 1
        else:
            if not data.is_initialized:
                raise AssertionError('Source path must be provided with uninitialized data module')
            self.seq_name, self.seq_set, self.seq_n_frames = data.seq_name, data.seq_set, data.seq_n_frames

            if self.start_frame_id < 0:
                self.start_frame_id = data.start_frame_id
            if self.end_frame_id < 0:
                self.end_frame_id = data.end_frame_id
        self.n_frames = self.end_frame_id - self.start_frame_id + 1

        self._logger.info('seq_set: {:s}'.format(self.seq_set))
        self._logger.info('seq_name: {:s}'.format(self.seq_name))
        self._logger.info('seq_n_frames: {:d}'.format(self.seq_n_frames))
        self._logger.info('start_frame_id: {:d}'.format(self.start_frame_id))
        self._logger.info('end_frame_id: {:d}'.format(self.end_frame_id))
        self._logger.info('n_frames: {:d}'.format(self.n_frames))

        if not read_img:
            # input pipeline is needed only for reading objects
            self._logger.info('Skipping image pipeline initialization')
            return True

        self.read_from_bin = self.params.read_from_bin

        if self.read_from_bin:
            if curr_path:
                bin_path = curr_path
            else:
                bin_fname = '{:s}_{:d}_{:d}'.format(self.seq_name, self.start_frame_id + 1,
                                                    self.end_frame_id + 1)
                bin_path = linux_path(self.params.db_root_path, self.seq_set,
                                      'Images', bin_fname) + '.bin'

            if not os.path.isfile(bin_path):
                self._logger.info('Binary data file {:s} does not exist'.format(bin_path))
                if curr_path:
                    return False
                self.read_from_bin = 0
            else:
                self._logger.info('Reading image data from binary file: {:s}'.format(bin_path))
                self.bin_fid = open(bin_path, 'rb')
                self.frame_size = tuple(np.fromfile(self.bin_fid, dtype=np.uint32, count=2))
                self.n_pix = self.frame_size[0] * self.frame_size[1]
                actual_file_size = os.path.getsize(bin_path)
                expected_file_size = self.n_frames * self.n_pix + 8
                if expected_file_size != actual_file_size:
                    self._logger.info('Size of binary file: {:d} does not match the expected size: {:d}'.format(
                        actual_file_size, expected_file_size))
                    self.read_from_bin = 0
                    self.bin_fid.close()
                else:
                    self.curr_frame = np.fromfile(self.bin_fid, dtype=np.uint8, count=self.n_pix).reshape(
                        (self.frame_size[1], self.frame_size[0]))
                    self.params.write_to_bin = 0

        if not self.read_from_bin:
            if not curr_path:
                curr_path = linux_path(self.params.db_root_path, self.seq_set,
                                       'Images', self.seq_name)
            if self.params.source_type == 1:
                self.source_path = curr_path + '.' + self.params.vid_fmt
            else:
                self.source_path = curr_path
                if self.params.img_fmt:
                    self.source_path = linux_path(self.source_path, self.params.img_fmt[0]
                                                  + '.' + self.params.img_fmt[1])

            self._logger.info('Getting images from: {:s}'.format(self.source_path))

            if self.params.source_type == 1:
                self._cap = cv2.VideoCapture()
                if not self._cap.open(self.source_path):
                    err_txt = 'Video file ' + self.source_path + ' could not be opened'
                    raise AssertionError(err_txt)
            else:
                self._cap = ImageSequenceCapture()  # type: ImageSequenceCapture
                if not self._cap.open(self.source_path):
                    err_txt = 'Image sequence ' + self.source_path + ' could not be read'
                    raise AssertionError(err_txt)
                assert self.seq_n_frames == self._cap.n_src_files, \
                    f"Mismatch between seq_n_frames: {self.seq_n_frames} and n_src_files: {self._cap.n_src_files}"

            if self.start_frame_id > 0:
                if cv2.__version__.startswith('2'):
                    cv_prop = cv2.cv.CV_CAP_PROP_POS_FRAMES
                else:
                    cv_prop = cv2.CAP_PROP_POS_FRAMES

                self._cap.set(cv_prop, self.start_frame_id)
                next_frame_id = self._cap.get(cv_prop)
                if next_frame_id != self.start_frame_id:
                    if self.params.source_type == 1:
                        self._logger.info('OpenCV VideoCapture set property functionality is not available so '
                                          'manually skipping {:d} frames'.format(self.start_frame_id))
                        for i in range(self.start_frame_id):
                            ret, _ = self._cap.read()
                            if not ret:
                                raise AssertionError('Frame {:d} could not be read'.format(self.frame_id + 1))
                    else:
                        raise AssertionError(
                            'Something weird going on in ImageSequenceCapture'.format(self.start_frame_id))
            # curr_frame_id = self._cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
            # self.logger.debug('curr_frame_id before initialize: {:f}'.format(curr_frame_id))
            ret, frame = self._cap.read()
            if not ret:
                raise AssertionError('Frame {:d} could not be read'.format(self.frame_id + 1))
            # curr_frame_id = self._cap.get(cv2.cv.CV_CAP_PROP_POS_FRAMES)
            # self.logger.debug('curr_frame_id after initialize: {:f}'.format(curr_frame_id))
            # cv2.imshow('Frame',frame)
            # if cv2.waitKey(0) == 27:
            # exit()
            self._logger.info('Input pipeline initialized successfully with frames of size: {:d} x {:d}'.format(
                frame.shape[1], frame.shape[0]))
            if self.params.convert_to_gs:
                self.curr_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
            else:
                self.curr_frame = frame

            if self.params.resize_factor != 1:
                self.curr_frame = cv2.resize(self.curr_frame, dsize=(0, 0), fx=self.params.resize_factor,
                                             fy=self.params.resize_factor)
                self._logger.info('Resizing input frames to : {:d} x {:d}'.format(
                    self.curr_frame.shape[1], self.curr_frame.shape[0]))

            # (width, height) format rather than the (n_rows, n_cols) of shape
            self.frame_size = (self.curr_frame.shape[1], self.curr_frame.shape[0])
            self.n_pix = self.frame_size[0] * self.frame_size[1]

            if self.params.write_to_bin:
                if self.params.source_type == 1:
                    bin_path = linux_path(os.path.dirname(self.source_path),
                                          os.path.splitext(os.path.basename(self.source_path))[0] + ".bin")
                else:
                    bin_path = curr_path + '.bin'
                self.bin_fid = open(bin_path, 'wb')
                self._logger.info('Writing image data to binary file: {:s}'.format(bin_path))
                np.array(self.frame_size, dtype=np.uint32).tofile(self.bin_fid)
                self.curr_frame.astype(np.uint8).tofile(self.bin_fid)

        if self.params.batch_mode:
            if not self._read_all_frames():
                return False

        self.is_initialized = True
        return True

    def update(self):
        """
        :rtype: bool
        """
        if self.frame_id >= self.n_frames - 1:
            return False

        if self.read_from_bin:
            self.curr_frame = np.fromfile(self.bin_fid, dtype=np.uint8, count=self.n_pix).reshape(
                (self.frame_size[1], self.frame_size[0]))
            self.frame_id += 1
            if self.frame_id == self.n_frames - 1:
                self.bin_fid.close()
        else:
            ret, curr_frame_rgb = self._cap.read()
            if not ret:
                raise AssertionError('Frame {:d} could not be read'.format(self.frame_id + 1))
                # raise IOError('End of sequence reached unexpectedly')
            if self.params.convert_to_gs:
                self.curr_frame = cv2.cvtColor(curr_frame_rgb, cv2.COLOR_BGR2GRAY)
                # self.curr_frame = cv2.cvtColor(curr_frame_rgb.astype(np.float32), cv2.COLOR_BGR2GRAY)
                # self.curr_frame = np.ceil(self.curr_frame).astype(np.uint8)
            else:
                self.curr_frame = curr_frame_rgb
            if self.params.resize_factor != 1:
                self.curr_frame = cv2.resize(self.curr_frame, dsize=(0, 0), fx=self.params.resize_factor,
                                             fy=self.params.resize_factor)
            self.frame_id += 1
            if self.params.write_to_bin:
                self.curr_frame.astype(np.uint8).tofile(self.bin_fid)
                if self.frame_id == self.n_frames - 1:
                    self.bin_fid.close()
        return True

    def _read_all_frames(self):
        """
        :rtype: bool
        """
        self._logger.info('Reading all frames...')
        start_t = time.time()
        self.all_frames = [None] * self.n_frames
        # print 'len(self.all_frames): ', len(self.all_frames)
        self.all_frames[self.frame_id] = self.curr_frame
        print_diff = max(1, int(5 * round(self.n_frames * 0.04 / 5)))
        while self.frame_id < self.n_frames - 1:
            if not self.update():
                sys.stdout.write('\n')
                return False
            # print 'self.frame_id: ', self.frame_id
            self.all_frames[self.frame_id] = self.curr_frame
            if (self.frame_id + 1) % print_diff == 0:
                sys.stdout.write('\rDone {:d}/{:d} frames'.format(
                    self.frame_id + 1, self.n_frames))
                sys.stdout.flush()
        end_t = time.time()
        sys.stdout.write('\rDone {:d}/{:d} frames\n'
                         'Time taken: {:f} secs\n'.format(
            self.n_frames, self.n_frames, end_t - start_t))
        sys.stdout.flush()
        return True

    def get_n_frames(self, path):
        return cv2.VideoCapture(path).get(cv2.CAP_PROP_FRAME_COUNT)

    # def clearDetections(self, build_index):
    #     self.detections = None

    def read_detections(self):
        """
        :rtype: bool
        """
        detections_params = copy.deepcopy(self.params.detections)
        if not detections_params.path:
            detections_params.path = linux_path(self.params.db_root_path, self.seq_set,
                                                detections_params.src_dir, self.seq_name + '.txt')

        self.detections = Detections(detections_params, self._logger)
        self.detections.initialize(self.seq_n_frames,
                                   self.start_frame_id, self.end_frame_id)

        ignored_regions = None
        if self.annotations is not None:
            ignored_regions = self.annotations.ignored_regions

        if not self.detections.read(
                frame_size=self.frame_size,
                ignored_regions=ignored_regions,
                resize_factor=self.params.resize_factor):
            self._logger.error('Failed to read detections')
            self.detections = None
            return False

        return True

    # def clearAnnotations(self, build_index):
    #     self.annotations = None

    def read_annotations(self):
        """
        :rtype: bool
        """
        annotations_params = copy.deepcopy(self.params.annotations)
        if not annotations_params.path:
            annotations_params.path = linux_path(self.params.db_root_path, self.seq_set,
                                                 annotations_params.src_dir, self.seq_name + '.txt')

        self.annotations = Annotations(annotations_params, self._logger)
        self.annotations.initialize(self.seq_n_frames, self.start_frame_id, self.end_frame_id)

        if not self.annotations.read(resize_factor=self.params.resize_factor, frame_size=self.frame_size):
            self._logger.error('Failed to read annotations')
            self.annotations = None
            return False

        return True

    # def clearTrackingResults(self, build_index):
    #     self.tracking_res = None

    def read_tracking_results(self, res_path):
        """
        :type res_path: str
        :type build_index: bool
        :rtype: bool
        """
        self.params.track_res.path = res_path
        self.track_res = TrackingResults(self.params.track_res, self._logger)
        self.track_res.initialize(self.seq_n_frames,
                                  self.start_frame_id, self.end_frame_id)
        if not self.track_res.read(self.frame_size, self.params.resize_factor):
            self._logger.error('Failed to read tracking results')
            self.track_res = None
            return False
        return True

    def get_frame(self):
        return self.curr_frame

    def get_all_frames(self):
        return self.all_frames

    def show_detections(self):
        for frame_id in range(self.n_frames):
            frame_disp = cv2.cvtColor(self.all_frames[frame_id], cv2.COLOR_GRAY2BGR)
            det_ids = self.detections.idx[frame_id]
            if det_ids is None:
                continue
            for det_id in det_ids:
                draw_box(frame_disp, self.detections.data[det_id, 2:6])
            cv2.imshow('Detections', frame_disp)
            if cv2.waitKey(1) == 27:
                exit()

    def show_annotations(self):
        for frame_id in range(self.n_frames):
            if self.all_frames[frame_id] is None:
                raise SystemError('frame is None')
            frame_disp = np.copy(self.all_frames[frame_id]).astype(np.uint8)
            # frame_disp = cv2.cvtColor(self.all_frames[frame_id], cv2.COLOR_GRAY2BGR)
            ann_ids = self.annotations.idx[frame_id]
            if ann_ids is None:
                continue
            for ann_id in ann_ids:
                draw_box(frame_disp, self.annotations.data[ann_id, 2:6])
            cv2.imshow('Annotations', frame_disp)
            if cv2.waitKey(1) == 27:
                exit()


from resnet import resnet
from MSA import MultiHeadAttention
from LayerNormalization import LayerNormalization
from loss import detr_loss, detr_loss_output
from keras.layers import Input, Conv2D, Embedding, Dropout, add, Reshape, Dense, Lambda, ReLU
from keras.models import Model
import tensorflow as tf
import keras.backend as K
import numpy as np
import math

def detr(input_shape=(512,512,3), pe='sine', n_classes=80, depth=50, dilation=False,
         emb_dim=256, enc_layers=6, dec_layers=6, max_boxes=100, max_target_boxes=20,
         mlp_dim=256, mode='test'):

    # inpt
    inpt = Input(input_shape)    # [b,h,w,c]
    mask = Input((input_shape[0],input_shape[1],1))   # [b,h,w,1]

    # back
    r50 = resnet(input_shape, depth=depth, dilation=dilation)
    feat = r50(inpt)    # stage 5 output
    stride = 32 // (2**int(dilation))
    feature_shape = (math.ceil(input_shape[0]/stride), math.ceil(input_shape[1]/stride))
    print('feature shape', feature_shape)
    feat_mask = Lambda(tf.image.resize_nearest_neighbor, arguments={'size':feature_shape})(mask)   # interpolate

    # reflect & pe
    x = Conv2D(emb_dim, 1, strides=1, padding='same', name='input_proj')(feat)   # [b,h,w,d]
    x = Reshape((feature_shape[0]*feature_shape[1], emb_dim))(x)                 # [b,hw,d]
    if pe=='sine':   # constant
        # feat_pe = PositionalEmbeddingSine(emb_dim, feature_shape)  # [1,h,w,d]
        feat_pe = Lambda(lambda x: PositionalEmbeddingSine(emb_dim, feature_shape), name='PESine')(x)
    elif pe=='learned':   # trainable weights
        feat_pe = PositionalEmbeddingLearned(emb_dim, feature_shape, name='PELearned')(x)
    feat_pe = Reshape((feature_shape[0]*feature_shape[1], emb_dim))(feat_pe)    # [1,hw,d]
    feat_mask = Reshape((feature_shape[0]*feature_shape[1],))(feat_mask)        # [b,hw]
    print('transformer feat', feat_pe, feat_mask)

    # transformer encoder: parse inputs
    for i in range(enc_layers):
        x = TransformerEncoderBlock(drop_rate=0.1)([x, feat_pe], mask=feat_mask)      # [b,hw,d]
    encoder_feats = x

    # transformer decoder: feed targets x is a zeros-variable initially, get updated through the decoder blocks
    x, target_pe = PrepareDecoderInput(max_boxes, emb_dim)(x)
    print('decoder pe', target_pe)

    for i in range(dec_layers):
        x = TransformerDecoderBlock(drop_rate=0.1)([x, target_pe, encoder_feats, feat_pe], key_mask=feat_mask)
    x = LayerNormalization()(x)    # norm no matter pre_norm/post_norm

    # head: mlp
    cls_output = Dense(n_classes, name='cls_pred', activation='softmax')(x)  # [b,N2,cls]
    box = Dense(mlp_dim, activation='relu', name='box_hidden_1')(x)
    box = Dense(mlp_dim, activation='relu', name='box_hidden_2')(box)
    box_output = Dense(4, activation='sigmoid', name='box_pred')(box)    # [b,N2,4]

    if mode=='test':
        model = Model([inpt,mask], [cls_output,box_output])
    else:
        gt = Input((max_target_boxes,n_classes+4))    # [b,N2,4], fg + bg
        loss = Lambda(detr_loss, arguments={'n_classes': n_classes})(
                      [cls_output,box_output,gt])
        model = Model([inpt,mask,gt], loss)

    return model

class PrepareDecoderInput(Model):
    def __init__(self, max_boxes=100, emb_dim=256, name='query_embed'):
        super(PrepareDecoderInput, self).__init__(name=name)
        self.emb = Embedding(max_boxes, emb_dim)
        self.max_boxes = max_boxes
        self.emb_dim = emb_dim

        # create variables
        self.target_indices = tf.expand_dims(tf.range(self.max_boxes), axis=0)  # [1,N1]
        self.target = tf.zeros((1, max_boxes, emb_dim))     # [1,N1,d], zeros

    def call(self, x):
        # x: encoder feats: [b,hw,d]
        b = tf.shape(x)[0]
        target_indices = tf.tile(self.target_indices, [b,1])    # [b,N1]
        target_pe = self.emb(target_indices)     # [b,N1,d], take corresponding word-vecs
        target = tf.tile(self.target, [b,1,1])   # [b,N1,d]
        return [target, target_pe]

    def compute_output_shape(self, input_shape):
        b, N, d = input_shape
        return [(b,self.max_boxes,d), (b,self.max_boxes,d)]

class TransformerEncoderBlock(Model):
    def __init__(self, attn_dim=256, ffn_dim=2048, drop_rate=0.1, norm_before=False):
        super(TransformerEncoderBlock, self).__init__()

        self.ln1 = LayerNormalization()
        self.msa = MultiHeadAttention(num_heads=8, model_size=attn_dim, name='self_att')
        self.drop1 = Dropout(drop_rate)

        self.ln2 = LayerNormalization()
        self.dense1 = Dense(ffn_dim)
        self.act1 = ReLU()
        self.dense2 = Dense(attn_dim)
        self.drop2 = Dropout(drop_rate)
        self.drop3 = Dropout(drop_rate)

        self.norm_before = norm_before

    def call(self, inputs, mask=None):
        if self.norm_before:
            return self.pre_norm(inputs, mask)
        else:
            return self.post_norm(inputs, mask)

    def pre_norm(self, inputs, mask=None):
        x, pe = inputs

        # id path
        inpt = x

        # residual path
        x = self.ln1(x)
        q = k = x + pe
        v = x
        x = self.msa([q,k,v], mask=mask)
        x = self.drop1(x)

        # add
        x = x + inpt

        # id path
        ffn_inpt = x

        # residual path
        x = self.ln2(x)
        x = self.drop2(self.act1(self.dense1(x)))
        x = self.drop3(self.dense2(x))

        # add
        x = x + ffn_inpt

        return x

    def post_norm(self, inputs, mask=None):
        x, pe = inputs

        # id path
        inpt = x

        # residual path
        q = k = x + pe
        v = x
        x = self.msa([q,k,v], mask=mask)
        x = self.drop1(x)

        # add
        x = x + inpt
        x = self.ln1(x)

        # id path
        ffn_inpt = x

        # residual path
        x = self.drop2(self.act1(self.dense1(x)))
        x = self.drop3(self.dense2(x))

        # add
        x = x + ffn_inpt
        x = self.ln2(x)

        return x

    def compute_output_shape(self, input_shape):
        return input_shape[0]

class TransformerDecoderBlock(Model):
    def __init__(self, attn_dim=256, ffn_dim=2048, drop_rate=0.1, norm_before=False):
        super(TransformerDecoderBlock, self).__init__()

        self.ln1 = LayerNormalization()
        self.msa1 = MultiHeadAttention(num_heads=8, model_size=attn_dim, name='self_att')   # self-att
        self.drop1 = Dropout(drop_rate)

        self.ln2 = LayerNormalization()
        self.msa2 = MultiHeadAttention(num_heads=8, model_size=attn_dim, name='mutual_att')   # mutual-att
        self.drop2 = Dropout(drop_rate)

        self.ln3 = LayerNormalization()
        self.dense1 = Dense(ffn_dim)
        self.act1 = ReLU()
        self.dense2 = Dense(attn_dim)
        self.drop3 = Dropout(drop_rate)
        self.drop4 = Dropout(drop_rate)

        self.norm_before = norm_before

    def call(self, inputs, mask=None, key_mask=None):
        # targets: decoder input
        # inputs: encoder output
        if self.norm_before:
            return self.pre_norm(inputs, key_mask=key_mask)
        else:
            return self.post_norm(inputs, key_mask=key_mask)

    def pre_norm(self, inputs, key_mask=None):
        x, target_pe, inputs, feat_pe = inputs

        # id path
        inpt = x

        # residual path
        x = self.ln1(x)
        q = k = x + target_pe
        v = x
        x = self.msa1([q,k,v])
        x = self.drop1(x)

        # add
        x = x + inpt

        # id path
        inpt = x

        # residual path
        x = self.ln2(x)
        q = x + target_pe
        k = inputs + feat_pe
        v = inputs
        x = self.msa2([q,k,v], key_mask=key_mask)
        x = self.drop2(x)

        # add
        x = x + inpt

        # id path
        ffn_inpt = x

        # residual path
        x = self.ln3(x)
        x = self.drop3(self.act1(self.dense1(x)))
        x = self.drop4(self.dense2(x))

        # add
        x = x + ffn_inpt

        return x

    def post_norm(self, inputs, key_mask=None):
        x, target_pe, inputs, feat_pe = inputs

        # id path
        inpt = x

        # residual path
        q = k = x + target_pe
        v = x
        x = self.msa1([q,k,v])
        x = self.drop1(x)

        # add
        x = x + inpt
        x = self.ln1(x)

        # id path
        inpt = x

        # residual path
        q = x + target_pe
        k = inputs + feat_pe
        v = inputs
        x = self.msa2([q,k,v], key_mask=key_mask)
        x = self.drop2(x)

        # add
        x = x + inpt
        x = self.ln2(x)

        # id path
        ffn_inpt = x

        # residual path
        x = self.drop3(self.act1(self.dense1(x)))
        x = self.drop4(self.dense2(x))

        # add
        x = x + ffn_inpt
        x = self.ln3(x)

        return x

    def compute_output_shape(self, input_shape):
        return input_shape[0]

def PositionalEmbeddingSine(emb_dim, feature_shape, temp=10000, normalize=True, eps=1e-6):
    # feature_shape: (h,w)
    # returns: [1,h,w,emd_dim] constant embedding, without weights, not trainable
    assert emb_dim%2==0, 'illegal embedding dim'
    h, w = feature_shape
    grid_x, grid_y = np.meshgrid(np.arange(w), np.arange(h))   # [h,w]
    grid_x, grid_y = grid_x+1, grid_y+1    # index start from 1
    if normalize:
        grid_x = grid_x / (w+eps) * 2 * math.pi
        grid_y = grid_y / (h+eps) * 2 * math.pi
    single_dim = np.arange(emb_dim//2)            # [half_dim,]
    single_dim = temp ** (4*(single_dim//2)/emb_dim)   # enlarge the unlinear range [1,1000]

    pe_x = np.tile(np.expand_dims(grid_x, axis=2), [1,1,emb_dim//2]) / single_dim   # [h,w,half_dim]
    pe_y = np.tile(np.expand_dims(grid_y, axis=2), [1,1,emb_dim//2]) / single_dim

    pe_x = np.stack([np.sin(pe_x[:,:,::2]), np.cos(pe_x[:,:,1::2])], axis=3).reshape((h,w,emb_dim//2))   # [h,w,half_dim]
    pe_y = np.stack([np.sin(pe_y[:,:,::2]), np.cos(pe_y[:,:,1::2])], axis=3).reshape((h,w,emb_dim//2))

    PE = np.concatenate([pe_y,pe_x], axis=2)    # [h,w,emb_dim]
    PE = K.constant(np.expand_dims(PE, axis=0))    # [1,h,w,emb_dim]

    return PE

class PositionalEmbeddingLearned(Model):
    # returns: [1,h,w,emd_dim] learnable embedding, with weights, trainable
    def __init__(self, emb_dim, feature_shape, name=None):
        super(PositionalEmbeddingLearned, self).__init__(name=name)
        assert emb_dim%2==0, 'illegal embedding dim'
        self.emb_x = Embedding(50, emb_dim//2)
        self.emb_y = Embedding(50, emb_dim//2)
        self.emb_dim = emb_dim
        self.feature_shape = feature_shape

    def call(self, inputs):
        h, w = self.feature_shape
        coord_x = tf.constant(np.arange(w).reshape((1,w)))   # constant, [1,w]
        coord_y = tf.constant(np.arange(h).reshape((1,h)))

        emb_x = self.emb_x(coord_x)   # [1,w,half_dim]
        emb_y = self.emb_y(coord_y)   # [1,h,half_dim]

        pe_x = tf.tile(emb_x, [h,1,1])
        pe_y = tf.tile(tf.transpose(emb_y, (1,0,2)), [1,w,1])

        PE = tf.concat([pe_x,pe_y], axis=2)   # [h,w,emb_dim]
        PE = tf.expand_dims(PE, axis=0)       # [1,h,w,emb_dim]

        return PE

    def compute_output_shape(self, input_shape):
        h,w = self.feature_shape
        return (None,h,w,self.emb_dim)

if __name__ == '__main__':

    pe = PositionalEmbeddingSine(256, (25,38), temp=10000, normalize=True, eps=1e-6)
    print(pe)

    pe_layer = PositionalEmbeddingLearned(128, (8,10))
    x = tf.ones((32,32))
    y = pe_layer(x)
    print(pe_layer.weights)  # The first call will create the weights

    model = detr(input_shape=(512,512,3), pe='learned', n_classes=92, depth=50, dilation=False,
                 emb_dim=256, enc_layers=6, dec_layers=6, max_boxes=100,
                 mode='train')
    model.summary()
    model.load_weights("weights/detr-r50.h5")

    # for l in model.layers:
    #     if 'decoderblock_1' in l.name:
    #         print(l.weights)


import collections
import math
import os
import random
import torch
from d2l import torch as d2l

d2l.DATA_HUB['ptb'] = (d2l.DATA_URL + 'ptb.zip', '319d85e578af0cdc590547f26231e4e31cdf1e42')

#@save
def read_ptb():
    """Load the PTB dataset into a list of text lines."""
    data_dir = d2l.download_extract('ptb')
    # Read the training set
    with open(os.path.join(data_dir, 'ptb.train.txt')) as f:
        raw_text = f.read()
    return [line.split() for line in raw_text.split('\n')]

def subsample(sentences, vocab):
    """Subsample high-frequency words."""
    # Exclude unknown tokens ('<unk>')
    sentences = [[token for token in line if vocab[token] != vocab.unk]
                 for line in sentences]
    counter = collections.Counter([
        token for line in sentences for token in line])
    num_tokens = sum(counter.values())

    # Return True if `token` is kept during subsampling
    def keep(token):
        return random.uniform(0, 1) < math.sqrt(1e-4 / counter[token] * num_tokens)

    return ([[token for token in line if keep(token)] for line in sentences],
            counter)
# def compare_counts(token):
#     return (f'# of "{token}": '
#             f'before={sum([l.count(token) for l in sentences])}, '
#             f'after={sum([l.count(token) for l in subsampled])}')

def get_centers_and_contexts(corpus, max_window_size):
    """Return center words and context words in skip-gram."""
    centers, contexts = [], []
    for line in corpus:
        # To form a "center word--context word" pair, each sentence needs to
        # have at least 2 words
        if len(line) < 2:
            continue
        centers += line
        for i in range(len(line)):  # Context window centered at `i`
            window_size = random.randint(1, max_window_size)
            indices = list(range(max(0, i - window_size),
                                 min(len(line), i + 1 + window_size)))
            indices.remove(i)
            contexts.append([line[idx] for idx in indices])
    return centers, contexts

def get_negatives(all_contexts, vocab, counter, K):
    """Return noise words in negative sampling."""
    # Sampling weights for words with indices 1, 2, ... (index 0 is the
    # excluded unknown token) in the vocabulary
    sampling_weights = [counter[vocab.to_tokens(i)]**0.75
                        for i in range(1, len(vocab))]
    all_negatives, generator = [], RandomGenerator(sampling_weights)
    for contexts in all_contexts:
        negatives = []
        while len(negatives) < len(contexts) * K:
            neg = generator.draw()
            # Noise words cannot be context words
            if neg not in contexts:
                negatives.append(neg)
        all_negatives.append(negatives)
    return all_negatives

#@save
def batchify(data):
    """Return a minibatch of examples for skip-gram with negative sampling."""
    max_len = max(len(c) + len(n) for _, c, n in data)
    centers, contexts_negatives, masks, labels = [], [], [], []
    for center, context, negative in data:
        cur_len = len(context) + len(negative)
        centers += [center]
        contexts_negatives += [context + negative + [0] * (max_len - cur_len)]
        masks += [[1] * cur_len + [0] * (max_len - cur_len)]
        labels += [[1] * len(context) + [0] * (max_len - len(context))]
    return (torch.tensor(centers).reshape((-1, 1)), torch.tensor(
        contexts_negatives), torch.tensor(masks), torch.tensor(labels))

#@save
def load_data_ptb(batch_size, max_window_size, num_noise_words):
    """Download the PTB dataset and then load it into memory."""
    num_workers = d2l.get_dataloader_workers()
    sentences = read_ptb()
    vocab = d2l.Vocab(sentences, min_freq=10)
    subsampled, counter = subsample(sentences, vocab)
    corpus = [vocab[line] for line in subsampled]
    all_centers, all_contexts = get_centers_and_contexts(
        corpus, max_window_size)
    all_negatives = get_negatives(
        all_contexts, vocab, counter, num_noise_words)

    class PTBDataset(torch.utils.data.Dataset):
        def __init__(self, centers, contexts, negatives):
            assert len(centers) == len(contexts) == len(negatives)
            self.centers = centers
            self.contexts = contexts
            self.negatives = negatives

        def __getitem__(self, index):
            return (self.centers[index], self.contexts[index],
                    self.negatives[index])

        def __len__(self):
            return len(self.centers)

    dataset = PTBDataset(all_centers, all_contexts, all_negatives)

    data_iter = torch.utils.data.DataLoader(dataset, batch_size, shuffle=True,
                                      collate_fn=batchify,
                                      num_workers=num_workers)
    return data_iter, vocab

class RandomGenerator:
    """Randomly draw among {1, ..., n} according to n sampling weights."""
    def __init__(self, sampling_weights):
        # Exclude
        self.population = list(range(1, len(sampling_weights) + 1))
        self.sampling_weights = sampling_weights
        self.candidates = []
        self.i = 0

    def draw(self):
        if self.i == len(self.candidates):
            # Cache `k` random sampling results
            self.candidates = random.choices(
                self.population, self.sampling_weights, k=10000)
            self.i = 0
        self.i += 1
        return self.candidates[self.i - 1]

#
# sentences = read_ptb()
# f'# sentences: {len(sentences)}'
# vocab = d2l.Vocab(sentences, min_freq=10)
# subsampled, counter = subsample(sentences, vocab)
# # d2l.show_list_len_pair_hist(['origin', 'subsampled'], '# tokens per sentence', 'count', sentences, subsampled)
# print(compare_counts('the'))
# print(compare_counts('join'))
# corpus = [vocab[line] for line in subsampled]
# print(corpus[:3])
# tiny_dataset = [list(range(7)), list(range(7, 10))]
# print('dataset', tiny_dataset)
# for center, context in zip(*get_centers_and_contexts(tiny_dataset, 2)):
#     print('center', center, 'has contexts', context)
#
#
# all_centers, all_contexts = get_centers_and_contexts(corpus, 5)
# print(f'# center-context pairs: {sum([len(contexts) for contexts in all_contexts])}')
#
# generator = RandomGenerator([2, 3, 4])
# print([generator.draw() for _ in range(10)])
#
# #@save
#
#
# all_negatives = get_negatives(all_contexts, vocab, counter, 5)
#
#
# x_1 = (1, [2, 2], [3, 3, 3, 3])
# x_2 = (1, [2, 2, 2], [3, 3])
# batch = batchify((x_1, x_2))
#
# names = ['centers', 'contexts_negatives', 'masks', 'labels']
# for name, data in zip(names, batch):
#     print(name, '=', data)
#
#
# names = ['centers', 'contexts_negatives', 'masks', 'labels']
# data_iter, vocab = load_data_ptb(512, 5, 5)
# for batch in data_iter:
#     for name, data in zip(names, batch):
#         print(name, 'shape:', data.shape)
#     break


import torch
from torch import nn
from .gaussian_smoothing import GaussianSmoothing

class SoftFrangiFilter2D(nn.Module):

    def __init__(self, channels, kernel_size, sigmas, beta, c, device):
        """
        Apply Soft Frangi filter on a 3d tensor.
        Arguments:
            channels (int, sequence): Number of channels of the input tensors. Output will
                have this number of channels as well.
            kernel_size (int, sequence): Size of the gaussian kernel.
            sigmas (list, sequence): List of standard deviations of the gaussian kernels.
            beta (float, sequence): Beta parameter of Frangi filter.
            beta (с, sequence): С parameter of Frangi filter.
                Default value is 2 (spatial).
        """
        super(SoftFrangiFilter2D, self).__init__()
        self.sigmas = sigmas
        self.gaus_filters = []
        self.beta = beta
        self.c = c
        for sigma in sigmas:
            self.gaus_filters.append([])
            for order in ['xx', 'yy', 'xy']:
                self.gaus_filters[-1].append(GaussianSmoothing(channels, kernel_size, sigma, 2, order, device))

    def _calc_frangi_response(self, xx, yy, xy):
        """
        Calculate Frangi filter response give the second order derivatives.
        Arguments:
            xx (torch.Tensor, sequence): (bs, channels, h, w), second order derivative on x-axis
            yy (torch.Tensor, sequence): (bs, channels, h, w), second order derivative on y-axis
            xy (torch.Tensor, sequence): (bs, channels, h, w), second order derivative on xy-axes
        """
        lambda_t1 = ((xx + yy) + torch.sqrt((xx - yy) ** 2 + 4 * xy ** 2 + 1e-6)) / 2
        lambda_t2 = ((xx + yy) - torch.sqrt((xx - yy) ** 2 + 4 * xy ** 2 + 1e-6)) / 2
        lambdas = torch.stack((lambda_t1, lambda_t2), dim=0)
        lambdas_abs, sorted_ind = torch.sort(torch.abs(lambdas), dim=0)
        lambda2_sign = (lambdas * sorted_ind).sum(dim=0)
        lambda1 = lambdas_abs[0]
        lambda2 = lambdas_abs[1]
        blobness = torch.zeros(lambda1.shape, device=lambda1.device)
        blobness[lambda2 != 0] = blobness[lambda2 != 0] + torch.exp(
            -(lambda1[lambda2 != 0] / lambda2[lambda2 != 0]) ** 2 / (2 * self.beta ** 2))
        hess_struc = (1 - torch.exp(-torch.sqrt(lambda1 ** 2 + lambda2 ** 2) ** 2 / (2 * self.c ** 2)))
        vness = blobness * hess_struc
        vness[lambda2_sign > 0] = vness[lambda2_sign > 0] * 0
        return vness, blobness, hess_struc, lambda1, lambda2

    def forward(self, img):
        """
        Apply Soft Frangi filter on a batch of images.
        Arguments:
            img (torch.Tensor, sequence): Tensor of shape (bs, channels, h, w)
        """
        frangi_resp = torch.zeros((len(self.sigmas),) + img.shape, dtype=torch.float32, device=img.device)
        for i in range(len(self.sigmas)):
            xx = self.gaus_filters[i][0](img)
            yy = self.gaus_filters[i][1](img)
            xy = self.gaus_filters[i][2](img)
            vness, _, _, _, _ = self._calc_frangi_response(xx, yy, xy)
            frangi_resp[i] = vness
        max_frangi_resp = (torch.softmax(frangi_resp, dim=0) * frangi_resp).sum(dim=0)
        return max_frangi_resp


r"""
Basic training script for PyTorch
"""

# Set up custom environment before nearly anything else is imported
# NOTE: this should be the first import (no not reorder)
from maskrcnn_benchmark.utils.env import setup_environment  # noqa F401 isort:skip

import argparse
import os
import json
import torch
from collections import defaultdict
from maskrcnn_benchmark.config import cfg
from maskrcnn_benchmark.utils.collect_env import collect_env_info
from maskrcnn_benchmark.utils.comm import synchronize, get_rank
from maskrcnn_benchmark.utils.imports import import_file
from maskrcnn_benchmark.utils.logger import setup_logger
from maskrcnn_benchmark.utils.miscellaneous import mkdir
from maskrcnn_benchmark.data.transforms import build_transforms
from maskrcnn_benchmark.data.build import build_dataset
from maskrcnn_benchmark.data.collate_batch import BatchCollator
from maskrcnn_benchmark.structures.segmentation_mask import SegmentationMask
from pycocotools import mask as maskUtils
import numpy as np
import pdb
import cv2
import math

def get_median(data):
    data = sorted(data)
    size = len(data)
    if size % 2 == 0:
        median = (data[size // 2] + data[size // 2 - 1]) / 2
        data[0] = median
    if size % 2 == 1:
        median = data[(size - 1) // 2]
        data[0] = median
    return data[0]
def chunks(arr, m):
    n = int(math.ceil(len(arr) / float(m)))
    return [arr[i:i + n] for i in range(0, len(arr), n)]
def annToMask(boxlist):
    """
    Convert annotation which can be polygons, uncompressed RLE to RLE.
    :return: binary mask (numpy 2D array)
    """
    h,w = boxlist.size
    masks = boxlist.get_field('masks')
    labels = boxlist.get_field('labels')
    # pdb.set_trace()
    index = list(np.nonzero(labels==1)[:,0])
    polygons = []
    for i, seg in enumerate(masks.polygons):
        if i in index:
            polygons.append(seg)
    RLES = []
    for segm in polygons:
        rles =  maskUtils.frPyObjects([p.numpy() for p in segm.polygons], h, w )
        rle = maskUtils.merge(rles)
        RLES.append(rle)

    return RLES

def split(cfg, num_fold):
    # when calculate the overlap ratio, set the sliding window overlap = 0
    paths_catalog = import_file(
        "maskrcnn_benchmark.config.paths_catalog", cfg.PATHS_CATALOG, True
    )
    DatasetCatalog = paths_catalog.DatasetCatalog
    dataset_list = cfg.DATASETS.TEST

    transforms = build_transforms(cfg,False)
    datasets = build_dataset(dataset_list, transforms, DatasetCatalog, False)
    collator = BatchCollator(cfg.DATALOADER.SIZE_DIVISIBILITY)
    data_loader = torch.utils.data.DataLoader(
        datasets[0],
        num_workers=1,
        collate_fn=collator,

    )

    name_dic = defaultdict(list)
    for i,  data in enumerate(data_loader,0):
        boxlists = data[1][0]

        masks = annToMask(boxlists)
        isCrowed = len(masks) * [0]
        iou = maskUtils.iou(masks, masks, isCrowed) - np.eye(len(masks))
        maxiou = np.max(iou,0)
        avgiou = np.mean(maxiou)
        name = datasets[0].get_img_info(i)['file_name']
        name_dic[name].append(avgiou)
    names = []
    overlapping = []
    for k, v in name_dic.items():
        names.append(k)
        overlapping.append(sum(v)/len(v))
    median = get_median(overlapping)
    # pdb.set_trace()
    hard = []
    easy = []
    for name, miou in zip(names, overlapping):
        if miou>median:
            hard.append(name)
        else:
            easy.append(name)

    # pdb.set_trace()
    easy_n  = chunks(easy, num_fold)
    hard_n = chunks(hard, num_fold)

    i = 1
    splitfile = {}
    for e,h in zip(easy_n,hard_n):
        splitfile[i] = {'easy': e, 'hard':h}
        i+=1

    with open('split.json', 'w') as f:
        json.dump(splitfile, f)

def main():

    parser = argparse.ArgumentParser(description="PyTorch Object Detection Training")
    parser.add_argument(
        "--config-file",
        default="",
        metavar="FILE",
        help="path to config file",
        type=str,
    )
    parser.add_argument("--local_rank", type=int, default=0)
    parser.add_argument(
        "--skip-test",
        dest="skip_test",
        help="Do not test the final model",
        action="store_true",
    )
    parser.add_argument(
        "opts",
        help="Modify config options using the command-line",
        default=None,
        nargs=argparse.REMAINDER,
    )
    cv2.setNumThreads(0)
    args = parser.parse_args()

    num_gpus = int(os.environ["WORLD_SIZE"]) if "WORLD_SIZE" in os.environ else 1
    args.distributed = num_gpus > 1

    if args.distributed:
        torch.cuda.set_device(args.local_rank)
        torch.distributed.init_process_group(
            backend="nccl", init_method="env://"
        )

    cfg.merge_from_file(args.config_file)
    cfg.merge_from_list(args.opts)
    cfg.freeze()

    output_dir = cfg.OUTPUT_DIR
    if output_dir:
        mkdir(output_dir)

    logger = setup_logger("maskrcnn_benchmark", output_dir, get_rank())
    logger.info("Using {} GPUs".format(num_gpus))
    logger.info(args)

    logger.info("Collecting env info (might take some time)")
    logger.info("\n" + collect_env_info())

    logger.info("Loaded configuration file {}".format(args.config_file))
    with open(args.config_file, "r") as cf:
        config_str = "\n" + cf.read()
        logger.info(config_str)
    logger.info("Running with config:\n{}".format(cfg))

    split(cfg, 3)

if __name__ == "__main__":
    main()


import json
import os
import random
import sys
from contextlib import closing

from boto3 import Session
from botocore.exceptions import BotoCoreError, ClientError

session = Session()
polly = session.client('polly')
outputFormat = 'mp3'
book = 'caterpillar'#'peter'
#book = 'peter'
voices = ('Amy', 'Emma', 'Joanna', 'Ivy', 'Joey', 'Justin', 'Kendra', 'Kimberly', 'Salli', 'Brian')
voices = ('Emma',)
base_path = os.path.join('catalyst', 'static', 'catalyst', book)
colors = [
    '#4A90E2', # Blue
    '#D0021B', # Red
    '#F5A623', # Orange
    '#F8E71C', # Yellow
    '#7ED321', # Green
    '#BD10E0', # Pink
    '#9013FE', # Purple
    '#50E3C2', # Teal
]
with open(os.path.join(base_path, 'english.json')) as f:
    english_texts = json.load(f)
with open(os.path.join(base_path, 'images.json')) as f:
    images = json.load(f)
for voiceId in voices:
    voice_path = os.path.join(base_path, voiceId)
    audio_path = os.path.join(voice_path, 'audio')
    for path in (voice_path, audio_path):
        if not os.path.exists(path):
            os.makedirs(path)
    pages = []
    audio_count = 1
    for page_count, text in enumerate(english_texts):
        image = images[page_count]
        page = {
            'text': text,
            'audio': '',
            'audio_info': [],
            'image': image,
            'color': '' if (text or image) else random.choice(colors),
        }
        pages.append(page)
        if text:
            response = polly.synthesize_speech(Text=text,
                                               VoiceId=voiceId,
                                               OutputFormat=outputFormat)
            with closing(response['AudioStream']) as stream:
                body = stream.read()
                with open(os.path.join(audio_path, '{}.mp3'.format(audio_count)), 'wb') as audio_file:
                    audio_file.write(body)
                page['audio'] = '{}.mp3'.format(audio_count)
                audio_count += 1
            response = polly.synthesize_speech(Text=page['text'],
                                               VoiceId=voiceId,
                                               OutputFormat='json',
                                               SpeechMarkTypes=['word'])
            with closing(response['AudioStream']) as stream:
                body = stream.read()
                # audio_info_dict = {}
                audio_info_list = []
                for audio_info_str in body.decode().split():
                    audio_info = json.loads(audio_info_str)
                    audio_info_list.append(audio_info)
                    # audio_info_dict[audio_info['time']] = audio_info
                page['audio_info'] = audio_info_list
                print(audio_info_list)

    with open(os.path.join(voice_path, 'meta.json'), 'w') as f:
        json.dump(pages, f)

    # with open(os.path.join(path, 'meta_audio.json'), 'w') as f:
    #     json.dump(audio_meta, f)
    # with open(os.path.join(path, 'meta.json'), 'w') as f:
    #     json.dump(meta, f)


"""
This module implements the Response class which is used to represent HTTP
responses in Scrapy.

See documentation in docs/topics/request-response.rst
"""
from __future__ import annotations

from ipaddress import IPv4Address, IPv6Address
from typing import (
    TYPE_CHECKING,
    Any,
    AnyStr,
    Callable,
    Dict,
    Generator,
    Iterable,
    List,
    Mapping,
    Optional,
    Tuple,
    Union,
    cast,
)
from urllib.parse import urljoin

from twisted.internet.ssl import Certificate

from scrapy.exceptions import NotSupported
from scrapy.http.headers import Headers
from scrapy.http.request import Request
from scrapy.link import Link
from scrapy.utils.trackref import object_ref

if TYPE_CHECKING:
    from scrapy.selector import SelectorList

class Response(object_ref):
    """An object that represents an HTTP response, which is usually
    downloaded (by the Downloader) and fed to the Spiders for processing.
    """

    attributes: Tuple[str, ...] = (
        "url",
        "status",
        "headers",
        "body",
        "flags",
        "request",
        "certificate",
        "ip_address",
        "protocol",
    )
    """A tuple of :class:`str` objects containing the name of all public
    attributes of the class that are also keyword parameters of the
    ``__init__`` method.

    Currently used by :meth:`Response.replace`.
    """

    def __init__(
        self,
        url: str,
        status: int = 200,
        headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
        body: bytes = b"",
        flags: Optional[List[str]] = None,
        request: Optional[Request] = None,
        certificate: Optional[Certificate] = None,
        ip_address: Union[IPv4Address, IPv6Address, None] = None,
        protocol: Optional[str] = None,
    ):
        self.headers: Headers = Headers(headers or {})
        self.status: int = int(status)
        self._set_body(body)
        self._set_url(url)
        self.request: Optional[Request] = request
        self.flags: List[str] = [] if flags is None else list(flags)
        self.certificate: Optional[Certificate] = certificate
        self.ip_address: Union[IPv4Address, IPv6Address, None] = ip_address
        self.protocol: Optional[str] = protocol

    @property
    def cb_kwargs(self) -> Dict[str, Any]:
        try:
            return self.request.cb_kwargs  # type: ignore[union-attr]
        except AttributeError:
            raise AttributeError(
                "Response.cb_kwargs not available, this response "
                "is not tied to any request"
            )

    @property
    def meta(self) -> Dict[str, Any]:
        try:
            return self.request.meta  # type: ignore[union-attr]
        except AttributeError:
            raise AttributeError(
                "Response.meta not available, this response "
                "is not tied to any request"
            )

    @property
    def url(self) -> str:
        return self._url

    def _set_url(self, url: str) -> None:
        if isinstance(url, str):
            self._url: str = url
        else:
            raise TypeError(
                f"{type(self).__name__} url must be str, " f"got {type(url).__name__}"
            )

    @property
    def body(self) -> bytes:
        return self._body

    def _set_body(self, body: Optional[bytes]) -> None:
        if body is None:
            self._body = b""
        elif not isinstance(body, bytes):
            raise TypeError(
                "Response body must be bytes. "
                "If you want to pass unicode body use TextResponse "
                "or HtmlResponse."
            )
        else:
            self._body = body

    def __repr__(self) -> str:
        return f"<{self.status} {self.url}>"

    def copy(self) -> Response:
        """Return a copy of this Response"""
        return self.replace()

    def replace(self, *args: Any, **kwargs: Any) -> Response:
        """Create a new Response with the same attributes except for those given new values"""
        for x in self.attributes:
            kwargs.setdefault(x, getattr(self, x))
        cls = kwargs.pop("cls", self.__class__)
        return cast(Response, cls(*args, **kwargs))

    def urljoin(self, url: str) -> str:
        """Join this Response's url with a possible relative url to form an
        absolute interpretation of the latter."""
        return urljoin(self.url, url)

    @property
    def text(self) -> str:
        """For subclasses of TextResponse, this will return the body
        as str
        """
        raise AttributeError("Response content isn't text")

    def css(self, *a: Any, **kw: Any) -> SelectorList:
        """Shortcut method implemented only by responses whose content
        is text (subclasses of TextResponse).
        """
        raise NotSupported("Response content isn't text")

    def jmespath(self, *a: Any, **kw: Any) -> SelectorList:
        """Shortcut method implemented only by responses whose content
        is text (subclasses of TextResponse).
        """
        raise NotSupported("Response content isn't text")

    def xpath(self, *a: Any, **kw: Any) -> SelectorList:
        """Shortcut method implemented only by responses whose content
        is text (subclasses of TextResponse).
        """
        raise NotSupported("Response content isn't text")

    def follow(
        self,
        url: Union[str, Link],
        callback: Optional[Callable] = None,
        method: str = "GET",
        headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
        body: Optional[Union[bytes, str]] = None,
        cookies: Optional[Union[dict, List[dict]]] = None,
        meta: Optional[Dict[str, Any]] = None,
        encoding: Optional[str] = "utf-8",
        priority: int = 0,
        dont_filter: bool = False,
        errback: Optional[Callable] = None,
        cb_kwargs: Optional[Dict[str, Any]] = None,
        flags: Optional[List[str]] = None,
    ) -> Request:
        """
        Return a :class:`~.Request` instance to follow a link ``url``.
        It accepts the same arguments as ``Request.__init__`` method,
        but ``url`` can be a relative URL or a ``scrapy.link.Link`` object,
        not only an absolute URL.

        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow`
        method which supports selectors in addition to absolute/relative URLs
        and Link objects.

        .. versionadded:: 2.0
           The *flags* parameter.
        """
        if encoding is None:
            raise ValueError("encoding can't be None")
        if isinstance(url, Link):
            url = url.url
        elif url is None:
            raise ValueError("url can't be None")
        url = self.urljoin(url)

        return Request(
            url=url,
            callback=callback,
            method=method,
            headers=headers,
            body=body,
            cookies=cookies,
            meta=meta,
            encoding=encoding,
            priority=priority,
            dont_filter=dont_filter,
            errback=errback,
            cb_kwargs=cb_kwargs,
            flags=flags,
        )

    def follow_all(
        self,
        urls: Iterable[Union[str, Link]],
        callback: Optional[Callable] = None,
        method: str = "GET",
        headers: Union[Mapping[AnyStr, Any], Iterable[Tuple[AnyStr, Any]], None] = None,
        body: Optional[Union[bytes, str]] = None,
        cookies: Optional[Union[dict, List[dict]]] = None,
        meta: Optional[Dict[str, Any]] = None,
        encoding: Optional[str] = "utf-8",
        priority: int = 0,
        dont_filter: bool = False,
        errback: Optional[Callable] = None,
        cb_kwargs: Optional[Dict[str, Any]] = None,
        flags: Optional[List[str]] = None,
    ) -> Generator[Request, None, None]:
        """
        .. versionadded:: 2.0

        Return an iterable of :class:`~.Request` instances to follow all links
        in ``urls``. It accepts the same arguments as ``Request.__init__`` method,
        but elements of ``urls`` can be relative URLs or :class:`~scrapy.link.Link` objects,
        not only absolute URLs.

        :class:`~.TextResponse` provides a :meth:`~.TextResponse.follow_all`
        method which supports selectors in addition to absolute/relative URLs
        and Link objects.
        """
        if not hasattr(urls, "__iter__"):
            raise TypeError("'urls' argument must be an iterable")
        return (
            self.follow(
                url=url,
                callback=callback,
                method=method,
                headers=headers,
                body=body,
                cookies=cookies,
                meta=meta,
                encoding=encoding,
                priority=priority,
                dont_filter=dont_filter,
                errback=errback,
                cb_kwargs=cb_kwargs,
                flags=flags,
            )
            for url in urls
        )


import datetime

from django.utils import timezone
from fora.tests.base import BaseTestCase
from forums.models import Forum
from mock import patch
from posts.models import Post
from threads.models import Thread

class PostModelTestCase(BaseTestCase):

    def setUp(self):
        self.forum = Forum.objects.create(name='Forum', description='Testing')
        self.thread = Thread.objects.create(subject='Thread', forum=self.forum)
        self.post = Post.objects.create(
            message='Post', thread=self.thread, parent=None)
        super(PostModelTestCase, self).setUp()

    def test_new_post(self):
        self.assertEqual(Post.objects.count(), 1)
        self.assertEqual(Post.objects.get(pk=1), self.post)

    def test_post_to_string(self):
        self.assertEqual(str(self.post), "o4")

    def test_post_to_unicode(self):
        self.assertEqual(unicode(self.post), u'o4')

    @patch.object(timezone, 'now', return_value=datetime.datetime(2015, 1, 1))
    def test_thread_created_date_and_updated_date(self, mock_now):
        p = Post.objects.create(
            message='Post', thread=self.thread, parent=None)
        self.assertEqual(p.created, datetime.datetime(2015, 1, 1))
        self.assertEqual(p.updated, datetime.datetime(2015, 1, 1))

    def test_post_depth(self):
        self.assertEqual(self.post.depth, 0)

        p = Post.objects.create(
            message='A Child', thread=self.thread, parent=self.post)
        self.assertEqual(p.depth, 1)


import numpy as np
import matplotlib.pyplot as plt
from sklearn import datasets
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from datetime import datetime

# Import classes
import sys
sys.path.insert(0,"..")
from linear_regression import Regression
from stochastic_gradient_descent import StochasticDescent
from logistic_regression import MultinomialRegression
from load_mnist import LoadMNIST

# Load MNIST dataset
dat = LoadMNIST()

# Logistic regression using sklearn  -----------------------
logreg = LogisticRegression()
logreg.fit(dat.x_train, dat.y_train)
pred = logreg.predict(dat.x_test)

# Checking accuracy
score = logreg.score(dat.x_test, dat.y_test)
print(score)

# Using our own logistic regression ------------------------

# Define parameters for SGD
batch_size = 2**6
epochs = 100
eta = 0.01
lam = 0.1

# Use our Logistic method to make prediction
mulreg = MultinomialRegression(dat.x_train, dat.y_train, batch_size=batch_size, epochs=epochs, t0=eta, lam=lam)
mulreg.fit()
pred2 = mulreg.predict(dat.x_test)

# Checking accuracy
score2 = mulreg.accuracy_score(pred2, dat.y_test)

print(dat.y_test[:30])
print(pred2[:30])
print(score2)

# Random sampling
def random_search(iterations, print_vals=False):
    """
    Performs random search for a given amount of iterations
    and writes out the best result to logistic_mark.txt.
    """
    # Set up parameter lists
    epochs = 500
    batch_sizes = [4, 16, 32, 64, 128]
    learning_rates = np.logspace(-2, 1, 10)
    gamma_array = np.linspace(0, 0.99, 10)
    penalties = np.logspace(-5, -1, 10)

    # Storage of parameters used and accuracy score
    eta_arr         = np.zeros(iterations)
    batch_size_arr  = np.zeros(iterations)
    gamma_arr       = np.zeros(iterations)
    lam_arr         = np.zeros(iterations)
    scores          = np.zeros(iterations)

    current_iteration = 1
    for i in range(iterations):
        # Fetch random values
        eta_arr[i]         = np.random.choice(learning_rates)
        batch_size_arr[i]  = np.random.choice(batch_sizes)
        gamma_arr[i]       = np.random.choice(gamma_array)
        lam_arr[i]         = np.random.choice(penalties)

        # Print for debugging
        if print_vals:
            print("eta: ", eta_arr[i])
            print("B: ", batch_size_arr[i])
            print("gamma: ", gamma_arr[i])
            print("lam: ", lam_arr[i])

        # Do logistic regression
        mulreg = MultinomialRegression(dat.x_train, dat.y_train,
        batch_size=batch_size_arr[i], epochs=epochs, t0=eta_arr[i], lam=lam_arr[i], gamma=gamma_arr[i])
        mulreg.fit()
        pred = mulreg.predict(dat.x_test)

        # Calculate accuracy
        scores[i] =  mulreg.accuracy_score(pred, dat.y_test)

        # Print progress
        print("Iteration: %i/%i"%(current_iteration, iterations))
        current_iteration += 1

    # Set nan values to 0
    scores = np.nan_to_num(scores)
    idx = np.argmax(scores)

    # Store best values in benchmark file
    filename = "../../benchmarks/logistic_mark.txt"
    outfile = open(filename, 'a')
    outfile.write("---- Random search: %i iterations ----" %iterations)
    outfile.write("\nAccuracy score: %f"    %scores[idx])
    outfile.write("\nLearning rate: %.3e"   %eta_arr[idx])
    outfile.write("\nBatch size: %i"        %batch_size_arr[idx])
    outfile.write("\nGamma: %.3f"           %gamma_arr[idx])
    outfile.write("\nPenalty: %.3e"         %lam_arr[idx])
    outfile.write("\n\n")

    print("Accuracy score: ", scores[idx])
    print("Learning rate: ", eta_arr[idx])
    print("Batch size: ", batch_size_arr[idx])
    print("Gamma: ", gamma_arr[idx])
    print("Penalty: ", lam_arr[idx])

random_search(100)


import cv2
import numpy as np
import pyautogui
import time

resolution = (1920,1080)
x = time.time()
fourcc = cv2.VideoWriter_fourcc(*'XVID')
out = cv2.VideoWriter('vid%s.avi'%(x),fourcc,20.0,(resolution))
fps = 60
prev = 0
while True:
    time_elapsed = time.time() - prev
    img = pyautogui.screenshot()
    if time_elapsed > 1.0/fps:
        prev = time.time()
        frame = np.array(img)
        frame = cv2.cvtColor(frame,cv2.COLOR_BGR2RGB)
        out.write(frame)
    cv2.waitKey(40)
cv2.destroyAllWindows()
cv2.release()


from Consts import Consts
from requests import post, get
from threading import Lock
from pickle import dump, load
from os.path import isfile
from datetime import datetime
import Networking

mutex = Lock()

class Ingest:
    def __init__(self):
        # Loading data from file if available
        if isfile(Consts.INGEST_ID_BACKUP):
            with open(Consts.INGEST_ID_BACKUP, 'rb') as backup:
                count = load(backup)
            self.__countid = count
        else:
            self.__countid = 0

    # Sending a new scan to the PROCESS module and returning the scan id
    # While also updating the list of scans
    def add_scan(self, content):
        # Locking the count id resource
        mutex.acquire()
        self.__countid += 1

        # Backing up count id
        with open(Consts.INGEST_ID_BACKUP, 'w+b') as backup:
            dump(self.__countid, backup)
        mutex.release()

        # Send a new scan task to the PROCESS module
        # Checking availability of the service
        if Networking.check_connection('127.0.0.1', Consts.PROCESS_PORT):
            # Adding the scan to the scans list
            post("http://127.0.0.1:" + str(Consts.PROCESS_PORT) +
                 "/addtoqeue/" + str(self.__countid) + "/" + str(content) + "/" + str(datetime.now()))
            return self.__countid

        # Go a step back as a call could not be made
        mutex.acquire()
        self.__countid -= 1
        mutex.release()

        # Return failure value
        return -1


from currency_converter import CurrencyConverter
from tkinter import *
import tkinter as tk

window = tk.Tk()

window.geometry('500x360')
a = CurrencyConverter()
def clicked():
    amount = int(e1.get())
    cur1 = e2.get()
    cur2 = e3.get()
    data = a.convert(amount,cur1,cur2)
    l5 = tk.Label(window, text = data,font = "Times 25 bold").place(x = 230, y= 290)

l1 = tk.Label(window, text = "Currency Converter", font = "Times 20 bold").place(x =100,y=30)
l2 = tk.Label(window, text="Enter amount here:",font = "Times 18 bold").place(x = 50, y = 80)
e1 = tk.Entry(window)

l3 = tk.Label(window, text="Enter currency here:",font = "Times 18 bold").place(x = 50, y = 130)
e2 = tk.Entry(window)

l4 = tk.Label(window, text="Enter require currency:",font = "Times 18 bold").place(x = 50, y = 180)
e3 = tk.Entry(window)

b1 = tk.Button(window, text = "click",command = clicked).place(x = 230,y = 240)
e1.place(x = 300, y = 90)
e2.place(x = 300, y = 140)
e3.place(x = 300, y = 190)
window.mainloop()

a = CurrencyConverter()
# print(a.convert(12,"USD","INR"))