from django.shortcuts import render, redirect
from .models import ImageList, Tester, Rating
import os
import csv
import logging

logger = logging.getLogger(__name__)

# 登录页面后先查看实验须知
def experiment_intro(request):
    return render(request, 'experiment_intro.html')

# 确认进入实验后提交被试编号和被试信息
#
def enter_info(request):
    if request.method == 'POST':
        age = request.POST.get('age')
        gender = request.POST.get('gender')
        subject_number = request.POST.get('subject_number')
        school_number = request.POST.get('school_number')
        account = request.POST.get("account")

        tester = Tester.objects.create(age=age, gender=gender, subject_number=subject_number, account=account, school_number=school_number)

        return redirect('experiment', tester_id=tester.id)  # 跳转到实验界面
    return render(request, 'enter_info.html')

def test_temp(request):
    return render(request, 'htmltemp.html')

def thank_you(request):
    return render(request, 'thank_you.html')

def experiment(request, tester_id):
    # 初始化tester
    tester = Tester.objects.get(pk=tester_id)

    # 根据被试输入的编号找到对应的图片列表
    subject_number = tester.subject_number
    image_names = ImageList.objects.get(name=f'image_list_{subject_number}.txt')
    print(image_names)

    # 当点击评分按钮后的反应
    if request.method == 'POST':
        # 处理用户提交的评分数据
        rating_value = request.POST.get('rating')
        rated_image_name = request.session.get('rated_image_name')
        print(f"rated_image_name: {rated_image_name}")
        logger.debug(f"rated_image_name : {rated_image_name}")
        # 保存评分到数据库
        rating = Rating.objects.create(user=tester, image=rated_image_name, rating=rating_value)

    # 从缓存中读取最新的图片列表（包含初始化
    image_list = request.session.get('image_list', image_names.get_image_names())
    # 从缓存中读取当前的图片序号
    no = request.session.get('no',0)
    # print(image_list)
    no = no + 1
    # 获取当前图片的名字和序号
    if no==150:
        current_image_name = "lowmeaning-pacth1.png"
    else:
        current_image_name = image_list[0] if image_list else None  # 默认为第一张图片
        # 更新图片列表，删除已经保存并即将递送显示的第一张图片
        image_list = image_list[1:]
    # 更新图片列表到缓存
    request.session['image_list'] = image_list
    request.session['rated_image_name'] = current_image_name
    request.session['no'] = no
    print(current_image_name)

    return render(request, "experiment.html", {'current_image_name': current_image_name, 'no': no})

def test_rate_single_image(request):
    csv_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'outcome.csv')
    print("here")

    if request.method == 'POST':
        # 处理用户提交的评分数据
        rating = request.POST.get('rating')
        rated_image_name = request.session.get('rated_image_name')

        # 保存评分到数据库
        if not os.path.exists(csv_file_path):
            # 如果不存在，创建一个新的csv文件并写入头部信息和当前数据
            with open(csv_file_path, 'w', newline='') as csvfile:
                fieldnames = ['rated_image_name', 'rating']
                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
                writer.writeheader()
                writer.writerow({'rated_image_name': rated_image_name, 'rating': rating})
        else:
            # 如果已存在，直接追加数据
            with open(csv_file_path, 'a', newline='') as csvfile:
                fieldnames = ['rated_image_name', 'rating']
                writer = csv.DictWriter(csvfile, fieldnames=fieldnames)

                # 将current_image_name和rating写入csv
                writer.writerow({'rated_image_name': rated_image_name, 'rating': rating})

    # 从缓存中读取最新的图片列表
    image_list = request.session.get('image_list', ['fine/4_1.png', 'fine/4_2.png', 'fine/4_3.png'])
    print(image_list)
    # 获取当前图片的名字
    current_image_name = image_list[0] if image_list else None  # 默认为第一张图片
    image_list = image_list[1:]  # 删除已经显示的第一张图片
    # 更新图片列表到缓存
    request.session['image_list'] = image_list
    request.session['rated_image_name'] = current_image_name
    print(current_image_name)

    return render(request, 'test_rate_single_image.html', {'current_image_name': current_image_name})


from warnings import catch_warnings
from numpy import character
import sqlalchemy as sql
from sqlalchemy.sql import text
import pandas as pd
import numpy as np

db_connection_str = 'mysql+pymysql://username:password@localhost:3306/AcademicWorld' #connect your loacal sql by your password and name
db_connection = sql.create_engine(db_connection_str)

def getTop5Professor():
    sql = "SELECT f.name, f.position, p.title, f.research_interest, f.photo_url, u.name FROM publication p JOIN faculty_publication fp on fp.publication_id = p.id JOIN faculty f on fp.faculty_id = f.id JOIN university u on f.university_id = u.id ORDER BY num_citations DESC LIMIT 5;"
    data = db_connection.execute(text(sql)).fetchall();
    data2 = [list(i) for i in data]
    return data2

def getTop5Areas():
    sql = "SELECT name, count FROM keyword JOIN (SELECT keyword_id, COUNT(keyword_id) as count FROM publication_keyword GROUP BY keyword_id ORDER BY COUNT(keyword_id) DESC LIMIT 5) temp ON id = temp.keyword_id;"
    data = db_connection.execute(text(sql)).fetchall();
    return data

def getTop5Article(keyword_name):
    sql = "SELECT title, num_citations FROM keyword f JOIN publication_keyword pk on f.id = pk.keyword_id JOIN publication p ON pk.publication_id = p.id WHERE f.name = :x ORDER BY num_citations DESC LIMIT 5;"
    data = db_connection.execute(text(sql), {'x': keyword_name}).fetchall()
    return data

def getPopularArticle():
    top5Areas = getTop5Areas()
    info = []
    character = [i[0] for i in top5Areas]
    parent = ["", "", "", "", ""]
    value = [i[1] for i in top5Areas]
    for i in top5Areas:
        area = i[0]
        article = getTop5Article(area)
        info.append(article)
        for j in article:
            character.append(j[0])
            parent.append(i[0])
            value.append(j[1])
    return character, parent, value

def insertCitaion(titleName):
    sql1 = "SELECT p.id FROM publication p WHERE title = :x;"
    data = db_connection.execute(text(sql1), {'x': titleName}).fetchall()
    if(len(data) == 0):
        return []
    try:
        sql2 = "INSERT INTO CitationList VALUES (:y);"
        db_connection.execute(text(sql2), {'y': data[0][0]})
    except:
        return [0]
    else:
        return data

def createTable():
    sql0 = "DROP TABLE IF EXISTS CitationList;"
    db_connection.execute(text(sql0))
    sql = "CREATE TABLE CitationList ( publication_id int primary key );"
    db_connection.execute(text(sql))

def getCitationList():
    sql = "SELECT title, name, year, num_citations FROM publication p JOIN CitationList c on p.id = c.publication_id JOIN faculty_publication fp ON p.id = fp.publication_id JOIN faculty f ON f.id = fp.faculty_id;"
    data = db_connection.execute(text(sql)).fetchall()
    data2 = [list(i) for i in data]
    data3 = list(map(list, zip(*data2)))
    return data3

def getTrend(topic_name):
    sql = "SELECT year, count(id) FROM (SELECT p.id, year, keyword_id FROM publication p JOIN publication_keyword pk ON p.id = pk.publication_id JOIN keyword k on pk.keyword_id = k.id WHERE k.name = :x) temp GROUP BY year ORDER BY year;"
    data = db_connection.execute(text(sql), {'x': topic_name}).fetchall()
    year = [i[0] for i in data]
    num = [i[1] for i in data]
    return year, num

def removeCitationList():
    if (getCitationList() == []):
        return;
    titles = getCitationList()[0]
    for j in titles:
        str = "UPDATE publication SET num_citations = num_Citations + 1 WHERE title = :x";
        db_connection.execute(text(str), {"x":j})
    sql = 'DELETE FROM CitationList;'
    db_connection.execute(text(sql))
    return;


from django.contrib import admin
from django.urls import path
from . import views
from .views import *

app_name = 'key'

urlpatterns = [
    path('', key_details, name='key_details'),
    path('keyword/', Add_keyword, name='keyword'),
    path('keyword_list/', Keyword_list, name='keyword_list'),
    path('keyword_list/keyword_que_list/<int:keyword_id>', keyword_que_list, name='keyword_que_list'),
    path('main/',main,name='main'),
    path('main/detail/<int:presentation_id>/',detailview, name='detailview'),
    path('main/deck/<int:presentation_id>/',deck, name='deck'),
    path('main/de/<int:keyword_id>/',de, name='de'),
    path('pdf/<int:presentation_id>/', views.GeneratePdf.as_view(), name='pdf'),
    path('keypdf/<int:keyword_id>/', views.KeyPdf.as_view(), name='keypdf'),
    path('ques/',views.manage, name='manage'),
    path('delete/<int:id>', views.que_delete, name='que_delete'),
    path('update/<int:id>', views.que_update, name='que_update'),
    path('delete/<int:detail_id>/', delete_slide, name='delete'),
    path('deletekeyword/<int:detail_id>/', delete_keyword_slide, name='deletekeyword'),
]


import pickle
import os
import numpy as np
import re
from multiprocessing import Pool
import instruction2vec
import eval_utils as utils
from collections import Counter

embed_info = {}
type_info = {
    'char': 0,
    'int': 1,
    'float': 2,
    'pointer': 3,
    'enum': 4,
    'struct': 5,
    'union': 6
}

def approximate_type(type_str):
    int_list = ['_Bool', 'unsigned int', 'int', 'long long int', 'long long unsigned int', 'unsigned short',
                'short unsigned int', 'short', 'long unsigned int', 'short int', 'long int']
    char_list = ['char', 'unsigned char', 'signed char']
    if type_str[-1] == '*' or type_str == 'func_ptr' or type_str.split()[0][-1] == '*':
        return 'pointer'
    elif type_str in int_list:
        return 'int'
    elif type_str[:5] == 'enum ':
        return 'enum'
    elif type_str in char_list:
        return 'char'
    elif type_str[:7] == 'struct ':
        return 'struct'
    elif type_str[:6] == 'union ':
        return 'union'
    elif type_str == 'double' or type_str == 'long double':
        return 'float'
    else:
        return type_str

def one_hot_encoding(label_id, class_num):
    temp = np.zeros(class_num)
    temp[label_id] = 1
    return temp

def get_single_num_args(folder_path, file_name, func_list, embed_dim, max_length, class_num, embd_type, vocab=None):
    def parse_instruction(ins):
        import re
        ins = re.sub('\s+', ', ', ins, 1)
        parts = ins.split(', ')
        return ','.join(parts)

    def parse_instruction_trans(ins):
        import re
        ins = re.sub('\s+', ', ', ins, 1)
        parts = ins.split(', ')
        operand = []
        token_lst = []
        if len(parts) > 1:
            operand = parts[1:]
        token_lst.append(parts[0])
        for i in range(len(operand)):
            symbols = re.split('([0-9A-Za-z]+)', operand[i])
            symbols = [s.strip() for s in symbols if s != '']
            token_lst.extend(symbols)
        token_lst = [t for t in token_lst if t]
        return ' '.join(token_lst)

    usable_encoder = utils.UsableTransformer(model_path="/path/to/palmtree/model", vocab_path="/path/to/palmtree/vocabulary")
    file_path = os.path.join(folder_path, file_name)
    extract_info = {}
    with open(file_path) as f:
        file_info = pickle.load(f)
    for func_name in func_list:
        func_tag = '%s#%s' % (file_name, func_name)
        extract_info[func_tag] = {}
        inst_bytes = file_info['functions'][func_name]['inst_bytes']
        inst_strings = file_info['functions'][func_name]['inst_strings']
        temp_data = []

        inst_strings = [parse_instruction_trans(str(ins)) for ins in inst_strings]
        if len(inst_strings) >= max_length:
            inst_strings = inst_strings[:max_length]
        temp_data = usable_encoder.encode(inst_strings)
        if temp_data.shape[0] < max_length:
            extract_info[func_tag]['length'] = temp_data.shape[0]
            temp_zero = np.zeros((max_length - temp_data.shape[0], embed_dim))
            temp_data = np.concatenate((temp_data, temp_zero), axis=0)
        else:
            extract_info[func_tag]['length'] = temp_data.shape[0]
        extract_info[func_tag]['data'] = temp_data
        extract_info[func_tag]['label'] = one_hot_encoding(file_info['functions'][func_name]['num_args'], class_num)
    return extract_info

def get_single_args_type(folder_path, file_name, func_list, embed_dim, max_length, class_num, arg_no):
    file_path = os.path.join(folder_path, file_name)
    extract_info = {}
    with open(file_path) as f:
        file_info = pickle.load(f)
    for func_name in func_list:
        func_tag = '%s#%s' % (file_name, func_name)
        extract_info[func_tag] = {}
        inst_bytes = file_info['functions'][func_name]['inst_bytes']
        temp_data = []
        for inst in inst_bytes:
            if str(inst) in embed_info:
                temp_data.append(embed_info[str(inst)]['vector'])
            else:
                temp_data.append([0.0] * embed_dim)
            if len(temp_data) >= max_length:
                break
        temp_data = np.asarray(temp_data)
        if temp_data.shape[0] < max_length:
            extract_info[func_tag]['length'] = temp_data.shape[0]
            temp_zero = np.zeros((max_length - temp_data.shape[0], embed_dim))
            temp_data = np.concatenate((temp_data, temp_zero), axis=0)
        else:
            extract_info[func_tag]['length'] = temp_data.shape[0]
        extract_info[func_tag]['data'] = temp_data
        temp_type = approximate_type(file_info['functions'][func_name]['args_type'][arg_no])
        extract_info[func_tag]['label'] = one_hot_encoding(type_info[temp_type], class_num)
    return extract_info

class Dataset(object):
    def __init__(self, data_folder, func_path, embed_path, thread_num, embed_dim, max_length, class_num, tag, embd_type):
        global embed_info
        self.data_folder = data_folder
        self.tag = tag #num_args or type#0
        if self.tag == 'num_args':
            pass
        else:
            self.arg_no = int(self.tag.split('#')[-1])
        self.thread_num = thread_num
        self.embed_dim = embed_dim
        self.max_length = max_length
        self.class_num = class_num
        self.embd_type = embd_type
        print(func_path)
        with open(func_path) as f:
            func_info = pickle.load(f)
        self.train_func_list = np.asarray(func_info['train'])
        self.train_num = len(self.train_func_list)
        print('Loaded train function information ... %s' % func_path)
        print('Train Function Number: %d' % self.train_num)
        self.func_list = np.asarray(func_info['test'])
        self.func_num = len(self.func_list)
        print('Loaded train function information ... %s' % func_path)
        print('Train Function Number: %d' % self.func_num)

        with open(embed_path) as f:
            embed_info = pickle.load(f)
        print('Loaded embed information ... %s' % embed_path)
        self.test_tag = True
        self._index_in_epoch = 0
        self._index_in_test = 0
        self._complete_epochs = 0

        # get vocabulary
        if self.embd_type == '1hot':
            counter = Counter()
            binaries = os.listdir(self.data_folder)
            for binary in binaries:
                print(binary)
                file_path = os.path.join(self.data_folder, binary)
                with open(file_path) as f:
                    file_info = pickle.load(f)
                print(len(file_info['functions']))
                for func in file_info['functions'].values():
                    counter.update(func['inst_strings'])
            self.vocabulary = sorted(counter, key=counter.get, reverse=True)[:5000]
            self.vocabulary.append('UNK')
        else:
            self.vocabulary = None

    def get_batch_data(self, batch_func_list):
        func_list = sorted(batch_func_list)
        binary_name = ''
        input_func_list = []
        batch_info = {}
        pool = Pool(self.thread_num)
        if self.tag == 'num_args':
            for whole_func_name in func_list:
                if binary_name == '':
                    binary_name = whole_func_name.split('#')[0]
                    input_func_list.append(whole_func_name.split('#')[1])
                else:
                    if binary_name == whole_func_name.split('#')[0]:
                        input_func_list.append(whole_func_name.split('#')[1])
                    else:
                        pool.apply_async(
                            get_single_num_args,
                            args= (self.data_folder, binary_name, input_func_list, self.embed_dim, self.max_length, self.class_num, self.embd_type, self.vocabulary),
                            callback= batch_info.update
                        )
                        binary_name = whole_func_name.split('#')[0]
                        input_func_list = [whole_func_name.split('#')[1]]
            if len(input_func_list) == 0:
                pass
            else:
                pool.apply_async(
                    get_single_num_args,
                    args=(self.data_folder, binary_name, input_func_list, self.embed_dim, self.max_length, self.class_num, self.embd_type, self.vocabulary),
                    callback=batch_info.update
                )
        else: #self.tag == 'type#0'
            for whole_func_name in func_list:
                if binary_name == '':
                    binary_name = whole_func_name.split('#')[0]
                    input_func_list.append(whole_func_name.split('#')[1])
                else:
                    if binary_name == whole_func_name.split('#')[0]:
                        input_func_list.append(whole_func_name.split('#')[1])
                    else:
                        pool.apply_async(
                            get_single_args_type,
                            args=(self.data_folder, binary_name, input_func_list, self.embed_dim, self.max_length,
                                  self.class_num, self.arg_no),
                            callback= batch_info.update
                        )
                        binary_name = whole_func_name.split('#')[0]
                        input_func_list = [whole_func_name.split('#')[1]]
            if len(input_func_list) == 0:
                pass
            else:
                pool.apply_async(
                    get_single_args_type,
                    args=(self.data_folder, binary_name, input_func_list, self.embed_dim, self.max_length, self.class_num, self.arg_no),
                    callback=batch_info.update
                )
        pool.close()
        pool.join()
        new_batch_data = {
            'data': [],
            'label': [],
            'length': [],
            'func_name':[]
        }
        for full_func_name in batch_info:
            new_batch_data['data'].append(batch_info[full_func_name]['data'])
            new_batch_data['label'].append(batch_info[full_func_name]['label'])
            new_batch_data['length'].append(batch_info[full_func_name]['length'])
            new_batch_data['func_name'].append(full_func_name)
        batch_info = {
            'data': np.asarray(new_batch_data['data'], dtype=np.float32),
            'label': np.asarray(new_batch_data['label'], dtype=np.float32),
            'length': np.asarray(new_batch_data['length'], dtype=np.float32),
            'func_name': np.asarray(new_batch_data['func_name'])
        }
        return batch_info

    def get_batch(self, batch_size):
        start = self._index_in_epoch
        # shuffle for the first round

        if self._complete_epochs == 0 and self._index_in_epoch == 0:
            perm0 = np.arange(self.train_num)
            np.random.shuffle(perm0)
            self.train_func_list = self.train_func_list[perm0]
        # go to the next epoch
        if start + batch_size > self.train_num:
            self._complete_epochs += 1
            rest_example_num = self.train_num - start
            rest_func_list = self.train_func_list[start:self.train_num]
            # shuffle for the new epoch
            perm = np.arange(self.train_num)
            np.random.shuffle(perm)
            self.train_func_list = self.train_func_list[perm]
            # start a new epoch
            start = 0
            self._index_in_epoch = batch_size - rest_example_num
            end = self._index_in_epoch
            new_func_list = self.train_func_list[start:end]
            func_list_batch = np.concatenate((rest_func_list, new_func_list), axis=0)
            train_batch = self.get_batch_data(func_list_batch)
            return train_batch
        else:  # process current epoch
            self._index_in_epoch += batch_size
            end = self._index_in_epoch
            func_list_batch = self.train_func_list[start:end]
            train_batch = self.get_batch_data(func_list_batch)
            return train_batch

    def get_test_batch(self, batch_size):
        start = self._index_in_test
        if start + batch_size >= self.func_num:
            self.test_tag = False
            func_list_batch = self.func_list[start:]
            test_batch = self.get_batch_data(func_list_batch)
            return test_batch
        else:
            self._index_in_test += batch_size
            end = self._index_in_test
            func_list_batch = self.func_list[start: end]
            test_batch = self.get_batch_data(func_list_batch)
            return test_batch


# !/usr/bin/python
# -*- coding:utf-8 -*-

import requests
import re
import csv

def getHtml(url):
    """
    get the webpage source code.
    :param url:
    :return:
    """
    headers = {
        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:58.0) Gecko/20100101 Firefox/58.0'
    }
    try:
        r = requests.get(url=url, headers=headers, timeout=10)
        r.raise_for_status()
        r.encoding = 'utf-8'
        return r.text
    except:
        return ''

def parsePage(html, infoList):
    """
    parse html,return commodity infomation list.
    :param html:
    :return: infoList
    """
    try:
        price = re.findall(r'"view_price":"(.*?)"', html)
        num = len(price)
        title = re.findall(r'"raw_title":"(.*?)"', html)
        sales = re.findall(r'"view_sales":"(\d+)人付款"', html)
        taobaoID = re.findall(r'"nick":"(.*?)"', html)[:num]
        location = re.findall(r'"item_loc":"(.*?)"', html)
        for i in range(num):
            infoList.append([title[i], price[i], sales[i], taobaoID[i], location[i]])
        return infoList
    except:
        return ''

def writeCommodity(infoList):
    """
    write the commodity list to the taobaoCommodity.csv file,
    :param infoList:
    :return:
    """
    with open('csv/书包.csv', 'w', encoding='utf-8_sig', newline='') as f:
        writer = csv.writer(f)
        writer.writerow(['名称', '价格', '付款人数', '淘宝ID', '店铺地址'])
        writer.writerows(infoList)

def main():
    commodity = '书包'
    depth = 101
    infoList = []
    for i in range(depth):
        try:
            i *= 44
            url = 'https://s.taobao.com/search?q={}&s={}'.format(commodity, i)
            html = getHtml(url)
            infoList = parsePage(html, infoList)
        except:
            continue
    infoList = sorted(infoList, key=lambda x:int(x[2]), reverse=True)   # 商品按销量降序排列
    writeCommodity(infoList)

if __name__ == '__main__':
    main()


"""Ontology Evaluator class for evaluating the cross-ontology mappings computed by OA models.
"""
from typing import List, Optional, Union

import pandas as pd

import bertmap
from bertmap.utils import uniqify
from pandas.core.frame import DataFrame

class OntoEvaluator:

    na_vals = bertmap.na_vals
    namespaces = bertmap.namespaces

    def __init__(
        self,
        pre_tsv: Union[str, DataFrame],
        ref_tsv: Union[str, DataFrame],
        ref_ignored_tsv: Optional[Union[str, DataFrame]] = None,
        threshold: float = 0.0,
    ):

        # filter the prediction mappings according to similarity scores
        self.pre = self.read_mappings(pre_tsv, threshold=threshold)
        self.id_maps = []  # mapping of same iris should be ignored
        self.identity_mapping_check()
        self.num_ignored = len(self.id_maps)

        # reference mappings and illegal mappings to be ignored
        self.ref = self.read_mappings(ref_tsv)
        if ref_ignored_tsv is None:
            self.ref_ignored = None
        else:
            self.ref_ignored = self.read_mappings(ref_ignored_tsv)

        # convert list to set for fast computation
        self.pre_set = set(self.pre)
        self.ref_set = set(self.ref)
        if self.ref_ignored:
            self.ref_ignored_set = set(self.ref_ignored)
            self.pre_unignored_set = self.pre_set - self.ref_ignored_set
            self.ref_unignored_set = self.ref_set - self.ref_ignored_set

        # compute Precision, Recall and Macro-F1
        try:
            self.P = self.precision()
            self.R = self.recall()
            self.F1 = self.f1()
        except:
            self.P = -1.0
            self.R = -1.0
            self.F1 = -1.0

    def precision(self) -> float:
        """
        % of predictions are correct:
            P = TP / (TP + FP)
        """
        if self.ref_ignored:
            num_pre_ignored = len(self.pre_set) - len(self.pre_unignored_set)
            self.num_ignored += num_pre_ignored
        # True Positive = the number of unignored prediction mappings that are True
        tp = len(self.pre_unignored_set.intersection(self.ref_set))
        # False Positive = the number of unignored prediction mappings that are False
        fp = len(self.pre_set) - tp - num_pre_ignored
        return tp / (tp + fp)

    def recall(self) -> float:
        """
        % of correct retrieved
            R = TP / (TP + FN)
        """
        if self.ref_ignored:
            num_ref_ignored = len(self.ref_set) - len(self.ref_unignored_set)
            self.num_ignored += num_ref_ignored
        # True Positive = the number of unignored reference mappings that are Positive
        tp = len(self.ref_unignored_set.intersection(self.pre_set))
        # False Negative = the number of unignored reference mappings that are Negative
        fn = len(self.ref_set) - tp - num_ref_ignored
        return tp / (tp + fn)

    def f1(self) -> float:
        return 2 * self.P * self.R / (self.P + self.R)

    def identity_mapping_check(self) -> None:
        for p_map in self.pre:
            if p_map.split("\t")[0] == p_map.split("\t")[1]:
                self.id_maps.append(p_map)
        self.pre = list(set(self.pre) - set(self.id_maps))
        if len(self.id_maps) > 0:
            print(f"detect and delete {len(self.id_maps)} mappings of identical iris ...")

    @classmethod
    def read_mappings(
        cls, mapping_file: Union[str, DataFrame], threshold: float = 0.0
    ) -> List[str]:
        """read unique mappings from tsv file or pandas.DataFrame"""
        if type(mapping_file) is DataFrame:
            _df = mapping_file
        else:
            _df = pd.read_csv(mapping_file, sep="\t", na_values=cls.na_vals, keep_default_na=False)
        mappings = [
            "\t".join(_df.iloc[i][:-1]) for i in range(len(_df)) if _df.iloc[i][-1] >= threshold
        ]
        return uniqify(mappings)


import os
import sys
import math

from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *

from lib.support_formats import FormatReader, PascalVocWriter, JsonWriter

DISTANCE_LIMIT = 4
CREATING, MOVING, ADJUSTING = list(range(3))
SUPPORT_FMTS = ["Pascal Voc", "Yolo", "Json"]
MIN_SCALE = 0.5
MAX_SCALE = 2.0

class PointCalculator(object):
    @staticmethod
    def pointInsideRect(imagePoint, rect):
        startPoint, endPoint = rect
        return (startPoint.x()-imagePoint.x()) * (endPoint.x()-imagePoint.x()) <= 0 and \
                 (startPoint.y()-imagePoint.y()) * (endPoint.y()-imagePoint.y()) <= 0

    @staticmethod
    def pointInsidePolygon(imagePoint, poly):
        qpoly = QPolygon(poly)
        return qpoly.containsPoint(imagePoint, Qt.OddEvenFill)

    @staticmethod
    def euclidDis(imagePoint, disPoint):
        return math.hypot(imagePoint.x() - disPoint.x(),
                        imagePoint.y() - disPoint.y())

    @staticmethod
    def rectDis(imagePoint, rect):
        return min([abs(imagePoint.x()-rect[0].x()),
                    abs(imagePoint.x()-rect[1].x()),
                    abs(imagePoint.y()-rect[0].y()),
                    abs(imagePoint.y()-rect[1].y())])

    @staticmethod
    def polyDis(imagePoint, poly):
        return min([PointCalculator.euclidDis(imagePoint, polyPoint) for polyPoint in poly])

    @staticmethod
    def nearestRectVertex(imagePoint, rects):
        rect_index = -1
        nearPoint = None
        distance = -1

        for i, rect in enumerate(rects):
            x1, x2, y1, y2 = rect[0].x(), rect[1].x(), rect[0].y(), rect[1].y()
            for vertexPoint in [QPoint(x1, y1), QPoint(x1, y2), QPoint(x2, y1), QPoint(x2, y2)]:
                current_distance = PointCalculator.euclidDis(imagePoint, vertexPoint)
                if distance == -1 or distance > current_distance:
                    distance = current_distance
                    nearPoint = vertexPoint
                    rect_index = i

        if distance <= DISTANCE_LIMIT:
            return nearPoint, rect_index
        return None, None

    @staticmethod
    def nearestVertex(imagePoint, shapes):
        shape_index = -1
        vertex_index = -1
        nearPoint = None
        distance = -1

        for i, shape in enumerate(shapes):
            for j, vertexPoint in enumerate(shape):
                current_distance = PointCalculator.euclidDis(imagePoint, vertexPoint)
                if distance == -1 or distance > current_distance:
                    distance = current_distance
                    nearPoint = vertexPoint
                    shape_index = i
                    vertex_index = j

        if distance <= DISTANCE_LIMIT:
            return nearPoint, shape_index, vertex_index
        return None, None, None

    @staticmethod
    def getReferPoint(imagePoint, rect):
        x, y = imagePoint.x(), imagePoint.y()
        x1, x2 = rect[0].x(), rect[1].x()
        y1, y2 = rect[0].y(), rect[1].y()
        nx = x1 if abs(x-x1) > abs(x-x2) else x2
        ny = y1 if abs(y-y1) > abs(y-y2) else y2
        return QPoint(nx, ny)

    @staticmethod
    def nearestRect(imagePoint, rects):
        index = -1
        distance = -1
        for i, rect in enumerate(rects):
            if PointCalculator.pointInsideRect(imagePoint, rect):
                distanceRect = PointCalculator.rectDis(imagePoint, rect)
                if distance == -1 or distance > distanceRect:
                    distance = distanceRect
                    index = i
        return index

    @staticmethod
    def nearestPoly(imagePoint, polys):
        index = -1
        distance = -1
        for i, poly in enumerate(polys):
            if PointCalculator.pointInsidePolygon(imagePoint, poly):
                distancePoly = PointCalculator.polyDis(imagePoint, poly)
                if distance == -1 or distance > distancePoly:
                    distance = distancePoly
                    index = i
        return index

class ShapeLabel(object):

    def __init__(self):
        super().__init__()
        # shape contour points
        self.currentShape = []
        self.editing_mode = None
        # Adjusting Mode
        self.adjustStatus = False
        self.nearPoint = None
        self.near_shape_index = -1
        self.near_vertex_index = -1
        # Moving Mode
        self.move_index = -1
        self.movePoint = None
        self.benchShape = []
        # shapes and shape class
        self.shapes = []
        self.classes = []

    def isAddPoint(self):
        return True

    def isAddShape(self):
        return True

    def addPoint(self, imagePoint):
        if self.isAddPoint():
            self.currentShape.append(imagePoint)

    def addShape(self):
        if self.isAddShape():
            self.shapes.append(self.currentShape)
            self.currentShape = []

    def clearShapes(self):
        self.shapes.clear()
        self.classes.clear()

    def createWithdrawal(self):
        if self.editing_mode == CREATING:
            if len(self.currentShape) > 0:
                self.currentShape.pop()

    def moveBench(self, imagePoint):
        if not self.benchShape:
            self.move_index = PointCalculator.nearestPoly(imagePoint, self.shapes)
            if self.move_index >= 0:
                self.movePoint = imagePoint
                self.benchShape = self.shapes[self.move_index]
        else:
            self.movePoint = None
            self.benchShape = None

    def moveUpdate(self, imagePoint):
        if self.move_index >= 0 and self.movePoint and self.benchShape:
            self.shapes[self.move_index] = \
                [imagePoint - self.movePoint + benchPoint for benchPoint in self.benchShape]

    def changeAdjustStatus(self):
        if not self.adjustStatus and self.nearPoint:
            self.adjustStatus = True
        elif self.adjustStatus:
            self.adjustStatus = False

    def adjustShape(self, imagePoint):
        if not self.adjustStatus:
            self.nearPoint, self.near_shape_index, self.near_vertex_index = PointCalculator.nearestVertex(imagePoint, self.shapes)
        else:
            if self.shapes and self.nearPoint:
                self.shapes[self.near_shape_index][self.near_vertex_index] = imagePoint

class RectLabel(ShapeLabel):

    def __init__(self):
        ShapeLabel.__init__(self)
        self.referPoint = None

    def isAddShape(self):
        if len(self.currentShape) == 2:
            return True
        return False

    def moveBench(self, imagePoint):
        if not self.benchShape:
            self.move_index = PointCalculator.nearestRect(imagePoint, self.shapes)
            if self.move_index >= 0:
                self.movePoint = imagePoint
                self.benchShape = self.shapes[self.move_index]
        else:
            self.movePoint = None
            self.benchShape = None

    def adjustShape(self, imagePoint):
        if not self.adjustStatus:
            self.nearPoint, self.near_shape_index = PointCalculator.nearestRectVertex(imagePoint, self.shapes)
            if self.shapes and self.nearPoint:
                self.referPoint = PointCalculator.getReferPoint(self.nearPoint, self.shapes[self.near_shape_index])
        else:
            if self.shapes and self.nearPoint:
                self.shapes[self.near_shape_index] = [imagePoint, self.referPoint]

class PolyLabel(ShapeLabel):

    def __init__(self):
        ShapeLabel.__init__(self)
        self.currentClosed = False

    def isClosePoint(self, imagePoint):
        return len(self.currentShape) >= 2 and PointCalculator.euclidDis(imagePoint, self.currentShape[0]) <= DISTANCE_LIMIT

    def addPoint(self, imagePoint):
        if not self.isClosePoint(imagePoint):
            self.currentShape.append(imagePoint)
        else:
            self.currentClosed = True

    def addShape(self):
        if self.currentClosed:
            self.shapes.append(self.currentShape)
            self.currentShape = []
            self.currentClosed = False

class LabelDialog(QDialog):

    def __init__(self, parent):
        super().__init__(parent)

        self.difficult = False
        self.name_list = []
        self.completer = QCompleter(self.name_list)
        self.writerType = None
        self.edit = QLineEdit()
        self.edit.setCompleter(self.completer)

        layout = QVBoxLayout()
        layout.addWidget(self.edit)
        self.diffCheckbox = QCheckBox('difficult', self)
        self.diffCheckbox.clicked.connect(self.changeDiffCheck)
        layout.addWidget(self.diffCheckbox)
        self.formatCombobox = QComboBox()
        for fmt in SUPPORT_FMTS:
            self.formatCombobox.addItem(fmt)
        self.formatCombobox.currentTextChanged.connect(self.changeWriter)
        self.writerType = SUPPORT_FMTS[0]
        layout.addWidget(self.formatCombobox)
        self.button_box = QDialogButtonBox(QDialogButtonBox.Ok | QDialogButtonBox.Cancel, Qt.Horizontal, self)
        self.button_box.accepted.connect(self.accept)
        self.button_box.rejected.connect(self.reject)
        layout.addWidget(self.button_box)
        self.setLayout(layout)

    def changeDiffCheck(self):
        if self.diffCheckbox.isChecked():
            self.difficult = True
        else:
            self.difficult = False

    def changeWriter(self, text):
        self.writerType = text

    def update(self, text):
        self.name_list.append(text)
        self.completer.model().setStringList(self.name_list)

    def popUp(self, text=''):
        self.edit.setText(text)
        self.edit.setSelection(0, len(text))
        self.edit.setFocus(Qt.PopupFocusReason)
        return self.edit.text() if self.exec_() else None

class ImageLabel(QLabel):

    def __init__(self):
        super().__init__()
        self.setAcceptDrops(True)

        self.tracking = False
        self.openStatus = False
        self.shape_mode = None
        self.editLabel = None
        self.labelDialog = LabelDialog(self)

        self.scale = 1
        self.delta_scale = 0.1
        self.labelPoint = QPoint()
        self.imagePoint = QPoint()

        self.image = QImage()
        self.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding)
        self.setAlignment(Qt.AlignCenter)
        self.setMouseTracking(True)
        self.setCursor(QCursor(Qt.PointingHandCursor))

        self.withdrawalShortcut = QShortcut(QKeySequence("Ctrl+Z"), self, activated=self.withdrawalShape)
        self.deleteShortcut = QShortcut(QKeySequence(Qt.Key_Delete), self, activated=self.deleteShape)

    def dragEnterEvent(self, event):
        if event.mimeData().hasUrls():
            event.accept()
        else:
            event.ignore()

    def dropEvent(self, event):
        if event.mimeData().hasUrls():
            localPath = event.mimeData().urls()[0].toLocalFile()
            self.parent().window().openImageFile(localPath)

    def resetStatus(self):
        self.tracking = False
        self.openStatus = False
        self.shape_mode = None
        self.editLabel = None

        self.scale = 1
        self.delta_scale = 0.1
        self.labelPoint = QPoint()
        self.imagePoint = QPoint()
        self.image = QImage()

    def withdrawalShape(self):
        if self.editLabel:
            self.editLabel.createWithdrawal()

    def deleteShape(self):
        if self.editLabel:
            if self.editLabel.editing_mode == MOVING:
                if self.editLabel.move_index >= 0:
                    self.editLabel.shapes.pop(self.editLabel.move_index)
                    self.editLabel.classes.pop(self.editLabel.move_index)
                    self.parent().window().shape_list.takeItem(self.editLabel.move_index)
                    self.editLabel.move_index = -1
            elif self.editLabel.editing_mode == ADJUSTING and self.editLabel.adjustStatus and self.editLabel.nearPoint:
                adjust_shape = self.editLabel.shapes[self.editLabel.near_shape_index]
                adjust_shape.pop(self.editLabel.near_vertex_index)
                self.editLabel.shapes[self.editLabel.near_shape_index] = adjust_shape
                self.editLabel.adjustStatus = False

    def loadLabel(self, labelName):
        data_dict = None
        if labelName and labelName.endswith(".xml"):
            data_dict = FormatReader.load_xml(labelName)
            data_dict['ext'] = ".xml"
        elif labelName and labelName.endswith(".json"):
            data_dict = FormatReader.load_json(labelName)
            data_dict['ext'] = ".json"
        return data_dict

    def loadShapes(self, data_dict):
        if data_dict and data_dict['ext'] == ".xml":
            for label, points, _, _, _ in data_dict['shapes']:
                x1, y1 = points[0]
                x2, y2 = points[2]
                self.editLabel.classes.append(label)
                self.editLabel.shapes.append([QPoint(x1, y1), QPoint(x2, y2)])
                self.parent().window().addShapeItem(label)
        elif data_dict and data_dict['ext'] == ".json":
            for shape_dict in data_dict['shapes']:
                self.editLabel.classes.append(shape_dict["label"])
                shape = []
                for x, y in shape_dict["points"]:
                    shape.append(QPoint(x, y))
                self.editLabel.shapes.append(shape)
                self.parent().window().addShapeItem(shape_dict["label"])

    def checkNullImage(self):
        return self.image.isNull()

    def checkImageRange(self, pos):
        return pos.x() >= 0 and pos.x() < self.image.width() and pos.y() >= 0 and \
            pos.y() < self.image.height()

    def resizeImage(self):
        pixmap = QPixmap.fromImage(self.image)
        scaled_pixmap = pixmap.scaled(self.scale * pixmap.size())
        self.setPixmap(scaled_pixmap)
        self.adjustSize()

    def zoomIn(self):
        if not self.checkNullImage():
            new_scale = self.scale + self.delta_scale
            if new_scale <= MAX_SCALE:
                self.scale = new_scale
                self.labelPoint = self.canvas_pos(self.imagePoint)
                self.resizeImage()

    def zoomOut(self):
        if not self.checkNullImage():
            new_scale = self.scale - self.delta_scale
            if new_scale >= MIN_SCALE:
                self.scale = new_scale
                self.labelPoint = self.canvas_pos(self.imagePoint)
                self.resizeImage()

    def offset_to_center(self):
        s = self.scale
        area = super().size()
        w, h = self.image.width() * s, self.image.height() * s
        aw, ah = area.width(), area.height()
        x = (aw - w) / (2 * s) if aw > w else 0
        y = (ah - h) / (2 * s) if ah > h else 0
        return QPoint(x, y)

    def image_pos(self, point):
        return point / self.scale - self.offset_to_center()

    def canvas_pos(self, point):
        return (point + self.offset_to_center()) * self.scale

    def canvas_shape(self, shape):
        return [self.canvas_pos(imagePoint) for imagePoint in shape]

    def getImageRect(self):
        pos = self.canvas_pos(QPoint(0, 0))
        return pos.x(), pos.y(), self.image.width(), self.image.height()

    def drawTrackingLine(self, painter):
        _, _, w0, h0 = self.getImageRect()
        painterpath = QPainterPath()
        painterpath.moveTo(self.canvas_pos(QPoint(0, self.imagePoint.y())))
        painterpath.lineTo(self.canvas_pos(QPoint(w0, self.imagePoint.y())))
        painterpath.moveTo(self.canvas_pos(QPoint(self.imagePoint.x(), 0)))
        painterpath.lineTo(self.canvas_pos(QPoint(self.imagePoint.x(), h0)))
        painter.drawPath(painterpath)

    def drawCreateRect(self, painter):
        startPoint = None
        endPoint = None

        if len(self.editLabel.currentShape) == 2:
            startPoint, endPoint = self.canvas_shape(self.editLabel.currentShape)
        elif len(self.editLabel.currentShape) == 1:
            startPoint = self.canvas_pos(self.editLabel.currentShape[0])

        if startPoint:
            painter.setPen(QPen(Qt.green, 2, Qt.SolidLine))
            painter.setBrush(QBrush(Qt.red, Qt.BDiagPattern))
            painterpath1 = QPainterPath()
            painterpath1.addRect(startPoint.x(), startPoint.y(),
                            self.labelPoint.x()-startPoint.x(),
                            self.labelPoint.y()-startPoint.y())
            painterpath1.moveTo(startPoint)
            painterpath1.lineTo(self.labelPoint)
            painter.drawPath(painterpath1)

            painter.setBrush(QBrush(Qt.green, Qt.SolidPattern))
            painterpath2 = QPainterPath()
            painterpath2.addEllipse(startPoint, 3, 3)
            painter.drawPath(painterpath2)
        if endPoint:
            painter.setPen(QPen(Qt.green, 2, Qt.SolidLine))
            painterpath2 = QPainterPath()
            painterpath2.addEllipse(endPoint, 3, 3)
            painter.drawPath(painterpath2)

    def drawCreatePoly(self, painter):
        painter.setPen(QPen(Qt.green, 2, Qt.SolidLine))
        painter.setBrush(QBrush(Qt.green, Qt.NoBrush))
        currentShape = self.canvas_shape(self.editLabel.currentShape)
        painterpath1 = QPainterPath()
        painterpath1.addPolygon(QPolygonF(currentShape))

        if len(currentShape) > 0:
            painterpath1.moveTo(currentShape[-1])
            painterpath1.lineTo(self.labelPoint)
        painter.drawPath(painterpath1)

        painter.setBrush(QBrush(Qt.green, Qt.SolidPattern))
        painterpath2 = QPainterPath()
        for labelPoint in currentShape:
            painterpath2.addEllipse(labelPoint, 3, 3)
        painter.drawPath(painterpath2)

    def drawAdjustRect(self, painter):
        startPoint, endPoint = self.canvas_shape(self.editLabel.shapes[self.editLabel.near_shape_index])
        painterpath1 = QPainterPath()
        painterpath1.addRect(startPoint.x(), startPoint.y(), endPoint.x()-startPoint.x(), endPoint.y()-startPoint.y())
        painter.setPen(QPen(Qt.green, 2, Qt.SolidLine))
        painter.setBrush(QBrush(QColor(0, 255, 0, 30), Qt.SolidPattern))
        painter.drawPath(painterpath1)

        painterpath2 = QPainterPath()
        painterpath2.addRect(startPoint.x()-5, startPoint.y()-5, 10, 10)
        painterpath2.addRect(endPoint.x()-5, startPoint.y()-5, 10, 10)
        painterpath2.addRect(startPoint.x()-5, endPoint.y()-5, 10, 10)
        painterpath2.addRect(endPoint.x()-5, endPoint.y()-5, 10, 10)
        painter.setPen(QPen(Qt.red, 2, Qt.SolidLine))
        painter.setBrush(QBrush(Qt.red, Qt.SolidPattern))
        painter.drawPath(painterpath2)

    def paintRect(self, painter):
        painter.setPen(QPen(Qt.green, 2, Qt.SolidLine))
        for i, shape in enumerate(self.editLabel.shapes):
            if PointCalculator.pointInsideRect(self.imagePoint, shape):
                painter.setBrush(QBrush(QColor(0, 255, 0, 30), Qt.SolidPattern))
            else:
                painter.setBrush(QBrush(QColor(0, 255, 0, 0), Qt.SolidPattern))
            if i == self.editLabel.move_index and self.editLabel.editing_mode == MOVING:
                painter.setBrush(QBrush(QColor(0, 0, 255, 30), Qt.SolidPattern))
            if i != self.editLabel.near_shape_index:
                startPoint, endPoint = self.canvas_shape(shape)
                painterpath0 = QPainterPath()
                painterpath0.addRect(startPoint.x(), startPoint.y(),
                                    endPoint.x()-startPoint.x(),
                                    endPoint.y()-startPoint.y())
                painter.drawPath(painterpath0)

        painter.setPen(QPen(Qt.blue, 2, Qt.SolidLine))
        painter.setBrush(QBrush(Qt.blue, Qt.SolidPattern))
        for i, shape in enumerate(self.editLabel.shapes):
            startPoint, endPoint = self.canvas_shape(shape)
            if i != self.editLabel.near_shape_index:
                painterpath1 = QPainterPath()
                painterpath1.addEllipse(QPoint(startPoint.x(), startPoint.y()), 3, 3)
                painterpath1.addEllipse(QPoint(startPoint.x(), endPoint.y()), 3, 3)
                painterpath1.addEllipse(QPoint(endPoint.x(), endPoint.y()), 3, 3)
                painterpath1.addEllipse(QPoint(endPoint.x(), startPoint.y()), 3, 3)
                painter.drawPath(painterpath1)

        if self.editLabel.editing_mode == CREATING:
            self.drawCreateRect(painter)

        if self.editLabel.editing_mode == ADJUSTING and self.editLabel.nearPoint:
            self.drawAdjustRect(painter)

    def drawCreateShape(self, painter):
        self.drawCreatePoly(painter)
        painter.setPen(QPen(Qt.green, 2, Qt.SolidLine))
        painter.setBrush(QBrush(Qt.red, Qt.BDiagPattern))
        if len(self.editLabel.currentShape) >= 1 and self.editLabel.isClosePoint(self.imagePoint):
            painter.setBrush(QBrush(Qt.green, Qt.SolidPattern))
            painterpath = QPainterPath()
            painterpath.addEllipse(self.canvas_pos(self.editLabel.currentShape[0]), 8, 8)
            painter.drawPath(painterpath)

    def drawAdjustShape(self, painter):
        shape = self.canvas_shape(self.editLabel.shapes[self.editLabel.near_shape_index])
        painter.setPen(QPen(Qt.green, 2, Qt.SolidLine))
        painter.setBrush(QBrush(QColor(0, 255, 0, 30), Qt.SolidPattern))
        painterpath1 = QPainterPath()
        painterpath1.addPolygon(QPolygonF(shape))
        painterpath1.moveTo(painterpath1.currentPosition())
        painterpath1.lineTo(QPointF(shape[0]))
        painter.drawPath(painterpath1)

        painter.setPen(QPen(Qt.red, 2, Qt.SolidLine))
        painter.setBrush(QBrush(Qt.red, Qt.SolidPattern))
        painterpath2 = QPainterPath()
        for vertexPoint in shape:
            painterpath2.addRect(vertexPoint.x()-5, vertexPoint.y()-5, 10, 10)
        painter.drawPath(painterpath2)

    def paintPoly(self, painter):
        painter.setPen(QPen(Qt.green, 2, Qt.SolidLine))
        for i, shape in enumerate(self.editLabel.shapes):
            if PointCalculator.pointInsidePolygon(self.imagePoint, shape):
                painter.setBrush(QBrush(QColor(0, 255, 0, 30), Qt.SolidPattern))
            else:
                painter.setBrush(QBrush(QColor(0, 255, 0, 0), Qt.SolidPattern))
            if i == self.editLabel.move_index and self.editLabel.editing_mode == MOVING:
                painter.setBrush(QBrush(QColor(0, 0, 255, 30), Qt.SolidPattern))
            if i != self.editLabel.near_shape_index:
                painterpath = QPainterPath()
                painterpath.addPolygon(QPolygonF(self.canvas_shape(shape)))
                painter.drawPath(painterpath)
                painterpath.lineTo(QPointF(self.canvas_pos(shape[0])))
                painter.drawPath(painterpath)

        painter.setPen(QPen(Qt.blue, 2, Qt.SolidLine))
        painter.setBrush(QBrush(Qt.blue, Qt.SolidPattern))
        for i, shape in enumerate(self.editLabel.shapes):
            if i != self.editLabel.near_shape_index:
                painterpath = QPainterPath()
                for vertexPoint in shape:
                    painterpath.addEllipse(self.canvas_pos(vertexPoint), 3, 3)
                painter.drawPath(painterpath)

        if self.editLabel.editing_mode == CREATING:
            self.drawCreateShape(painter)

        if self.editLabel.editing_mode == ADJUSTING and self.editLabel.nearPoint:
            self.drawAdjustShape(painter)

    def paintShape(self, painter):
        if self.shape_mode == "Rect":
            if not self.editLabel and self.openStatus:
                self.editLabel = RectLabel()
                self.parent().window().createRadiobox.setEnabled(True)
                self.parent().window().moveRadiobox.setEnabled(True)
                self.parent().window().adjustRadiobox.setEnabled(True)
                self.parent().window().createRadiobox.setChecked(True)
                self.parent().window().polyRadio.setEnabled(False)
                self.parent().window().setLabelName()
                self.labelDialog.formatCombobox.setCurrentIndex(0)
                self.openStatus = False
            elif self.editLabel and self.openStatus:
                self.editLabel.clearShapes()
                self.parent().window().setLabelName()
                self.openStatus = False
            self.paintRect(painter)
        elif self.shape_mode == "Poly":
            if not self.editLabel and self.openStatus:
                self.editLabel = PolyLabel()
                self.parent().window().createRadiobox.setEnabled(True)
                self.parent().window().moveRadiobox.setEnabled(True)
                self.parent().window().adjustRadiobox.setEnabled(True)
                self.parent().window().createRadiobox.setChecked(True)
                self.parent().window().rectRadio.setEnabled(False)
                self.parent().window().setLabelName()
                self.labelDialog.formatCombobox.setCurrentIndex(2)
                self.openStatus = False
            elif self.editLabel and self.openStatus:
                self.editLabel.clearShapes()
                self.parent().window().setLabelName()
                self.openStatus = False
            self.paintPoly(painter)

    def paintEvent(self, event):
        super().paintEvent(event)
        if self.image.width() and self.checkImageRange(self.imagePoint):
            painter = QPainter(self)
            if self.tracking:
                painter.setPen(QPen(Qt.black, 2, Qt.SolidLine))
                self.drawTrackingLine(painter)
                painter.drawText(QRect(self.labelPoint.x(), self.labelPoint.y(), 80, -60),
                                Qt.AlignCenter, "(%d, %d)" % (self.imagePoint.x(), self.imagePoint.y()))
            self.paintShape(painter)

        self.update()

    def mousePressEvent(self, event):
        labelPoint = event.pos()
        imagePoint = self.image_pos(labelPoint)
        if self.checkImageRange(imagePoint):
            self.labelPoint = labelPoint
            self.imagePoint = imagePoint

            if self.shape_mode == "Rect" and self.editLabel:
                if self.editLabel.editing_mode == CREATING:
                    self.editLabel.addPoint(self.imagePoint)
                elif self.editLabel.editing_mode == MOVING:
                    self.editLabel.moveBench(self.imagePoint)
                elif self.editLabel.editing_mode == ADJUSTING:
                    self.editLabel.changeAdjustStatus()
            elif self.shape_mode == "Poly" and self.editLabel:
                if self.editLabel.editing_mode == CREATING:
                    self.editLabel.addPoint(self.imagePoint)
                elif self.editLabel.editing_mode == MOVING:
                    self.editLabel.moveBench(self.imagePoint)
                elif self.editLabel.editing_mode == ADJUSTING:
                    self.editLabel.changeAdjustStatus()

        super().mousePressEvent(event)
        self.update()

    def mouseMoveEvent(self, event):
        labelPoint = event.pos()
        imagePoint = self.image_pos(labelPoint)
        if self.checkImageRange(imagePoint):
            self.labelPoint = labelPoint
            self.imagePoint = imagePoint

            if self.image.width() and self.checkImageRange(self.imagePoint):
                self.parent().window().status.showMessage("X: %d Y: %d" % (self.imagePoint.x(), self.imagePoint.y()))
            else:
                self.parent().window().status.showMessage("")
            if self.shape_mode == "Rect" and self.editLabel:
                if self.editLabel.editing_mode == MOVING:
                    self.editLabel.moveUpdate(self.imagePoint)
                elif self.editLabel.editing_mode == ADJUSTING:
                    self.editLabel.adjustShape(self.imagePoint)
            if self.shape_mode == "Poly" and self.editLabel:
                if self.editLabel.editing_mode == MOVING:
                    self.editLabel.moveUpdate(self.imagePoint)
                elif self.editLabel.editing_mode == ADJUSTING:
                    self.editLabel.adjustShape(self.imagePoint)

        super().mouseMoveEvent(event)
        self.update()

    def mouseReleaseEvent(self, event):
        labelPoint = event.pos()
        imagePoint = self.image_pos(labelPoint)
        if self.checkImageRange(imagePoint):
            self.labelPoint = labelPoint
            self.imagePoint = imagePoint

            if self.shape_mode == "Rect" and self.editLabel:
                if self.editLabel.editing_mode == CREATING:
                    if len(self.editLabel.currentShape) == 2:
                        text = self.labelDialog.popUp()
                        self.editLabel.addShape()
                        if not text or len(text) == 0:
                            self.editLabel.shapes.pop()
                        else:
                            self.editLabel.classes.append(text)
                            self.parent().window().addShapeItem(text)
                            self.labelDialog.update(text)
            elif self.shape_mode == "Poly" and self.editLabel:
                if self.editLabel.editing_mode == CREATING:
                    if self.editLabel.isClosePoint(self.imagePoint):
                        text = self.labelDialog.popUp()
                        self.editLabel.addShape()
                        if not text or len(text) == 0:
                            self.editLabel.shapes.pop()
                        else:
                            self.editLabel.classes.append(text)
                            self.parent().window().addShapeItem(text)
                            self.labelDialog.update(text)

        super().mouseReleaseEvent(event)
        self.update()

class MainWindow(QMainWindow):

    def __init__(self):
        super().__init__()
        self.version = "1.0.0"
        self.fileName = None
        self.labelName = None
        self.setMouseTracking(False)
        self.setMenuBar()
        self.setToolBar()
        self.setScrollArea()
        self.setPropertyDock()
        self.setTrackingDock()
        self.setShapeDock()
        self.setStatusBar()
        self.resize(800, 600)

    def setMenuBar(self):
        menubar = self.menuBar()
        fileMenu = menubar.addMenu('&File')
        openAction = QAction("&Open...", self, shortcut="Ctrl+O", triggered=self.openImageDialog)
        saveAction = QAction("&Save...", self, shortcut="Ctrl+S", triggered=self.saveLabelDialog)
        fileMenu.addAction(openAction)
        fileMenu.addAction(saveAction)

    def setToolBar(self):
        self.toolbar = self.addToolBar('')
        self.addToolBar(Qt.TopToolBarArea, self.toolbar)
        zoomInAction = QAction("&Zoom In...", self, shortcut="Ctrl++", triggered=self.zoomIn)
        zoomOutAction = QAction("&Zoom Out...", self, shortcut="Ctrl+-", triggered=self.zoomOut)

        zoomInButton = QToolButton()
        zoomInButton.setText("Zoom In")
        zoomInButton.setCheckable(True)
        zoomInButton.setAutoExclusive(True)
        zoomInButton.setToolButtonStyle(self.toolbar.toolButtonStyle())
        zoomInButton.setDefaultAction(zoomInAction)
        self.toolbar.addWidget(zoomInButton)

        zoomOutButton = QToolButton()
        zoomOutButton.setText("Zoom Out")
        zoomOutButton.setCheckable(True)
        zoomOutButton.setAutoExclusive(True)
        zoomOutButton.setToolButtonStyle(self.toolbar.toolButtonStyle())
        zoomOutButton.setDefaultAction(zoomOutAction)
        self.toolbar.addWidget(zoomOutButton)

    def zoomIn(self):
        self.imageLabel.zoomIn()

    def zoomOut(self):
        self.imageLabel.zoomOut()

    def setScrollArea(self):
        scroll = QScrollArea()
        self.imageLabel = ImageLabel()
        scroll.setWidget(self.imageLabel)
        scroll.setWidgetResizable(True)
        self.setCentralWidget(scroll)
        scroll.setVisible(True)

    def setPropertyDock(self):
        self.propertyDock = QDockWidget("Image Properties", self)
        self.tableView = QTableView()
        self.model = QStandardItemModel(4, 2)
        self.model.setHorizontalHeaderLabels(['Property', 'Value'])
        self.tableView.setModel(self.model)
        self.tableView.horizontalHeader().setStretchLastSection(True)
        self.tableView.horizontalHeader().setSectionResizeMode(QHeaderView.Stretch)
        self.tableView.setAlternatingRowColors(True)
        self.nameCombox = QComboBox()
        self.nameCombox.addItem("File Name(including path)")
        self.nameCombox.addItem("File Name(pure name)")
        self.nameCombox.addItem("Directory Name")
        self.nameText = QLabel()
        item21 = QStandardItem("Image Width")
        item31 = QStandardItem("Image Height")
        item41 = QStandardItem("Image Depth")
        self.tableView.setIndexWidget(self.model.index(0, 0), self.nameCombox)
        self.tableView.setIndexWidget(self.model.index(0, 1), self.nameText)
        self.model.setItem(1, 0, item21)
        self.model.setItem(2, 0, item31)
        self.model.setItem(3, 0, item41)
        self.propertyDock.setWidget(self.tableView)
        self.addDockWidget(Qt.RightDockWidgetArea, self.propertyDock)

    def setTrackingDock(self):
        self.trackingDock = QDockWidget("Image Tracking", self)
        self.trackingWidget = QWidget()
        whole_layout = QVBoxLayout()

        self.trackCheckbox = QCheckBox('Tracking', self)
        self.trackCheckbox.clicked.connect(self.changeTrackingMode)
        whole_layout.addWidget(self.trackCheckbox)

        operationGroup = QGroupBox('Operate Mode:', self)
        operation_layout = QVBoxLayout()

        self.createRadiobox = QRadioButton('Creating', self)
        self.createRadiobox.toggled.connect(self.changeEditMode)
        self.createRadiobox.setEnabled(False)
        operation_layout.addWidget(self.createRadiobox)

        self.moveRadiobox = QRadioButton('Moving', self)
        self.moveRadiobox.toggled.connect(self.changeEditMode)
        self.moveRadiobox.setEnabled(False)
        operation_layout.addWidget(self.moveRadiobox)

        self.adjustRadiobox = QRadioButton('Adjusting', self)
        self.adjustRadiobox.toggled.connect(self.changeEditMode)
        self.adjustRadiobox.setEnabled(False)
        operation_layout.addWidget(self.adjustRadiobox)

        operationGroup.setLayout(operation_layout)
        whole_layout.addWidget(operationGroup)

        shapeGroup = QGroupBox('Shape Mode:', self)
        shape_layout = QVBoxLayout()
        self.rectRadio = QRadioButton('Rect', self)
        self.rectRadio.clicked.connect(lambda checked, text=self.rectRadio.text():self.changeShapeMode(checked, text))
        shape_layout.addWidget(self.rectRadio)

        self.polyRadio = QRadioButton('Poly', self)
        self.polyRadio.clicked.connect(lambda checked, text=self.polyRadio.text():self.changeShapeMode(checked, text))
        shape_layout.addWidget(self.polyRadio)

        shapeGroup.setLayout(shape_layout)
        whole_layout.addWidget(shapeGroup)
        self.trackingWidget.setLayout(whole_layout)

        self.trackingDock.setWidget(self.trackingWidget)
        self.addDockWidget(Qt.RightDockWidgetArea, self.trackingDock)

    def setShapeDock(self):
        self.shapeDock = QDockWidget("Image Shapes", self)
        shapeWidget = QWidget()
        shape_layout = QVBoxLayout()
        shape_layout.setContentsMargins(0, 0, 0, 0)
        self.shape_list = QListWidget()
        shape_layout.addWidget(self.shape_list)
        shapeWidget.setLayout(shape_layout)
        self.shapeDock.setWidget(shapeWidget)
        self.addDockWidget(Qt.RightDockWidgetArea, self.shapeDock)

    def changeTrackingMode(self, checked):
        if checked:
            self.imageLabel.tracking = True
            return
        self.imageLabel.tracking = False

    def changeEditMode(self, checked):
        if self.createRadiobox.isChecked():
            if self.imageLabel.editLabel:
                self.imageLabel.editLabel.editing_mode = CREATING
        elif self.moveRadiobox.isChecked():
            if self.imageLabel.editLabel:
                self.imageLabel.editLabel.editing_mode = MOVING
        elif self.adjustRadiobox.isChecked():
            if self.imageLabel.editLabel:
                self.imageLabel.editLabel.editing_mode = ADJUSTING

    def changeShapeMode(self, checked, text):
        self.imageLabel.shape_mode = text
        if text == "Rect":
            self.imageLabel.labelDialog.writerType = SUPPORT_FMTS[0]
        elif text == "Poly":
            self.imageLabel.labelDialog.writerType = SUPPORT_FMTS[2]

    def setStatusBar(self):
        self.status = self.statusBar()

    def resetStatus(self):
        self.trackCheckbox.setChecked(False)
        self.createRadiobox.setAutoExclusive(False)
        self.createRadiobox.setChecked(False)
        self.createRadiobox.setAutoExclusive(True)
        self.moveRadiobox.setAutoExclusive(False)
        self.moveRadiobox.setChecked(False)
        self.moveRadiobox.setAutoExclusive(True)
        self.adjustRadiobox.setAutoExclusive(False)
        self.adjustRadiobox.setChecked(False)
        self.adjustRadiobox.setAutoExclusive(True)
        self.createRadiobox.setEnabled(False)
        self.moveRadiobox.setEnabled(False)
        self.adjustRadiobox.setEnabled(False)
        self.polyRadio.setAutoExclusive(False)
        self.polyRadio.setChecked(False)
        self.polyRadio.setAutoExclusive(True)
        self.polyRadio.setEnabled(True)
        self.rectRadio.setAutoExclusive(False)
        self.rectRadio.setChecked(False)
        self.rectRadio.setAutoExclusive(True)
        self.rectRadio.setEnabled(True)
        self.shape_list.clear()

    def openImageFile(self, fileName):
        if fileName:
            self.saveLabelDialog()
            self.imageLabel.resetStatus()
            self.resetStatus()
            self.fileName = fileName
            self.imageLabel.image = QImage(fileName)
            if self.imageLabel.checkNullImage():
                QMessageBox.information(self, "Warning", "Cannot load %s." % fileName)
                return

            self.imageLabel.setPixmap(QPixmap.fromImage(self.imageLabel.image))
            self.imageLabel.adjustSize()
            self.setTable()
            self.imageLabel.openStatus = True

    def openImageDialog(self):
        options = QFileDialog.Options()
        fileName, _ = QFileDialog.getOpenFileName(self, "Open Image File", "", "All Files (*)", options=options)
        self.openImageFile(fileName)

    def loadLabel(self):
        if os.path.exists(self.labelName):
            data_dict = self.imageLabel.loadLabel(self.labelName)
            self.imageLabel.loadShapes(data_dict)

    def addShapeItem(self, shape_class):
        shape_item = QListWidgetItem()
        shape_item.setText(shape_class)
        shape_item.setFlags(shape_item.flags() | Qt.ItemIsUserCheckable)
        shape_item.setCheckState(Qt.Checked)
        self.shape_list.addItem(shape_item)

    def setLabelName(self):
        if self.fileName and self.imageLabel and \
            self.imageLabel.shape_mode == "Rect" and \
            self.imageLabel.labelDialog.writerType == SUPPORT_FMTS[0]:
            self.labelName = self.fileName.split(".")[0] + ".xml"
        elif self.fileName and self.imageLabel and \
            self.imageLabel.shape_mode == "Poly" and \
            self.imageLabel.labelDialog.writerType == SUPPORT_FMTS[2]:
            self.labelName = self.fileName.split(".")[0] + ".json"
        self.loadLabel()

    def saveLabelDialog(self):
        if self.labelName:
            result = QMessageBox.question(self,
                      "Save before exit",
                      "Would you like to save the label format?",
                      QMessageBox.Yes| QMessageBox.No)

            if result == QMessageBox.Yes:
                if not os.path.exists(self.labelName):
                    options = QFileDialog.Options()
                    name, _ = QFileDialog.getSaveFileName(self, "Save File", self.labelName, "All Files (*)", options=options)
                    if name:
                        self.saveLabel(name)
                else:
                    self.saveLabel(self.labelName)

    def getFileName(self):
        path, fn = os.path.split(self.fileName)
        return {"File Name(including path)": self.fileName,
                "File Name(pure name)": fn,
                "Directory Name": path}

    def setTable(self):
        item22 = QStandardItem("{}".format(self.imageLabel.image.width()))
        item32 = QStandardItem("{}".format(self.imageLabel.image.height()))
        item42 = QStandardItem("{}".format(self.imageLabel.image.depth()))
        self.nameText.setText(self.getFileName()[str(self.nameCombox.currentText())])
        self.nameCombox.activated[str].connect(self.changeNameText)
        self.model.setItem(1, 1, item22)
        self.model.setItem(2, 1, item32)
        self.model.setItem(3, 1, item42)

        if not self.propertyDock.isVisible():
            self.propertyDock.show()

    def changeNameText(self, text):
        self.nameText.setText(self.getFileName()[text])

    def saveLabel(self, name):
        if self.imageLabel.editLabel and len(self.imageLabel.editLabel.classes) > 0:
            folder_path = os.path.dirname(self.fileName)
            folder_name = os.path.split(folder_path)[-1]
            file_name = os.path.basename(self.fileName)
            shape = [self.imageLabel.image.height(),
                    self.imageLabel.image.width(),
                    1 if self.imageLabel.image.isGrayscale() else 3]

            if self.imageLabel.shape_mode == "Rect" and\
                self.imageLabel.labelDialog.writerType == SUPPORT_FMTS[0]:
                writer = PascalVocWriter(folder_name, file_name, shape,
                                        local_img_path=self.fileName)
                difficult = self.imageLabel.labelDialog.difficult
                for className, rectPoint in zip(self.imageLabel.editLabel.classes,
                                                self.imageLabel.editLabel.shapes):
                    startP, endP = rectPoint
                    xmin, xmax = sorted([int(startP.x()), int(endP.x())])
                    ymin, ymax = sorted([int(startP.y()), int(endP.y())])
                    writer.add_bnd_box(xmin, ymin, xmax, ymax, className, difficult)

                writer.save(target_file=name)
            elif self.imageLabel.shape_mode == "Poly" and\
                self.imageLabel.labelDialog.writerType == SUPPORT_FMTS[2]:
                writer = JsonWriter(folder_name, file_name, shape,
                                        local_img_path=self.fileName)

                shape_list = []
                for className, shape in zip(self.imageLabel.editLabel.classes,
                                                self.imageLabel.editLabel.shapes):
                    shape_dict = {}
                    shapePoints = []
                    for vertexPoint in shape:
                        shapePoints.append([vertexPoint.x(), vertexPoint.y()])
                    shape_dict["label"] = className
                    shape_dict["points"] = shapePoints
                    shape_dict["group_id"] = writer.group_id
                    shape_dict["shape_type"] = writer.shape_type
                    shape_dict["flags"] = writer.flags
                    shape_list.append(shape_dict)

                writer.save(shape_list, name)

    def closeEvent(self, event):
        self.saveLabelDialog()
        event.accept()

if __name__ == "__main__":
    app = QApplication(sys.argv)
    app.setApplicationName("PanyImage")
    win = MainWindow()
    win.show()
    app.exec_()


#!/usr/bin/env python
# coding: utf-8

from google_play_scraper import app
import pandas as pd
import pandahouse as ph
from clickhouse_driver import Client
from tqdm import tqdm

bundle_list = pd.read_csv('/home/v.roizen/scripts/parsing/rustore_bundles.csv')

bundle_list = list(set(bundle_list['bundle_id']))

def get_key(d, value):
    for k, v in d.items():
        if v == value:
            return k

colmn=['bundle_id','google_play_version']
df=pd.DataFrame(columns=colmn)

for i in tqdm(range(len(bundle_list))):
    try:
        result = app(
            bundle_list[i],
            lang='en', # defaults to 'en'
            country='us' # defaults to 'us'
        )
        google_play_version=result.get('version')
        bundle_id=bundle_list[i]
    except:
        google_play_version='App not found'
        bundle_id=bundle_list[i]

    row = [bundle_id, google_play_version]
    new_df = pd.DataFrame([row], columns=colmn)
    df = pd.concat([df, new_df], ignore_index = True)

df.to_csv('/home/v.roizen/scripts/parsing/google_play.csv')


"""
Both t-tests and ANOVA have a major drawback: the population that is being
sampled must follow a normal distribution. In many applications, this is not
too restrictive because many real-world population values follow a normal
distribution, or some rules, such as the central limit theorem, allow us to
analyze some related data. However, it is simply not true that all possible
population values follow a normal distribution in any reasonable way. For these
(thankfully, rare) cases, we need some alternative test statistics to use as
replacements for t-tests and ANOVA.

This module illustrates how to use Wilcoxon rank-sum and the Kruskal-Wallis
test for differences between two (or more, in the latter case) populations.
"""
from scipy import stats
from numpy.random import default_rng

rng = default_rng(12345)

sample_A = rng.uniform(2.5, 4.5, size=22)
sample_B = rng.uniform(3.0, 4.4, size=25)
sample_C = rng.uniform(3.0, 4.4, size=30)

significance = 0.05
statistic, p_value = stats.kruskal(sample_A, sample_B, sample_C)

print(f"Statistic: {statistic}, p-value: {p_value}")
# Statistic: 5.09365664638392, p-value: 0.07832970895845669

if p_value <= significance:
    print("Accept H0: all medians equal")
else:
    print("There are differences between population medians")
# There are differences between population medians

_, p_A_B = stats.ranksums(sample_A, sample_B)
_, p_A_C = stats.ranksums(sample_A, sample_C)
_, p_B_C = stats.ranksums(sample_B, sample_C)

if p_A_B > significance:
    print(f"Significance differences between A and B, p-value: {p_A_B}")
# Significant differences between A and B, p-value 0.08808151166219029

if p_A_C > significance:
    print(f"Significance differences between A and C, p-value: {p_A_C}")
# Significant differences between A and C, p-value 0.4257804790323789

if p_B_C > significance:
    print(f"Significance differences between B and C, p-value: {p_B_C}")
else:
    print(f"No significant differences between B and C, p-value: {p_B_C}")
# No significant differences between B and C, p-value 0.037610047044153536


from bs4 import BeautifulSoup
import requests

html = requests.get("http://localhost:8000/index.html")
html_soup = BeautifulSoup(html.content,"html.parser")
judul = html_soup.find('p',class_='judul').text
paragraf = html_soup.find('p',class_ ='paragraf').text

print(judul)
print(paragraf)


from django.shortcuts import render
from github.api_client import GithubApiClient

def organization_repos(request):
    """Shows the repositories of a organization"""
    if 'organization' in request.POST:
        organization = request.POST['organization']
        api_client = GithubApiClient()
        repositories = api_client.get_repositories_by_organization(organization)
        context = {'organization': organization, 'repositories': repositories}
    else:
        context = {'organization': None}

    return render(request, 'repos/organization_repos.html', context)

def stars_repos(request):
    """Shows the selected amount of repositories with more stars in github"""
    number_of_repos = ''
    if 'number_of_repos' in request.POST:
        number_of_repos = request.POST['number_of_repos']
    if number_of_repos == '':
        number_of_repos = 10
    api_client = GithubApiClient()
    repositories = api_client.get_repositories_by_stars(number_of_repos)
    context = {
        'number_of_repos': number_of_repos,
        'repositories': repositories
    }
    return render(request, 'repos/stars_repos.html', context)


from django.shortcuts import render, get_object_or_404, redirect, reverse
from django.contrib import messages
from django.contrib.auth.decorators import login_required
from django.core.exceptions import ValidationError
from django.core.validators import validate_email
from django.core.mail import send_mail
from .models import NewletterSubscribers

def newsletter(request):
    """ Add user newsletter signups to the database """

    if request.method == 'POST':
        # get email address from form
        email = request.POST['newsletter']
        # if user already has already subscripbed
        if NewletterSubscribers.objects.filter(email=email).exists():
            messages.error(request, 'Already subscribed')
            return redirect('/')
        # if new email
        else:
            # saves form to database
            NewletterSubscribers.objects.create(email=email)
            # send email confirmation to user
            send_mail(
                'North Yeast Brewing',
                'Hey, You have subscriped to the North Yeast Brewing newletter.\
                 We will send you all the lastest events and offers',
                'northyeastbrewing.com',
                [email],
                fail_silently=False,
            )

        context = {
            'email': email,
        }

    return render(request, 'newsletter/newsletter_success.html', context)

@login_required
def send_newsletter(request):
    """" Gets user input then sends a email to newsletter subscriber\
        with vaild email addresss """
    if not request.user.is_superuser:
        messages.error(request, 'Sorry, you must be admin to add products.')
        return redirect(reverse('home'))
    if request.method == 'POST':
        # Get data from form
        subject = request.POST['subject']
        message = request.POST['message']

        # Get emal list from database
        email_list = []
        invalid_email = 0
        sent_emails = 0

        for subscriber in NewletterSubscribers.objects.all():
            email_list.append(subscriber.email)

        for email in email_list:
            # vaildate email and send to vaid email address
            try:
                validate_email(email)
            except ValidationError as e:
                invalid_email = invalid_email + 1
            else:
                sent_emails = sent_emails + 1
                send_mail(
                    subject,
                    message,
                    'northyeastbrewing.com',
                    [email],
                    fail_silently=False,
                )
        messages.success(request, f"Sent newsletter to {sent_emails} \
                         email address. { invalid_email} could not sent ")

    return render(request, 'newsletter/send_newsletter.html')

@login_required
def view_email_list(request):
    """ Allow admin to view email list """
    if not request.user.is_superuser:
        messages.error(request, 'Sorry, you must be admin to edit emails.')
        return redirect(reverse('home'))
    email_list = NewletterSubscribers.objects.all()

    context = {
        'email_list': email_list,
    }

    return render(request, 'newsletter/view_email_list.html', context)

@login_required()
def delete_mail(request, email_id):
    """ Allow admin to delete email subscriber from list """
    if not request.user.is_superuser:
        messages.error(request, 'Sorry, you must be admin\
             to delete newsletter emails.')
        return redirect(reverse('home'))
    # find email in database
    email = get_object_or_404(NewletterSubscribers, pk=email_id)
    # delete email address from databese
    email.delete()
    # get new mailing list from database
    email_list = NewletterSubscribers.objects.all()

    context = {
        'email_list': email_list,
    }
    return render(request, 'newsletter/view_email_list.html', context)

@login_required()
def edit_mail(request, email_id):
    """ Allow admin to edit and delete from email list """

    if request.method == 'POST':
        updated_email = request.POST['updated-email']
        email = get_object_or_404(NewletterSubscribers, pk=email_id)
        email.email = updated_email
        email.save()

    email_list = NewletterSubscribers.objects.all()
    context = {
        'email_list': email_list,
    }

    return render(request, 'newsletter/view_email_list.html', context)


import urllib
from urllib.request import urlopen as uReq
from bs4 import BeautifulSoup as soup

import os

from newspaper import Article, Source

user_agent = 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.7) Gecko/2009021910 Firefox/3.0.7'
headers={'User-Agent':user_agent,}

list_of_newspapers = ['20Minutos', 'ElMundo', 'ElPais', 'LaVanguardia']
list_of_url = ['https://www.20minutos.es/nacional/', 'https://www.elmundo.es/espana', 'https://elpais.com/espana', 'https://www.lavanguardia.com/politica']

for i,url in enumerate(list_of_url):

  newspaper_name = list_of_newspapers[i]

  # Opens up the connection and grabs the requested webpage.

  request = urllib.request.Request(url, None, headers)

  news_connection = uReq(request)

  html = news_connection.read()
  news_connection.close()

  news_soup = soup(html, "html.parser")

  # Finds the articles on the main page.
  containers = news_soup.find_all("article")

  # Creates a .csv file of the articles.

  if not os.path.isdir("newsPapers/" + newspaper_name):
    os.makedirs("newsPapers/" + newspaper_name)

  # Loops through all the articles on the technology page.

  try:
    for container in containers:
      article_link = str(container.a['href'])
      if newspaper_name == "ElPais":
        article_link = 'https://elpais.com' + article_link
      article = Article(article_link, language = 'es')
      article.download()
      article.parse()

      # Gets the data from the website.
      article_title = article.title

      article_authors = str(article.authors)

      date = article.publish_date

      year, month, day = str(date).split()[0].split("-")

      article_text = article.text.replace("\n"," ")

      article.nlp()
      article_keywords = str(article.keywords)
      artcle_summary = article.summary.replace("\n"," ")

      if not os.path.isdir("newsPapers/" +  newspaper_name + "/" + year + "-" + month + "-" + day):
        os.makedirs("newsPapers/" + newspaper_name + "/" + year + "-" + month + "-" + day)

      file_name = article_title.replace(u'\xa0', u' ').replace("|","").replace('"',"").replace("?","").replace("¿","").replace("'","").replace(":"," ").replace("/","")

      fp = open("newsPapers/" + newspaper_name + "/" + year + "-" + month + "-" + day + "/" + file_name + ".txt","w", encoding="utf-8")
      fp.write("title: " + article_title + "\n")
      fp.write("nlp_summary: " + artcle_summary + "\n")
      fp.write("text: " + article_text + "\n")
      fp.write("nlp_keywords: " + article_keywords + "\n")
      fp.write("link: " + article_link + "\n")
      fp.write("authors: " + article_authors + "\n")

      fp.close()

  except Exception as e:
    print(e)
    print(article_link)


# encoding=utf-8

from urllib import urlencode
from oauth2 import Client
from urlparse import urlunparse
import json

class GeoAPI:

  ''' field '''
  client = None
  import hosts
  api = hosts.API_HOST + "/geo/"

  ''' constractor '''
  def __init__(self, client):
    self.client = client

  ''' each API method '''
  def id(self,place_id):
    url = self.api+'id/'+place_id+'.json'
    st,res = self.client.request(url,'GET')
    return st,json.loads(res)

  def nearby_place(self, option):
    url = self.api+'nearby_place.json?'+urlencode(option)
    st,res = self.client.request(url,'GET')
    return st,json.loads(res)

  def reverse_geocode(self,lat,long,accuracy=None,granularity=None,max_results=None,callback=None):
    option = {'lat':lat,'long':long}
    if accuracy: option['accuracy'] = accuracy
    if granularity: option['granularity'] = granularity
    if max_results: option['max_results'] = max_results
    if callback: option['callback'] = callback
    url = self.api+'reverse_geocode.json?'+urlencode(option)
    st,res = self.client.request(url,'GET')
    return st,json.loads(res)

  def search(self,lat=None,long=None,query=None,ip=None,granularity=None,accuracy=None,max_results=None,contained_within=None,street_address=None,callback=None):
    option = {}
    if lat: option['lat'] = lat
    if long: option['long'] = long
    if query: option['query'] = query
    if ip: option['ip'] = ip
    if granularity: option['granularity'] = granularity
    if accuracy: option['accuracy'] = accuracy
    if max_results: option['max_results'] = max_results
    if contained_within: option['contained_within'] = contained_within
    if street_address: option['attribute:street_address'] = street_address
    if callback: option['callback'] = callback
    url = self.api + 'search.json?' + urlencode(option)
    st,res = self.client.request(url,'GET')
    return st,json.loads(res)

  def similar_place(self,lat,long,name,contained_within=None,street_address=None,callback=None):
    option = {'lat':lat,'long':long,'name':name}
    if contained_within: option['contained_within'] = contained_within
    if street_address: option['attribute:street_address'] = street_address
    if callback: option['callback'] = None
    url = self.api + 'similar_place.json?' + urlencode(option)
    st,res = self.client.request(url,'GET')
    return st,json.loads(res)

  def place(self,name,contained_within,token,lat,long,street_address=None,callback=None):
    option = {'name':name,'lat':lat,'long':long,'token':token}
    option['contained_within'] = contained_within
    if street_address: option['attribute:street_address'] = street_address
    if callback: option['callback'] = callback
    url = self.api + 'place.json?' + urlencode(option)
    st,res = self.client.request(url,'GET')
    return st,json.loads(res)


import torch
import torch.nn as nn
from torch.nn import functional as F

import requests

# Save the link to the data, to later be able to delete it from the directory to save storage
link_to_data = f"https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt"
filename = "tiny_shakespeare.txt"

# HYPERPARAMETERS
batch_size = 32
block_size = 8 # what is the maximum context length for predictions?
max_iters = 3000
eval_interval = 300
learning_rate = 1e-2
device = "cpu"
eval_iters = 200
n_embed = 32

torch.manual_seed(1337)

# GET DATA
def download_data(link, filename):
    r = requests.get(link_to_data)

    with open(filename, "w") as f:
        f.write(r.text)
# download_data(link=link_to_data, filename=filename)

# LOAD DATA
with open(filename, "r", encoding="utf-8") as f:
    text = f.read()

# CREATE ENCODING AND DECODING FUNCTIONS
chars = sorted(list(set(text)))
vocab_size = len(chars)
stoi = {s: i for i, s in enumerate(chars)}
itos = {i: s for i, s in enumerate(chars)}
encode = lambda s: [stoi[c] for c in s]
decode = lambda l: ''.join([itos[i] for i in l])

# CREATE TRAIN AND VAL SPLITS
data = torch.tensor(encode(text), dtype=torch.long)
n = int(0.9 * len(data))
train_data = data[:n]
val_data = data[n:]

# FUNCTION TO GET A BATCH OF DATA
def get_batch(split: str = "train"):
    data = train_data if split == "train" else val_data
    ix = torch.randint(len(data) - block_size, (batch_size, ))
    x = torch.stack([data[i: i+block_size] for i in ix])
    y = torch.stack([data[i+1: i+block_size+1] for i in ix])
    x, y = x.to(device), y.to(device)
    return x, y

# FUNCTIONS TO ESTIMATE LOSS
@torch.no_grad()
def estimate_loss():
    out = {}
    model.eval()

    for split in ["train", "val"]:
        losses = torch.zeros(eval_iters)

        for i in range(eval_iters):
            # Get a batch of data
            X, y = get_batch(split)
            # Make predictions
            logits, loss = model(X, y)
            losses[i] = loss.item()

        out[split] = losses.mean()
    model.train()
    return out

# CREATE A SIMPLE BASELINE BIGRAM MODEL
class BigramLanguageModel(nn.Module):
    def __init__(self):
        super().__init__()
        self.token_embedding_table = nn.Embedding(vocab_size, n_embed)
        self.pos_embedding_table = nn.Embedding(block_size, n_embed)
        self.lm_head = nn.Linear(n_embed, vocab_size)

    def forward(self, idx, targets=None):
        B, T = idx.shape

        # idx and targets are both (B,T) tensor of integers
        tok_emb = self.token_embedding_table(idx) # (B,T,C)
        pos_emb = self.pos_embedding_table(torch.arange(T, device=device)) # (T, C)
        x = tok_emb + pos_emb # (B, T, C)
        logits = self.lm_head(x) # (B, T, vocab_size)

        if targets is None:
            loss = None
        else:
            B, T, C = logits.shape
            logits = logits.view(B*T, C)
            targets = targets.view(B*T)
            loss = F.cross_entropy(logits, targets)

        return logits, loss

    def generate(self, idx, max_new_tokens):
        # idx is (B, T) array of indices in the current context
        for _ in range(max_new_tokens):
            # get the predictions
            logits, loss = self(idx)
            # focus only on the last time step
            logits = logits[:, -1, :] # becomes (B, C)
            # apply softmax to get probabilities
            probs = F.softmax(logits, dim=-1) # (B, C)
            # sample from the distribution
            idx_next = torch.multinomial(probs, num_samples=1) # (B, 1)
            # append sampled index to the running sequence
            idx = torch.cat((idx, idx_next), dim=1) # (B, T+1)
        return idx

model = BigramLanguageModel().to(device)

# CREATE A PYTORCH OPTMIZER
optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)

# TRAIN THE MODEL
for i in range(max_iters):

    if i % eval_interval == 0:
        losses = estimate_loss()
        print(f"Epoch: {i}/{max_iters}: train loss: {losses['train']:.4f} | val loss: {losses['val']:.4f}")

    # Sample a batch of data
    xb, yb = get_batch()

    # Make predictions
    logits, loss = model(xb, yb)
    optimizer.zero_grad(set_to_none=True)
    loss.backward()
    optimizer.step()

# GENERATE TEXT FROM THE MODEL
context = torch.zeros((1, 1), dtype=torch.long)
print(decode(model.generate(context, max_new_tokens=500)[0].tolist()))


from __future__ import unicode_literals

import logging

from mopidy import core

import pykka

from .input import InputThread, Handler

logger = logging.getLogger(__name__)

class VolumeHandler(Handler):
  def __init__(self, device_fn, event_type, event_code, actor_ref):
    super(VolumeHandler, self).__init__(device_fn, event_type, event_code)
    self.actor_proxy=actor_ref.proxy()

  def handle(self, event):
    logger.debug("Event for volume {0}".format(event.value))
    self.actor_proxy.change_volume(event.value)

class PlaylistHandler(Handler):
  def __init__(self, device_fn, event_type, event_code, actor_ref):
    super(PlaylistHandler, self).__init__(device_fn, event_type, event_code)
    self.actor_proxy=actor_ref.proxy()

  def handle(self, event):
    logger.debug("Event for playlist {0}".format(event.value))
    self.actor_proxy.change_playlist(event.value)

class MuteHandler(Handler):
  def __init__(self, device_fn, event_type, event_code, actor_ref, longpress=5):
    super(MuteHandler, self).__init__(device_fn, event_type, event_code)
    self.actor_proxy=actor_ref.proxy()
    self.longpress=longpress

  def handle(self, event):
    if (event.value==1):
      self.timestamp=event.sec
    if (event.value==0):
      diff=event.sec-self.timestamp
      if (diff>self.longpress):
        logger.debug("Longpress mute")
        self.actor_proxy.halt()
      else:
        logger.debug("Toggle mute")
        self.actor_proxy.toggle_mute()

class InputFrontend(pykka.ThreadingActor, core.CoreListener):
    def __init__(self, config, core):
        super(InputFrontend, self).__init__()
        self.config = config['headless']
        self.core=core

        self.playlists = None
        self.selected_playlist = None
        if self.config["volume_max"] is None:
            self.config["volume_max"]=100

    def on_start(self):
        self.reload_playlists()
        self.change_playlist(0)

        self.inputthread=InputThread()
        self.inputthread.registerHandler(VolumeHandler(self.config["volume_device"], "EV_REL", self.config["volume_axis"], self.actor_ref))
        self.inputthread.registerHandler(PlaylistHandler(self.config["playlist_device"], "EV_REL", self.config["playlist_axis"], self.actor_ref))
        self.inputthread.registerHandler(MuteHandler(self.config["mute_device"], "EV_KEY", self.config["mute_key"], self.actor_ref))
        self.inputthread.start()

    def on_stop(self):
        self.inputthread.stop()

    def halt(self):
        print("Goodbye")

    def change_volume(self, value):
      volume=self.core.mixer.get_volume().get()+value*self.config["volume_step"]
      if volume<0:
          volume=0
      elif volume>self.config["volume_max"]:
          volume=self.config["volume_max"]

      logger.debug("Volume changed: {0}".format(volume))
      self.core.mixer.set_volume(volume)

    def toggle_mute(self):
        mute = not self.core.mixer.get_mute().get()
        logger.debug("Muted: {0}".format(mute))
        self.core.mixer.set_mute(mute)
        if (mute):
            self.core.playback.stop()
        else:
            self.change_playlist(0)

    def change_playlist(self, value):
        self.selected_playlist= (self.selected_playlist+value) % len(self.playlists)
        logger.debug("Change playlist: {0}".format(self.selected_playlist))
        self.core.tracklist.clear()
        logger.debug("Playing {0}".format(self.playlists[self.selected_playlist].uri))
        tracks = self.core.playlists.get_items(self.playlists[self.selected_playlist].uri).get()
        logger.debug(tracks)
        track_uris = [track.uri for track in tracks]
        logger.debug("Tracks: {0}".format(track_uris))
        self.core.tracklist.add(uris=track_uris)
        self.core.playback.play()

    def reload_playlists(self):
        self.playlists = []
        for playlist in self.core.playlists.as_list().get():
            self.playlists.append(playlist)
            logger.debug(playlist)
        self.selected_playlist = 0
        logger.debug("Found {0} playlists.".format(len(self.playlists)))