import gc
import re
import torch
import os
from models.models import create_pipeline, current_model_is_in_painting, models_memory_checker
from images.latents import create_latents_noise, latents_to_pil
from exceptions.exceptions import CancelException
from utils.settings import get_setting
from utils.images import pil_as_dict, pil_from_dict, inpaint_fill_image
from models.my_gfpgan import gfpgan_dwonload_model, gfpgan_restore_faces
from models.paths import LORA_DIR

from dexpert import progress, progress_canceled, progress_title

REPORT_PREFIX = 'Text To Image'
REPORT_PREFIXES = {
    'txt2img': 'Text to Image',
    'inpaint2img': 'Inpaint',
    'img2img': 'Image to Image',
}

def set_prefix(text):
    global REPORT_PREFIX
    REPORT_PREFIX = text

def report(message):
    progress_title(f'[{REPORT_PREFIX}] - {message}')

def get_lora_path(lora: str) -> str:
    for d in (LORA_DIR, get_setting('add_lora_dir', '')):
        for e in ('.safetensors', '.ckpt'):
            filepath = os.path.join(d, f'{lora}{e}')
            if os.path.exists(filepath):
                return filepath
    return None

def parse_prompt_loras(prompt: str):
    lora_re = re.compile('<lora:[^:]+:[^>]+>')
    lora_list = re.findall(lora_re, prompt)

    lora_items = []
    for lora in lora_list:
        lora = lora.replace('<', '').replace('>', '')
        p = lora.split(':')
        if len(p) != 3:
            continue
        p = [p[1], p[2]]
        try:
            weight = float(p[1])
        except Exception:
            report(f"Invalid lora weigth {p[1]}")
            continue
        filepath = get_lora_path(p[0])
        if not filepath:
            report(f"Lora not found: {p[0]}")
            continue
        lora_items.append([filepath, weight])
    return re.sub(lora_re, '', prompt), lora_items

@torch.no_grad()
def _run_pipeline(pipeline_type, params):
    device = get_setting('device', 'cuda')
    restore_faces = params.get('restore_faces')
    if restore_faces:
        gfpgan_dwonload_model()
        restore_faces = True

    prompt, lora_list = parse_prompt_loras(params['prompt'])
    negative = params['negative']

    if len(negative or '') < 2:
        negative = None

    seed = params['seed']
    model = params["model"]
    cfg = params["cfg"]
    steps = params["steps"]
    width = params["width"]
    height = params["height"]
    reload_model = params.get("reload_model", False)
    input_image = params.get("image")
    input_mask = params.get("mask")
    inpaint_mode = params.get("inpaint_mode", "original")
    controlnets = params.get("controlnets", [])

    if width % 8 != 0:
        width += 8 - width % 8

    if height % 8 != 0:
        height += 8 - height % 8

    variation_enabled = params.get('var_stren', 0) > 0
    var_stren = params.get("var_stren", 0)
    subseed = params['variation'] if variation_enabled else None

    shape = (4, height // 8, width // 8 )
    latents_noise = create_latents_noise(shape, seed, subseed, var_stren)

    generator = None if seed == -1  else torch.Generator(device=device).manual_seed(seed)

    if pipeline_type == 'img2img' and input_mask is not None:
        pipeline_type = 'inpaint2img'

    set_prefix(REPORT_PREFIXES.get(pipeline_type, "Text to Image"))

    report("started")

    report("creating the pipeline")
    pipeline = create_pipeline(pipeline_type, model, controlnets=controlnets, lora_list=lora_list, reload_model=reload_model)
    report("pipeline created")

    if pipeline_type == 'inpaint2img':
        if not current_model_is_in_painting():
            return {
                "error": "The current model is not for inpainting"
            }
    elif current_model_is_in_painting():
        return {
            "error": "The current model is a inpainting model"
        }

    def progress_preview(step, timestep, latents):
        progress(step, steps, latents_to_pil(step, pipeline.vae, latents))
        if progress_canceled():
            raise CancelException()

    additional_args = {}
    if pipeline_type == 'txt2img':
        additional_args = {
            'width': width,
            'height': height,
            'latents': latents_noise,
        }
        if len(controlnets):
            images = []
            conds = []
            for c in controlnets:
                if c['strength'] < 0:
                    c['strength'] = 0
                if c['strength'] > 2.0:
                    c['strength'] = 2.0
                images.append(pil_from_dict(c['image']))
                conds.append(c['strength'])
            if len(images) == 1:
                images = images[0]
            if len(conds) == 1:
                conds = conds[0]
            additional_args['image'] = images
            additional_args['controlnet_conditioning_scale'] = conds
    elif pipeline_type == 'img2img':
        additional_args = {
            'image': pil_from_dict(input_image),
            'strength': params['strength'],
        }
        if len(controlnets):
            additional_args['width'] = width
            additional_args['height'] = height
            images = []
            conds = []
            for c in controlnets:
                if c['strength'] < 0:
                    c['strength'] = 0
                if c['strength'] > 2.0:
                    c['strength'] = 2.0
                images.append(pil_from_dict(c['image']))
                conds.append(c['strength'])
            if len(images) == 1:
                images = images[0]
            if len(conds) == 1:
                conds = conds[0]
            additional_args['controlnet_conditioning_image'] = images
            additional_args['controlnet_conditioning_scale'] = conds
    elif pipeline_type == 'inpaint2img':
        if not current_model_is_in_painting():
            return {
                "error": "The current model is not for in painting"
            }
        image = pil_from_dict(input_image)
        mask = pil_from_dict(input_mask)

        if inpaint_mode != 'original':
            image = inpaint_fill_image(image, mask)

        additional_args = {
            'image': image,
            'mask_image': mask,
            'width': width,
            'height': height,
            'latents': latents_noise,
        }
        if len(controlnets):
            images = []
            conds = []
            for c in controlnets:
                if c['strength'] < 0:
                    c['strength'] = 0
                if c['strength'] > 2.0:
                    c['strength'] = 2.0
                images.append(pil_from_dict(c['image']))
                conds.append(c['strength'])
            if len(images) == 1:
                images = images[0]
            if len(conds) == 1:
                conds = conds[0]
            additional_args['controlnet_conditioning_image'] = images
            additional_args['controlnet_conditioning_scale'] = conds

    pipeline.to(device)
    latents_noise.to(device)
    report("generating the variation" if variation_enabled else "generating the image")
    with torch.inference_mode(), torch.autocast(device):
        result = pipeline(
            prompt,
            negative_prompt=negative,
            guidance_scale=cfg,
            num_inference_steps=steps,
            generator=generator,
            callback=progress_preview,
            **additional_args,
        ).images[0]

    if restore_faces:
        progress(99, 100, pil_as_dict(result))
        result = gfpgan_restore_faces(result)

    report("image generated")
    return pil_as_dict(result)

def run_pipeline(mode: str, params: dict):
    progress(0, 100, {})

    try:
        data = _run_pipeline(mode, params)
    except CancelException:
        print("Image generation canceled")
        data = {"error": "Operation canceled by the user"}
    gc.collect()

    progress(100, 100, {})

    return data


import argparse
import logging
from tools import *
from main import get_best_tpn
import datetime

def run(args):
    """collect parameters"""
    try:  # for split domains
        test = int(args.d.split('/')[-2])
        args.d_name = args.d.split('/')[-3]
    except:
        args.d_name = args.d.split('/')[-2]
    args.p_name = args.p.split('/')[-1].split('.')[0]
    args.d = os.path.join(os.getcwd(), 'benchmarks', args.d)
    args.p = os.path.join(os.getcwd(), 'benchmarks', args.p)
    args.n = int(args.n)
    params = [args.p_type, args.d, args.p, args.d_name, args.p_name, args.n, args.n, 'Python', 'C0', 'K-First',
              [None, None], args.planner, args.verbose, args.timeout, ['Full', 'Semi'], ['Strict', 'Loose'],
              args.ground, 'Gecode']
    method_list = []

    """OUTPUT DIR"""
    DIRECTORIES["PLAN_OUTPUT_DIR"] = os.path.join(os.getcwd(), 'output/planner_output', args.p_type, args.d_name,
                                                  args.p_name, str(args.n))
    if not os.path.exists(DIRECTORIES["PLAN_OUTPUT_DIR"]):
        os.makedirs(DIRECTORIES["PLAN_OUTPUT_DIR"])

    """RUN"""
    name = '_'.join([args.p_type, str(args.n), args.d_name, args.p_name])
    log_name = 'logs/' + '_'.join([name, datetime.datetime.now().strftime("%d-%m %H:%M").replace(' ', '_')])
    logging.basicConfig(filename=log_name + '.log', filemode='w', format='%(name)s - %(levelname)s - %(message)s',
                        level=logging.INFO)
    tpns, msg, naive_tpn = get_best_tpn(params)
    if tpns:
        for regime_method in tpns:
            tpn_name = '_'.join([name, regime_method])
            if tpns[regime_method]:
                tpns[regime_method].name = name
                save_object(tpns[regime_method], tpn_name, DIRECTORIES['TPN_DIR'])
                method_list.append([tpn_name, 'TPN generated'])
            else:
                save_object(naive_tpn, tpn_name, DIRECTORIES['TPN_DIR'])
                method_list.append([tpn_name, 'Time limit reached'])
    else:
        name = '_'.join([name, msg.upper()])
        save_object(naive_tpn, name, DIRECTORIES['TPN_DIR'])
        method_list.append([name, msg])

    return method_list

def get_prob_name(problem):
    if problem[-5:] == '.pddl':
        return problem[:-5]  # remove pddl extension
    else:
        return problem

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Automatically generate a TPN for input temporal planning problem.')
    parser.add_argument("-p_type", help="PDDL problem type", default='Temporal')
    parser.add_argument("-d", help="PDDL domain file path from benchmarks folder", default=None)
    parser.add_argument('-p', type=str, help='problem file path from benchmarks folder')
    parser.add_argument("-n", help="number of plans", default=None)
    parser.add_argument("-ground", help="grounded domain", default=False)
    parser.add_argument('-planner', type=str, help='which planner to use', default='optic_plus')
    parser.add_argument('-verbose', type=bool, help='verbose for planner', default=False)
    parser.add_argument('-timeout', type=int, help='COP timeout (sec), default is 5 min (300 sec)', default=30 * 60)
    args = parser.parse_args()

    run(args)

    # -p_type Classical -d Classical/driverlog/domain.pddl -n 2 -p Classical/driverlog/p01.pddl
    # -p_type Temporal -d Temporal/parking/domain.pddl -n 2 -p Temporal/parking/p15-9-3.pddl
    # -p_type Temporal -d Temporal/quantum_circuit/quantum_circuit_01/domain.pddl -n 2 -p Temporal/quantum_circuit/quantum_circuit_01/problem_n4_i1_u1.0_P1_V2.pddl


# This file requiere that you have ran 'Demo_psnr_all_nets.m' to create the file
# 'add_more_samples/ll_psnr.mat'

# This script plots the psnr-values for the various networks.

import numpy as np;
import matplotlib.pyplot as plt;
from scipy.io import loadmat;
from os.path import join

dest = 'add_more_samples';
data = loadmat(join(dest, 'all_psnr.mat'));

mean_psnr_ell50 = data['ell50_psnr_mean'];
mean_psnr_med50 = data['med50_psnr_mean'];
#mean_psnr_med143 = data['med143_psnr_mean'];

views = data['nbr_views'];

lwidth = 2;
fsize  = 20;

p_max = max(max(mean_psnr_ell50), max(mean_psnr_med50)); #, max(mean_psnr_med143));
p_min = min(min(mean_psnr_ell50), min(mean_psnr_med50)); #, min(mean_psnr_med143));

fig = plt.figure();
plt.plot([50,50],      [0.92*p_min, 1.04*p_max], 'r--', linewidth=lwidth);
#plt.plot([143,143],    [1.02*p_min, 0.92*p_max], 'r--', linewidth=lwidth);
plt.plot(views, mean_psnr_ell50,  'b*-', linewidth=lwidth, ms=12, label='Ell 50');
plt.plot(views, mean_psnr_med50,  'm*-', linewidth=lwidth, ms=12, label='Med 50');
#plt.plot(views, mean_psnr_med143, 'y*-', linewidth=lwidth, ms=12, label='med143');

plt.legend(fontsize=fsize, loc=(0.65,0.6));
plt.axis([0, views[-1], 0.92*p_min, 1.04*p_max]);

plt.xlabel('Number of angles', fontsize=fsize);
plt.ylabel('Avrage PSNR', fontsize=fsize);
#plt.title('FBPConvNet', fontsize=fsize);
fig.savefig(join(dest, 'FBPConvNet_add_more_samples.png'),
            bbox_inches='tight');


from typing import Tuple, Optional, Union
from packaging import version

from torch import Tensor
from torch.nn import functional as F
from torch.nn import Module

from brevitas import torch_version
from brevitas.quant_tensor import QuantTensor
from brevitas.export.onnx.manager import ONNXBaseManager
from brevitas.export.manager import _set_layer_export_handler, _set_layer_export_mode

from .handler.base import StdQOpONNXQuantLayerHandler
from .handler.parameter import StdQOpONNXQuantConv2dHandler
from .handler.parameter import StdQOpONNXQuantAdd2dHandler
from .handler.parameter import StdQOpONNXQuantConv1dHandler
from .handler.parameter import StdQOpONNXQuantLinearHandler
from .handler.act import StdQOpONNXQuantReLUHandler
from .handler.act import StdQOpONNXQuantHardTanhHandler
from .handler.act import StdQOpONNXQuantIdentityHandler
from .handler.act import StdQOpONNXQuantTanhHandler
from .handler.act import StdQOpONNXQuantSigmoidHandler
from .handler.pool import StdQOpONNXQuantMaxPool1d
from .handler.pool import StdQOpONNXQuantMaxPool2d
from .. import OPSET
from ..function import QuantizeLinearFn, DequantizeLinearFn

class StdQOpONNXManager(ONNXBaseManager):
    target_name = 'StdQOpONNX'

    _fn_to_cache = [
        F.relu,
        F.relu6,
        F.hardtanh,
        F.max_pool1d,
        F.max_pool2d,
        F.max_pool3d,
        F.adaptive_max_pool1d,
        F.adaptive_max_pool2d,
        F.adaptive_max_pool3d,
    ]

    handlers = [
        StdQOpONNXQuantConv1dHandler,
        StdQOpONNXQuantConv2dHandler,
        StdQOpONNXQuantAdd2dHandler,
        StdQOpONNXQuantLinearHandler,
        StdQOpONNXQuantReLUHandler,
        StdQOpONNXQuantHardTanhHandler,
        StdQOpONNXQuantIdentityHandler,
        StdQOpONNXQuantTanhHandler,
        StdQOpONNXQuantSigmoidHandler,
        StdQOpONNXQuantMaxPool1d,
        StdQOpONNXQuantMaxPool2d]

    onnx_passes = [
        # remove unused graph inputs & initializers
        "eliminate_unused_initializer"]

    @classmethod
    def solve_enable_onnx_checker(cls, export_kwargs):
        if torch_version >= version.parse('1.5.0'):
            export_kwargs['enable_onnx_checker'] = True

    @classmethod
    def _trace_fn_dispatcher(cls, fn, input, *args, **kwargs):
        cached_io = cls._fn_cache.pop(0)
        if cached_io is not None:
            cached_inp, cached_out = cached_io
            if cached_inp is not None:
                deq_kwargs = StdQOpONNXQuantLayerHandler.dequant_symbolic_kwargs_from_cached_io(
                    cached_inp)
                input = DequantizeLinearFn.apply(input, *deq_kwargs.values())
            output = fn(input, *args, **kwargs)
            if cached_out is not None:
                q_kwargs = StdQOpONNXQuantLayerHandler.quant_symbolic_kwargs_from_cached_io(
                    cached_out)
                output = QuantizeLinearFn.apply(output, *q_kwargs.values())
        else:
            output = fn(input, *args, **kwargs)
        return output

    @classmethod
    def set_export_mode(cls, module: Module, enabled: bool):
        _set_layer_export_mode(module, enabled)

    @classmethod
    def set_export_handler(cls, module: Module):
        _set_layer_export_handler(cls, module)

    @classmethod
    def export_onnx(
            cls,
            module: Module,
            input_shape: Tuple[int, ...] = None,
            export_path: Optional[str] = None,
            input_t: Optional[Union[Tensor, QuantTensor]] = None,
            disable_warnings=True,
            **kwargs):
        output = super().export_onnx(
            module, input_shape, export_path, input_t,
            disable_warnings=disable_warnings, opset_version=OPSET, **kwargs)
        return output


import numpy as np
import pandas as pd
from krwordrank.word import KRWordRank
from soynlp.noun import LRNounExtractor_v2
from konlpy.tag import Okt
from collections import Counter
import copy

# 영어 숫자를 제외한 한국어 댓글를 통해 키워드를 추출
# soynlp, krwordsrank는 비지도 학습이므로 댓글 수가 적으면 정확한 키워드 추출이 안될 수 있다(댓글 50개이하..?)
# 댓글 적은거는 분석 안해주기로

class keywordExtractor: # 키워드 추출 클래스

  def __init__(self, common_words=[]):
    self.common_words = common_words

  def set_common_words(self, common_words):
    self.common_words = common_words

  def get_comments_from_excel(self,link):
    data = pd.read_excel(link)  # 엑셀 파일 읽기
    self.data = copy.deepcopy(data)   # 깊은 복사
    data['comment'] = data['comment'].str.replace('[^ㄱ-ㅎㅏ-ㅣ가-힣 ]','', regex=True)   # 한국어를 제외한 문자 제거(숫자, 영어도 모두 제거)
    data['comment'] = data['comment'].str.replace('^ +','', regex=True)
    data['comment'].replace('',np.nan, inplace=True)
    data = data.dropna(how='any')
    self.comments = data['comment'].values.tolist() # df to list
    return self

  def keywords_from_krwordsrank(self):
    # krwordsrank를 통한 키워드 추출
    wordrank_extractor = KRWordRank()
    # 영어 포함 시 normalize 해주어야함
    keywords, rank, graph = wordrank_extractor.extract(self.comments)
    return keywords

  def keywords_from_soynlp(self):
    # soynlp를 통한 키워드 추출
    # verbose : 함수 실행시 상세 정보를 출력할 것인지
    noun_extractor = LRNounExtractor_v2(verbose=False, extract_compound=True)
    keywords = noun_extractor.train_extract(self.comments)
    keywords = sorted(keywords.items(), reverse=True, key= lambda x : x[1])   # 빈도수 순으로 정렬(빈도수 높은 순으로)
    keywords = dict(((key, value) for key, value in keywords))
    # 딕셔너리로 변환
    # 반환값 : 빈도수 순으로 정렬된 키워드 딕셔너리(키워드 : 빈도수)
    return keywords

  def keywords_from_konlpy(self):
    # konlpy를 통한 키워드 추출
    data = copy.deepcopy(self.comments)   # 깊은 복사
    okt = Okt()
    for i in range(len(data)):   # 토큰화 n번
      data[i] = okt.nouns(data[i])  #명사 추출
    data = sum(data,[])  # 리스트 합치기(2차원 -> 1차원)
    keywords = dict(Counter(data))  #토큰 빈도수 계산
    keywords = sorted(keywords.items(), reverse=True, key= lambda x : x[1]) # 빈도수 순으로 정렬(빈도수 높은 순으로)
    keywords = dict(((key, value) for key, value in keywords))
    # 딕셔너리로 변환
    # 반환값 : 빈도수 순으로 정렬된 키워드 딕셔너리(키워드 : 빈도수)
    return keywords

  def get_best5_keywords(self, keywords, one=False):
    # best 5 키워드 뽑기
    # common_words나 한글자 단어는 키워드에서 제외
    # 파라미터 one은 한글자 단어를 제외시킬건지(True:제외, False:제외x)
    # 한글자 단어를 제외시키면 real 한글자 키워드를 뽑지 못하는 issue(ex : 깡, 진)
    keywords = [*keywords]
    # 딕셔너리의 인덱스값만을 뽑아 배열로 만듬
    if(one == True):
      keywords = [word for word in keywords if len(word)!=1]
      # 한글자 단어 키워드에서 제외
    best_5_keywords = [word for word in keywords if not word in self.common_words][:5]
    # common_words 제외 & 상위 5개 키워드 추출
    # 반환값 : 상위 5개 키워드
    return best_5_keywords

  def get_comments_related_to_keyword(self,keyword, n=None):
    # 키워드에 대한 관련 댓글 추출
    data = self.data.sort_values(by='num_likes', ascending=False)
    # 좋아요 순으로 정렬
    comments = data['comment'].values.tolist()
    related_comments = [] #관련 댓글
    for i in range(len(comments)):
      if(len(related_comments) == n):
        break
      else:
        if(comments[i].find(keyword)!=-1):
          markedword = "<mark>"+keyword+"</mark>"
          comments[i] =comments[i].replace(keyword, markedword)
          related_comments.append(comments[i])
    return related_comments

  def method_comparison(self, link):
    # soynlp, krwordsrank, konlpy 결과 비교
    self.get_comments_from_excel(link)
    keywords = [self.keywords_from_konlpy(), self.keywords_from_soynlp(), self.keywords_from_krwordsrank()]
    print('konlpy best 5 :'+', '.join(self.get_best5_keywords(keywords[0],True)))
    print('soynlp best 5 :'+', '.join(self.get_best5_keywords(keywords[1],True)))
    print('krwordsrank best 5 :'+', '.join(self.get_best5_keywords(keywords[2],False)))

  def get_comments_related_to_best5keywords(self, func, one=True, n=10):
    # best5 키워드를 추출하고 그에 관한 관련댓글 추출
    # 파라미터 func 사용할 키워드 추출 함수 설정
    if(func == self.keywords_from_krwordsrank):
      # func가 krwordsrank를 통한 키워드 추출일 경우 one 파라미터를 False로 수정한다
      one = False
    comments = []  #best5 키워드의 관련 댓글들 n개*5가 담길 리스트
    best_5_keywords = self.get_best5_keywords(func(),one) #best5 키워드
    for i in best_5_keywords:
      comments.append(self.get_comments_related_to_keyword(i,n))
    # best5 키워드와 관련 댓글들을 반환
    return best_5_keywords, comments


from django.contrib import admin
from django.shortcuts import reverse
from django.utils.html import format_html
from django import forms
from ckeditor.widgets import CKEditorWidget
from ckeditor_uploader.widgets import CKEditorUploadingWidget
from django import forms
from . import models
from . import tasks
from openpyxl import load_workbook
from import_export import resources, fields
from import_export.admin import ImportExportModelAdmin
from import_export.widgets import ForeignKeyWidget
from tablib import Dataset
from collections import OrderedDict

class ArticleCreateForm(forms.ModelForm):

    class Meta:
        model = models.Article
        fields = 'title', 'author', 'info', 'body', 'pub_date'

        widgets = {
            'body': CKEditorUploadingWidget(),
            'info': CKEditorWidget(),
            }

class ArticleAttachInlineAdmin(admin.TabularInline):
    model = models.Attach
    fields = 'name', 'attach', 'created'
    readonly_fields = 'created',
    extra = 0

@admin.register(models.Article)
class ArticleAdmin(admin.ModelAdmin):
    list_display = 'id', 'title', 'author', 'pub_date', 'created', 'updated'
    list_display_links = 'title',
    search_fields = 'title', 'author', 'pub_date', 'info', 'body'
    date_hierarchy = 'created'
    readonly_fields = 'id', 'created', 'updated'

    form = ArticleCreateForm

    inlines = ArticleAttachInlineAdmin,

class AreaInlineAdmin(admin.TabularInline):
    model = models.Area
    fields = 'id', 'name',
    readonly_fields = 'id',
    extra = 0

@admin.register(models.Province)
class ProvinceAdmin(admin.ModelAdmin):
    list_display = 'id', 'name'
    list_display_links = 'name',
    search_fields = 'name',

    inlines = AreaInlineAdmin,

class AreaResource(resources.ModelResource):
    province = fields.Field(column_name='province', attribute='province',
                            widget=ForeignKeyWidget(models.Province, 'name'))

    __header_map = {'所属省份': 'province', '区域名称': 'name'}

    class Meta:
        model = models.Area
        exclude = 'id',
        import_id_fields = ('province', 'name')
        clean_model_instances = True

    def get_export_headers(self):
        headers = []
        for field in self.get_export_fields():
            headers.append(self.Meta.model._meta.get_field(field.column_name).verbose_name)
        return headers

    def dehydrate_province(self, obj: models.ProvinceDataRecord):
        return obj.province.name

    def get_or_init_instance(self, instance_loader, row):
        params = {}
        for key in instance_loader.resource.get_import_id_fields():
            field = instance_loader.resource.fields[key]
            params[field.attribute] = field.clean(row)
        try:
            obj = self.get_queryset().get(**params)
            return obj, False
        except models.Area.DoesNotExist:
            headers = [field.column_name for field in self.get_export_fields()]
            for key in headers:
                field = instance_loader.resource.fields[key]
                params[field.attribute] = field.clean(row)
            obj = models.Area.objects.create(**params)
            return obj, True

    def import_data(self, dataset, dry_run=False, raise_errors=False,
                    use_transactions=None, collect_failed_rows=False, **kwargs):
        headers = []
        for header in dataset.headers:
            if header in self.__header_map:
                headers.append(self.__header_map[header])
            else:
                headers.append(header)

        _dataset = Dataset(headers=headers)
        for row in dataset:
            _dataset.append(row)
        dataset = _dataset
        return super(self.__class__, self).import_data(
            dataset=dataset, dry_run=dry_run, raise_errors=raise_errors,
            use_transactions=use_transactions,
            collect_failed_rows=collect_failed_rows, **kwargs
            )

@admin.register(models.Area)
class AreaAdmin(ImportExportModelAdmin):
    resource_class = AreaResource
    list_display = 'id', 'province', 'name'
    list_display_links = 'name',
    search_fields = 'name', 'province',

city_verbose_to_header_map = OrderedDict([("省份", "province"), ('地区', 'area'), ('年度', 'year'),
        ('规模以上工业企业R&D经费支出(万元)', 'r_d'), ('单位GDP能耗(吨/万元)', 'per_unit_gdp'), ('乡-城人均年收入比(/)', 'rural_urban'),
        ('城镇居民人均可支配收入(元)', 'urban_per'), ('农村居民人均可支配收入(元)', 'rural_per'), ('生活垃圾无害化处理率(%)', 'garbage'),
        ('平均万人拥有公共汽车(/)', 'bus_per'), ('城镇生活污水集中处理率(%)', 'urban_sewage'), ('平均受教育年限(年)', 'edu_years'),
        ('死亡率(%)', 'mortality'), ('养老保险覆盖率(%)', 'pension_cov'), ('医疗保险覆盖率(%)', 'medical_cov'),
        ('失业保险覆盖率(%)', 'unemployment_cov'), ('PM2.5年平均浓度(微克/立方米)', 'pm25'), ('二氧化硫排放量(万吨)', 'so2_emissions'),
        ('单位GDP二氧化碳排放量(千克/元)', 'co2_per_gdp'), ('化学需氧量排放量(万吨)', 'cod_emissions'), ('氨氮排放量(万吨)', 'nh_emissions'),
        ('人均耗水量(立方米)', 'water_per'), ('单位GDP用水量(立方米/万元)', 'water_per_gdp'), ('播种面积占比(%)', 'planting_area'),
        ('人均生态足迹(万吨碳/万人)', 'ef_per'), ('绿色创新', 'green_innovation'), ('能源利用', 'energy_use'), ('帕尔玛比率', 'parma_ratio'),
        ('收入', 'income'), ('基础设施建设', 'infrastructure'), ('教育', 'education'), ('预期寿命', 'life_expectancy'),
        ('社会保障', 'social_security'), ('大气污染', 'air_pollution'), ('温室气体排放', 'greenhouse'), ('氮排放', 'nitrogen'),
        ('取水量', 'water_withdrawal'), ('土地利用', 'land_use'), ('生态足迹', 'EF'), ('绿色经济', 'city_green_economy'),
        ('可持续发展', 'city_sustainable'), ('GEP+', 'city_gep_plus'),
        ])
city_header_to_verbose_map = OrderedDict(
    [('province', '省份'), ('area', '地区'), ('year', '年度'), ('r_d', '规模以上工业企业R&D经费支出(万元)'),
        ('per_unit_gdp', '单位GDP能耗(吨/万元)'), ('rural_urban', '乡-城人均年收入比(/)'), ('urban_per', '城镇居民人均可支配收入(元)'),
        ('rural_per', '农村居民人均可支配收入(元)'), ('garbage', '生活垃圾无害化处理率(%)'), ('bus_per', '平均万人拥有公共汽车(/)'),
        ('urban_sewage', '城镇生活污水集中处理率(%)'), ('edu_years', '平均受教育年限(年)'), ('mortality', '死亡率(%)'),
        ('pension_cov', '养老保险覆盖率(%)'), ('medical_cov', '医疗保险覆盖率(%)'), ('unemployment_cov', '失业保险覆盖率(%)'),
        ('pm25', 'PM2.5年平均浓度(微克/立方米)'), ('so2_emissions', '二氧化硫排放量(万吨)'), ('co2_per_gdp', '单位GDP二氧化碳排放量(千克/元)'),
        ('cod_emissions', '化学需氧量排放量(万吨)'), ('nh_emissions', '氨氮排放量(万吨)'), ('water_per', '人均耗水量(立方米)'),
        ('water_per_gdp', '单位GDP用水量(立方米/万元)'), ('planting_area', '播种面积占比(%)'), ('ef_per', '人均生态足迹(万吨碳/万人)'),
        ('green_innovation', '绿色创新'), ('energy_use', '能源利用'), ('parma_ratio', '帕尔玛比率'), ('income', '收入'),
        ('infrastructure', '基础设施建设'), ('education', '教育'), ('life_expectancy', '预期寿命'), ('social_security', '社会保障'),
        ('air_pollution', '大气污染'), ('greenhouse', '温室气体排放'), ('nitrogen', '氮排放'), ('water_withdrawal', '取水量'),
        ('land_use', '土地利用'), ('EF', '生态足迹'), ('city_green_economy', '绿色经济'), ('city_sustainable', '可持续发展'),
        ('city_gep_plus', 'GEP+')

        ])

models.Area.objects.filter(province__name='', name='')

class AreaForeignWidget(ForeignKeyWidget):

    def clean(self, value, row=None, *args, **kwargs):
        return self.model.objects.get(
            name=row["area"], province__name=row["province"],
        )

class CityDataRecordResource(resources.ModelResource):

    province = fields.Field()
    area = fields.Field(
        column_name='area', attribute='area',
        widget=AreaForeignWidget(models.Area, 'name')
        )

    class Meta:
        model = models.CityDataRecord
        exclude = 'id',
        verbose_name = True
        import_fields = city_header_to_verbose_map.keys()
        export_fields = city_header_to_verbose_map.keys()
        import_id_fields = ('area', 'year')
        clean_model_instances = True

    def dehydrate_province(self, obj: models.CityDataRecord):
        return obj.area.province.name

    def dehydrate_area(self, obj: models.CityDataRecord):
        return obj.area.name

    def get_export_headers(self):
        headers = city_verbose_to_header_map.keys()
        return headers

    def get_or_init_instance(self, instance_loader, row):
        params = {}
        for key in instance_loader.resource.get_import_id_fields():
            field = instance_loader.resource.fields[key]
            params[field.attribute] = field.clean(row)
        try:
            obj = self.get_queryset().get(**params)
            return obj, False
        except models.CityDataRecord.DoesNotExist:
            headers = [field.column_name for field in self.get_export_fields()]
            for key in headers:
                field = instance_loader.resource.fields[key]
                if field.attribute:
                    params[field.attribute] = field.clean(row)
            print(params)
            obj = models.CityDataRecord.objects.create(**params)
            return obj, True

    def import_data(self, dataset, dry_run=False, raise_errors=False,
                    use_transactions=None, collect_failed_rows=False, **kwargs):

        headers = []
        for header in dataset.headers:
            if header in city_verbose_to_header_map:
                headers.append(city_verbose_to_header_map[header])
            else:
                headers.append(header)

        _dataset = Dataset(headers=headers)
        for row in dataset:
            _dataset.append(row)
        dataset = _dataset

        return super(self.__class__, self).import_data(
            dataset=dataset, dry_run=dry_run, raise_errors=raise_errors,
            use_transactions=use_transactions, collect_failed_rows=collect_failed_rows, **kwargs)

@admin.register(models.CityDataRecord)
class CityDataRecordAdmin(ImportExportModelAdmin):
    resource_class = CityDataRecordResource
    list_display = 'id', 'area', 'year',
    list_display_links = 'area', 'year',
    # readonly_fields = 'area', 'year', 'r_d', 'per_unit_gdp', 'rural_urban', 'urban_per', 'rural_per', 'garbage', \
    #                   'bus_per', 'urban_sewage', 'edu_years', 'mortality', 'pension_cov', 'medical_cov',\
    #                   'unemployment_cov', 'pm25', 'so2_emissions', 'co2_per_gdp', 'cod_emissions',\
    #                   'nh_emissions', 'water_per', 'water_per_gdp', 'planting_area', 'ef_per',\
    #                   'green_innovation', 'energy_use', 'parma_ratio', 'income', 'infrastructure', 'education',\
    #                   'life_expectancy', 'social_security', 'air_pollution', 'greenhouse', 'nitrogen', \
    #                   'water_withdrawal', 'land_use', 'EF', 'city_green_economy', 'city_sustainable', 'city_gep_plus'
    autocomplete_fields = 'area',
    search_fields = 'area__province__name', 'area__name', 'year'
    fieldsets = (
        ('地区年份', {
            'fields': (('area',), ('year',), )
            }),
        ('绿色经济基础数据', {
            'fields': (('r_d', ),
                       ('per_unit_gdp',), ('rural_urban', ),
                       ('urban_per', ), ('rural_per', ),
                       ('garbage', ), ('bus_per', ),
                       ('urban_sewage', ), ('edu_years', ),
                       ('mortality', ), ('pension_cov', ),
                       ('medical_cov', ), ('unemployment_cov', ),
                       ('pm25', ), ('so2_emissions', ),)
            }),
        ('可持续发展基础数据', {
            'fields': (('co2_per_gdp', ), ('cod_emissions', ),
                       ('nh_emissions', ), ('water_per', ),
                       ('water_per_gdp', ), ('planting_area', ),
                       ('ef_per', ),
                       )
            }

        ),
        ('高级信息', {
            'classes': ('collapse',),
            'fields': (('green_innovation',), ('energy_use',), ('parma_ratio',), ('income',), ('infrastructure',),
                       ('education',), ('life_expectancy',), ('social_security',), ('air_pollution',), ('greenhouse',),
                       ('nitrogen',), ('water_withdrawal',), ('land_use',), ('EF',), ('city_green_economy',),
                       ('city_sustainable',), ('city_gep_plus',)
            ),

            })

    )

class ProvinceDataRecordCreateForm(forms.ModelForm):

    class Meta:
        model = models.ProvinceDataRecord
        fields = ('province', 'year', 'r_d', 'r_d_target', 'per_unit_gdp', 'per_unit_gdp_target',
                  'rural_urban', 'rural_urban_target', 'urban_per', 'urban_per_target', 'rural_per',
                  'rural_per_target', 'garbage', 'garbage_target', 'bus_per', 'bus_per_target',
                  'urban_sewage', 'urban_sewage_target', 'edu_years', 'edu_years_target', 'mortality',
                  'mortality_target', 'pension_cov', 'pension_cov_target', 'medical_cov',
                  'medical_cov_target', 'unemployment_cov', 'unemployment_cov_target',
                  'pm25', 'pm25_target', 'so2_emissions', 'so2_emissions_target', 'co2_per_gdp',
                  'co2_per_gdp_target', 'cod_emissions', 'cod_emissions_target', 'nh_emissions',
                  'nh_emissions_target', 'water_per', 'water_per_target', 'water_per_gdp',
                  'water_per_gdp_target', 'planting_area', 'planting_area_target', 'ef_per',
                  'ef_per_target',
                  # 'attach'
                  )
        # exclude = ('green_innovation', 'energy_use', 'parma_ratio', 'income',
        #            'infrastructure', 'education', 'life_expectancy', 'social_security',
        #            'air_pollution', 'greenhouse', 'nitrogen', 'water_withdrawal', 'land_use',
        #            'EF', 'city_green_economy', 'city_sustainable', 'city_gep_plus')

    def clean_attach(self):
        attach = self.cleaned_data.get('attach')
        province = self.cleaned_data.get('province')
        if isinstance(province, models.Province):
            models.import_excel_validator(attach, province_obj=province)
        return attach

class ProvinceDataRecordResource(resources.ModelResource):
    province = fields.Field(
        column_name='province', attribute='province', widget=ForeignKeyWidget(models.Province, 'name'))

    __header_map = {
        '省份': 'province', '年度': 'year', '企业 R&D 内部经费支出(亿元)': 'r_d', '可再生能源供给占比(%)':
            'renewable_energy_per', '单位GDP能耗(吨/万元)': 'per_unit_gdp',
        '乡-城人均年收入比(/)': 'rural_urban', '城镇居民人均可支配收入(元)': 'urban_per',
        '农村居民人均可支配收入(元)': 'rural_per', '生活垃圾无害化处理率(%)': 'garbage',
        '平均万人拥有公共汽车(/)': 'bus_per', '城镇生活污水集中处理率(%)': 'urban_sewage',
        '平均受教育年限(年)': 'edu_years', '死亡率(%)': 'mortality', '养老保险覆盖率(%)': 'pension_cov',
        '医疗保险覆盖率(%)': 'medical_cov', '失业保险覆盖率(%)': 'unemployment_cov',
        'PM2.5年平均浓度(微克/立方米)': 'pm25', '二氧化硫排放量(万吨)': 'so2_emissions',
        '单位GDP二氧化碳排放量(千克/元)': 'co2_per_gdp', '化学需氧量排放量(万吨)': 'cod_emissions',
        '氨氮排放量(万吨)': 'nh_emissions', '人均耗水量(立方米)': 'water_per',
        '单位GDP用水量(立方米/万元)': 'water_per_gdp', '播种面积占比(%)': 'planting_area',
        '人均生态足迹(万吨碳/万人)': 'ef_per', '企业 R&D 内部经费支出的目标值': 'r_d_target',
        '可再生能源供给占比的目标值': 'renewable_energy_per_target', '单位GDP能耗的目标值': 'per_unit_gdp_target',
        '乡-城人均年收入比的目标值': 'rural_urban_target', '城镇居民人均可支配收入的目标值': 'urban_per_target',
        '农村居民人均可支配收入的目标值': 'rural_per_target', '生活垃圾无害化处理率的目标值': 'garbage_target',
        '平均万人拥有公共汽车的目标值': 'bus_per_target', '城镇生活污水集中处理率的目标值': 'urban_sewage_target',
        '平均受教育年限的目标值': 'edu_years_target', '死亡率的目标值': 'mortality_target',
        '养老保险覆盖率的目标值': 'pension_cov_target', '医疗保险覆盖率的目标值': 'medical_cov_target',
        '失业保险覆盖率的目标值': 'unemployment_cov_target', 'PM2.5年平均浓度的目标值': 'pm25_target',
        '二氧化硫排放量的目标值': 'so2_emissions_target', '单位GDP二氧化碳排放量的目标值': 'co2_per_gdp_target',
        '化学需氧量排放量的目标值': 'cod_emissions_target', '氨氮排放量的目标值': 'nh_emissions_target',
        '人均耗水量的目标值': 'water_per_target', '单位GDP用水量的目标值': 'water_per_gdp_target',
        '播种面积占比的目标值': 'planting_area_target', '人均生态足迹的目标值': 'ef_per_target',
        '绿色创新': 'green_innovation', '可再生能源供给': 'renewable_energy',
        '能源利用': 'energy_use', '帕尔玛比率': 'parma_ratio', '收入': 'income',
        '基础设施建设': 'infrastructure', '教育': 'education',
        '预期寿命': 'life_expectancy', '社会保障': 'social_security',
        '大气污染': 'air_pollution', '温室气体排放': 'greenhouse', '氮排放': 'nitrogen',
        '取水量': 'water_withdrawal', '土地利用': 'land_use',
        '生态足迹': 'EF',
        '绿色经济': 'city_green_economy', '可持续发展': 'city_sustainable',
        'GEP+': 'city_gep_plus'}

    class Meta:
        model = models.ProvinceDataRecord
        exclude = 'id', 'status'
        verbose_name = True
        import_id_fields = ('province', 'year')
        clean_model_instances = True

    def get_export_headers(self):
        headers = []
        for field in self.get_export_fields():
            headers.append(self.Meta.model._meta.get_field(field.column_name).verbose_name)
        return headers

    def dehydrate_province(self, obj: models.ProvinceDataRecord):
        return obj.province.name

    def get_or_init_instance(self, instance_loader, row):
        params = {}
        for key in instance_loader.resource.get_import_id_fields():
            field = instance_loader.resource.fields[key]
            params[field.attribute] = field.clean(row)
        try:
            obj = self.get_queryset().get(**params)
            return obj, False
        except models.ProvinceDataRecord.DoesNotExist:
            headers = [field.column_name for field in self.get_export_fields()]
            for key in headers:
                field = instance_loader.resource.fields[key]
                params[field.attribute] = field.clean(row)
            obj = models.ProvinceDataRecord.objects.create(**params)
            return obj, True

    def import_data(self, dataset, dry_run=False, raise_errors=False,
                    use_transactions=None, collect_failed_rows=False, **kwargs):
        headers = []
        for header in dataset.headers:
            if header in self.__header_map:
                headers.append(self.__header_map[header])
            else:
                headers.append(header)

        _dataset = Dataset(headers=headers)
        for row in dataset:
            _dataset.append(row)
        dataset = _dataset
        return super(self.__class__, self).import_data(
            dataset=dataset, dry_run=dry_run, raise_errors=raise_errors,
            use_transactions=use_transactions,
            collect_failed_rows=collect_failed_rows, **kwargs
            )

@admin.register(models.ProvinceDataRecord)
class ProvinceDataRecordAdmin(ImportExportModelAdmin):
    resource_class = ProvinceDataRecordResource
    list_display = 'id', 'province', 'year', 'calc_url'
    list_display_links = 'province', 'year',
    # readonly_fields = 'green_innovation', 'renewable_energy', 'energy_use', 'parma_ratio', 'income', 'infrastructure', 'education',\
    #                   'life_expectancy', 'social_security', 'air_pollution', 'greenhouse', 'nitrogen', \
    #                   'water_withdrawal', 'land_use', 'EF', 'city_green_economy', 'city_sustainable', 'city_gep_plus',\
    #                   'calc_url',

    readonly_fields = 'calc_url',

    def calc_url(self, ins):
        url = reverse('work:process', kwargs=dict(province_name=ins.province.name, year=ins.year))
        html = f' 计算'
        return format_html(html)

    calc_url.short_description = '执行计算'

    form = ProvinceDataRecordCreateForm
    fieldsets = (
        ('省份年份', {
            'fields': (('province',), ('year',), )
            }),
        ('绿色经济基础数据', {
            'fields': (('r_d', 'r_d_target'), ('renewable_energy_per', 'renewable_energy_per_target'),
                       ('per_unit_gdp', 'per_unit_gdp_target'), ('rural_urban', 'rural_urban_target'),
                       ('urban_per', 'urban_per_target'), ('rural_per', 'rural_per_target'),
                       ('garbage', 'garbage_target'), ('bus_per', 'bus_per_target'),
                       ('urban_sewage', 'urban_sewage_target'), ('edu_years', 'edu_years_target'),
                       ('mortality', 'mortality_target'), ('pension_cov', 'pension_cov_target'),
                       ('medical_cov', 'medical_cov_target'), ('unemployment_cov', 'unemployment_cov_target'),
                       ('pm25', 'pm25_target'), ('so2_emissions', 'so2_emissions_target'),)
            }),
        ('可持续发展基础数据', {
            'fields': (('co2_per_gdp', 'co2_per_gdp_target'), ('cod_emissions', 'cod_emissions_target'),
                       ('nh_emissions', 'nh_emissions_target'), ('water_per', 'water_per_target'),
                       ('water_per_gdp', 'water_per_gdp_target'), ('planting_area', 'planting_area_target'),
                       ('ef_per', 'ef_per_target'),
                       )
            }

        ),
        ('高级信息', {
            'classes': ('collapse',),
            'fields': (('green_innovation',), ('renewable_energy',), ('energy_use',), ('parma_ratio',), ('income',), ('infrastructure',),
                       ('education',), ('life_expectancy',), ('social_security',), ('air_pollution',), ('greenhouse',),
                       ('nitrogen',), ('water_withdrawal',), ('land_use',), ('EF',), ('city_green_economy',),
                       ('city_sustainable',), ('city_gep_plus',)
            ),

            })

    )
    search_fields = 'province',
    autocomplete_fields = 'province',

admin.site.site_header = "数据管理后台"
admin.site.site_title = "数据管理后台"
admin.site.index_title = "欢迎来到数据管理后台"


#!/usr/bin/env python3

"""The setup script."""

from setuptools import setup, find_packages

with open('README.rst') as readme_file:
    readme = readme_file.read()

with open('HISTORY.rst') as history_file:
    history = history_file.read()

requirements = [ ]

test_requirements = ['pytest>=3', ]

setup(
    author="Jonathan Velasco",
    author_email='velasco_jonathan@hotmail.com',
    maintainer="CSC - IT Center for Science :  ElmerFEM Developers",
    python_requires='>=3.7',
    classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)',
        'Natural Language :: English',
        'Programming Language :: Python :: 3',
        'Programming Language :: Python :: 3.7',
        'Programming Language :: Python :: 3.8',
        'Programming Language :: Python :: 3.9',
    ],
    description="Python module for creating the circuit simulation definitions for Elmer FEM. The circuit definitions enable easy setup of coils (e.g. massive, stranded, and foil) in 2D and 3D for magnetodynamics applications.",
    install_requires=requirements,
    license="GNU Lesser General Public License v3",
    long_description=readme + '\n\n' + history,
    include_package_data=True,
    keywords='elmer_circuitbuilder',
    name='elmer_circuitbuilder',
    #packages=find_packages(include=['elmer_circuitbuilder', 'elmer_circuitbuilder.*']),
    test_suite='tests',
    tests_require=test_requirements,
    url='https://github.com/ElmerCSC/elmer_circuitbuilder.git',
    version='v1.0.0',
    zip_safe=False,
    extras_require={"dev":["pytest>=3.7"], },
    py_modules=["elmer_circuitbuilder"],
    package_dir={'': 'elmer_circuitbuilder'},

)


from PygameWidgets.Widgets.UpdateWidget import UpdateWidget
import pygame
from .UpdateWidget import UpdateWidget
from pygame import Rect

class ProgressBar(UpdateWidget):

    def __init__(this, window, x, y, w, h, backgroundColor, foregroundColor, valuePointer, packetreader):
        super().__init__(window, x, y, w, h, backgroundColor, valuePointer, packetreader)
        this.foregroundColor = foregroundColor

    def draw(this):
        super().draw()

        # Draw Border
        x, y = this.x - this.w//2, this.y - this.h//2
        w, h = this.w, this.h
        borderRect = Rect(x, y, w, h)
        pygame.draw.rect(this.window, this.foregroundColor, borderRect, 2, 10)

        # Draw Progress
        x, y = this.x - this.w//2, this.y - this.h//2
        w, h = this.w/100 * this.value//40000, this.h
        progressRect = Rect(x, y, w, h)
        pygame.draw.rect(this.window, this.foregroundColor, progressRect, 0, 10)

        return (this.x, this.y, this.w, this.h)


'''
    Commands examples:
    python3 main.py -t (Scan all files in assets/text/*)
    python3 main.py -t -p assets/text/a.txt (Scan file of assets/text/a.txt)
'''
import json
import os
import requests
from com_function.get_local_storage import get_token

class Type():
    text = "text"

# API of Text Audit
url_list = {
    Type.text: 'https://aip.baidubce.com/rest/2.0/solution/v1/text_censor/v2/user_defined?access_token='
}

# Check text audit

def cli_text_audit(args):
    if args.path:
        path = args.path
        output_result_tojson(path, Type.text)

    else:
        for file_path in os.listdir('assets/text'):
            path = 'assets/text/' + file_path
            output_result_tojson(path, Type.text)

# Output json result to json file
def output_result_tojson(path, type: Type):
    token = get_token('Auth')
    url = url_list[type] + token
    file_handler = open(path, 'r')
    list_of_lines = file_handler.readlines()
    resultDict = {}
    for index, line in enumerate(list_of_lines):
        meta_data = line.strip()
        para = {"text": meta_data}
        r = requests.post(url, data=para)
        resultDict[meta_data] = r.json()
        print('all %s, current' % len(list_of_lines), index+1)
    with open('output/%s-result.json' % path[12:], "w", encoding='utf-8') as outfile:
        outfile.write(json.dumps(resultDict, ensure_ascii=False, indent=4))

    print('Done. Please check jason file!')


from __future__ import absolute_import

import collections
import uuid

from dolfin import Expression, Constant, FunctionSpace, VectorFunctionSpace, \
    interpolate
from . import Material

__all__ = ["CompositeMaterial"]

class CellExpr(Expression):
  def __init__(self, attr, domains, base_material, materials):
    self._attr          = attr
    self._domains       = domains
    self._base_material = base_material
    self._materials     = materials

  def eval_cell(self, result, x, cell):
    try:
      material = self._materials[self._domains[cell.index]]
    except KeyError:
      material = self._base_material

    try:
      result[0] = getattr(material, self._attr)
    except AttributeError:
      result[0] = 0.0

class VectorCellExpr(Expression):
  def __init__(self, attr, domains, base_material, materials):
    self._attr          = attr
    self._domains       = domains
    self._base_material = base_material
    self._materials     = materials

  def eval_cell(self, result, x, cell):
    try:
      material = self._materials[self._domains[cell.index]]
    except KeyError:
      material = self._base_material

    try:
      values = getattr(material, self._attr)
      result[0] = values[0]
      result[1] = values[1]
      result[2] = values[2]
    except (AttributeError, TypeError):
      result[0] = 0.0
      result[1] = 0.0
      result[2] = 0.0
  def value_shape(self):
    return (3,)

class CompositeMaterial(object):
  def __init__(self, state, base_material = Material()):
    self._state         = state
    self._base_material = base_material
    self._cache         = {}
    self._materials     = collections.OrderedDict()
    self.uuid           = uuid.uuid4()

  def __setitem__(self, index, value):
    self._materials[index] = value
    self._cache = {}
    self.uuid = uuid.uuid4()
    return value

  def __getitem__(self, index):
    return self._materials[index]

  def __delitem__(self, index):
    del self._materials[index]
    self._cache = {}
    self.uuid = uuid.uuid4()

  def __setattr__(self, attr, value):
    if attr[0] == '_' or attr == 'uuid':
      return super(CompositeMaterial, self).__setattr__(attr, value)
    else:
      self._cache = {}
      self.uuid = uuid.uuid4()
      return setattr(self._base_material, attr, value)

  def __getattr__(self, attr):
    if attr in self._cache:
      return self._cache[attr]

    materials = {}
    for domain, material in self._materials.items():
      ids = self._state.domain_ids(domain)
      for id in ids:
        if materials.has_key(id):
          materials[id] = Material(materials[id], material)
        else:
          materials[id] = material

    # scalar or vector parameter
    V    = None
    expr = None
    if len(self._materials) == 0:
      self._cache[attr] = Constant(getattr(self._base_material, attr))
    else:
      for material in [self._base_material] + self._materials.values():
        try:
          value = getattr(material, attr)
          if isinstance(value, tuple) or isinstance(value, list):
            assert len(value) == 3
            V    = VectorFunctionSpace(self._state.mesh, 'DG', 0)
            expr = VectorCellExpr(attr, self._state.cell_domains, self._base_material, materials)
          else:
            V    = FunctionSpace(self._state.mesh, 'DG', 0)
            expr = CellExpr(attr, self._state.cell_domains, self._base_material, materials)
          break
        except AttributeError:
          continue

      self._cache[attr] = interpolate(expr, V)

    return self._cache[attr]


from app import config, mongodb
import smtplib, ssl
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart

context = ssl.create_default_context()

def send_email(receiver_email, subject, message):
    with smtplib.SMTP_SSL(config['GMAIL_SMTP'], config['GMAIL_PORT'], context=context) as server:
        server.login(config['GMAIL_ADDRESS'], config['GMAIL_PASSWORD'])
        message["Subject"] = subject
        message["From"] = config['GMAIL_ADDRESS']
        message["To"] = receiver_email
        server.sendmail(config['GMAIL_ADDRESS'], receiver_email, message.as_string())

def verify_code_email(user):
    message = MIMEMultipart("alternative")
    code = mongodb.pass_decrypt(user['verification']['code'])
    url_uuid = str(user['verification']['uuid'])
    url_id = str(user['_id'])
    if config['FLASK_ENV'] == 'development':
        web_url = 'localhost:5000'
    elif config['FLASK_ENV'] == 'development':
        web_url = 'quantumportfolio.herokuapp.com'
    url = f'http://{web_url}/verify?a={url_uuid}&b={url_id}'
    text = f"""\
Hello,

To begin using your account and optimizing a quantum portfolio, please use the following \
verification code to verify your account at the url below within the next 10 minutes:

Code: {code}
URL: {url}

QuantumPortfolio
"""
    html = f"""\

Hello,

            To begin using your account and optimizing a quantum portfolio, please use the following
            verification code to verify your account at the url below within the next 10 minutes:

            Code: {code}

            URL: {url}

"""
    part1 = MIMEText(text, "plain")
    part2 = MIMEText(html, "html")
    message.attach(part1)
    message.attach(part2)
    send_email(user['email'], 'QuantumPortfolio Account Verification', message)


from setuptools import setup, find_packages

def read(filename):
    return [requirement.strip() for requirement in open(filename).readlines()]

setup(
    name="school-system",
    version="0.1.0", # major, minor, patch
    description="a school system API",
    packages=find_packages(),
    include_package_data=True,
    install_requires=read("requirements.txt"),
    extras_require={"dev": read("requirements-dev.txt")}
    )


from sockjs.tornado import SockJSConnection
import json
import tornado.web
import tornado.gen
from .utils import get_redis

class Connection(SockJSConnection):
    clients = set()
    available_experiments = {}
    def send_error(self, message, error_type=None):
        """
        Standard format for all errors
        """
        return self.send(json.dumps({
            'type': 'error' if not error_type else '%s_error' % error_type,
            'data': {
                'message': message
            }
        }))

    def send_message(self, message, data_type):
        """
        Standard format for all messages
        """
        return self.send(json.dumps({
            'type': data_type,
            'data': message
        }))

    def on_experiments_updated(self):
        self.send_message(
            {
                "available_experiments": Connection.available_experiments
            },
            'experiments_list_update' # what to name the channel
        )

    @tornado.gen.coroutine
    def on_open(self, request):
        """
        Choose what model updates to listen to in visualizer:
        """
        self.authenticated = True
        self.experiment_uuid = None
        # choose the channel
        self.on_experiments_updated()
        self.clients.add(self)

    def on_message(self, msg):
        # For every incoming message, broadcast it to all clients
        # self.broadcast(self.clients, msg)
        if len(msg) > 2:
            try:
                json_msg = json.loads(msg)
            except Exception as e:
                print ("Error parsing JSON from client: " + e)
                return
            if "experiment_uuid" in json_msg:
                # new participant to this channel:
                self.experiment_uuid = json_msg["experiment_uuid"]
        else:
            pass

    def on_close(self):
        # If client disconnects, remove him from the clients list
        self.clients.remove(self)

    @classmethod
    def announce_new_experiments(cls, experiments):
        Connection.available_experiments = experiments
        for client in cls.clients:
            if client.authenticated:
                client.on_experiments_updated()

    @classmethod
    def pubsub_message(cls, msg):
        PREFIX = 'updates_'
        if not msg.channel.startswith(PREFIX) or msg.kind == 'psubscribe':
            return
        try:
            experiment_uuid = msg.channel[len(PREFIX):]
            data = json.loads(msg.body)
            data_type = data["type"]
        except Exception as e:
            print('WARNING: exception JSON parsing incoming update: ', e)
            return

        if data_type == 'heartbeat':
            # only used by server - do not pass to website
            return

        for client in cls.clients:
            if client.authenticated and client.experiment_uuid == experiment_uuid:
                # here is what redis sends
                client.send(msg.body)


import pandas as pd
from yelpapi import YelpAPI

from config import api_key

class YelpBusinessResults:

    default_api_key = api_key

    def __init__(
        self,
        search_term,
        location,
        business_types,
        key=None
        ):
        self.search_term = search_term
        self.location = location
        self.business_types = business_types
        if key is None:
            key = self.default_api_key
        self.key = key

        self.output_list = []

        # call the `_get_businesses()` method to fill in the above `output_list`
        self._get_businesses()

    def _get_businesses(self):

        limit = 50

        with YelpAPI(self.key) as yelp_api:

            # grab the first page of results, also use this to get the total number of results
            city_data = yelp_api.search_query(
                term=self.search_term,
                location=self.location,
                limit=limit # look at adding the business_types to this initial request
                )

            num_results = city_data['total']
            pages = (num_results // limit) + 1

            self._parse_response(city_data['businesses'])

            # pull the remaining pages of data, not past 1000 results (this is the Yelp-imposed max number of results)
            for n in range(1, pages):
                offset = limit * n
                if offset < 1000:
                    city_data = yelp_api.search_query(
                        term=self.search_term,
                        location=self.location,
                        limit=limit,
                        offset=offset
                        )
                    self._parse_response(city_data['businesses'])

    def _parse_response(self, businesses_list):
        for i,c in enumerate(businesses_list):
            try:
                country = businesses_list[i]['location']['country']
                business_type = businesses_list[i]['categories'][0]['alias']
                original_search_city = self.location.split(',')[0]

                if country == 'US' and business_type in self.business_types:
                    yelp_id = businesses_list[i]['id']
                    name = businesses_list[i]['name']
                    lat = businesses_list[i]['coordinates']['latitude']
                    long = businesses_list[i]['coordinates']['longitude']
                    yelp_city = businesses_list[i]['location']['city']
                    is_closed = businesses_list[i]['is_closed']
                    rating = businesses_list[i]['rating']
                    zip_code = businesses_list[i]['location']['zip_code']
                    state = businesses_list[i]['location']['state']

                    response_data = {
                        'yelp_id':yelp_id,
                        'name':name,
                        'lat':lat,
                        'lng': long,
                        'original_search_city':original_search_city,
                        'yelp_city':yelp_city,
                        'business`_type':business_type,
                        'is_closed':is_closed,
                        'rating':rating,
                        'zip_code':zip_code,
                        'state':state
                        }
                    self.output_list.append(response_data)
            except IndexError:
                print(i, c)


from django import forms

class KelasForm(forms.Form):
    nama = forms.CharField(max_length=100)

class MuridForm(forms.Form):
    namadepan = forms.CharField(max_length=100)
    namabelakang = forms.CharField(max_length=100)

class BaseMuridFormSet(forms.formsets.BaseFormSet):

    def clean(self):

        if any(self.errors):
            return

        namadepans = []
        namabelakangs = []
        duplikat = False

        for form in self.forms:
            if form.cleaned_data:
                namadepan = form.cleaned_data['namadepan']
                namabelakang = form.cleaned_data['namabelakang']

                if namadepan:
                    if namadepan in namadepans:
                        duplikat = True
                    namadepans.append(namadepan) #ingat, namadepan untuk satu objek,
                                                                             # namadepans untuk list
                    if namabelakang in namabelakangs:
                        duplikat = True
                    namabelakangs.append(namabelakang)

                if duplikat:
                    raise forms.ValidationError("Nama depan atau belakang terduplikasi",
                                                code="duplikat")

MuridFormSet = forms.formset_factory(MuridForm,
                                     formset  = BaseMuridFormSet)


# -*- coding:utf-8 -*-

import os
import sys

import numpy as np
import matplotlib.pyplot as plt
import matplotlib.patches as patches

MAX_LENGTH = 3
#COLORS = ['magenta', 'red', 'blue', 'yellow', 'cyan', 'green']
#COLORS = ['red', 'blue', 'cyan', 'green']
COLORS = ['red', 'blue', 'green']

class Square(object):
    def __init__(self):
        self.width  = np.random.randint(1, MAX_LENGTH + 1)
        self.height = np.random.randint(1, MAX_LENGTH + 1)
        self.color_id = np.random.choice([_ for _ in range(len(COLORS))])

        self.x = None
        self.y = None

    def set_xy(self, x, y):
        self.x = x; self.y = y

    def add_rectangle_to_ax(self, ax):
        r = patches.Rectangle(xy = (self.x, self.y),
                              width = self.width, height = self.height,
                              edgecolor = 'black',
                              facecolor = COLORS[self.color_id]
                              )
        ax.add_patch(r)

class Squares(object):
    def __init__(self):
        self.__data = []

    def add_square(self):
        self.__data.append(Square())

    def set_xys(self, xys):
        # xys = [(x, y), ....]
        assert(len(xys) == len(self.__data))
        for xy, square in zip(xys, self.__data):
            square.set_xy(xy[0], xy[1])

    def add_rectangles_to_ax(self, ax):
        for square in self.__data:
            square.add_rectangle_to_ax(ax)

    def get_s_id_list_with_c_id(self, c_id):
        ret = []
        for s_id, square in enumerate(self.__data):
            if square.color_id == c_id:
                ret.append(s_id)
        return ret

    def __iter__(self):
        return self.__data.__iter__()

    def __getitem__(self, idx):
        return self.__data[idx]

    def __len__(self):
        return len(self.__data)

if __name__ == '__main__':

    ss = Squares()
    for _ in range(2):
        ss.add_square()
    ss.set_xys([(10, 10), (20, 20)])

    fig = plt.figure()
    ax = fig.add_subplot(1,1,1)
    ax.set_xlim([-1, 51])
    ax.set_ylim([-1, 51])
    r = patches.Rectangle(xy = (0, 0),
                          width = 50, height = 50,
                          edgecolor = 'black',
                          facecolor = 'white'
    )
    ax.add_patch(r)
    ss.add_rectangles_to_ax(ax)
    plt.savefig('test.png')


"""
Check whether the primer sequences are in the sequenes.
"""

import logging
import re

# pylint: disable=C0103
# Invalid constant name
logger = logging.getLogger('pipe.primer')

def primer(configs, seqs):
    """Primer check.

    Check whether any of the primer sequences appear in the contig. The primer
    sequences are defined in Primer -> primers options.
    """
    # Logging
    logger.info('Checking primers')

    # Forward
    primers_fwd = re.split(';\n?', configs['primers'])

    # Reverse complement (translate then reverse the order)
    table = str.maketrans('ATGC', 'TACG')
    primers_rev = [p.translate(table)[::-1] for p in primers_fwd]

    # All the primer sequences
    primers = primers_fwd + primers_rev

    # Query data dict (qseqid -> SeqRecord)
    qry = seqs.qry

    # Search primer sequences in all contigs
    for qseqid in qry:
        primers_found = []
        for p in primers:
            indx = qry[qseqid].seq.find(p)
            if indx != -1:
                primers_found.append("{}({})".format(p, indx+1))

        if primers_found:
            seqs.call[qseqid]['primer'] = "Yes"
        else:
            seqs.call[qseqid]['primer'] = "No"

        seqs.info[qseqid]['primer'] = primers_found

    with open(configs['file_out'], 'w') as fh_o:
        # Header
        print("Contig ID\tSequence\tPrimer", file=fh_o)

        for qid in seqs.qids:
            print("{}\t{}\t{}".format(qid,
                                      ";".join(seqs.info[qid]['primer']),
                                      seqs.call[qid]['primer']),
                  file=fh_o)


import os
from lxml import objectify

from openlp.core.ui.wizard import WizardStrings
from .songimport import SongImport

class PowerPraiseImport(SongImport):
    """
    The :class:`PowerpraiseImport` class provides OpenLP with the
    ability to import Powerpraise song files.
    """
    def do_import(self):
        self.import_wizard.progress_bar.setMaximum(len(self.import_source))
        for file_path in self.import_source:
            if self.stop_import_flag:
                return
            self.import_wizard.increment_progress_bar(WizardStrings.ImportingType % os.path.basename(file_path))
            root = objectify.parse(open(file_path, 'rb')).getroot()
            self.process_song(root)

    def process_song(self, root):
        self.set_defaults()
        self.title = str(root.general.title)
        verse_order_list = []
        verse_count = {}
        for item in root.order.item:
            verse_order_list.append(str(item))
        for part in root.songtext.part:
            original_verse_def = part.get('caption')
            # There are some predefined verse defitions in PowerPraise, try to parse these
            if original_verse_def.startswith("Strophe") or original_verse_def.startswith("Teil"):
                verse_def = 'v'
            elif original_verse_def.startswith("Refrain"):
                verse_def = 'c'
            elif original_verse_def.startswith("Bridge"):
                verse_def = 'b'
            elif original_verse_def.startswith("Schluss"):
                verse_def = 'e'
            else:
                verse_def = 'o'
            verse_count[verse_def] = verse_count.get(verse_def, 0) + 1
            verse_def = '%s%d' % (verse_def, verse_count[verse_def])
            verse_text = []
            for slide in part.slide:
                if not hasattr(slide, 'line'):
                    continue  # No content
                for line in slide.line:
                    verse_text.append(str(line))
            self.add_verse('\n'.join(verse_text), verse_def)
            # Update verse name in verse order list
            for i in range(len(verse_order_list)):
                if verse_order_list[i].lower() == original_verse_def.lower():
                    verse_order_list[i] = verse_def

        self.verse_order_list = verse_order_list
        if not self.finish():
            self.log_error(self.import_source)


import torch

from src.common import get_rays, raw2outputs_nerf_color, sample_pdf

class Renderer(object):
    def __init__(self, cfg, args, slam, points_batch_size=500000, ray_batch_size=50000):
        self.prefix_keys = ["", "2_", "3_"]
        self.num_sensors = cfg["num_sensors"]
        self.num_features = cfg["num_features"]

        self.patch_size = cfg["patch_size"]
        self.unused_list = ["error", "rays_o", "rays_d", "color", "local"]
        if self.num_features == 2:
            self.unused_list += ["dx", "dy", "normal"]
        elif self.num_features == 7:
            pass
        else:
            raise Exception("Number of features != 2 or 7")

        self.ray_batch_size = ray_batch_size
        self.points_batch_size = points_batch_size

        self.lindisp = cfg["rendering"]["lindisp"]
        self.perturb = cfg["rendering"]["perturb"]
        self.N_samples = cfg["rendering"]["N_samples"]
        self.N_surface = cfg["rendering"]["N_surface"]
        self.N_importance = cfg["rendering"]["N_importance"]

        self.scale = cfg["scale"]
        self.occupancy = cfg["occupancy"]
        self.bound = slam.bound

        self.H, self.W, self.fx, self.fy, self.cx, self.cy = (
            slam.H,
            slam.W,
            slam.fx,
            slam.fy,
            slam.cx,
            slam.cy,
        )

    def eval_points(self, p, d_feats, c_feats, decoders, c=None, stage="color", device="cuda:0"):
        """Evaluate the occupancy and/or color value for the points.

        Args:
            p (tensor, N*3): Point coordinates.
            d_feats (tensor, N*Y*2): Tensor of added features for uncertainty evaluation.
            c_feats (tensor, N*Z*2): Tensor of added features for color uncertainty evaluation.
            decoders (nn.module decoders): Decoders.
            c (dicts, optional): Feature grids. Defaults to None.
            stage (str, optional): Query stage, corresponds to different levels. Defaults to 'color'.
            device (str, optional): CUDA device. Defaults to 'cuda:0'.

        Returns:
            ret (tensor): occupancy (and color) value of input points.
        """

        p_split = torch.split(p, self.points_batch_size)

        bound = self.bound

        rets = []

        for i, pi in enumerate(p_split):
            # mask for points out of bound
            mask_x = (pi[:, 0] < bound[0][1]) & (pi[:, 0] > bound[0][0])
            mask_y = (pi[:, 1] < bound[1][1]) & (pi[:, 1] > bound[1][0])
            mask_z = (pi[:, 2] < bound[2][1]) & (pi[:, 2] > bound[2][0])
            mask = mask_x & mask_y & mask_z

            pi = pi.unsqueeze(0)
            ret, var = decoders(pi, c_grid=c, stage=stage, d_feats=d_feats, c_feats=c_feats)
            ret = ret.squeeze(0)
            if len(ret.shape) == 1 and ret.shape[0] == 4:
                ret = ret.unsqueeze(0)

            ret[~mask, 3] = 100
            rets.append(ret)

        ret = torch.cat(rets, dim=0)

        return ret, var

    def render_batch_ray(self, c, decoders, rays_d, rays_o, batch_dict, device, stage):
        """Render color, depth and uncertainty of a batch of rays.

        Args:
            c (dict): feature grids.
            decoders (nn.module): decoders.
            rays_d (tensor, N*3): rays direction.
            rays_o (tensor, N*3): rays origin.
            batch_dict (dict): Includes various features. sensor depth image, normal, color...
            device (str): device name to compute on.
            stage (str): query stage.

        Returns:
            depth (tensor): rendered depth.
            d_var (tensor): rendered sensor uncertainty.
            uncertainty (tensor): rendered model uncertainty.
            color (tensor): rendered color.

        """

        N_samples = self.N_samples
        N_surface = self.N_surface
        N_importance = self.N_importance

        N_rays = rays_o.shape[0]

        features = list(batch_dict.keys())

        c_features = [feat for feat in features if (feat[-1].isdigit() and feat.startswith("color"))]

        # only keeps expanded patch of features
        features = [
            feat for feat in features if (feat[-1].isdigit() and not any(feat.startswith(s) for s in self.unused_list))
        ]

        # filters out additional sensors
        features = [feat for feat in features if not feat.startswith("2")]

        gt_depth_samples = batch_dict["depth"].reshape(-1, 1).repeat(1, N_samples)
        for i in range(1, self.num_sensors):
            prefix = self.prefix_keys[i]
            gt_depth_samples += batch_dict[f"{prefix}depth"].reshape(-1, 1).repeat(1, N_samples)
        gt_depth_samples /= self.num_sensors
        near = gt_depth_samples * 0.01

        with torch.no_grad():
            det_rays_o = rays_o.clone().detach().unsqueeze(-1)  # (N, 3, 1)
            det_rays_d = rays_d.clone().detach().unsqueeze(-1)  # (N, 3, 1)
            t = (self.bound.unsqueeze(0).to(device) - det_rays_o) / det_rays_d  # (N, 3, 2)
            far_bb, _ = torch.min(torch.max(t, dim=2)[0], dim=1)
            far_bb = far_bb.unsqueeze(-1)
            far_bb += 0.01

        # in case the bound is too large
        max_depth = torch.max(batch_dict["depth"] * 1.2)
        for i in range(1, self.num_sensors):
            prefix = self.prefix_keys[i]
            max_depth = max((max_depth, torch.max(batch_dict[f"{prefix}depth"] * 1.2)))
        far = torch.clamp(far_bb, 0, max_depth)  # NOTE: may cause vis to be incomplete if all depth measurements are 0
        if far.mean() == 0:
            far = torch.ones_like(far) * far_bb

        # add points selected near surface
        if N_surface > 0:
            # since we want to colorize even on regions with no depth sensor readings,
            # meaning colorize on interpolated geometry region,
            # we sample all pixels (not using depth mask) for color loss.
            # Therefore, for pixels with non-zero depth value, we sample near the surface,
            # since it is not a good idea to sample 16 points near (half even behind) camera,
            # for pixels with zero depth value, we sample uniformly from camera to max_depth.

            z_val_surface_list = []

            for i in range(self.num_sensors):
                depth_key = f"{self.prefix_keys[i]}depth"
                gt_none_zero_mask = batch_dict[depth_key] > 0
                gt_none_zero = batch_dict[depth_key][gt_none_zero_mask]
                gt_none_zero = gt_none_zero.unsqueeze(-1)
                gt_depth_surface = gt_none_zero.repeat(1, N_surface)
                t_vals_surface = torch.linspace(0.0, 1.0, steps=N_surface).double().to(device)
                # emperical range 0.05*depth
                z_vals_surface_depth_none_zero = 0.95 * gt_depth_surface * (
                    1.0 - t_vals_surface
                ) + 1.05 * gt_depth_surface * (t_vals_surface)
                z_vals_surface = torch.zeros(batch_dict[depth_key].shape[0], N_surface).to(device).double()
                gt_none_zero_mask = gt_none_zero_mask.squeeze(-1)
                z_vals_surface[gt_none_zero_mask, :] = z_vals_surface_depth_none_zero
                near_surface = 0.001
                far_surface = torch.max(batch_dict[depth_key])
                z_vals_surface_depth_zero = near_surface * (1.0 - t_vals_surface) + far_surface * (t_vals_surface)

                z_vals_surface_depth_zero = z_vals_surface_depth_zero.unsqueeze(0).repeat((~gt_none_zero_mask).sum(), 1)

                z_vals_surface[~gt_none_zero_mask, :] = z_vals_surface_depth_zero

                z_val_surface_list.append(z_vals_surface)

            z_vals_surface = torch.cat(z_val_surface_list, -1)

        t_vals = torch.linspace(0.0, 1.0, steps=N_samples, device=device)

        if not self.lindisp:
            z_vals = near * (1.0 - t_vals) + far * (t_vals)
        else:
            z_vals = 1.0 / (1.0 / near * (1.0 - t_vals) + 1.0 / far * (t_vals))

        if self.perturb > 0.0:
            # get intervals between samples
            mids = 0.5 * (z_vals[..., 1:] + z_vals[..., :-1])
            upper = torch.cat([mids, z_vals[..., -1:]], -1)
            lower = torch.cat([z_vals[..., :1], mids], -1)
            # stratified samples in those intervals
            t_rand = torch.rand(z_vals.shape).to(device)
            z_vals = lower + (upper - lower) * t_rand

        if N_surface > 0:
            z_vals, _ = torch.sort(torch.cat([z_vals, z_vals_surface.double()], -1), -1)

        pts = rays_o[..., None, :] + rays_d[..., None, :] * z_vals[..., :, None]  # [N_rays, N_samples+N_surface, 3]

        pointsf = pts.reshape(-1, 3)

        features_list = []
        c_features_list = []
        c_num_features = 0

        for i in range(self.num_sensors):
            features_list.append([])
            prefix = self.prefix_keys[i]
            num_features = 0

            for feature in features:
                if batch_dict[feature].shape[-1] == 3:
                    num_features += 3
                    features_list[-1].append(batch_dict[f"{prefix}{feature}"].reshape(-1, 3))
                else:
                    num_features += 1
                    features_list[-1].append(batch_dict[f"{prefix}{feature}"].reshape(-1, 1))

        for feature in c_features:
            c_num_features += 3
            c_features_list.append(batch_dict[feature].reshape(-1, 3))

        d_feats_list = []

        for i in range(self.num_sensors):
            d_feats_list.append(torch.cat(features_list[i], 1).to(device).float())
            d_feats_list[-1] = d_feats_list[-1].reshape(-1, num_features)

        d_feats = torch.cat([feats.unsqueeze(2) for feats in d_feats_list], 2)
        c_feats = torch.cat(c_features_list, 1).to(device).float()
        c_feats = c_feats.reshape(-1, c_num_features)

        raw, var = self.eval_points(pointsf, d_feats, c_feats, decoders, c, stage, device)

        raw = raw.reshape(N_rays, N_samples + self.num_sensors * N_surface, -1)

        depth, var_out, uncertainty, color, weights = raw2outputs_nerf_color(
            raw,
            var,
            z_vals,
            rays_d,
            occupancy=self.occupancy,
            device=device,
        )

        if N_importance > 0:
            z_vals_mid = 0.5 * (z_vals[..., 1:] + z_vals[..., :-1])
            z_samples = sample_pdf(
                z_vals_mid,
                weights[..., 1:-1],
                N_importance,
                det=(self.perturb == 0.0),
                device=device,
            )
            z_samples = z_samples.detach()
            z_vals, _ = torch.sort(torch.cat([z_vals, z_samples], -1), -1)

            pts = rays_o[..., None, :] + rays_d[..., None, :] * z_vals[..., :, None]
            pts = pts.reshape(-1, 3)
            raw, var = self.eval_points(pts, decoders, c, stage, device)
            raw = raw.reshape(N_rays, N_samples + N_importance + N_surface, -1)

            depth, uncertainty, color, weights = raw2outputs_nerf_color(
                raw, var, z_vals, rays_d, occupancy=self.occupancy, device=device
            )

        return depth, var_out, uncertainty, color

    def render_img(self, c, decoders, c2w, frame_dict, feature_maps, device, stage):
        """Render out depth, uncertainty, and color images.

        Args:
            c (dict): feature grids.
            decoders (nn.module): decoders.
            c2w (tensor): camera to world matrix of current frame.
            device (str): device name to compute on.
            stage (str): query stage.
            gt_depth (tensor, optional): sensor depth image. Defaults to None.

        Returns:
            depth (tensor, H*W): rendered depth image.
            uncertainty (tensor, H*W): rendered uncertainty image.
            color (tensor, H*W*3): rendered color image.
        """

        img_dict = {}

        with torch.no_grad():
            H = self.H
            W = self.W

            rays_o, rays_d = get_rays(H, W, self.fx, self.fy, self.cx, self.cy, c2w, device)
            rays_o = rays_o.reshape(-1, 3)
            rays_d = rays_d.reshape(-1, 3)

            depth_list = []
            d_var_list = []
            uncertainty_list = []
            color_list = []

            ray_batch_size = self.ray_batch_size

            for feature in feature_maps:
                if frame_dict[feature].shape[-1] == 3:
                    img_dict[feature] = frame_dict[feature].reshape(-1, 3)
                else:
                    img_dict[feature] = frame_dict[feature].reshape(-1)
                for n in range(self.patch_size):
                    for m in range(self.patch_size):
                        mlp_index = n * self.patch_size + m
                        temp_feature_map = torch.roll(
                            frame_dict[feature],
                            shifts=(n - self.patch_size // 2, m - self.patch_size // 2),
                            dims=(0, 1),
                        )
                        if frame_dict[feature].shape[-1] == 3:
                            img_dict[f"{feature}_{mlp_index}"] = temp_feature_map.reshape(-1, 3)
                        else:
                            img_dict[f"{feature}_{mlp_index}"] = temp_feature_map.reshape(-1)

            img_dict["rays_o"] = rays_o.float()
            img_dict["rays_d"] = rays_d.float()

            batch_dict = {}

            for i in range(0, rays_d.shape[0], ray_batch_size):
                rays_d_batch = rays_d[i : i + ray_batch_size]
                rays_o_batch = rays_o[i : i + ray_batch_size]

                for feature in feature_maps + ["rays_o", "rays_d"]:
                    batch_dict[feature] = img_dict[feature][i : i + ray_batch_size]
                    for mlp_index in range(self.patch_size**2):
                        if feature not in ["rays_o", "rays_d"]:
                            batch_dict[f"{feature}_{mlp_index}"] = img_dict[f"{feature}_{mlp_index}"][
                                i : i + ray_batch_size
                            ]
                ret = self.render_batch_ray(c, decoders, rays_d_batch, rays_o_batch, batch_dict, device, stage)

                depth, d_var, uncertainty, color = ret
                depth_list.append(depth.double())
                d_var_list.append(d_var.double())
                uncertainty_list.append(uncertainty.double())
                color_list.append(color)

            depth = torch.cat(depth_list, dim=0)
            d_var = torch.cat(d_var_list, dim=0)
            uncertainty = torch.cat(uncertainty_list, dim=0)
            color = torch.cat(color_list, dim=0)

            depth = depth.reshape(H, W)
            d_var = d_var.reshape(H, W, -1)
            uncertainty = uncertainty.reshape(H, W)
            color = color.reshape(H, W, 3)
            return depth, d_var, uncertainty, color


import os
from IPython.display import display, Image

root = '../notMNIST_small'
data_folders = [os.path.join(root, d) for d in sorted(os.listdir(root))]
folder = data_folders[0]

# Display 10 images in notebook
for file in os.listdir(folder)[:10]:
    display(Image(filename=os.path.join(folder, file)))


import psycopg2

class Database:
    def __init__(self, db_name) -> None:
        self.name = db_name
        self.cursor = None
        self.conn = self.__try_connect(db_name, autocommit=True)
        self.cursor = self.conn.cursor()

    @staticmethod
    def __try_connect(db_name, *, autocommit):
        try:
            conn = psycopg2.connect(dbname=db_name, host="localhost", port=5432)
        except (Exception, psycopg2.DatabaseError) as error:
            print(f"Database connection failed: {error}")
            raise error
        conn.autocommit = autocommit
        return conn


#!/usr/bin/python

import sys
import config
import os.path
import dependency
from optparse import OptionParser

def run(files):
    for input_filename in files:
        f = config.get_current_file(input_filename, '.pre')
        if os.path.isfile(f):
            os.remove(f)

        f = config.get_current_file(input_filename, '.norm')
        if os.path.isfile(f):
            os.remove(f)

        f = config.get_current_file(input_filename, '.sort')
        if os.path.isfile(f):
            os.remove(f)

        f = config.get_current_file(input_filename, '.pv')
        if os.path.isfile(f):
            os.remove(f)

        f = config.get_current_file(input_filename, '.cat_pv')
        if os.path.isfile(f):
            os.remove(f)

if __name__ == "__main__":
    parser = OptionParser()
    parser.add_option('-c', '--column',
        help='Column to sum', type=int, default=4)

    options, arguments = parser.parse_args()

    # Filename on the command line have precedence over configuration
    files = config.input_files
    if arguments:
        files = arguments

    run(files)


from __future__ import print_function, division

from keras.layers import Input, Dense, Flatten, Dropout, Reshape
from keras.layers import BatchNormalization, Activation, Conv2D, Conv2DTranspose
from keras.layers.advanced_activations import LeakyReLU
from keras.models import Model
from keras.optimizers import Adam

from keras.datasets import cifar10
import keras.backend as K

import matplotlib.pyplot as plt
import cv2
import sys
import numpy as np
import os
from sklearn.model_selection import train_test_split

img_data_list=[]
def get_generator(input_layer):

    hid = Dense(128 * 16 * 16, activation='relu')(input_layer)
    hid = BatchNormalization(momentum=0.9)(hid)
    hid = LeakyReLU(alpha=0.1)(hid)
    hid = Reshape((16, 16, 128))(hid)

    hid = Conv2D(128, kernel_size=5, strides=1,padding='same')(hid)
    hid = BatchNormalization(momentum=0.9)(hid)
    #hid = Dropout(0.5)(hid)
    hid = LeakyReLU(alpha=0.1)(hid)

    hid = Conv2DTranspose(128, 4, strides=2, padding='same')(hid)
    hid = BatchNormalization(momentum=0.9)(hid)
    hid = LeakyReLU(alpha=0.1)(hid)

    hid = Conv2D(128, kernel_size=5, strides=1, padding='same')(hid)
    hid = BatchNormalization(momentum=0.9)(hid)
    #hid = Dropout(0.5)(hid)
    hid = LeakyReLU(alpha=0.1)(hid)

    hid = Conv2D(128, kernel_size=5, strides=1, padding='same')(hid)
    hid = BatchNormalization(momentum=0.9)(hid)
    hid = LeakyReLU(alpha=0.1)(hid)

    hid = Conv2D(3, kernel_size=5, strides=1, padding="same")(hid)
    out = Activation("tanh")(hid)

    model = Model(input_layer, out)
    model.summary()

    return model, out

def get_discriminator(input_layer):

#Requires the input layer as input, outputs the model and the final layer

    hid = Conv2D(128, kernel_size=1, strides=1, padding='same')(input_layer)
    hid = BatchNormalization(momentum=0.9)(hid)
    hid = LeakyReLU(alpha=0.1)(hid)

    hid = Conv2D(128, kernel_size=2, strides=2, padding='same')(hid)
    hid = BatchNormalization(momentum=0.9)(hid)
    hid = LeakyReLU(alpha=0.1)(hid)

    hid = Conv2D(128, kernel_size=2, strides=2, padding='same')(hid)
    hid = BatchNormalization(momentum=0.9)(hid)
    hid = LeakyReLU(alpha=0.1)(hid)

    hid = Conv2D(128, kernel_size=2, strides=2, padding='same')(hid)
    hid = BatchNormalization(momentum=0.9)(hid)
    hid = LeakyReLU(alpha=0.1)(hid)

    hid = Flatten()(hid)
    hid = Dropout(0.4)(hid)
    out = Dense(1, activation='sigmoid')(hid)

    model = Model(input_layer, out)

    model.summary()

    return model, out

from keras.preprocessing import image

def load_lfw():
        data_path = './lfw/'
        data_dir_list = os.listdir(data_path)
        label_list_name=[]
        for dataset in data_dir_list:
            img_list = os.listdir(data_path+'/'+ dataset)
            for img in img_list:
                label_list_name.append(dataset)
                input_img=cv2.imread(data_path + '/'+ dataset + '/'+ img )
                input_img_resize=cv2.resize(input_img,(256,256))
                img_data_list.append(input_img_resize)

        img_data = np.asarray(img_data_list)
        img_data = img_data.astype('float32')
        img_data = np.expand_dims(img_data, -1)
        img_data.shape
        return img_data, label_list_name

def load_jaffe():
        data_path = './jaffe/'
        data_dir_list = os.listdir(data_path)
        for dataset in data_dir_list:
            img_list=os.listdir(data_path+'/'+ dataset)
            print ('Loaded the images of dataset-'+'{}\n'.format(dataset))
            for img in img_list:
                input_img=cv2.imread(data_path + '/'+ dataset + '/'+ img )
                input_img=cv2.cvtColor(input_img, cv2.COLOR_BGR2GRAY)
                input_img_resize=cv2.resize(input_img,(256,256))
                img_data_list.append(input_img_resize)

        img_data = np.asarray(img_data_list)
        img_data = img_data.astype('float32')
        img_data = np.expand_dims(img_data, -1)
        img_data.shape

        num_of_samples = img_data.shape[0]
        labels = np.ones((num_of_samples,),dtype='int64')

        labels[0:29]=0 #30
        labels[30:58]=1 #29
        labels[59:90]=2 #32
        labels[91:121]=3 #31
        labels[122:151]=4 #30
        labels[152:182]=5 #31
        labels[183:]=6 #30

        names = ['ANGRY','DISGUST','FEAR','HAPPY','NEUTRAL','SAD','SURPRISE']
        return img_data, labels

def generate_noise(n_samples, noise_dim):
  X = np.random.normal(0, 1, size=(n_samples, noise_dim))
  return X

def show_imgs(batchidx):
  noise = generate_noise(9, 100)
  gen_imgs = generator.predict(noise)

  fig, axs = plt.subplots(3, 3)
  count = 0
  for i in range(3):
    for j in range(3):
      # Dont scale the images back, let keras handle it
      img = image.array_to_img(gen_imgs[count], scale=True)
      axs[i,j].imshow(img)
      axs[i,j].axis('off')
      count += 1
  plt.show()
  plt.close()

# GAN creation
img_input = Input(shape=(256,256,3))
discriminator, disc_out = get_discriminator(img_input)
discriminator.compile(optimizer=Adam(0.0002, 0.5), loss='binary_crossentropy', metrics=['accuracy'])

discriminator.trainable = False

noise_input = Input(shape=(100,))
generator, gen_out = get_generator(noise_input)

gan_input = Input(shape=(100,))
x = generator(gan_input)
gan_out = discriminator(x)
gan = Model(gan_input, gan_out)
gan.summary()

gan.compile(optimizer=Adam(0.0002, 0.5), loss='binary_crossentropy')

BATCH_SIZE = 16

# # Get training images
faces, labels = load_lfw()

X_train, xtest,ytrain,ytest = train_test_split(faces, labels, test_size=0.2,shuffle=True)

X_train = (X_train.astype(np.float32) - 127.5) / 127.5

num_batches = int(BATCH_SIZE / 2)

N_EPOCHS = 50
for epoch in range(N_EPOCHS):

  cum_d_loss = 0.
  cum_g_loss = 0.

  for batch_idx in range(num_batches):
    # Get the next set of real images to be used in this iteration
    images = X_train[batch_idx*BATCH_SIZE : (batch_idx+1)*BATCH_SIZE]

    noise_data = generate_noise(BATCH_SIZE, 100)
    generated_images = generator.predict(noise_data)

    # Train on soft labels (add noise to labels as well)
    noise_prop = 0.05 # Randomly flip 5% of labels

    # Prepare labels for real data
    true_labels = np.zeros((BATCH_SIZE, 1)) + np.random.uniform(low=0.0, high=0.1, size=(BATCH_SIZE, 1))
    flipped_idx = np.random.choice(np.arange(len(true_labels)), size=int(noise_prop*len(true_labels)))
    true_labels[flipped_idx] = 1 - true_labels[flipped_idx]

    # Train discriminator on real data
    d_loss_true = discriminator.train_on_batch(images, true_labels)

    # Prepare labels for generated data
    gene_labels = np.ones((BATCH_SIZE, 1)) - np.random.uniform(low=0.0, high=0.1, size=(BATCH_SIZE, 1))
    flipped_idx = np.random.choice(np.arange(len(gene_labels)), size=int(noise_prop*len(gene_labels)))
    gene_labels[flipped_idx] = 1 - gene_labels[flipped_idx]

    # Train discriminator on generated data
    d_loss_gene = discriminator.train_on_batch(generated_images, gene_labels)

    d_loss = 0.5 * np.add(d_loss_true, d_loss_gene)
    cum_d_loss += d_loss

    # Train generator
    noise_data = generate_noise(BATCH_SIZE, 100)
    g_loss = gan.train_on_batch(noise_data, np.zeros((BATCH_SIZE, 1)))
    cum_g_loss += g_loss

  print('  Epoch: {}, Generator Loss: {}, Discriminator Loss: {}'.format(epoch+1, cum_g_loss/num_batches, cum_d_loss/num_batches))
  show_imgs("epoch" + str(epoch))


# -*- coding: utf-8 -*-
# @Project: Spider_Hero
# @Author: little fly
# @File name: 6.多协程2
# @Create time: 2020/8/15 0:02
import requests
import asyncio
import time
import aiohttp

async def get_page(url):
    async with aiohttp.ClientSession() as session:
        # get()、post():
        # headers,params/data,proxy='http://ip:port'
        async with await session.get(url) as response:
            # text()返回字符串形式的响应数据
            # read()返回的二进制形式的响应数据
            # json()返回的就是json对象
            # 注意：获取响应数据操作之前一定要使用await进行手动挂起
            page_text = await response.text()
            print(page_text)

if __name__ == '__main__':
    urls = [
        'http://127.0.0.1:5000/zhx',
        'http://127.0.0.1:5000/lst',
        'http://127.0.0.1:5000/zxy'
    ]
    start_time = time.time()
    tasks = []
    for url in urls:
        c = get_page(url)
        task = asyncio.ensure_future(c)
        tasks.append(task)
    loop = asyncio.get_event_loop()
    # 需要将任务列表封装到wait中
    loop.run_until_complete(asyncio.wait(tasks))
    print('all_time:', time.time() - start_time)


import torch
import torch.nn as nn
import torch.optim as optim
from tqdm import tqdm
import pickle
import argparse
import torch.nn.functional as F
from sklearn.model_selection import train_test_split
import numpy as np
import itertools
from base_model import Model

# Take available models, evaluate RMSE on random sample of dataset
# Slow_fusion x [vanilla_rnn lstm lstmn transformer_abs]
def eval(model, data, device):
    criterion = nn.MSELoss()
    rmse = 0
    i = 0
    model.eval()
    with torch.no_grad():
        for ex, rt_score in data:
            i += 1
            out = model(ex.to(device))
            rmse_i = criterion(out.squeeze(), rt_score.to(device))
            rmse += rmse_i.item()

    return (rmse/i) ** 0.5

def eval_model(img_net, seq_net, device, data):
    model = Model(img_net, seq_net).to(device)
    model_name = f"SEQ_{img_net}_IMG_{seq_net}"
    model_path = f"./trained_models_new/{model_name}_checkpoint.pt"
    print("LOADING MODEL")
    model.load_state_dict(torch.load(model_path, map_location=device))
    print("MODEL LOADED - EVAL STARTING")
    return eval(model, data, device)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", "--device", type=str, required=True)
    args = parser.parse_args()

    device = torch.device(args.device)

    chunked_data = pickle.load(open("./converter/trailers_chunked.p", "rb"))
    data = pickle.load(open("./converter/trailers_normal.p", "rb"))
    labels = pickle.load(open("./converter/scores.p", "rb"))
    chunked_data, data, labels = list(map(lambda x: torch.from_numpy(x).float(), (chunked_data, data, labels)))

    chunked_data = list(zip(chunked_data, labels))
    data = list(zip(data, labels))

    hold_seq_fixed = "lstm"
    cnn_options = ['early_fusion', 'late_fusion', 'slow_fusion', 'vanilla_cnn']

    results = []

    for cnn in cnn_options:
        chunked_needed = cnn in frozenset(["slow_fusion", "early_fusion", "late_fusion"])
        res = eval_model(cnn, hold_seq_fixed, device, chunked_data if chunked_needed else data)
        results.append(((cnn, hold_seq_fixed), res))

    cnn_f = "slow_fusion"
    seq_v = ["vanilla_rnn", "lstm", "lstmn", "transformer_abs", "lstm_stack"]
    for seq in seq_v:
        chunked_needed = cnn_f in frozenset(["slow_fusion", "early_fusion", "late_fusion"])
        res = eval_model(cnn_f, seq, device, chunked_data if chunked_needed else data)
        results.append(((cnn_f, seq), res))

    pickle.dump(results, open(f"./results.p", "wb"))

if __name__ == "__main__" : main()


import os
import shutil
import threading
from tkinter import filedialog, messagebox
from cryptography.fernet import Fernet

class Core:
    def __init__(self, open_dialog) -> None:
        if open_dialog is True:
            self.path = filedialog.askopenfilename()
        elif open_dialog is False:
            pass

    @staticmethod
    def encrypt_file(path_file):
        def start_encryption():
            # Usando chave
            # <--------------------------------------------------------------->
            fernet = Fernet(key_gen)
            with open(path_mod1, "rb") as file:
                data_origem = file.read()
            # <--------------------------------------------------------------->

            # Encriptar arquivo
            # <--------------------------------------------------------------->
            encrypted = fernet.encrypt(data_origem)
            with open(path_mod1, "wb") as file:
                file.write(encrypted)
            # <--------------------------------------------------------------->

        path_mod1 = path_file.replace(" ", "_")
        path_mod2 = path_mod1.replace("/", " ")
        os.rename(path_file, path_mod1)
        folder = "Keys"

        try:
            os.mkdir(folder)
        except FileExistsError:
            pass

        # Criando chave privada e salvando
        # <--------------------------------------------------------------->
        key_gen = Fernet.generate_key()
        name = f"KeyFile [{path_mod2.split()[-1]}].key"
        with open(name, "wb") as file_key:
            file_key.write(key_gen)
            messagebox.showinfo("Chave", f"Chave criada para o arquivo {path_file}")
        try:
            shutil.move(name, folder)
            messagebox.showwarning("Atenção", "Arquivo Key movido para pasta /Keys.")
        except Exception as e:
            if "already exists" in e.__str__():
                messagebox.showwarning("Erro", "Arquivo key existente na pasta Keys, o arquivo foi salvo na pasta raiz do programa.")
            else:
                messagebox.showerror("Erro", e.__str__())
        # <--------------------------------------------------------------->

        # Executando uma thread
        messagebox.showinfo("", "Começando a encriptar.")
        thrd = threading.Thread(target=start_encryption)
        thrd.start()

    @staticmethod
    def decrypt_file(path_file, private_key):
        try:
            # Abrindo o arquvo key
            # <---------------------------------->
            with open(private_key, "rb") as key_file:
                key = key_file.read()
            # <---------------------------------->

            # Lendo o arquivo encriptado
            # <---------------------------------->
            fernet = Fernet(key)
            with open(path_file, "rb") as file:
                file_encrypted = file.read()
            # <---------------------------------->

            # Descriptando
            # <---------------------------------->
            decrypted = fernet.decrypt(file_encrypted)
            with open(path_file, "wb") as file:
                file.write(decrypted)
                messagebox.showinfo("Sucesso", "Arquivo Desencriptado.")
            # <---------------------------------->

        except Exception:
            messagebox.showerror("Erro", "Private Key ou caminho invalido.")

    def return_path(self):
        return self.path