import ase.io
from ase.build import fcc100, add_adsorbate
from ase.constraints import FixAtoms
from ase.optimize import BFGS
from ase.parallel import parprint

from gpaw import GPAW, FermiDirac

from catlearn.optimize.mlneb import MLNEB

"""
Toy model for the diffusion of a Au atom on an Al(111) surface.
This example contains:
1. Optimization of the initial and final end-points of the reaction path with
GPAW
2. NEB optimization using our machine-learning surrogate model and GPAW.
"""

# 1. Structural relaxation.

# Setup calculator:
calc_args = {'mode': 'lcao',
             'h': 0.18,
             'basis': 'dzp',
             'symmetry': 'off',
             'xc': 'PBE',
             'txt': open('gpaw.txt', 'a'),
             'occupations': FermiDirac(0.03)}

# 1.1. Structures:
# 2x2-Al(001) surface with 3 layers and an
# Au atom adsorbed in a hollow site:
slab = fcc100('Al', size=(2, 2, 3))
add_adsorbate(slab, 'Au', 1.7, 'hollow')
slab.center(axis=2, vacuum=4.0)
slab.set_calculator(GPAW(**calc_args))

# Fix second and third layers:
mask = [atom.tag > 1 for atom in slab]
slab.set_constraint(FixAtoms(mask=mask))

# 1.2. Optimize initial and final end-points.
# Initial end-point:
qn = BFGS(slab, trajectory='initial.traj')
qn.run(fmax=0.01)

# Final end-point:
slab[-1].x += slab.get_cell()[0, 0] / 2
qn = BFGS(slab, trajectory='final.traj')
qn.run(fmax=0.01)

# # Define number of images:
n_images = 15

# 2.B. NEB using CatLearn
neb_catlearn = MLNEB(start='initial.traj',
                     end='final.traj',
                     ase_calc=GPAW(**calc_args),
                     n_images=n_images,
                     interpolation='idpp', restart=False)
neb_catlearn.run(fmax=0.05, trajectory='ML-NEB.traj', full_output=True)

# 3. Summary of the results #################################################
# ML-NEB:
atoms_catlearn = ase.io.read('evaluated_structures.traj', ':')
n_eval_catlearn = len(atoms_catlearn) - 2
parprint('Number of function evaluations CatLearn:', n_eval_catlearn)


import pytest
from lambda_utils.tests.unit.utils import make_aws_event

from nrlf.core.errors import NRLF_TO_SPINE_4XX_ERROR, RequestValidationError
from nrlf.core.event_parsing import fetch_body_from_event
from nrlf.core.model import APIGatewayProxyEventModel

@pytest.mark.parametrize(
    "event, expected_body",
    [
        (make_aws_event(body='{"test": "test"}'), {"test": "test"}),
        (make_aws_event(body="{}"), {}),
    ],
)
def test_parsing_body_from_event(event, expected_body):
    event_model = APIGatewayProxyEventModel(**event)
    actual_body = fetch_body_from_event(event_model)
    assert actual_body == expected_body

@pytest.mark.parametrize(
    ["event", "error_message"],
    [
        [make_aws_event(body="12345"), "Body is not expected json type"],
        [make_aws_event(body="null"), "Body is not expected json type"],
        [make_aws_event(body="true"), "Body is not expected json type"],
        [make_aws_event(body="{not json]"), "Body is not valid json"],
    ],
)
def test_parsing_body_from_event_errors(event, error_message):
    with pytest.raises(RequestValidationError) as e:
        event_model = APIGatewayProxyEventModel(**event)
        fetch_body_from_event(event_model)
    assert str(e.value) == error_message
    assert e.type in NRLF_TO_SPINE_4XX_ERROR


import optuna
import kevin_toolbox.nested_dict_list as ndl

def sample_from_feasible_domain(var, trial: optuna.trial.BaseTrial, pre_name=""):
    """
        使用试验 trial 基于输入中的定义域 feasible_domain 部分进行参数采样和替换。
            遍历输入中的所有元素，找出符合 <feasible_domain> 格式要求的记录了参数定义域的元素，
            然后使用输入的试验实例 trial 结合参数定义域采样出对应的参数，最后用采样出来的参数替换掉原来的定义域。

        参数：
            inputs:             <list/dict> 当其中的元素满足 <feasible_domain> 格式要求，将进行采样与替换。
                                    <feasible_domain> 格式要求：
                                        1. 是一个 dictionary
                                        2. 包含 "p_type"  字段
                                            "p_type" 表示定义域类型，常见值包括："float" "int" "categorical" 等
                                        3. 根据不同的定义域类型，给出定义域的参数
                                            比如，p_type="categorical" 时应该包含可选值列表 "choices"
                                            p_type="float"或者"int" 时应该包含最大、最小、间隔值、坐标轴类型 "high" "low" "step" "log" 等
                                        更多参见 https://optuna.readthedocs.io/zh_CN/latest/reference/generated/optuna.trial.Trial.html#optuna.trial.Trial
                                        中的 suggest_xxx() 函数。
            trial:              <optuna.trial.BaseTrial> 试验
            pre_name:           <string> 采样出的参数在试验 trial 中注册的名称的前缀

        返回：
            var, name_ls
                var 是采样后的结果
                name_ls 是被采样的节点名称，注意并不带 pre_name 前缀

        实例：
            对于输入 inputs={
                            "thr":[
                                "interval_thr": {
                                    "p_type": "categorical",
                                    "choices": [
                                      1000,
                                      2000,
                                      5000,
                                      10000
                                    ]
                                },
                                "iou_thr": {
                                    "p_type": "float",
                                    "low": 0,
                                    "high": 1.0000001,
                                    "step": 0.05
                                },
                                "connection": {
                                    "p_type": "categorical",
                                    "choices": {
                                        "skip": lambda x:x,
                                        "and":  lambda x,y : x and y,
                                        "or":  lambda x,y : x or y,
                                    }
                                },
                            ]
                        }
            可能返回的采样结果是 res={"thr":[{"interval_thr":1000}, {"iou_thr":0.6}, {"connection":lambda x:x} ], }。
            当 pre_name="my" 时，这些参数在 trial 中注册的名称分别是 "my:thr@0:interval_thr"，"my:thr@1:iou_thr" 和 "my:thr@1:connection"。
            特别地，
                - 对于字典形式的 choices，其中保存在 trial 中的取值是其键 key 而非 value。
                - 对于list形式，但含有非支持类型的  choices，其中保存在 trial 中的取值是元素的 index。
            这些名称的含义详见 get_value()。
    """

    name_ls=[]

    def func(idx, v):
        nonlocal name_ls
        p_type = v.pop("p_type")
        kwargs = v
        choice_values = None
        if p_type == "categorical":
            # optuna 目前的类别元素仅支持 None, bool, int, float 和 str 类型
            #   对于其他类型的元素，比如 list 和 dict，需要替换成对应 index_ls 或者 key_ls
            #   然后再根据建议的 index 或者 key 到 list 和 dict 中取值
            assert "choices" in kwargs
            if isinstance(kwargs["choices"], (list, tuple,)) and \
                    not all([isinstance(i, (bool, int, float, str,)) or i is None for i in kwargs["choices"]]):
                choice_values = kwargs["choices"]
                kwargs["choices"] = list(range(len(kwargs["choices"])))
            elif isinstance(kwargs["choices"], (dict,)):
                choice_values = kwargs["choices"]
                kwargs["choices"] = list(kwargs["choices"].keys())

        v = eval(f'trial.suggest_{p_type}(name=name, **kwargs)',
                 {"trial": trial, "name": f'{pre_name}{idx}', "kwargs": kwargs})
        if choice_values is not None:
            v = choice_values[v]
        name_ls.append(idx)
        return v

    var = ndl.traverse(var=var, match_cond=lambda _, __, v: isinstance(v, (dict,)) and "p_type" in v,
                       action_mode="replace", converter=func, b_use_name_as_idx=True)

    return var, name_ls


import urllib.request
import requests
import json
import os
#
# url= "http://172.16.100.221:5000/get_file?file_path=/opt/alpha/appdata/autotest/UT/FFRct_Adapters_UT/Data&file_name=4433121_img.zip"
# # file = urllib.request.urlopen(url)
# file = urllib.request.urlretrieve(url, "./hahah")
# # new_filename = "jjjnn" + '.' + "zip"  # 修改了上传的文件名
# # file.save(os.path.join("./", new_filename))
# print(file)

data = {
  "method": "read_dicom",
  "ver": "2.0",
  "requestId": 54321,
  'data':{"ser_id":"1.3.12.2.1107.5.1.4.73757.30000017091200222701600058375",
  	'dicomPath': '/opt/alpha/appdata/autotest/UT/FFRct_Adapters_UT/Data/Li_Yan/'
  }
}
url = r"http://127.0.0.1:7000/"
#
response = requests.post(url, json=data)
# print(response)
#
print(response.text)

# url = r"http://172.16.100.221:5000/get_file?file_path=../file&file_name=1.3.12.2.1107.5.1.4.73757.30000017091200222701600058375_mask.raw"
#
# response = requests.get(url)
# response.raise_for_status()
# playFile = open('RomeoAndJuliet.raw', 'wb')
# for chunk in response.iter_content(100000):
#     playFile.write(chunk)
# playFile.close()

# print(response.text)


"""GIDDY: GeospatIal Distribution DYnamics

Giddy is an open-source python library for the analysis of dynamics of
longitudinal spatial data. Originating from the spatial dynamics module
in PySAL (Python Spatial Analysis Library), it is under active development
for the inclusion of many newly proposed analytics that consider the
role of space in the evolution of distributions over time and has
several new features including inter- and intra-regional decomposition
of mobility association and local measures of exchange mobility in
addition to space-time LISA and spatial markov methods. Give
giddy a try if you are interested in space-time analysis!

"""

DOCLINES = __doc__.split("\n")

from setuptools import setup
from distutils.command.build_py import build_py

setup(name='giddy',  # name of package
      version='1.0.2',
      description=DOCLINES[0],
      long_description="\n".join(DOCLINES[2:]),
      url='https://github.com/pysal/giddy',
      maintainer='Wei Kang',
      maintainer_email='weikang9009@gmail.com',
      test_suite='nose.collector',
      tests_require=['nose'],
      keywords='spatial statistics, spatiotemporal analysis',
      classifiers=[
        'Development Status :: 5 - Production/Stable',
        'Intended Audience :: Science/Research',
        'Intended Audience :: Developers',
        'Intended Audience :: Education',
        'Topic :: Scientific/Engineering',
        'Topic :: Scientific/Engineering :: GIS',
        'License :: OSI Approved :: BSD License',
        'Programming Language :: Python',
        'Programming Language :: Python :: 2.7',
        'Programming Language :: Python :: 3.4',
        'Programming Language :: Python :: 3.5',
        'Programming Language :: Python :: 3.6'
        ],
      license='3-Clause BSD',
      packages=['giddy'],
      install_requires=['mapclassify', 'esda'],
      zip_safe=False,
      cmdclass={'build.py': build_py})


from django.db import models
from django.db.models import Sum, F
from django.core.validators import MinValueValidator, MaxValueValidator
from phonenumber_field.modelfields import PhoneNumberField
from functools import reduce
import copy

class Restaurant(models.Model):
    name = models.CharField(
        'название',
        max_length=50
    )
    address = models.CharField(
        'адрес',
        max_length=100,
        blank=True,
    )
    contact_phone = models.CharField(
        'контактный телефон',
        max_length=50,
        blank=True,
    )

    class Meta:
        verbose_name = 'ресторан'
        verbose_name_plural = 'рестораны'

    def __str__(self):
        return self.name

class ProductQuerySet(models.QuerySet):
    def available(self):
        products = (
            RestaurantMenuItem.objects
            .filter(availability=True)
            .values_list('product')
        )
        return self.filter(pk__in=products)

class ProductCategory(models.Model):
    name = models.CharField(
        'название',
        max_length=50
    )

    class Meta:
        verbose_name = 'категория'
        verbose_name_plural = 'категории'

    def __str__(self):
        return self.name

class Product(models.Model):
    name = models.CharField(
        'название',
        max_length=50
    )
    category = models.ForeignKey(
        ProductCategory,
        verbose_name='категория',
        related_name='products',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
    )
    price = models.DecimalField(
        'цена',
        max_digits=8,
        decimal_places=2,
        validators=[MinValueValidator(0)]
    )
    image = models.ImageField(
        'картинка'
    )
    special_status = models.BooleanField(
        'спец.предложение',
        default=False,
        db_index=True,
    )
    description = models.TextField(
        'описание',
        max_length=200,
        blank=True,
    )

    objects = ProductQuerySet.as_manager()

    class Meta:
        verbose_name = 'товар'
        verbose_name_plural = 'товары'

    def __str__(self):
        return self.name

class RestaurantMenuItem(models.Model):
    restaurant = models.ForeignKey(
        Restaurant,
        related_name='menu_items',
        verbose_name="ресторан",
        on_delete=models.CASCADE,
    )
    product = models.ForeignKey(
        Product,
        on_delete=models.CASCADE,
        related_name='menu_items',
        verbose_name='продукт',
    )
    availability = models.BooleanField(
        'в продаже',
        default=True,
        db_index=True
    )

    class Meta:
        verbose_name = 'пункт меню ресторана'
        verbose_name_plural = 'пункты меню ресторана'
        unique_together = [
            ['restaurant', 'product']
        ]

    def __str__(self):
        return f"{self.restaurant.name} - {self.product.name}"

class OrderQuerySet(models.QuerySet):
    def get_order_price(self):
        return self.annotate(total_price=Sum(F('items__price')))

    def get_available_restaurants(self):
        product_restaurant_menu = RestaurantMenuItem.objects.select_related('product', 'restaurant')
        for order in self:
            prepares_product_restaurants = []
            for order_product in order.items.all():
                prepares_product_restaurants.append([rest_item.restaurant for rest_item in product_restaurant_menu
                                                    if order_product.product_id == rest_item.product.pk])
            cooking_restaurants = reduce(set.intersection, map(set, prepares_product_restaurants))
            order.cooking_restaurants = copy.deepcopy(cooking_restaurants)
        return self

class Order(models.Model):
    ORDER_STATUS = [
        ('UNPROCESSED', 'Не обработан'),
        ('PROCESSED', 'Готовится'),
        ('DONE', 'Выполнен')
    ]

    PAYMENT_METHOD = [
        ('CASH', 'Наличные'),
        ('ONLINE', 'Онлайн')
    ]

    firstname = models.CharField(max_length=100, verbose_name='Имя')
    lastname = models.CharField(max_length=100,  verbose_name='Фамилия')
    phonenumber = PhoneNumberField(verbose_name='Номер телефона', db_index=True)
    address = models.CharField(max_length=200, verbose_name='Адрес')
    comment = models.TextField(verbose_name='Комментарий', blank=True)
    registered_at = models.DateTimeField(auto_now_add=True, db_index=True, verbose_name='Дата создания')
    called_at = models.DateTimeField(db_index=True, verbose_name='Дата Звонка', null=True, blank=True)
    delivery_at = models.DateTimeField(db_index=True, verbose_name='Дата Доставки', null=True, blank=True)
    status = models.CharField(
        max_length=20,
        choices=ORDER_STATUS,
        db_index=True,
        verbose_name='Статус Заказа',
        default='UNPROCESSED')
    payment_method = models.CharField(
        max_length=20,
        db_index=True,
        choices=PAYMENT_METHOD,
        verbose_name='Способ оплаты')
    selected_restaurant = models.ForeignKey(
        Restaurant,
        on_delete=models.CASCADE,
        null=True,
        blank=True,
        related_name='orders',
        verbose_name='Готовящий ресторан')

    objects = OrderQuerySet.as_manager()

    class Meta:
        verbose_name = 'Заказ'
        verbose_name_plural = 'Заказы'

    def __str__(self):
        return f"{self.firstname} {self.lastname}, {self.address}"

class OrderItem(models.Model):
    product = models.ForeignKey(
        Product,
        related_name='order_items',
        on_delete=models.CASCADE,
        verbose_name='Товар')
    order = models.ForeignKey(
        Order,
        on_delete=models.CASCADE,
        related_name='items',
        verbose_name='Заказ')
    quantity = models.IntegerField(
        default=1,
        db_index=True,
        validators=[MinValueValidator(1), MaxValueValidator(100)],
        verbose_name='Количество')
    price = models.DecimalField(
        max_digits=10,
        decimal_places=2,
        validators=[MinValueValidator(0)],
        db_index=True,
        verbose_name='Общая стоимость товара'
    )

    class Meta:
        verbose_name = 'Заказанный товар'
        verbose_name_plural = 'Заказанные товары'

    def __str__(self):
        return f'{self.product}'


import torch.nn as nn
import torch
import numpy as np
import string
from utils import jaccard, AverageMeter, cal_jaccard, cal_accucary, load_model
from tqdm import tqdm
import time
from scipy.special import softmax

def KSLoss(preds, target):
    pred_cdf = torch.cumsum(torch.softmax(preds, dim=1), dim=1)
    target_cdf = torch.cumsum(target, dim=1)
    error = (target_cdf - pred_cdf)**2
    return torch.mean(error)

def custLoss(preds, target):
    pred = torch.argmax(torch.softmax(preds, axis=1), axis=1)
    target = torch.argmax(target, axis=1)
    return torch.sum((target-pred)**2)

def EntropyLoss(o1, o2, t1, t2):
    t1 = torch.argmax(t1, axis=1)
    t2 = torch.argmax(t2, axis=1)
    CE = nn.CrossEntropyLoss()
    l1 = CE(o1, t1)
    l2 = CE(o2, t2)

    return l1 + l2

def BcwLoss(o1, o2, t1, t2):
    bcw = nn.BCEWithLogitsLoss()
    l1 = bcw(o1, t1)
    l2 = bcw(o2, t2)
    return l1 + l2

def loss_fn(*loss):
    return sum(loss)

def trn_loop_fn(data_loader, model, optimzer, device):
    model.train()
    tk = tqdm(data_loader, total=len(data_loader))
    losses = AverageMeter()

    for i, d in enumerate(tk):
        ids = d['ids']
        mask_ids = d['mask_ids']
        token_type_ids = d['token_type_ids']
        target_start_idx = d['target_start_idx']
        target_end_idx = d['target_end_idx']
        offsets = d['offsets']
        orig_sentiment = d['orig_sentiment']
        orig_sele_text = d['orig_sele_text']
        orig_text = d['orig_text']
        targ_sentiment = d['targ_sentiment']

        ids = ids.to(device, dtype=torch.long)
        mask_ids = mask_ids.to(device, dtype=torch.long)
        token_type_ids = token_type_ids.to(device, dtype=torch.long)
        target_start_idx = target_start_idx.to(device, dtype=torch.float)
        target_end_idx = target_end_idx.to(device, dtype=torch.float)
        targ_sentiment = targ_sentiment.to(device, dtype=torch.long)

        optimzer.zero_grad()
        o1, o2 = model(ids, mask_ids, token_type_ids)

        #loss = EntropyLoss(o1, o2, target_start_idx, target_end_idx)
        loss = BcwLoss(o1, o2, target_start_idx, target_end_idx)
        #loss = loss_fn(loss1, loss2)

        loss.backward()
        optimzer.step()
        losses.update(loss.item(), ids.size(0))
        tk.set_postfix(loss=losses.avg)

def eval_loop_fn(data_loader, model,  device, df_type, rm_length):
    model.eval()

    fin_output_start = []
    fin_output_end = []
    fin_output_sentiment = []
    fin_offset = []
    fin_orig_sentiment = []
    fin_orig_selected = []
    fin_orig_text = []
    fin_targ_sentiment = []

    for bi, d in enumerate(data_loader):
        ids = d['ids']
        mask_ids = d['mask_ids']
        token_type_ids = d['token_type_ids']
        target_start_idx = d['target_start_idx']
        target_end_idx = d['target_end_idx']
        offsets = d['offsets']
        orig_sentiment = d['orig_sentiment']
        orig_sele_text = d['orig_sele_text']
        orig_text = d['orig_text']
        targ_sentiment = d['targ_sentiment']

        with torch.no_grad():
            ids = ids.to(device, dtype=torch.long)
            mask_ids = mask_ids.to(device, dtype=torch.long)
            token_type_ids = token_type_ids.to(device, dtype=torch.long)
            target_start_idx = target_start_idx.to(device, dtype=torch.long)
            target_end_idx = target_end_idx.to(device, dtype=torch.long)
            targ_sentiment = targ_sentiment.to(device, dtype=torch.long)

            o1, o2 = model(ids, mask_ids, token_type_ids)
            fin_output_start.append(o1.cpu().detach().numpy())
            fin_output_end.append(o2.cpu().detach().numpy())

            fin_offset.append(offsets)
            fin_orig_sentiment.extend(orig_sentiment)
            fin_orig_selected.extend(orig_sele_text)
            fin_orig_text.extend(orig_text)
            fin_targ_sentiment.append(targ_sentiment.cpu().detach().numpy())

    fin_output_start = np.vstack(fin_output_start)
    fin_output_end = np.vstack(fin_output_end)
    fin_offset = np.vstack(fin_offset)
    fin_targ_sentiment = np.concatenate(fin_targ_sentiment)

    if df_type == 'val' or df_type == 'trn':
        jaccard_score = cal_jaccard(fin_output_start, fin_output_end, fin_offset, fin_orig_sentiment, fin_orig_selected, fin_orig_text, rm_length)
        return jaccard_score, fin_output_start, fin_output_end

def pred_loop_fn(data_loader, device, file_path):
    fin_output_start = []
    fin_output_end = []
    fin_offset = []
    fin_orig_sentiment = []
    fin_orig_selected = []
    fin_orig_text = []

    tk = tqdm(data_loader, total=len(data_loader))
    fin_output_string = []

    for bi, d in enumerate(tk):
        ids = d['ids']
        mask_ids = d['mask_ids']
        token_type_ids = d['token_type_ids']
        offsets = d['offsets']
        orig_sentiment = d['orig_sentiment']
        orig_text = d['orig_text']

        with torch.no_grad():
            ids = ids.to(device, dtype=torch.long)
            mask_ids = mask_ids.to(device, dtype=torch.long)
            token_type_ids = token_type_ids.to(device, dtype=torch.long)

            model1, model2, model3, model4, model5 = load_model(file_path)
            output_start1, output_end1 = model1(ids, mask_ids, token_type_ids)
            output_start2, output_end2 = model2(ids, mask_ids, token_type_ids)
            output_start3, output_end3 = model3(ids, mask_ids, token_type_ids)
            output_start4, output_end4 = model4(ids, mask_ids, token_type_ids)
            output_start5, output_end5 = model5(ids, mask_ids, token_type_ids)

            output_start = output_start1 + output_start2 + output_start3 + output_start4 + output_start5
            output_end = output_end1 + output_end2 + output_end3 + output_end4 + output_end5

            output_start = softmax(output_start.cpu().detach().numpy(), axis=1)
            output_end = softmax(output_end.cpu().detach().numpy(), axis=1)

            fin_output_start.append(output_start)
            fin_output_end.append(output_end)
            fin_offset.append(offsets)
            fin_orig_sentiment.extend(orig_sentiment)
            fin_orig_text.extend(orig_text)

    fin_output_start = np.vstack(fin_output_start)
    fin_output_end = np.vstack(fin_output_end)
    fin_offset = np.vstack(fin_offset)

    for i in range(fin_output_start.shape[0]):
        output_start = fin_output_start[i]
        output_start = np.argmax(output_start)
        output_end = fin_output_end[i]
        output_end = np.argmax(output_end)
        offset = fin_offset[i]
        orig_sentiment = fin_orig_sentiment[i]
        orig_text = fin_orig_text[i]

        if output_start > output_end:
            output_end = output_start

        output_string = ""
        for j in range(output_start, output_end+1):
            output_string += orig_text[offset[j][0]:offset[j][1]]
            if (ix+1) < len(offset) and offset[ix][1] < offset[ix+1][0]:
               final_output += " "

        if orig_sentiment == 'neutral':
            output_string = orig_text

        fin_output_string.append(output_string)

    return fin_output_string


import os, sys
from numpy.lib.npyio import save
import vtk, vtktools
import u2r
import numpy as np
import matplotlib.pyplot as plt
import random
from nirom_dd_tools import *
import copy

def get_clean_vtk_file(filename):
    "Removes fields and arrays from a vtk file, leaving the coordinates/connectivity information."
    vtu_data = vtktools.vtu(filename)
    clean_vtu = vtktools.vtu()
    clean_vtu.ugrid.DeepCopy(vtu_data.ugrid)
    fieldNames = clean_vtu.GetFieldNames()
    # remove all fields and arrays from this vtu
    for field in fieldNames:
        clean_vtu.RemoveField(field)
        fieldNames = clean_vtu.GetFieldNames()
        vtkdata = clean_vtu.ugrid.GetCellData()
        arrayNames = [vtkdata.GetArrayName(i) for i in range(vtkdata.GetNumberOfArrays())]
    for array in arrayNames:
        vtkdata.RemoveArray(array)
    return clean_vtu

def plot_grid(grid_origin, grid_width, nx, ny):
    # include plot of entire coordinates with grid
    # plt.figure(figsize=(9,9))
    plt.plot(coordinates[:, 0], coordinates[:, 1], 'g.', ms=0.3,
             label='angle = {}'.format(random_angle))  # corrdinates

    for d in range(ny + 1):
        if d % 4 == 0:
            plt.plot([grid_origin[0], grid_origin[0] + grid_width[0]],
                     [grid_origin[1] + d * ddx[1], grid_origin[1] + d * ddx[1]], 'k-', lw=1.2)  # horizontal
            if ny == nx:
                plt.plot([grid_origin[0] + d * ddx[1], grid_origin[0] + d * ddx[1]],
                         [grid_origin[1], grid_origin[1] + grid_width[1]], 'k-', lw=1.2)  # vertical

    if ny != nx:
        for d in range(nx + 1):  # vertical
            if d % 4 == 0:
                plt.plot([grid_origin[0] + d * ddx[0], grid_origin[0] + d * ddx[0]],
                         [grid_origin[1], grid_origin[1] + grid_width[1]], 'k-', lw=1.2)  # vertical

def sample_(mesh, grid_origin):
    """
    Returns a snapshot matrix of shape (5,nx*ny) and
    snapshot_ae of shape (5,nx,ny) with given
    mesh and central grid origin for the starshape grid formation
    """

    s_ae = np.zeros((nx, ny))

    value_grid = u2r.simple_interpolate_from_mesh_to_grid(mesh, x_all, x_ndgln, ddx, grid_origin, nx, ny, nz,
                                                          zeros_beyond_mesh, nEl, nloc, nNodes, nScalar, nDim, 1)
    s_ae[:, :] = value_grid.reshape((nx, ny))

    return s_ae

def rotate_mesh(angle):
    theta = np.radians(angle)

    # shift coordinates so that they are centred at (0,0)
    # for i in range(coordinates.shape[0]):
    #     coordinates[i][0] -= 1.5
    #     coordinates[i][1] -= 1.5

    new_mesh = np.zeros(coordinates.shape)

    for i in range(coordinates.shape[0]):
        new_mesh[i][0] = (coordinates[i][0] - 1.5) * np.cos(theta) - (coordinates[i][1] - 1.5) * np.sin(theta)
        new_mesh[i][1] = (coordinates[i][0] - 1.5) * np.sin(theta) + (coordinates[i][1] - 1.5) * np.cos(theta)

    # rotate the velocity field as well

    return new_mesh

# snapshot_data_location = 'F:/new_VTU_POD/time_data_new/'
snapshot_data_location = 'F:/new_VTU_POD/time_data_new/'
snapshot_file_base = 'Flow_past_buildings_'

field_names = ['Velocity', 'VelocityAbsorption']
nFields = len(field_names)

xlength = 3.0
ylength = 3.0

grid_width = [0.48, 0.48]
# spacing inside small grid
nx = int(grid_width[0] * 100)
ny = nx
nz = 1
ddx = np.array((0.01, 0.01))

# get a vtu file (any will do as the mesh is not adapted)
#filename = snapshot_data_location + snapshot_file_base + '400.vtu'

nTime = 100
start = 50
Ssnapshot_ae = np.zeros((nTime,6 * 6, nx, ny, 2))
Ssnapshot_ae_info = np.zeros((nTime,6 * 6, nx, ny, 1))

for iTime in range(nTime):
    filename = snapshot_data_location + snapshot_file_base + str(start+iTime)+'.vtu'
    print(filename)
    representative_vtu = vtktools.vtu(filename)
    coordinates_org = representative_vtu.GetLocations()  # coordinates of the nodes
    coordinates = coordinates_org

    nNodes = coordinates.shape[0]  # vtu_data.ugrid.GetNumberOfPoints()
    nEl = representative_vtu.ugrid.GetNumberOfCells()
    # print('nEl', nEl, type(nEl), 'nNodes', nNodes)
    nloc = 3  # number of local nodes, ie three nodes per element (in 2D)
    # nScalar = 2 # dimension of fields , 2 = u and v
    nScalar = 1
    nDim = 2

    # get global node numbers
    x_ndgln = np.zeros((nEl * nloc), dtype=int)
    for iEl in range(nEl):
        n = representative_vtu.GetCellPoints(iEl) + 1
        x_ndgln[iEl * nloc:(iEl + 1) * nloc] = n

    # min_x = min(coordinates[:, 0])
    # max_x = max(coordinates[:, 0])
    # min_y = min(coordinates[:, 1])
    # max_y = max(coordinates[:, 1])
    # print(min_x,max_x,min_y,max_y)

    print("Generating starshape grids")
    zeros_beyond_mesh = 0
    start_point = [0, 2.5]

    nIter = 6

    for iIter in range(nIter):

        velocity_field = representative_vtu.GetField(field_names[0])[:, :nDim]  # field name 0 is velocity field
        start_point[0] = 0

        for iGrid in range(6):

            # rotate mesh and velocity
            coordinates = coordinates_org
            #coordinates = rotate_mesh(rangle)
            x_all = np.transpose(coordinates[:, 0:nDim])
            #velocity_field_g = rotate_vel(rangle, velocity_field)
            va_field = representative_vtu.GetField(field_names[1])[:,0]

            # starshape for u_vel
            u_mesh = np.zeros((1, nNodes, 1))
            u_mesh[:, :, 0] = np.transpose(velocity_field[:, 0])
            u_sae = sample_(u_mesh, start_point)
            # Ssnapshots_matrix[:nx*ny,5*iGrid:5*iGrid+5] = u_smatrix
            Ssnapshot_ae[iTime, iIter * 6 + iGrid, :, :, 0] = u_sae

            # starshape for v_vel
            v_mesh = np.zeros((1, nNodes, 1))
            v_mesh[:, :, 0] = np.transpose(velocity_field[:, 1])
            v_sae = sample_(v_mesh, start_point)
            # Ssnapshots_matrix[nx*ny:nx*ny*2,5*iGrid:5*iGrid+5] = v_smatrix
            Ssnapshot_ae[iTime, iIter * 6 + iGrid, :, :, 1] = v_sae

            # starshape for va
            va_mesh = np.zeros((1, nNodes, 1))
            va_mesh[:, :, 0] = np.transpose(va_field)
            va_sae = sample_(va_mesh, start_point)
            # Ssnapshots_matrix[nx*ny*2:nx*ny*3,5*iGrid:5*iGrid+5] = va_smatrix
            Ssnapshot_ae_info[iTime, iIter * 6 + iGrid, :, :, 0] = va_sae

            start_point[0] = start_point[0] + 0.48

        start_point[1] = start_point[1] - 0.48

# Scale values
# min_vel = np.amin(Ssnapshots_matrix[:nx*ny*2,:]) #minimum among u and v velocity
# max_vel = np.amax(Ssnapshots_matrix[:nx*ny*2,:])
min_vel = np.amin(Ssnapshot_ae)
max_vel = np.amax(Ssnapshot_ae)
vel_scaling = 1 / (max_vel - min_vel)
min_va = np.amin(Ssnapshot_ae_info)
max_va = np.amax(Ssnapshot_ae_info)

va_scaling = 1 / (max_va - min_va)
# scale snapshots for ae
Ssnapshot_ae = vel_scaling * (Ssnapshot_ae - min_vel)
Ssnapshot_ae_info = va_scaling* (Ssnapshot_ae_info - min_va)
print("min_vel = ", min_vel, " , max_vel = ", max_vel, " , vel_scaling = ", vel_scaling)

print("min_va = ", min_va, " , max_va = ", max_va, " , va_scaling = ", va_scaling)

# np.save("regular_time_velocity.npy", Ssnapshot_ae)
# np.save("regular_time_info.npy",Ssnapshot_ae_info)


from flask import Flask, request
from flask.views import MethodView
from flask_cors import CORS

import assistance as asst

app = Flask(__name__)
CORS(app)

class PhChaserAPI(MethodView):

    def get(self):
        code = request.args.get("CODE")
        phone = request.args.get("PHONE")
        return asst.Tracker.take_by_code(code, phone)

app.add_url_rule('/PhChaser', view_func=PhChaserAPI.as_view('PhChaser'))

class PyCountriesAPI(MethodView):

    def get(self, iso):
        if not iso:
            return asst.PyCountries.take()
        # Country Code
        return asst.PyCountries.take_code_by(iso)

PyCountriesVIEW = PyCountriesAPI.as_view('PyCountries')
app.add_url_rule('/PyCountries', view_func=PyCountriesVIEW, defaults={'iso': None})
app.add_url_rule('/PyCountries/<string:iso>/Code', view_func=PyCountriesVIEW,
                 methods=['GET', 'PUT', 'DELETE'])

class PyCountriesCodeAPI(MethodView):

    def get(self):
        return asst.PyCountries.take_code()

PyCountriesCodeVIEW = PyCountriesCodeAPI.as_view('PyCountriesCode')
app.add_url_rule('/PyCountriesCode', view_func=PyCountriesCodeVIEW)

if __name__ == '__main__':
    app.run(debug=True)


import os

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'devproj.settings')

import django,names,random

django.setup()
from interactWBL.models import User, Academic, Course, Competency, CourseTarget, Student, Enrolment, Assignment, Submission, Reflection, PersonalCompetency, CompetencyEndorsement

def populate():
    # have existing competencies ready in a list and calculate how many there are
    competencies_list = list(Competency.objects.all())
    comps_no = len(competencies_list)

    # academic first names
    academics_names = ['Mechelle', 'Evelia', 'Sylvie', 'Stephania', 'Octavio', 'Shona', 'Spencer', 'Natisha', 'Guadalupe',
             'Lea', ]

    # course names for courses to be created
    course_names = ['Programming', 'Cyber Security Fundamentals', 'Systems and Networks', 'Enterprise Cyber Security',
                    'Database Theory and Application', 'Software Engineering', 'Software Project Managements',
                    'Internet Technology', 'Advanced Programming', 'Algorithms and Data Structures',]

    # collect existing students in a list and determine their number
    students_list = list(Student.objects.all())
    studs_no = len(students_list)
    print("there are " + str(studs_no) + " students")

    # create 10 users, then associate 10 academics with the 10 users & create 10 courses, 1 for taught by each academic

    i = 0
    for n in academics_names:
        # create a user with name from the list above
        u = User.objects.get_or_create(username=n.lower(), password='')[0]
        u.set_password('securepassword')
        u.save()
        # then associate an academic profile with that user
        a = Academic.objects.get_or_create(user=u)[0]
        u.first_name = academics_names[i]
        u.last_name = names.get_last_name()
        u.email = academics_names[i].lower() + '.'+ u.last_name.lower()+'@gla.ac.uk'
        u.save()
        # after the academic profile is fully set up, create a course taught by that academic

        c = Course.objects.get_or_create(teacher=a, year = random.randint(1,5), name=course_names[i],
                                         lecture_recordings="http://www.echo360.com",missed_lecture_procedure="The exact procedure will be announced later, but please make sure to message me if you have any pressing doubts at the moment.",
                                         ILOs="The course aims to provide students with mathematical/quantitative skills and knowledge that consititute the foundation for techniques and instruments in both microeconomic theory and intertemporal macroeconomics (such as multivariate calculus and integration, constrained optimisation, differential equations, dynamic programming methods, functional analysis), and to demonstrate various mathematical techniques are applied to economic problems.",
                                         moodle='https://moodle.org/',
                                         description="The aim of this course is to provide students with a comprehensive overview of web application development. It will provide students with the skills to design and develop distributed web applications in a disciplined manner, using a range of tools and technologies. It will also strengthen their understanding of the context and rationale of distributed systems.")[0]
        i += 1 # this is so we iterate through academic and course names at the same time

        """
                 this next sections deals with randomly assigning competency objectives to the created courses
                 We select a competency at random from the list and add about 8-9 targets to the most recent course
        """

        for j in range (1,10):
            t = CourseTarget.objects.get_or_create(course=c,competency=competencies_list[random.randint(0,comps_no-1)])[0]

        """
                this next sections deals with enrolling students in courses (according to the year of the course)
        """
    course_list = list(Course.objects.all())
    for course in course_list:
        print(course)
        print("now enrolling:")
        for s in students_list:
            if s.year == course.year:
                e = Enrolment.objects.get_or_create(course=course, student=s)[0]
                e.save()
                print(e)
            else:
                pass
    print("Generating assignments...")
    print("Generating submissions...")
    print("Generating reflections and personal competencies...")
    for course in course_list[0:6]:
        # for half of the courses determine the students enrolled
        enrollments  = list(Enrolment.objects.filter(course= course))
        students_in_course =[]
        # for each course, create an assignment
        assignment = Assignment.objects.get_or_create(name=course.name + " project", course=course,
                                                      description="Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.",)[0]
        # add associate 4 competencies with this assignment
        for i in range(1, 5):
            random_competency = competencies_list[random.randint(0, comps_no - 1)]
            assignment.competencies.add(random_competency)
        # from the list of enrollments, obtain the list of all students enrolled in this course
        for e in enrollments:
            students_in_course.append(e.student)

        #for each student in the course, create a weekly reflection by that student
        for student in students_in_course:
            weekly_reflection = Reflection.objects.get_or_create(student=student, course=course,type='weekly',assignment=assignment,
                                                                 content="Diam sit amet nisl suscipit adipiscing bibendum est ultricies. In pellentesque massa placerat duis ultricies lacus. Cursus mattis molestie a iaculis at erat. Libero enim sed faucibus turpis in eu mi bibendum. Purus in massa tempor nec feugiat nisl pretium fusce id. Sit amet purus gravida quis. Nisl pretium fusce id velit ut tortor. Eu consequat ac felis donec et odio pellentesque. Diam sollicitudin tempor id eu nisl nunc mi ipsum. Faucibus interdum posuere lorem ipsum. Vitae nunc sed velit dignissim sodales ut. Aenean pharetra magna ac placerat vestibulum lectus. Dictum non consectetur a erat nam at lectus urna. Nunc sed velit dignissim sodales. Eget nunc lobortis mattis aliquam faucibus purus in massa tempor. Fames ac turpis egestas integer eget aliquet nibh. Sit amet massa vitae tortor. Potenti nullam ac tortor vitae. Odio aenean sed adipiscing diam. In cursus turpis massa tincidunt dui ut ornare.")[0]
            # print info to show to the user while they are waiting for pop script to complete:
            print("created assignment: "+assignment.__str__())
            # for each weekly reflection associated with that assignment, associate some competencies with them
            for i in range(1,4):
                random_competency = competencies_list[random.randint(0,comps_no-1)]
                weekly_reflection.competencies.add(random_competency)
                # also create a PersonalCompetency object to track the student's progress with the competencies targeted by the assignment
                personal_competency = PersonalCompetency.objects.get_or_create(student=student, progress='1', competency=random_competency)[0]
                personal_competency.assignments.add(assignment)
            # for each assignment, create a submission
            submission = Submission.objects.get_or_create(student=student, assignment=assignment, course=course)[0]
            submission.file.name='codesubmission.txt'
            submission.save()
            print(submission)

    print('Printing created academics' + '\n')
    for a in Academic.objects.all():
        print(a.__str__() + '\n')

    print('Printing created courses' + '\n')
    for c in Course.objects.all():
        print(c.__str__() + '\n')

if __name__ == '__main__':
    print('Starting academic, course and course targets population script:' + '\n')
    populate()


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import OrderedDict

from paddle import fluid
from paddle.fluid.param_attr import ParamAttr
from paddle.fluid.regularizer import L2Decay

__all__ = ['MultiClassNMS', 'YOLOv3Head']

class MultiClassNMS(object):
    # __op__ = fluid.layers.multiclass_nms
    def __init__(self, background_label, keep_top_k, nms_threshold, nms_top_k, normalized, score_threshold):
        super(MultiClassNMS, self).__init__()
        self.background_label = background_label
        self.keep_top_k = keep_top_k
        self.nms_threshold = nms_threshold
        self.nms_top_k = nms_top_k
        self.normalized = normalized
        self.score_threshold = score_threshold

class YOLOv3Head(object):
    """Head block for YOLOv3 network

    Args:
        norm_decay (float): weight decay for normalization layer weights
        num_classes (int): number of output classes
        ignore_thresh (float): threshold to ignore confidence loss
        label_smooth (bool): whether to use label smoothing
        anchors (list): anchors
        anchor_masks (list): anchor masks
        nms (object): an instance of `MultiClassNMS`
    """

    def __init__(self,
                 norm_decay=0.,
                 num_classes=80,
                 ignore_thresh=0.7,
                 label_smooth=True,
                 anchors=[[10, 13], [16, 30], [33, 23], [30, 61], [62, 45], [59, 119], [116, 90], [156, 198],
                          [373, 326]],
                 anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
                 nms=MultiClassNMS(
                     background_label=-1,
                     keep_top_k=100,
                     nms_threshold=0.45,
                     nms_top_k=1000,
                     normalized=True,
                     score_threshold=0.01),
                 weight_prefix_name=''):
        self.norm_decay = norm_decay
        self.num_classes = num_classes
        self.ignore_thresh = ignore_thresh
        self.label_smooth = label_smooth
        self.anchor_masks = anchor_masks
        self._parse_anchors(anchors)
        self.nms = nms
        self.prefix_name = weight_prefix_name

    def _conv_bn(self, input, ch_out, filter_size, stride, padding, act='leaky', is_test=True, name=None):
        conv = fluid.layers.conv2d(
            input=input,
            num_filters=ch_out,
            filter_size=filter_size,
            stride=stride,
            padding=padding,
            act=None,
            param_attr=ParamAttr(name=name + ".conv.weights"),
            bias_attr=False)

        bn_name = name + ".bn"
        bn_param_attr = ParamAttr(regularizer=L2Decay(self.norm_decay), name=bn_name + '.scale')
        bn_bias_attr = ParamAttr(regularizer=L2Decay(self.norm_decay), name=bn_name + '.offset')
        out = fluid.layers.batch_norm(
            input=conv,
            act=None,
            is_test=is_test,
            param_attr=bn_param_attr,
            bias_attr=bn_bias_attr,
            moving_mean_name=bn_name + '.mean',
            moving_variance_name=bn_name + '.var')

        if act == 'leaky':
            out = fluid.layers.leaky_relu(x=out, alpha=0.1)
        return out

    def _detection_block(self, input, channel, is_test=True, name=None):
        assert channel % 2 == 0, \
            "channel {} cannot be divided by 2 in detection block {}" \
            .format(channel, name)

        conv = input
        for j in range(2):
            conv = self._conv_bn(
                conv, channel, filter_size=1, stride=1, padding=0, is_test=is_test, name='{}.{}.0'.format(name, j))
            conv = self._conv_bn(
                conv, channel * 2, filter_size=3, stride=1, padding=1, is_test=is_test, name='{}.{}.1'.format(name, j))
        route = self._conv_bn(
            conv, channel, filter_size=1, stride=1, padding=0, is_test=is_test, name='{}.2'.format(name))
        tip = self._conv_bn(
            route, channel * 2, filter_size=3, stride=1, padding=1, is_test=is_test, name='{}.tip'.format(name))
        return route, tip

    def _upsample(self, input, scale=2, name=None):
        out = fluid.layers.resize_nearest(input=input, scale=float(scale), name=name)
        return out

    def _parse_anchors(self, anchors):
        """
        Check ANCHORS/ANCHOR_MASKS in config and parse mask_anchors

        """
        self.anchors = []
        self.mask_anchors = []

        assert len(anchors) > 0, "ANCHORS not set."
        assert len(self.anchor_masks) > 0, "ANCHOR_MASKS not set."

        for anchor in anchors:
            assert len(anchor) == 2, "anchor {} len should be 2".format(anchor)
            self.anchors.extend(anchor)

        anchor_num = len(anchors)
        for masks in self.anchor_masks:
            self.mask_anchors.append([])
            for mask in masks:
                assert mask < anchor_num, "anchor mask index overflow"
                self.mask_anchors[-1].extend(anchors[mask])

    def _get_outputs(self, input, is_train=True):
        """
        Get YOLOv3 head output

        Args:
            input (list): List of Variables, output of backbone stages
            is_train (bool): whether in train or test mode

        Returns:
            outputs (list): Variables of each output layer
        """

        outputs = []

        # get last out_layer_num blocks in reverse order
        out_layer_num = len(self.anchor_masks)
        if isinstance(input, OrderedDict):
            blocks = list(input.values())[-1:-out_layer_num - 1:-1]
        else:
            blocks = input[-1:-out_layer_num - 1:-1]
        route = None
        for i, block in enumerate(blocks):
            if i > 0:  # perform concat in first 2 detection_block
                block = fluid.layers.concat(input=[route, block], axis=1)
            route, tip = self._detection_block(
                block, channel=512 // (2**i), is_test=(not is_train), name=self.prefix_name + "yolo_block.{}".format(i))

            # out channel number = mask_num * (5 + class_num)
            num_filters = len(self.anchor_masks[i]) * (self.num_classes + 5)
            block_out = fluid.layers.conv2d(
                input=tip,
                num_filters=num_filters,
                filter_size=1,
                stride=1,
                padding=0,
                act=None,
                param_attr=ParamAttr(name=self.prefix_name + "yolo_output.{}.conv.weights".format(i)),
                bias_attr=ParamAttr(
                    regularizer=L2Decay(0.), name=self.prefix_name + "yolo_output.{}.conv.bias".format(i)))
            outputs.append(block_out)

            if i < len(blocks) - 1:
                # do not perform upsample in the last detection_block
                route = self._conv_bn(
                    input=route,
                    ch_out=256 // (2**i),
                    filter_size=1,
                    stride=1,
                    padding=0,
                    is_test=(not is_train),
                    name=self.prefix_name + "yolo_transition.{}".format(i))
                # upsample
                route = self._upsample(route)

        return outputs, blocks

    def get_prediction(self, outputs, im_size):
        """
        Get prediction result of YOLOv3 network

        Args:
            outputs (list): list of Variables, return from _get_outputs
            im_size (Variable): Variable of size([h, w]) of each image

        Returns:
            pred (Variable): The prediction result after non-max suppress.

        """
        boxes = []
        scores = []
        downsample = 32
        for i, output in enumerate(outputs):
            box, score = fluid.layers.yolo_box(
                x=output,
                img_size=im_size,
                anchors=self.mask_anchors[i],
                class_num=self.num_classes,
                conf_thresh=self.nms.score_threshold,
                downsample_ratio=downsample,
                name=self.prefix_name + "yolo_box" + str(i))
            boxes.append(box)
            scores.append(fluid.layers.transpose(score, perm=[0, 2, 1]))

            downsample //= 2

        yolo_boxes = fluid.layers.concat(boxes, axis=1)
        yolo_scores = fluid.layers.concat(scores, axis=2)
        pred = fluid.layers.multiclass_nms(
            bboxes=yolo_boxes,
            scores=yolo_scores,
            score_threshold=self.nms.score_threshold,
            nms_top_k=self.nms.nms_top_k,
            keep_top_k=self.nms.keep_top_k,
            nms_threshold=self.nms.nms_threshold,
            background_label=self.nms.background_label,
            normalized=self.nms.normalized,
            name="multiclass_nms")
        return pred


from typing import DefaultDict
import numpy as np
from numpy.core.fromnumeric import size
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset
import pandas as pd
from sklearn.metrics import roc_auc_score,log_loss
import os
import random
from utils import Config
import wandb
import collections
import math
import scipy.stats
from torch.optim import Adagrad

if __name__ == "__main__":

    # preprocess
    print(f'num_feat:{num_categorial_feat}')
    examplar_size = data_num//task_num
    criterion = nn.BCELoss()
    wconfig = {
        "weight_decay":weight_decay,
        "batch_win_size":Config.batch_win_size,
        "lr":lr,
        "pretrain_epoch":epoch_num,
        "finetune_epoch":epoch_num,
        "patience":PATIENCE,
        "task_num":task_num,
        "batch_size":batch_size,
        "hidden_dim":HID_DIM,
        "device":device
    }
    wandb.init(project="old_sampler", entity="ludlethwang",config=wconfig,name=f"icg")

    # training & evaulation for all configuration
    for ifrandom, oldrate in [(0,2)]:
        model = BaseModel(sizes).to(device)
        embeds = [eb.weight.detach() for eb in model.embeddings]
        print(f'model param nums: {sum([param.numel() for param in model.parameters()])}')
        print(f"batch_size:{batch_size}")
        optimizer = Adagrad(
            model.parameters(),
            lr=lr,
            initial_accumulator_value=1e-8,
            weight_decay=weight_decay
        )
        examplar_features, examplar_labels = [f[:int(data_num/task_num*Config.batch_win_size)] for f in categorial_features], labels[:int(data_num/task_num*Config.batch_win_size)]
        guards = [torch.zeros([sizes[_]+1],dtype=torch.long) for _ in range(num_categorial_feat)]
        overall_auc = []
        overall_logloss = []
        overall_auc_old = []
        overall_logloss_old = []
        overall_auc_notold = []
        overall_logloss_notold = []

        # incremental learning train and evaluate $win_size tasks one by one
        for task_idx in range(Config.batch_win_size, task_num-1):
            global_auc = 0.0
            global_logloss = 10.0
            global_auc_old = 0.0
            global_logloss_old = 10.0
            global_auc_notold = 0.0
            global_logloss_notold = 10.0
            global_patience = PATIENCE
            patience = global_patience
            print(f'running task {task_idx}')

            # sampling the examplar training set
            dataloader,test_loader,guards,examplar_features,examplar_labels = olddata_sampler(
                task_idx,ifrandom,oldrate,1,examplar_features,examplar_labels,guards
                )
            _,test_loader_old,__,___,____ = olddata_sampler(task_idx,ifrandom,oldrate,0,None,None,None)
            _,test_loader_notold,__,___,____ = olddata_sampler(task_idx,ifrandom,oldrate,2,None,None,None)
            for epoch in range(epoch_num):
                model.train()
                total_loss = []

                #training process
                for features, batchlabel in dataloader:
                    features = features.to(device)
                    batchlabel = batchlabel.to(device)
                    prediction =model(features)
                    if if_guard:
                        last_embeds = embeds
                        embeds = [eb.weight.detach().clone() for eb in model.embeddings]
                        reg_l = model.reg(guards,last_embeds)
                        # print(reg_l)
                    else:
                        reg_l = 0
                    loss = criterion(prediction.view(-1),batchlabel)+reg_l
                    optimizer.zero_grad()
                    loss.backward()
                    optimizer.step()
                    total_loss.append(loss.item())
                train_loss = sum(total_loss)/len(total_loss)
                if (epoch+1)%2==0:
                    print(epoch,"train loss:",train_loss)

                    # evaluation for each 2 epochs
                    with torch.no_grad():
                        model.eval()
                        auc,logloss = evaluate(task_idx+1,model)
                        print("evaluation auc,logloss on", task_idx+1,"is",auc,logloss)
                        auc_old,logloss_old = evaluate_spec(test_loader_old,model)
                        print("evaluation auc,logloss on", task_idx+1,"is",auc_old,logloss_old)
                        auc_notold,logloss_notold = evaluate_spec(test_loader_notold,model)
                        print("evaluation auc,logloss on", task_idx+1,"is",auc_notold,logloss_notold)
                        if auc_old<global_auc_old or logloss_old>global_logloss_old:
                            if not patience:
                                overall_auc.append(global_auc)
                                overall_logloss.append(global_logloss)
                                overall_auc_old.append(global_auc_old)
                                overall_logloss_old.append(global_logloss_old)
                                overall_auc_notold.append(global_auc_notold)
                                overall_logloss_notold.append(global_logloss_notold)
                                break
                            patience-=1
                        else:
                            patience = global_patience
                            global_auc, global_logloss = auc, logloss
                            global_auc_old, global_logloss_old = auc_old, logloss_old
                            global_auc_notold, global_logloss_notold = auc_notold, logloss_notold

                        if epoch == epoch_num-1:
                            overall_auc.append(global_auc)
                            overall_logloss.append(global_logloss)
                            overall_auc_old.append(global_auc_old)
                            overall_logloss_old.append(global_logloss_old)
                            overall_auc_notold.append(global_auc_notold)
                            overall_logloss_notold.append(global_logloss_notold)
            wandb.log({"train loss":train_loss})
            wandb.log({"auc":global_auc})
            wandb.log({"logloss":global_logloss})

        #logging to wandb
        wandb.log({"train loss":train_loss})
        wandb.log({"auc":sum(overall_auc)/len(overall_auc)})
        wandb.log({"logloss":sum(overall_logloss)/len(overall_logloss)})


import sys
import os
import astropy.io.fits as pyfits
import numpy

from podi_definitions import *

import scipy.interpolate

def subtract_pupilghost(filename_hdu, output_file, profile_file, operation="subtract", verbose=True):

    # Make the routine flexible in handling both a filename or HDUList as input parameter
    if (type(filename_hdu) is str):
        if (os.path.isfile(fitsfile)):
            hdulist = pyfits.open(fitsfile)
        else:
            return
    else:
        hdulist = filename_hdu

    filter = hdulist[0].header['FILTER']

    if (os.path.isfile(profile_file) and not operation=="ignore"):
        profile_data = numpy.loadtxt(profile_file)

        # Loop over all extensions
        # For now only use the first one, hard enough
        for ota_id in range(1, len(hdulist)):

            ota = hdulist[ota_id]
            extname = ota.header["EXTNAME"]

            if (filter in pupilghost_centers):
                if (extname in pupilghost_centers[filter]):
                    center_x, center_y = pupilghost_centers[filter][extname]

                    if (verbose): stdout_write("\rCorrecting pupil-ghost in extension %s ..." % (extname))
                    # Create the pixel array and create all distances
                    #        buffer = numpy.zeros(shape=(ota.data.shape[1], ota.data.shape[0]))
                    x, y = numpy.indices(ota.data.shape)
                    dx = x - center_x
                    dy = y - center_y
                    d = numpy.sqrt(dx*dx + dy*dy)

                    # Next load the profile, and fit a spline that we can use to evaluate it
                    profile = scipy.interpolate.interp1d(profile_data[:,0]*4, profile_data[:,1], bounds_error=False, fill_value=0)
                    fit = profile(d.ravel())

                    if (operation == "replace"):
                        hdulist[ota_id].data = fit.reshape(ota.data.shape).transpose()
                    else:
                        hdulist[ota_id].data -= fit.reshape(ota.data.shape).transpose()

                    ota.header.add_history("Pupilghost: profile in %s" % profile_file)
                    ota.header.add_history("Pupilghost: center @ %d,%d" % (center_x, center_y))

    if (output_file is not None):
        # Now all the work is done, all final data is stored in img2, write results to new file
        if (verbose): stdout_write(" writing output ...")
        clobberfile(outputfile)
        hdulist.writeto(outputfile)
        if (verbose): stdout_write(" done!\n\n")
    else:
        if (verbose): stdout_write(" done, handing on data!\n\n")
        return hdulist

if __name__ == "__main__":

    # Read in the input parameters
    fitsfile = sys.argv[1]
    profile_file = sys.argv[2]
    outputfile = sys.argv[3]

    operation = cmdline_arg_set_or_default("-op", "subtract")

    subtract_pupilghost(fitsfile, outputfile, profile_file, operation=operation, verbose=True)


#
import glob
import os
import argparse

from PIL import Image
import matplotlib.pyplot as plt

import torch.optim as optim
import torch.backends.cudnn as cudnn
from torch.utils.data import Dataset, DataLoader
import torchvision.transforms as transforms

from sklearn.metrics import roc_curve, roc_auc_score, auc
from sklearn.model_selection import  GroupKFold

from models import *
from bilinear_layers import *

import utils

class oaDataset(Dataset):
    def __init__(self, root_path, transform):
        # store filenames
        self.roi_root = root_path
        self.flist = glob.glob(os.path.join(self.roi_root, "**"))
        self.transform = transform

    def __len__(self):
        # return size of dataset
        return len(self.flist)

    def __getitem__(self, idx):
        fname = self.flist[idx]
        image, min_jsw, jsw_des, fjsw, kl, jsize = np.load(fname, allow_pickle=True)  # PIL image

        if kl > 1:
            grade =1
        else:
            grade = 0

        image = Image.fromarray(image.astype('uint8'), 'L')
        image = self.transform(image)

        jsw = np.array(jsw_des)
        # noise = np.random.normal(0, 1, 221)
        # jsw = jsw + noise
        jsw = torch.from_numpy(jsw).float()

        return image, jsw, grade

if __name__ == "__main__":

    parser = argparse.ArgumentParser()
    parser.add_argument('--oai_assesment', default='/../DATA/OAI/kXR_SQ_BU00_SAS/kxr_sq_bu00.sas7bdat')
    parser.add_argument('--model', default=combined)
    args = parser.parse_args()

    SEED = 42
    MAX_EPOCH = 100

    np.random.seed(SEED)
    torch.manual_seed(SEED);
    use_cuda = torch.cuda.is_available()
    device = torch.device("cuda:0" if use_cuda else "cpu")
    cudnn.benchmark = True

    cv = GroupKFold(n_splits=5)

    train_transforms = transforms.Compose([
        transforms.Resize(size=(56,56)),
        transforms.RandomCrop(size= (48,48)),
        transforms.ToTensor(),
    ])

    eval_transforms = transforms.Compose([
        transforms.Resize(size=(48,48)),
        transforms.ToTensor(),
    ])

    plt.figure(figsize=(10, 10))

    train_root = '../DATA/CROPS/OAI_00m_tm_fjsw_standardized/'
    test_root = '../DATA/CROPS/MOST_00m_tm_fjsw_standardized/'

    train_dataloader = DataLoader(oaDataset(train_root, train_transforms),
                                  batch_size=64,
                                  shuffle=True,
                                  num_workers=8,
                                  drop_last=True)
    test_dataloader = DataLoader(oaDataset(test_root, eval_transforms),
                                 batch_size=64,
                                 num_workers=8,
                                 drop_last=True)

    net = args.model().to(device)

    # Loss and optimizer
    criterion = nn.BCEWithLogitsLoss()
    optimizer = optim.SGD(net.parameters(), lr=0.1, momentum=0.9, nesterov=True)

    best_acc = 0.0
    best_epoch = None

    fpr = []
    tpr = []

    for epoch in range(MAX_EPOCH):

        optimizer,lr = utils.adjust_learning_rate(optimizer,epoch,0.00001, 0.01, 8)

        # Training
        train_loss = utils.train_epoch(epoch, net, optimizer, train_dataloader, criterion)
        # Validating
        val_loss, preds, truth = utils.validate_epoch(net, test_dataloader, criterion)

        auc_val = roc_auc_score(truth, preds)
        print(epoch + 1, train_loss, val_loss, auc_val)

        if auc_val > best_acc:
            best_acc = auc_val
            best_epoch = epoch
            fpr, tpr, thresholds = roc_curve(truth, preds)

    plt.plot(fpr, tpr, lw=2, alpha=0.8,
             label='ROC (AUC = %0.2f)' % (best_acc))
    plt.show()

    print("Best AUC:", best_acc)


import socket
import json

ListenSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ListenSocket.bind(('127.0.0.1', 5701))
ListenSocket.listen(100)

HttpResponseHeader = '''HTTP/1.1 200 OK\r\n
Content-Type: text/html\r\n\r\n
'''

def recv_end(the_socket):
    total_data = bytes()
    while True:
        data = the_socket.recv(1024)
        total_data += data
        if total_data[-2] == 125 and total_data[-1] == 10:
            break
    return total_data.decode(encoding='utf-8')

def request_to_json(msg):
    for i in range(len(msg)):
        if msg[i] == '{' and msg[-1] == '\n':
            return json.loads(msg[i:])
    return None

# 需要循环执行，返回值为json格式
def rev_msg():  # json or None
    Client, Address = ListenSocket.accept()
    Request = recv_end(Client)
    rev_json = request_to_json(Request)
    Client.sendall((HttpResponseHeader).encode(encoding='utf-8'))
    Client.close()
    return rev_json


import sys
import vtk
import numpy as np
import cv2
import py_ct_reg as ct
import os
import HandEyeCalLogic as he
import pydicom as dicom
from eval_dereflection_cl import Evaluator
from PyQt5 import QtCore, QtGui, QtWidgets
from vtkMainWindow_ui import Ui_MainWindow
from skimage import measure
from pycpd import AffineRegistration
#import matplotlib.pyplot as plt
import csv
import matlab.engine
from vtk.qt.QVTKRenderWindowInteractor import QVTKRenderWindowInteractor
from sksurgerynditracker.nditracker import NDITracker

POSITIONER_PRIMARY_ANGLE_INCREMENT = 1
POSITIONER_SECONDARY_ANGLE_INCREMENT = 1

# Change thresholds based on trial and error with extracted fiducial images
THRESHOLD_VALUE_1 = 0.3
THRESHOLD_VALUE_2 = 0.25
REORDER_INDEX = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 11, 10]

FRAME_INTERVAL_MS = 1000

# Might need to change this based on how many COM ports show up on in device manager
NUM_PORTS = 2
SPHERE_RADIUS = 5

class DeepDecompositionViewer(QtWidgets.QMainWindow, Ui_MainWindow):
    def __init__(self, parent=None):
        super().__init__()
        ckptsdir = "C:/Users/iahmedf/Documents/em-fluoro-artifact-removal/checkpoints"

        # Output directory for image files and tracking data
        self.outdir = "C:/Users/iahmedf/Documents/em-fluoro-artifact-removal/image_files"
        os.mkdir(f"{self.outdir}/preprocessed")
        os.mkdir(f"{self.outdir}/fiducials")
        os.mkdir(f"{self.outdir}/removed")
        self.trackingFile = open(f'{self.outdir}/trackingdata.txt', 'w')
        self.carmFile = open(f'{self.outdir}/carmdata.txt', 'w')

        self.eval = Evaluator(ckptsdir)
        self.setupUi(self)

        # Tracker widget setup
        self.ren = vtk.vtkRenderer()
        self.qvtkwin = QVTKRenderWindowInteractor(self.vtkWidget)
        self.qvtkwin.GetRenderWindow().AddRenderer(self.ren)
        self.ren.SetBackground(0.1, 0.2, 0.4)
        self.iren = self.qvtkwin.GetRenderWindow().GetInteractor()
        self.vtkGridLayout.addWidget(self.qvtkwin, 0, 0)
        self.updateTimer = QtCore.QTimer()

        # 3D overlay on fluoro stream

        # Tracker setup
        self.tracker = None
        self.isTrackerInitialized = False
        self.trackerSettings = {
            "tracker type": "aurora",
            "ports to probe": NUM_PORTS,
            "verbose": True
        }
        self.trackedTipSource = vtk.vtkSphereSource()
        self.trackedTipActor = vtk.vtkActor()
        self.trackedTipMapper = vtk.vtkPolyDataMapper()
        self.trackedTipTransform = vtk.vtkTransform()

        self.centers_original_ref = np.zeros((12, 2))
        self.fiducialWorldPts = []
        self.intCalMat = np.eye(3)
        self.trackerCTRegMat = np.eye(4)

        # Transformations between CT and tracker (loaded as CSV)
        self.trackerCTRegTransform = vtk.vtkTransform()
        self.CTTrackerRegTransform = vtk.vtkTransform()

        # Transformation from CT to fluoro (calculated per frame)
        self.fluoroCTRegTransform = vtk.vtkTransform()

        self.fluoroTransform = vtk.vtkTransform()
        self.fluoroCubeSource = vtk.vtkCubeSource()
        self.fluoroCubeActor = vtk.vtkActor()
        self.fluoroCubeMapper = vtk.vtkPolyDataMapper()

        self.connectSignalsSlots()
        self.setupVtkObjects()

        # Fluoro stream
        self.fluoroStream = None
        self.isStreaming = False
        self.fluoroFrameCtr = 0

        self.eng = matlab.engine.start_matlab()

        self.updateTimer.timeout.connect(self.updateScene)
        self.updateTimer.start(FRAME_INTERVAL_MS)

    def __del__(self):
        # Closes file with tracking data (which was opened in constructor)
        self.trackingFile.close()

    def setupVtkObjects(self):
        """Initializes and connects VTK objects"""
        self.fluoroCubeSource.SetCenter(0, 0, 0)
        self.fluoroCubeSource.SetXLength(100)
        self.fluoroCubeSource.SetYLength(100)
        self.fluoroCubeSource.SetZLength(100)

        self.trackedTipTransform.Identity()
        self.trackedTipSource.SetCenter(0, 0, 0)
        self.trackedTipSource.SetRadius(SPHERE_RADIUS)
        self.trackedTipMapper.SetInputConnection(self.trackedTipSource.GetOutputPort())
        self.trackedTipActor.SetMapper(self.trackedTipMapper)

        # Initialize all transforms to identity
        self.trackerCTRegTransform.Identity()
        self.CTTrackerRegTransform.Identity()
        self.fluoroCTRegTransform.Identity()
        self.fluoroTransform.Identity()

        # Mapper/actor for fluoro camera cube
        self.fluoroCubeMapper.SetInputConnection(self.fluoroCubeSource.GetOutputPort())
        self.fluoroCubeActor.SetMapper(self.fluoroCubeMapper)

        # Transformation chain for fluoro camera cube
        # If I set this up correctly, this transforms the fluoro into EM space
        self.fluoroTransform.PostMultiply()
        self.fluoroTransform.Concatenate(self.fluoroCTRegTransform)
        self.fluoroTransform.Concatenate(self.CTTrackerRegTransform)

    def connectSignalsSlots(self):
        """Connects QT buttons/toggles with functions"""
        self.browseFiducialFileButton.clicked.connect(self.browseFiducialFile)
        self.browseIntCalButton.clicked.connect(self.browseIntCal)
        self.browseCarmCalDicom1Button.clicked.connect(self.browseCarmCalDicom1)
        self.browseCarmCalDicom2Button.clicked.connect(self.browseCarmCalDicom2)
        self.browseTrackerCTRegButton.clicked.connect(self.browseTrackerCTReg)
        self.browseSimDicomButton.clicked.connect(self.browseSimDicom)
        self.browseRefCentroidsButton.clicked.connect(self.browseRefCentroids)

        self.trackerToggle.toggled.connect(self.startTracker)
        self.streamToggle.toggled.connect(self.handleStreamToggle)
        self.confirmParametersButton.clicked.connect(self.confirmParameters)
        self.runCarmCalButton.clicked.connect(self.runCarmCal)
        self.runSimButton.clicked.connect(self.runSimSingleImg)
    def browseRefCentroids(self):
        """Browsing window for CSV file containing reference centroids"""
        fname, d = QtWidgets.QFileDialog.getOpenFileName()
        self.refCentroidsField.setText(fname)
    def browseImage(self):
        """Browsing window for preprocessed image"""
        fname, d = QtWidgets.QFileDialog.getOpenFileName()
        self.imageField.setText(fname)
        self.preImageLabel.setPixmap(QtGui.QPixmap(fname))
    def browseFiducialFile(self):
        """Browsing window for fiducial world points (predetermined from CT)"""
        fname, d = QtWidgets.QFileDialog.getOpenFileName()
        self.fiducialFileField.setText(fname)
    def browseIntCal(self):
        """Browsing window for file containing intrinsic calibration matrix"""
        fname, d = QtWidgets.QFileDialog.getOpenFileName()
        self.intCalFileField.setText(fname)
    def browseCarmCalDicom1(self):
        """Browsing window for Axis 1 DICOM file for C-arm calibration"""
        fname, d = QtWidgets.QFileDialog.getOpenFileName()
        self.carmCalDicom1Field.setText(fname)
    def browseCarmCalDicom2(self):
        """Browsing window for Axis 2 DICOM file for C-arm calibration"""
        fname, d = QtWidgets.QFileDialog.getOpenFileName()
        self.carmCalDicom2Field.setText(fname)
    def browseTrackerCTReg(self):
        """Browsing window for CT-Tracking registration extrinsic matrix"""
        fname, d = QtWidgets.QFileDialog.getOpenFileName()
        self.trackerCTRegFileField.setText(fname)
    def browseSimDicom(self):
        """Browsing window for DICOM file for simulated workflow"""
        fname, d = QtWidgets.QFileDialog.getOpenFileName()
        self.simDicomField.setText(fname)
    def processImage(self):
        """Calls eval_dereflection algorithm"""
        outfile_fiducials, outfile_removed = self.eval.run_eval(self.imageField.text())
        self.postImageLabel.setPixmap(QtGui.QPixmap(outfile_removed))
        #self.getCarmPose()

    def startTracker(self):
        """ Starts NDI Aurora (magnetic) or Polaris (optical) tracker and sets up VTK tracked objects"""
        if self.trackerToggle.isChecked():
            if not self.isTrackerInitialized:
                try:
                    self.tracker = NDITracker(self.trackerSettings)
                    self.isTrackerInitialized = True
                    self.tracker.use_quaternions = False

                except:
                    print("Unable to connect to NDI Tracked device")
                    self.isTrackerInitialized = False
                    self.trackerToggle.setChecked(False)
            if self.isTrackerInitialized:
                self.tracker.start_tracking()
                self.trackedTipActor.SetUserTransform(self.trackedTipTransform)
                self.ren.AddActor(self.trackedTipActor)
                self.qvtkwin.GetRenderWindow().Render()

        else:
            if self.isTrackerInitialized:
                self.isTrackerInitialized = False
                self.tracker.stop_tracking()
                self.ren.RemoveActor(self.trackedTipActor)
                self.qvtkwin.GetRenderWindow().Render()
                print("Tracking Stopped")

    def confirmParameters(self):
        """Add fiducials to VTK widget scene"""
        self.readFiducialFile()
        self.readRefCentroidsFile()
        self.addFiducialsToScene()
        self.readIntCalFromFile()
        self.readTrackerCtRegFromFile()

    def readRefCentroidsFile(self):
        """Reads reference pixel centroids from file"""
        refcentroids_fname = self.refCentroidsField.text()
        with open(refcentroids_fname, 'r') as f:
            lines = f.readlines()
            for i in range(12):
                line_arr = lines[i].split(',')
                self.centers_original_ref[i, 0] = float(line_arr[0].strip())
                self.centers_original_ref[i, 1] = float(line_arr[1].strip())

    def readIntCalFromFile(self):
        """Reads fluoro intrinsic calibration from file"""

        intcal_fname = self.intCalFileField.text()

        with open(intcal_fname, 'r') as f:
            lines = f.readlines()
            for i in range(3):
                line_arr = lines[i].split(',')
                for j in range(3):
                    self.intCalMat[i, j] = float(line_arr[j].strip())

    def readTrackerCtRegFromFile(self):
        """Reads registration between CT and tracker from file"""
        extcal_fname = self.trackerCTRegFileField.text()
        with open(extcal_fname, 'r') as f:
            lines = f.readlines()
            for i in range(4):
                line_arr = lines[i].split(',')
                for j in range(4):
                    self.trackerCTRegMat[i, j] = float(line_arr[j].strip())

        # Saves transformations in both directions
        # Choose based on whether you want 3D visualization to be in EM or CT space
        self.trackerCTRegTransform.SetMatrix(np.reshape(self.trackerCTRegMat, 16))
        CTTrackerRegMat = np.linalg.inv(self.trackerCTRegMat)
        self.CTTrackerRegTransform.SetMatrix(np.reshape(CTTrackerRegMat, 16))

    def addFiducialsToScene(self):
        """Reads fiducials from CSV file, transforms them to EM space, and adds them to scene"""
        sphSrc = vtk.vtkSphereSource()
        sphSrc.SetRadius(5)
        sphSrc.SetCenter(0, 0, 0)
        for pt in self.fiducialWorldPts:
            tfrm = vtk.vtkTransform()
            tfrm.PostMultiply()
            ptMat = np.eye(4)
            ptMat[0:3, 3] = pt
            tfrm.Concatenate(np.reshape(ptMat, 16))

            # Applies transformation to put each fiducial into EM tracking space
            tfrm.Concatenate(self.CTTrackerRegTransform)
            sphAct = vtk.vtkActor()
            sphMap = vtk.vtkPolyDataMapper()

            sphMap.SetInputConnection(sphSrc.GetOutputPort())
            sphAct.SetMapper(sphMap)
            sphAct.SetUserTransform(tfrm)
            self.ren.AddActor(sphAct)

    def getCarmPose(self, img):

        """Gets C-arm pose from fiducial image
        Uses reference centroids provided from CSV file"""

        img_float = img/float(np.amax(img))
        img_float = img_float.astype(np.float32)

        # Threshold Segmentation

        # Change threshold based on
        bw = img_float < 0.35
        L = measure.label(bw, connectivity=2)
        S = measure.regionprops(L)
        S_area = [S_i.area for S_i in S]
        S_area_arr = np.array(S_area)
        S_area_idx = np.where((S_area_arr >= 200) & (S_area_arr <= 600))
        S_area_idx += np.ones(len(S_area_idx))
        bw2 = np.isin(L, S_area_idx)
        bw2flt = bw2.astype(np.float32)
        cv2.imshow("bw2flt", bw2flt)
        cv2.waitKey(0)
        L2 = measure.label(bw2, connectivity=2)

        C = measure.regionprops(L2)
        centers_original = np.array([[C_i.centroid[1], C_i.centroid[0]] for C_i in C])

        # Registration with reference centroids
        reg = AffineRegistration(X=self.centers_original_ref, Y=centers_original)
        TY, (B_reg, t_reg) = reg.register()
        delta = (self.centers_original_ref - TY)
        delta_rmse = delta[:, 0] * delta[:, 0] + delta[:, 1] * delta[:, 1]
        reorderIndex = []
        if delta_rmse.all() < 1:
            reorderIndex = np.arange(len(self.centers_original_ref))
        else:
            for i in range(len(self.centers_original_ref)):
                delta_temp = TY - self.centers_original_ref[i, :]
                delta_temp_rmse = delta_temp[:, 0] * delta_temp[:, 0] + delta_temp[:, 1] * delta_temp[:, 1]
                I = np.argmin(delta_temp_rmse)
                reorderIndex.append(I)

        centers = centers_original[reorderIndex, :]

        worldPt_T = np.transpose(np.array(self.fiducialWorldPts))
        imgPt = np.transpose(np.array(centers))
        radialDistortion = np.array([0.0, 0.0])
        tangentialDistortion = np.array([0.0, 0.0])
        worldPts2 = np.array(self.fiducialWorldPts[:, 0:2], dtype=float)
        cameraParams = self.eng.cameraParameters('IntrinsicMatrix', np.transpose(self.intCalMat), 'RadialDistortion', radialDistortion, 'TangentialDistortion', tangentialDistortion, 'WorldPoints', worldPts2);

        # MATLAB function calls
        [rotationMatrix, translationVector] = self.eng.extrinsics(np.array(centers, dtype=float), worldPts2, cameraParams, nargout=2)
        [orientation, location] = self.eng.extrinsicsToCameraPose(rotationMatrix, translationVector, nargout=2)

        # Assigns MATLAB results to VTK transforms
        xfrmMat = np.empty((4, 4))
        xfrmMat[0:3, 0:3] = orientation
        xfrmMat[0:3, 3] = location

        self.fluoroCTRegTransform.SetMatrix(np.reshape(xfrmMat, 16))

        # Writes C-arm pose (calculated from fiducials - so I believe this pose is in CT space) to text file
        self.trackingFile.write(f"Frame: {self.fluoroFrameCtr}\n")
        self.trackingFile.write(f"Matrix:\n[[{xfrmMat[0][0, 0]}, {xfrmMat[0][0, 1]}, {xfrmMat[0][0, 2]}, {xfrmMat[0][0, 3]}]\n[{xfrmMat[0][1, 0]}, {xfrmMat[0][1, 1]}, {xfrmMat[0][1, 2]}, {xfrmMat[0][1, 3]}]\n[{xfrmMat[0][2, 0]}, {xfrmMat[0][2, 1]}, {xfrmMat[0][2, 2]}, {xfrmMat[0][2, 3]}]\n[{xfrmMat[0][3, 0]}, {xfrmMat[0][3, 1]}, {xfrmMat[0][3, 2]}, {xfrmMat[0][3, 3]}]]\n\n")

        # If I set up the transforms right, this is the pose of the C-arm in EM tracking space
        self.fluoroTransform.Update()

        return xfrmMat

    def updateScene(self):
        """Full workflow (streaming, artifact removal, pose calculation)"""
        if self.isStreaming:
            # Read frame
            #_, img_pre = self.fluoroStream.read()
            img_pre = cv2.imread("C:/Users/iahmedf/Documents/em-fluoro-artifact-removal/datasets_old/Syn3/val/defocused/C/7822.png")
            img_pre = cv2.cvtColor(img_pre, cv2.COLOR_BGR2RGB)
            self.fluoroFrameCtr += 1

            # Show frame in Qt
            #qimg_pre = self.cv2ToQPixmap(img_pre, QtGui.QImage.Format.Format_RGB888)
            #self.preImageLabel.setPixmap(QtGui.QPixmap.fromImage(qimg_pre))

            # Save frame
            cv2.imwrite(f"{self.outdir}/preprocessed/{self.fluoroFrameCtr}.png", img_pre)

            # Artifact removal
            img_fiducials, img_removed = self.eval.run_eval(img_pre, self.outdir, self.fluoroFrameCtr)

            img_removed = cv2.cvtColor(img_removed, cv2.COLOR_RGB2BGR)
            img_fiducials = cv2.cvtColor(img_fiducials, cv2.COLOR_RGB2BGR)

            #qimg_fiducials = self.cv2ToQPixmap(img_fiducials, QtGui.QImage.Format.Format_RGB888)
            qimg_removed = self.cv2ToQPixmap(img_pre, QtGui.QImage.Format.Format_RGB888)
            self.fluoroImgLabel.setPixmap(QtGui.QPixmap.fromImage(qimg_removed))
            #self.fiducialsImageLabel.setPixmap(QtGui.QPixmap.fromImage(qimg_fiducials))

            # Calculate C-arm pose
            self.getCarmPose(img_fiducials)
            #print(self.fluoroCubeActor.GetUserMatrix())
        if self.isTrackerInitialized:
            port_handles, time_stamps, frame_numbers, tracking, tracking_quality = self.tracker.get_frame()
            print("read frame")
            self.trackedTipTransform.SetMatrix(np.reshape(tracking[0], 16))
            self.trackedTipTransform.Update()
            self.ToolTxLCD.display(tracking[0][0, 3])
            self.ToolTyLCD.display(tracking[0][1, 3])
            self.ToolTzLCD.display(tracking[0][2, 3])

            # Frame column of CSV shows -1 if not video streaming; shows the fluoro frame number if streaming
            if self.isStreaming:
                curFrame = self.fluoroFrameCtr
            else:
                curFrame = -1
            self.trackingFile.write(f"Frame: {curFrame}\n")
            self.trackingFile.write(f"Matrix:\n[[{tracking[0][0, 0]}, {tracking[0][0, 1]}, {tracking[0][0, 2]}, {tracking[0][0, 3]}]\n[{tracking[0][1, 0]}, {tracking[0][1, 1]}, {tracking[0][1, 2]}, {tracking[0][1, 3]}]\n[{tracking[0][2, 0]}, {tracking[0][2, 1]}, {tracking[0][2, 2]}, {tracking[0][2, 3]}]\n[{tracking[0][3, 0]}, {tracking[0][3, 1]}, {tracking[0][3, 2]}, {tracking[0][3, 3]}]]\n\n")
        self.ren.ResetCameraClippingRange()
        self.qvtkwin.GetRenderWindow().Render()

    def cv2ToQPixmap(self, cv2img, format):
        """Converts cv2 image to QPixmap format"""
        height, width, bytesPerComponent = cv2img.shape
        bytesPerLine = bytesPerComponent * width
        return QtGui.QImage(cv2img.data, width, height, bytesPerLine, format)

    def readFiducialFile(self):
        """Reads list of fiducial positions from CSV file"""
        fname = self.fiducialFileField.text()

        with open(fname, 'r') as f:
            lines = f.readlines()
            for line in lines:
                line_arr = line.split(',')
                self.fiducialWorldPts.append([float(line_arr[1].strip()), float(line_arr[2].strip()), float(line_arr[3].strip())])
        self.fiducialWorldPts = np.array(self.fiducialWorldPts, dtype=np.float32)

    def handleStreamToggle(self):
        """Starts/stops video stream"""
        if self.streamToggle.isChecked():
            self.isStreaming = True
            self.fluoroStream = cv2.VideoCapture(0)
            self.fluoroCubeActor.SetUserTransform(self.fluoroCTRegTransform)
            self.ren.AddActor(self.fluoroCubeActor)
            #self.updateTimer.timeout.connect(self.updateScene)

        else:
            self.isStreaming = False
            self.fluoroStream = None

    # The following 3 functions are not used in the actual workflow - they were just used to simulate/test parts of the workflow
    def runCarmCal(self):

        fname1 = self.carmCalDicom1Field.text()
        fname2 = self.carmCalDicom2Field.text()
        fiducial_fname = ct.extractFiducials(fname1, fname2)
        worldpts_fname = self.fiducialFileField.text()
        print(self.intCalMat)
        ct.computeCarmPose(self.intCalMat, fiducial_fname, worldpts_fname)

    def runSimSingleImg(self):
        imgname = self.simDicomField.text()
        img = cv2.imread(imgname)

        inv = self.getCarmPose(img)
        print(inv)
    def updateScene_streamingtest(self):
        """Just tests streaming and artifact removal (no C-arm pose calculation)"""
        if self.isStreaming:
            _, img_pre = self.fluoroStream.read()
            img_pre = cv2.cvtColor(img_pre, cv2.COLOR_BGR2RGB)
            self.fluoroFrameCtr += 1
            qimg_pre = self.cv2ToQPixmap(img_pre, QtGui.QImage.Format.Format_RGB888)
            self.preImageLabel.setPixmap(QtGui.QPixmap.fromImage(qimg_pre))
            cv2.imwrite(f"{self.outdir}/preprocessed/{self.fluoroFrameCtr}.png", img_pre)

            img_fiducials, img_removed = self.eval.run_eval(img_pre, self.outdir, self.fluoroFrameCtr)
            cv2.imshow("removed rgb", img_removed)
            cv2.waitKey(0)
            img_removed = cv2.cvtColor(img_removed, cv2.COLOR_RGB2BGR)
            img_fiducials = cv2.cvtColor(img_fiducials, cv2.COLOR_RGB2BGR)

            img_removed = cv2.cvtColor(img_removed, cv2.COLOR_BGR2RGB)
            img_fiducials = cv2.cvtColor(img_fiducials, cv2.COLOR_BGR2RGB)
            qimg_fiducials = self.cv2ToQPixmap(img_fiducials, QtGui.QImage.Format.Format_RGB888)
            qimg_removed = self.cv2ToQPixmap(img_removed, QtGui.QImage.Format.Format_RGB888)
            self.removedImageLabel.setPixmap(QtGui.QPixmap.fromImage(qimg_removed))
            self.fiducialsImageLabel.setPixmap(QtGui.QPixmap.fromImage(qimg_fiducials))
        self.ren.ResetCameraClippingRange()
        self.qvtkwin.GetRenderWindow().Render()
    def runSimNoRemoval(self):
        # Takes in a DICOM file and calculates the C-arm pose for every frame, then creates visualization

        # Reads in DICOM and parameters
        dcmname = self.simDicomField.text()
        info = dicom.dcmread(dcmname)
        angleIncrement = info.PositionerSecondaryAngleIncrement
        uniqueAngles = np.unique(angleIncrement)
        numUniqueAngles = len(uniqueAngles)

        # Sets up visualization
        self.fluoroCubeActor.SetUserTransform(self.fluoroCTRegTransform)
        self.ren.AddActor(self.fluoroCubeActor)

        # Saves all C-arm positions
        translationVectorTotal = np.zeros((3, numUniqueAngles-20))

        # Calculates reference centroids
        indexRef = int(np.ceil(numUniqueAngles/2.0)) - 1
        framesIndex = np.where(angleIncrement == uniqueAngles[indexRef])
        frameIndex = int(np.ceil(np.median(framesIndex)))
        refimg = info.pixel_array[frameIndex]

        img_float = refimg/float(np.amax(refimg))
        img_float = img_float.astype(np.float32)

        # Threshold Segmentation
        bw = img_float < 0.3
        bwint = bw.astype(np.float32)
        L = measure.label(bw, connectivity=2)
        S = measure.regionprops(L)
        S_area = [S_i.area for S_i in S]
        S_area_arr = np.array(S_area)
        S_area_idx = np.where((S_area_arr >= 200) & (S_area_arr <= 2000))
        S_area_idx += np.ones(len(S_area_idx))
        bw2 = np.isin(L, S_area_idx)
        bw2flt = bw2.astype(np.float32)
        L2 = measure.label(bw2flt, connectivity=2)

        C = measure.regionprops(L2)
        self.centers_original_ref = np.array([[C_i.centroid[1], C_i.centroid[0]] for C_i in C])

        # Finds centroids and calculates pose for each frame
        for indexRef in range(numUniqueAngles - 20):
            framesIndex = np.where(angleIncrement == uniqueAngles[indexRef])
            frameIndex = int(np.ceil(np.median(framesIndex)))
            print(f"--frameIndex {frameIndex}--")
            img = info.pixel_array[frameIndex]

            inv = self.getCarmPose(img)
            translationVectorTotal[:, indexRef] = inv[0:3, 3]
            self.ren.ResetCameraClippingRange()
            self.qvtkwin.GetRenderWindow().Render()
        # fig = plt.figure()
        # ax = fig.add_subplot(projection='3d')
        # ax.scatter(translationVectorTotal[0, :], translationVectorTotal[1, :], translationVectorTotal[2, :], s=10, c='b', marker='s')
        # ax.set_xlim3d(-100, 100)
        with open('translationVectorTotal.csv', 'w', newline='') as f:
            writer = csv.writer(f)
            for t in np.transpose(translationVectorTotal):
                writer.writerow(t)

        #plt.show()

if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    window = DeepDecompositionViewer()
    window.show()
    window.iren.Initialize()
    sys.exit(app.exec())


import math
import matplotlib.pyplot as plt

n= 100
def cal_aprox_pi(n):

    pi= 0

    for i in range(n):
        numerador = (2**i)*(math.factorial(i)**2)
        denomidador = math.factorial(2*i+1)
        fraction = numerador/denomidador

        pi += fraction
    pi = 2*pi

    return pi
list_euler = []
Err_list_euler = []

for i in range(1, n):
    aprox_pi = cal_aprox_pi(i)
    list_euler.append(aprox_pi)

    Err_real = math.pi - aprox_pi
    Err_abs = abs(Err_real)
    Err_relativ = abs(Err_real/math.pi)

    Err_list_euler.append(Err_relativ)
    print("%d \t %f \t %f \t %f \t %.4f " % (i, aprox_pi, Err_real, Err_abs,Err_relativ*100 ))

plt.figure("Serie de Euler")
plt.plot([0, n], [math.pi, math.pi], "g-")
plt.plot(list_euler , "bo-", label="Serie de Euler")
plt.xlabel("n")
plt.ylabel(u"$\pi$")
plt.grid(ls="dashed")
plt.legend()

plt.figure("Gráfica del Error")
plt.plot(Err_list_euler, "bo-", label=" Serie de Euler")
plt.xlabel("n")
plt.ylabel(u"$\% error$")
plt.grid(ls="dashed")
plt.legend()

plt.show()


#!/usr/bin/env python
# -*- coding: utf-8 -*-
import cv2
if __name__ == '__main__':
    # KCF
    tracker = cv2.TrackerKCF_create()
    # Boosting
    #tracker = cv2.TrackerBoosting_create()
    # MedianFlow
    # tracker = cv2.TrackerMedianFlow_create()

    cap = cv2.VideoCapture(2)
    while True:
        ret, frame = cap.read()
        if not ret:
            continue
        bbox = cv2.selectROI(frame, False)
        ok = tracker.init(frame, bbox)
        cv2.destroyAllWindows()
        break
    while True:
        ret, frame = cap.read()
        # トラッカーをアップデートする
        track, bbox = tracker.update(frame)
        if track:
            # Tracking success
            p1 = (int(bbox[0]), int(bbox[1]))
            p2 = (int(bbox[0] + bbox[2]), int(bbox[1] + bbox[3]))
            cv2.rectangle(frame, p1, p2, (0,255,0), 2, 1)
        else :
            cv2.putText(frame, "Failure", (10,50), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1, cv2.LINE_AA);
        # 加工済の画像を表示する
        cv2.imshow("Tracking", frame)

        # キー入力を1ms待って、k が27（ESC）だったらBreakする
        k = cv2.waitKey(1)
        if k == 27 :
            break

# キャプチャをリリースして、ウィンドウをすべて閉じる
cap.release()
cv2.destroyAllWindows()


from pyhanlp import HanLP
from tqdm import tqdm
import os

def segmentKey():
    input_dir = "../../dataset/result/2-extract/queries.result"
    output_dir_prefix = "../dataset/result/3-segment/hanlp/normal_segmented_"
    keywords = []
    output_dirs = []
    with open("../../keyword.txt", "r", encoding="utf8") as keyword_file, open(input_dir, "r", encoding="utf8") as input_file:
        for line in keyword_file:
            keyword = line.strip()
            HanLP.Config.CustomDictionary.add(keyword)  # 添加用户自定义词汇
            keywords.append(keyword)
            output_dirs.append(output_dir_prefix + keyword + ".result")

        # 检查文件是否存在或不为空
        for output_file in output_dirs:
            if os.path.exists(output_file) and os.path.getsize(output_file) > 0:
                with open(output_file, 'w', encoding="utf8") as output_file:
                    # 清空文件内容
                    output_file.truncate()

        # 寻找关键字并分词
        for line in tqdm(input_file, desc="Processing", ncols=100):
            for keyword in keywords:
                if keyword in line.strip():
                    word_list = HanLP.segment(line.strip())  # 使用HanLP进行分词
                    word_list = [term.word for term in word_list]  # 获取分词结果中的词语部分
                    with open(output_dir_prefix + keyword + ".result", "a", encoding="utf8") as output_file:
                        output_file.write(" ".join(word_list) + "\n")  # 写入换行符保持原始行结构

if __name__ == '__main__':
    segmentKey()


from flask import jsonify, abort, make_response, request, Blueprint
from .models import parcels

NOT_FOUND = 'data NOT found'
BAD_REQUEST = 'Bad request'

v1_blueprint = Blueprint(
    'v1', __name__
)

def _get_order(id):
    return [parcel for parcel in parcels if parcel['id'] == id]

def _entry_exists_for_delivery_order(item_name):
    return [parcel for parcel in parcels if parcel["item_name"] == item_name]

@v1_blueprint.errorhandler(404)
def not_found(error):
    return make_response(jsonify({'error': NOT_FOUND}), 404)

@v1_blueprint.errorhandler(400)
def bad_request(error):
    return make_response(jsonify({'error': BAD_REQUEST}), 400)

@v1_blueprint.route('/api/v1')
def index():
    return jsonify({"welcome": "crackers ni wale wase"})

@v1_blueprint.route('/api/v1/parcels', methods=['POST'])
def post_delivery_order():
    item_id = 0
    if not request.json or 'item_name' not in request.json:
        abort(400)
    orders = parcels
    if len(orders) == 0:
        item_id = 1
    else:
        item_id = parcels[-1].get("id") + 1

    owner_id = request.json.get('owner_id')
    item_name = request.json.get('item_name')
    origin = request.json.get('origin')
    pickup_location = request.json.get('pickup_location')
    price = request.json.get('price')
    status = request.json.get('status')

    if _entry_exists_for_delivery_order(item_name):
        return jsonify({'message': "entry exists"})
    parcel = {
        "id": item_id,
        "owner_id": owner_id,
        "item_name": item_name,
        "origin": origin,
        "pickup_location": pickup_location,
        "price": price,
        "status": status
    }
    parcels.append(parcel)
    return jsonify({'parcel': parcel}), 201

@v1_blueprint.route('/api/v1/parcels', methods=['GET'])
def get_all_parcel_orders():
    orders = parcels
    if len(orders) == 0:
        return jsonify({'message': 'no orders have been posted'})
    return jsonify({'all orders': parcels})

# get a particular parcel delivery order
@v1_blueprint.route('/api/v1/parcels/<int:id>', methods=['GET'])
def get_particular_parcel_order(id):
    order = _get_order(id)
    if not order:
        abort(404)
    return jsonify({'order': order})

# get parcel orders belonging to a particular user
@v1_blueprint.route('/api/v1/users/<int:id>/parcels', methods=['GET'])
def get_user_parcel_delivery_orders(id):
    parcel = [parcel for parcel in parcels if parcel['owner_id'] == id]
    if not parcel:
        abort(404)
    return jsonify({"parcels": parcel})

@v1_blueprint.route('/api/v1/parcels/<int:id>/cancel', methods=['PUT'])
def cancel_delivery_order(id):
    if not request.json:
        abort(400)
    orders = _get_order(id)
    if not orders:
        abort(404)
    elif orders[0]['status'] == "delivered":
        return jsonify({"error": "order has already been delivered!!"})
    elif orders[0]['status'] == "cancelled":
        return jsonify({"error": "order has already been cancelled!!"})
    orders[0]['status'] = "cancelled"
    return jsonify({'delivery order': orders})