import os
import argparse
import torch
from torch import nn
import torch.backends.cudnn as cudnn
from torch.utils.data import DataLoader
from hrcnet import PoseHighResolutionNet
from dataset import potsdam
from loss import Edge_loss
import time
from scipy import io
import logging
logging.basicConfig(filename='hrnet_v11.log', level=logging.INFO)

# 0=impervious surfacescd
# 1=building
# 2=low vegetation
# 3=tree
# 4=car
# 5=background

os.environ["CUDA_VISIBLE_DEVICES"] = "0,1"
os.environ.setdefault('RANK', '0')
os.environ.setdefault('WORLD_SIZE', '1')
os.environ.setdefault('MASTER_ADDR', '202.204.54.125')
os.environ.setdefault('MASTER_PORT', '29555')

class FullModel(nn.Module):
  """
  Distribute the loss on multi-gpu to reduce
  the memory cost in the main gpu.
  You can check the following discussion.
  https://discuss.pytorch.org/t/dataparallel-imbalanced-memory-usage/22551/21
  """
  def __init__(self, model, loss, edge_loss):
    super(FullModel, self).__init__()
    self.model = model
    self.loss = loss
    self.edge_loss = edge_loss

  def forward(self, inputs, labels):
    output, edge_out, se_out = self.model(inputs)
    return output

def get_world_size():
    if not torch.distributed.is_initialized():
        return 1
    return torch.distributed.get_world_size()

def get_rank():
    if not torch.distributed.is_initialized():
        return 0
    return torch.distributed.get_rank()

class params():
    def __init__(self):
        self.number_of_classes = 6
        self.TRAIN_BATCH_SIZE_PER_GPU = 8
        self.VAL_BATCH_SIZE_PER_GPU = 16
        self.learning_rate = 0.01
        "hrnet48"
        self.STAGE2 = {'NUM_MODULES': 1,
                        'NUM_BRANCHES': 2,
                        'NUM_BLOCKS': [4,4],
                        'NUM_CHANNELS': [48,96],
                        'BLOCK':'BASIC',
                        'FUSE_METHOD': 'SUM'}
        self.STAGE3 = {'NUM_MODULES': 4,
                       'NUM_BRANCHES': 3,
                       'NUM_BLOCKS': [6, 6, 6],
                       'NUM_CHANNELS': [48, 96, 192],
                       'BLOCK': 'BASIC',
                       'FUSE_METHOD': 'SUM'}
        self.STAGE4 = {'NUM_MODULES': 3,
                       'NUM_BRANCHES': 4,
                       'NUM_BLOCKS': [3, 3, 3, 3],
                       'NUM_CHANNELS': [48, 96, 192, 384],
                       'BLOCK': 'BASIC',
                       'FUSE_METHOD': 'SUM'}
        # "hrnet32"
        # self.STAGE2 = {'NUM_MODULES': 1,
        #                'NUM_BRANCHES': 2,
        #                'NUM_BLOCKS': [4, 4],
        #                'NUM_CHANNELS': [32, 64],
        #                'BLOCK': 'BASIC',
        #                'FUSE_METHOD': 'SUM'}
        # self.STAGE3 = {'NUM_MODULES': 4,
        #                'NUM_BRANCHES': 3,
        #                'NUM_BLOCKS': [4, 4, 4],
        #                'NUM_CHANNELS': [32, 64, 128],
        #                'BLOCK': 'BASIC',
        #                'FUSE_METHOD': 'SUM'}
        # self.STAGE4 = {'NUM_MODULES': 3,
        #                'NUM_BRANCHES': 4,
        #                'NUM_BLOCKS': [4, 4, 4, 4],
        #                'NUM_CHANNELS': [32, 64, 128, 256],
        #                'BLOCK': 'BASIC',
        #                'FUSE_METHOD': 'SUM'}
        # "hrnet32_cut"
        # self.STAGE2 = {'NUM_MODULES': 1,
        #                'NUM_BRANCHES': 2,
        #                'NUM_BLOCKS': [4, 4],
        #                'NUM_CHANNELS': [32, 64],
        #                'BLOCK': 'BASIC',
        #                'FUSE_METHOD': 'SUM'}
        # self.STAGE3 = {'NUM_MODULES': 1,
        #                'NUM_BRANCHES': 3,
        #                'NUM_BLOCKS': [6, 6, 6],
        #                'NUM_CHANNELS': [32, 64, 128],
        #                'BLOCK': 'BASIC',
        #                'FUSE_METHOD': 'SUM'}
        # self.STAGE4 = {'NUM_MODULES': 1,
        #                'NUM_BRANCHES': 4,
        #                'NUM_BLOCKS': [3, 3, 3, 3],
        #                'NUM_CHANNELS': [32, 64, 128, 256],
        #                'BLOCK': 'BASIC',
        #                'FUSE_METHOD': 'SUM'}

def parse_args():
    parser = argparse.ArgumentParser(description='Train segmentation network')
    parser.add_argument("--local_rank", type=int, default=0)
    parser.add_argument('opts',
                        help="Modify config options using the command-line",
                        default=None,
                        nargs=argparse.REMAINDER)
    args2 = parser.parse_args()
    return args2

args = params()

args2 = parse_args()

dataset_dir = "../dataset"

cudnn.benchmark = True
cudnn.deterministic = False
cudnn.enabled = True
distributed = True
device = torch.device(('cuda:{}').format(args2.local_rank))
PoseHighResolutionNet = PoseHighResolutionNet(args)

# flops, params = profile(PoseHighResolutionNet, (1, 3, 384, 384))
# print('flops:{}G'.format(flops/1e9), 'params:{}M'.format(params/(1024*1024)))

if distributed:
    torch.cuda.set_device(args2.local_rank)
    torch.distributed.init_process_group(
        backend="nccl", init_method="env://",
    )

potsdam_test = potsdam(base_dir=dataset_dir, train=False)
val_sampler = None
dataloader_test = DataLoader(
    potsdam_test,
    batch_size=args.VAL_BATCH_SIZE_PER_GPU,
    shuffle=False,
    num_workers=4,
    pin_memory=True,
    sampler=val_sampler)

seg_criterion = nn.CrossEntropyLoss(ignore_index=255)
edge_criterion = Edge_loss()

PoseHighResolutionNet = FullModel(PoseHighResolutionNet, seg_criterion, edge_criterion)
PoseHighResolutionNet = nn.SyncBatchNorm.convert_sync_batchnorm(PoseHighResolutionNet)
PoseHighResolutionNet = PoseHighResolutionNet.to(device)
PoseHighResolutionNet = nn.parallel.DistributedDataParallel(
    PoseHighResolutionNet, device_ids=[args2.local_rank], output_device=args2.local_rank)

optimizer = torch.optim.SGD([{'params':
                                  filter(lambda p: p.requires_grad,
                                         PoseHighResolutionNet.parameters()),
                                  'lr': args.learning_rate}],
                                lr=args.learning_rate,
                                momentum=0.9,
                                weight_decay=0.0005,
                                nesterov=False,
                                )

def test():
    PoseHighResolutionNet.eval()
    with torch.no_grad():
        model_state_file = "model/edge1_se_hrnet48_3463_cut_acfpn3463_384_72_sem_gcnet_stage4_best_result_397.pkl"
        if os.path.isfile(model_state_file):
            logging.info("=> loading checkpoint '{}'".format(model_state_file))
            checkpoint = torch.load(model_state_file, map_location=lambda storage, loc: storage)
            PoseHighResolutionNet.load_state_dict(checkpoint)
        n = 0
        for i, sample in enumerate(dataloader_test):
            images, labels = sample['image'], sample['label']
            images, labels = images.cuda(), labels.cuda()
            labels = labels.long().squeeze(1)
            start = time.clock()
            logits = PoseHighResolutionNet(images, labels)
            time_cost = time.clock() - start
            print("time_cost:{}".format(time_cost))
            logits = logits.argmax(dim=1)
            logits = logits.cpu().detach().numpy()
            for j in range(logits.shape[0]):
                n += 1
                io.savemat("network_output/" + str(n) + ".mat", {'network_output': logits[j]})

if __name__ == '__main__':
    test()


"""
Module with reading functionalities for isochrones.
"""

import configparser
import os
import warnings

from typing import Optional, Tuple

import h5py
import numpy as np

from scipy.interpolate import griddata
from typeguard import typechecked

from species.core import box
from species.read import read_model
from species.util import read_util

class ReadIsochrone:
    """
    Class for reading isochrone data from the database.
    """

    @typechecked
    def __init__(self, tag: str) -> None:
        """
        Parameters
        ----------
        tag : str
            Database tag of the isochrone data.

        Returns
        -------
        NoneType
            None
        """

        self.tag = tag

        config_file = os.path.join(os.getcwd(), "species_config.ini")

        config = configparser.ConfigParser()
        config.read(config_file)

        self.database = config["species"]["database"]

        with h5py.File(self.database, "r") as h5_file:
            if f"isochrones/{self.tag}" not in h5_file:
                raise ValueError(f"There is no isochrone data stored "
                                 f"with the selected tag \'{tag}\'.")

    @typechecked
    def get_isochrone(
        self,
        age: float,
        masses: np.ndarray,
        filters_color: Optional[Tuple[str, str]] = None,
        filter_mag: Optional[str] = None,
    ) -> box.IsochroneBox:
        """
        Function for selecting an isochrone.

        Parameters
        ----------
        age : float
            Age (Myr) at which the isochrone data is interpolated.
        masses : np.ndarray
            Masses (:math:`M_\\mathrm{J}`) at which the isochrone
            data is interpolated.
        filters_color : tuple(str, str), None
            Filter names for the color as listed in the file with the
            isochrone data. Not selected if set to ``None`` or if only
            evolutionary tracks are available.
        filter_mag : str, None
            Filter name for the absolute magnitude as listed in the
            file with the isochrone data. Not selected if set to
            ``None`` or if only evolutionary tracks are available.

        Returns
        -------
        species.core.box.IsochroneBox
            Box with the isochrone.
        """

        age_points = np.full(masses.shape[0], age)  # (Myr)

        color = None
        mag_abs = None

        index_teff = 2
        index_logg = 4

        # Read isochrone data

        with h5py.File(self.database, "r") as h5_file:
            model = h5_file[f"isochrones/{self.tag}/evolution"].attrs["model"]
            evolution = np.asarray(h5_file[f"isochrones/{self.tag}/evolution"])

            if model == "baraffe":
                filters = list(h5_file[f"isochrones/{self.tag}/filters"])
                magnitudes = np.asarray(h5_file[f"isochrones/{self.tag}/magnitudes"])

                # Convert the h5py list of filters from bytes to strings
                for i, item in enumerate(filters):
                    if isinstance(item, bytes):
                        filters[i] = item.decode("utf-8")

        if model == "baraffe":
            if filters_color is not None:
                index_color_1 = filters.index(filters_color[0])
                index_color_2 = filters.index(filters_color[1])

            if filter_mag is not None:
                index_mag = filters.index(filter_mag)

            if filters_color is not None:
                mag_color_1 = griddata(
                    points=evolution[:, 0:2],
                    values=magnitudes[:, index_color_1],
                    xi=np.stack((age_points, masses), axis=1),
                    method="linear",
                    fill_value="nan",
                    rescale=False,
                )

                mag_color_2 = griddata(
                    points=evolution[:, 0:2],
                    values=magnitudes[:, index_color_2],
                    xi=np.stack((age_points, masses), axis=1),
                    method="linear",
                    fill_value="nan",
                    rescale=False,
                )

                color = mag_color_1 - mag_color_2

            if filter_mag is not None:
                mag_abs = griddata(
                    points=evolution[:, 0:2],
                    values=magnitudes[:, index_mag],
                    xi=np.stack((age_points, masses), axis=1),
                    method="linear",
                    fill_value="nan",
                    rescale=False,
                )

        teff = griddata(
            points=evolution[:, 0:2],
            values=evolution[:, index_teff],
            xi=np.stack((age_points, masses), axis=1),
            method="linear",
            fill_value="nan",
            rescale=False,
        )

        logg = griddata(
            points=evolution[:, 0:2],
            values=evolution[:, index_logg],
            xi=np.stack((age_points, masses), axis=1),
            method="linear",
            fill_value="nan",
            rescale=False,
        )

        return box.create_box(
            boxtype="isochrone",
            model=self.tag,
            filters_color=filters_color,
            filter_mag=filter_mag,
            color=color,
            magnitude=mag_abs,
            teff=teff,
            logg=logg,
            masses=masses,
        )

    @typechecked
    def get_color_magnitude(
        self,
        age: float,
        masses: np.ndarray,
        model: str,
        filters_color: Tuple[str, str],
        filter_mag: str,
        adapt_logg: bool = False,
    ) -> box.ColorMagBox:
        """
        Function for calculating color-magnitude
        combinations from a selected isochrone.

        Parameters
        ----------
        age : float
            Age (Myr) at which the isochrone data is interpolated.
        masses : np.ndarray
            Masses (:math:`M_\\mathrm{J}`) at which the isochrone
            data is interpolated.
        model : str
            Atmospheric model used to compute the synthetic photometry.
        filters_color : tuple(str, str)
            Filter names for the color as listed in the file with the
            isochrone data. The filter names should be provided in the
            format of the SVO Filter Profile Service.
        filter_mag : str
            Filter name for the absolute magnitude as listed in the
            file with the isochrone data. The value should be equal
            to one of the ``filters_color`` values.
        adapt_logg : bool
            Adapt :math:`\\log(g)` to the upper or lower boundary of
            the atmospheric model grid whenever the :math:`\\log(g)`
            that has been calculated from the isochrone mass and
            radius lies outside the available range of the synthetic
            spectra. Typically :math:`\\log(g)` has only a minor
            impact on the broadband magnitudes and colors.

        Returns
        -------
        species.core.box.ColorMagBox
            Box with the color-magnitude data.
        """

        isochrone = self.get_isochrone(
            age=age, masses=masses, filters_color=None, filter_mag=None
        )

        model1 = read_model.ReadModel(model=model, filter_name=filters_color[0])
        model2 = read_model.ReadModel(model=model, filter_name=filters_color[1])

        param_bounds = model1.get_bounds()

        if model1.get_parameters() == ["teff", "logg", "feh"]:
            if model == "sonora-bobcat":
                iso_feh = float(self.tag[-4:])
            else:
                iso_feh = 0.0

        elif model1.get_parameters() != ["teff", "logg"]:
            raise ValueError(
                "Creating synthetic colors and magnitudes from "
                "isochrones is currently only implemented for "
                "models with only Teff and log(g) as free parameters. "
                "Please contact Tomas Stolker if additional "
                "functionalities are required."
            )

        else:
            iso_feh = None

        mag1 = np.zeros(isochrone.masses.shape[0])
        mag2 = np.zeros(isochrone.masses.shape[0])
        radius = np.zeros(isochrone.masses.shape[0])

        for i, mass_item in enumerate(isochrone.masses):
            model_param = {
                "teff": isochrone.teff[i],
                "logg": isochrone.logg[i],
                "mass": mass_item,
                "distance": 10.0,
            }

            if iso_feh is not None:
                model_param['feh'] = iso_feh

            radius[i] = read_util.get_radius(
                model_param["logg"], model_param["mass"]
            )  # (Rjup)

            if np.isnan(isochrone.teff[i]):
                mag1[i] = np.nan
                mag2[i] = np.nan

                warnings.warn(
                    f"The value of Teff is NaN for the following "
                    f"isochrone sample: {model_param}. Setting "
                    f"the magnitudes to NaN."
                )

            else:
                for item_bounds in param_bounds:
                    if model_param[item_bounds] < param_bounds[item_bounds][0]:
                        if adapt_logg and item_bounds == "logg":
                            warnings.warn(
                                f"The log(g) is {model_param[item_bounds]} but the "
                                f"lower boundary of the model grid is "
                                f"{param_bounds[item_bounds][0]}. Adapting "
                                f"log(g) to {param_bounds[item_bounds][0]} since "
                                f"adapt_logg=True."
                            )

                            model_param["logg"] = param_bounds["logg"][0]

                        else:
                            mag1[i] = np.nan
                            mag2[i] = np.nan

                            warnings.warn(
                                f"The value of {item_bounds} is "
                                f"{model_param[item_bounds]}, which is below "
                                f"the lower bound of the model grid "
                                f"({param_bounds[item_bounds][0]}). Setting the "
                                f"magnitudes to NaN for the following isochrone "
                                f"sample: {model_param}."
                            )

                    elif model_param[item_bounds] > param_bounds[item_bounds][1]:
                        if adapt_logg and item_bounds == "logg":
                            warnings.warn(
                                f"The log(g) is {model_param[item_bounds]} but "
                                f"the upper boundary of the model grid is "
                                f"{param_bounds[item_bounds][1]}. Adapting "
                                f"log(g) to {param_bounds[item_bounds][1]} "
                                f"since adapt_logg=True."
                            )

                            model_param["logg"] = param_bounds["logg"][1]

                        else:
                            mag1[i] = np.nan
                            mag2[i] = np.nan

                            warnings.warn(
                                f"The value of {item_bounds} is "
                                f"{model_param[item_bounds]}, which is above "
                                f"the upper bound of the model grid "
                                f"({param_bounds[item_bounds][1]}). Setting the "
                                f"magnitudes to NaN for the following isochrone "
                                f"sample: {model_param}."
                            )

                if not np.isnan(mag1[i]):
                    mag1[i], _ = model1.get_magnitude(model_param)
                    mag2[i], _ = model2.get_magnitude(model_param)

        if filter_mag == filters_color[0]:
            abs_mag = mag1

        elif filter_mag == filters_color[1]:
            abs_mag = mag2

        else:
            raise ValueError(
                "The argument of filter_mag should be equal to "
                "one of the two filter values of filters_color."
            )

        return box.create_box(
            boxtype="colormag",
            library=model,
            object_type="model",
            filters_color=filters_color,
            filter_mag=filter_mag,
            color=mag1 - mag2,
            magnitude=abs_mag,
            names=None,
            sptype=masses,
            mass=masses,
            radius=radius,
            iso_tag=self.tag,
        )

    @typechecked
    def get_color_color(
        self,
        age: float,
        masses: np.ndarray,
        model: str,
        filters_colors: Tuple[Tuple[str, str], Tuple[str, str]],
    ) -> box.ColorColorBox:
        """
        Function for calculating color-magnitude combinations from a
        selected isochrone.

        Parameters
        ----------
        age : float
            Age (Myr) at which the isochrone data is interpolated.
        masses : np.ndarray
            Masses (:math:`M_\\mathrm{J}`) at which the isochrone
            data is interpolated.
        model : str
            Atmospheric model used to compute the synthetic photometry.
        filters_colors : tuple(tuple(str, str), tuple(str, str))
            Filter names for the colors as listed in the file with the
            isochrone data. The filter names should be provided in the
            format of the SVO Filter Profile Service.

        Returns
        -------
        species.core.box.ColorColorBox
            Box with the color-color data.
        """

        isochrone = self.get_isochrone(
            age=age, masses=masses, filters_color=None, filter_mag=None
        )

        model1 = read_model.ReadModel(model=model, filter_name=filters_colors[0][0])
        model2 = read_model.ReadModel(model=model, filter_name=filters_colors[0][1])
        model3 = read_model.ReadModel(model=model, filter_name=filters_colors[1][0])
        model4 = read_model.ReadModel(model=model, filter_name=filters_colors[1][1])

        if model1.get_parameters() == ["teff", "logg", "feh"]:
            if model == "sonora-bobcat":
                iso_feh = float(self.tag[-4:])
            else:
                iso_feh = 0.0

        elif model1.get_parameters() != ["teff", "logg"]:
            raise ValueError(
                "Creating synthetic colors and magnitudes from "
                "isochrones is currently only implemented for "
                "models with only Teff and log(g) as free parameters. "
                "Please contact Tomas Stolker if additional "
                "functionalities are required."
            )

        else:
            iso_feh = None

        mag1 = np.zeros(isochrone.masses.shape[0])
        mag2 = np.zeros(isochrone.masses.shape[0])
        mag3 = np.zeros(isochrone.masses.shape[0])
        mag4 = np.zeros(isochrone.masses.shape[0])
        radius = np.zeros(isochrone.masses.shape[0])

        for i, mass_item in enumerate(isochrone.masses):
            model_param = {
                "teff": isochrone.teff[i],
                "logg": isochrone.logg[i],
                "mass": mass_item,
                "distance": 10.0,
            }

            if iso_feh is not None:
                model_param['feh'] = iso_feh

            radius[i] = read_util.get_radius(
                model_param["logg"], model_param["mass"]
            )  # (Rjup)

            if np.isnan(isochrone.teff[i]):
                mag1[i] = np.nan
                mag2[i] = np.nan
                mag3[i] = np.nan
                mag4[i] = np.nan

                warnings.warn(
                    f"The value of Teff is NaN for the following isochrone "
                    f"sample: {model_param}. Setting the magnitudes to NaN."
                )

            else:
                for item_bounds in model1.get_bounds():
                    if model_param[item_bounds] < model1.get_bounds()[item_bounds][0]:
                        mag1[i] = np.nan
                        mag2[i] = np.nan
                        mag3[i] = np.nan
                        mag4[i] = np.nan

                        warnings.warn(
                            f"The value of {item_bounds} is "
                            f"{model_param[item_bounds]}, which is "
                            f"below the lower bound of the model grid "
                            f" ({model1.get_bounds()[item_bounds][0]}). "
                            f"Setting the magnitudes to NaN for the "
                            f"following isochrone sample: {model_param}."
                        )

                    elif (
                        model_param[item_bounds] > model1.get_bounds()[item_bounds][1]
                    ):
                        mag1[i] = np.nan
                        mag2[i] = np.nan
                        mag3[i] = np.nan
                        mag4[i] = np.nan

                        warnings.warn(
                            f"The value of {item_bounds} is "
                            f"{model_param[item_bounds]}, which is above "
                            f"the upper bound of the model grid "
                            f"({model1.get_bounds()[item_bounds][1]}). "
                            f"Setting the magnitudes to NaN for the "
                            f"following isochrone sample: {model_param}."
                        )

                if (
                    not np.isnan(mag1[i])
                    and not np.isnan(mag2[i])
                    and not np.isnan(mag3[i])
                    and not np.isnan(mag4[i])
                ):
                    mag1[i], _ = model1.get_magnitude(model_param)
                    mag2[i], _ = model2.get_magnitude(model_param)
                    mag3[i], _ = model3.get_magnitude(model_param)
                    mag4[i], _ = model4.get_magnitude(model_param)

        return box.create_box(
            boxtype="colorcolor",
            library=model,
            object_type="model",
            filters=filters_colors,
            color1=mag1 - mag2,
            color2=mag3 - mag4,
            names=None,
            sptype=masses,
            mass=masses,
            radius=radius,
            iso_tag=self.tag,
        )


import torch
import arcsim
import gc
import time
import json
import sys
import gc
import os
import numpy as np

from datetime import datetime
now = datetime.now()
timestamp = datetime.now().strftime('%Y-%m-%d_%H:%M:%S')

print(sys.argv)
if len(sys.argv)==1:
	out_path = 'default_out'
else:
	out_path = sys.argv[1]
if not os.path.exists(out_path):
	os.mkdir(out_path)

def save_config(config, file):
	with open(file,'w') as f:
		json.dump(config, f)

with open('conf/rigidcloth/trampoline/trampoline.json','r') as f:
	config = json.load(f)
"""
matfile = config['cloths'][0]['materials'][0]['data']
with open(matfile,'r') as f:
	matconfig = json.load(f)
new_streching = np.array(matconfig['stretching'])*1e-2
matconfig['stretching'] = tuple(map(tuple, new_streching))
print(matconfig)
save_config(matconfig, matfile)
"""

# save_config(matconfig, sys.argv[1]+'/mat.json')
# save_config(matconfig, sys.argv[1]+'/orimat.json')
# config['cloths'][0]['materials'][0]['data'] = sys.argv[1]+'/mat.json'
# config['end_time']=20
save_config(config, out_path+'/conf.json')

torch.set_num_threads(8)
spf = config['frame_steps']
scalev=1

def reset_sim(sim, epoch):
	arcsim.init_physics(out_path+'/conf.json',out_path+'/out%d'%epoch,False)
	print(sim.obstacles[0].curr_state_mesh.dummy_node.x)

def get_loss(ans, param_g):
	loss = torch.zeros([1],dtype=torch.float64)

	return loss

def run_sim(steps,sim,param_g):

	for obstacle in sim.obstacles:
		for node in obstacle.curr_state_mesh.nodes:
			node.m    *= 0.1

	# sim.obstacles[2].curr_state_mesh.dummy_node.x = param_g[1]
	print("step")

	sim.cloths[0].materials[0].stretching = sim.cloths[0].materials[0].stretching*0.3

	dv = []
	dv.append(torch.zeros([3],dtype=torch.float64))
	dv.append(param_g)
	dv.append(torch.zeros([1],dtype=torch.float64))
	dv = torch.cat(dv)

	for step in range(30):
		#if (step%10==0):
		#	sim.obstacles[0].curr_state_mesh.dummy_node.v += dv*np.exp(-step/30)

		if (step==13):

			for obstacle in sim.obstacles:
				for node in obstacle.curr_state_mesh.nodes:
					node.m    *= 0.04

			sim.cloths[0].materials[0].stretching = sim.cloths[0].materials[0].stretching*10

		print(step)
		arcsim.sim_step()

	cnt = 0

	ans = sim.obstacles[0].curr_state_mesh.dummy_node.x
	ans = ans

	# ans = torch.tensor([0, 0, 0],dtype=torch.float64)
	# for node in sim.cloths[0].mesh.nodes:
	# 	cnt += 1
	# 	ans = ans + node.x
	# ans /= cnt
	loss = get_loss(ans, param_g)

	return loss, ans

def do_train(cur_step,optimizer,sim,param_g):
	epoch = 0
	while True:
		steps=4*25*spf
		reset_sim(sim, epoch)
		st = time.time()
		loss, ans = run_sim(steps, sim, param_g)
		en0 = time.time()
		# loss = get_loss(steps,sim)
		optimizer.zero_grad()

		# print('step={}'.format(cur_step))
		# print('loss={}'.format(loss.data))
		# f.write('step {}: loss={}\n'.format(cur_step, loss.data))
		# print('step {}: loss={}\n'.format(cur_step, loss.data))

		#loss.backward(retain_graph=True)

		# if loss<8e-2:
		# 	break
		# dgrad, stgrad, begrad = torch.autograd.grad(loss, [density, stretch, bend])
		en1 = time.time()
		# print(ans)
		# asdf.asdf
		print("=======================================")
		# print(param_g.data)
		# print(param_g.grad.data)
		f.write('epoch {}:  loss={} \n'.format(epoch,  loss.data))
		print('epoch {}:  loss={} \n'.format(epoch, loss.data))

		print('forward time={}'.format(en0-st))
		print('backward time={}'.format(en1-en0))

		optimizer.step()
		if epoch>=0:
			quit()
		epoch = epoch + 1
		# break

with open(out_path+('/log%s.txt'%timestamp),'w',buffering=1) as f:
	tot_step = 1
	sim=arcsim.get_sim()
	# reset_sim(sim)

	param_g = torch.zeros([2],dtype=torch.float64, requires_grad=True)

	lr = 0.3
	momentum = 0.4
	f.write('lr={} momentum={}\n'.format(lr,momentum))
	optimizer = torch.optim.SGD([{'params':param_g,'lr':lr}],momentum=momentum)
	# optimizer = torch.optim.Adadelta([density, stretch, bend])
	for cur_step in range(tot_step):
		do_train(cur_step,optimizer,sim,param_g)

print("done")


"""Base and child classes to handle light curve fitting

Author: Joe Filippazzo
Email: jfilippazzo@stsci.edu
"""

from bokeh.plotting import figure, show
import numpy as np
import pandas as pd

from .fitters import lmfitter
from .models import Model, CompositeModel
from ..utils import COLORS

class LightCurve(Model):
    def __init__(self, time, flux, unc=None, parameters=None, units='MJD', name='My Light Curve'):
        """
        A class to store the actual light curve

        Parameters
        ----------
        time: sequence
            The time axis in days, [MJD or BJD]
        flux: sequence
            The flux in electrons (not ADU)
        unc: sequence
            The uncertainty on the flux
        parameters: str, object (optional)
            The orbital parameters of the star/planet system,
            may be a path to a JSON file or a parameter object
        units: str
            The time units
        name: str
            A name for the object
        """
        # Initialize the model
        super().__init__()

        # Check data
        if len(time) != len(flux):
            raise ValueError('Time and flux axes must be the same length.')

        # Set the data arrays
        if unc is not None:
            if len(unc) != len(time):
                raise ValueError('Time and unc axes must be the same length.')

            self.unc = unc

        else:
            self.unc = np.array([np.nan] * len(self.time))

        # Set the time and flux axes
        self.time = time
        self.flux = flux

        # Set the units
        self.units = units
        self.name = name

        # Place to save the fit results
        self.results = []

    def fit(self, model, fitter='lmfit', **kwargs):
        """Fit the model to the lightcurve

        Parameters
        ----------
        model: ExoCTK.lightcurve_fitter.models.Model
            The model to fit to the data
        fitter: str
            The name of the fitter to use
        """
        # Empty default fit
        fit_model = None

        # Make sure the model is a CompositeModel
        if not isinstance(model, type(CompositeModel)):
            model = CompositeModel([model])

        if fitter == 'lmfit':

            # Run the fit
            fit_model = lmfitter(self.time, self.flux, model, self.unc, **kwargs)

        else:
            raise ValueError("{} is not a valid fitter.".format(fitter))

        # Store it
        if fit_model is not None:
            self.results.append(fit_model)

    def plot(self, fits=True, draw=True):
        """Plot the light curve with all available fits

        Parameters
        ----------
        fits: bool
            Plot the fit models
        draw: bool
            Show the figure, else return it

        Returns
        -------
        bokeh.plotting.figure
            The figure
        """
        # Make the figure
        fig = figure(width=800, height=400)

        # Draw the data
        fig.circle(self.time, self.flux, legend=self.name)

        # Plot fit models
        if fits and len(self.results) > 0:
            for model in self.results:
                model.plot(self.time, fig=fig, color=next(COLORS))

        # Format axes
        fig.xaxis.axis_label = str(self.units)
        fig.yaxis.axis_label = 'Flux'

        # Draw or return
        if draw:
            show(fig)
        else:
            return fig

    def reset(self):
        """Reset the results"""
        self.results = []

class LightCurveFitter:
    def __init__(self, time, flux, model):
        """Fit the model to the flux cube

        Parameters
        ----------
        time:
            1D or 2D time axes
        flux:
            2D flux
        """
        self.flux = np.ones(100)
        self.time = np.arange(100)
        self.results = pd.DataFrame(names=('fit_number', 'wavelength', 'P',
                                           'Tc', 'a/Rs', 'b', 'd', 'ldcs',
                                           'e', 'w', 'model_name', 'chi2'))

    def run(self):
        """Run the model fits"""
        pass

    # Method to return sliced results table
    def master_slicer(self, value, param_name='wavelength'):
        return self.results.iloc[self.results[param_name] == value]


import numpy as np
import os

import cPickle as pickle

import scipy.ndimage as ndimage

class ReplayMemory:
	"""
	"""

	def __init__(self, memory_size=1000000, height=84, width=84):
		"""
		Create a recorder to record the dataset
		"""

		self.width = width
		self.height = height

		# Buffers to store the data
		self.frames = np.ones((memory_size, height, width), np.uint8)
		self.actions = np.ones((memory_size,), np.uint8)
		self.rewards = np.ones((memory_size,))
		self.terminal = np.ones((memory_size,), np.bool)

		self.memory_size = memory_size

		# The current index of the buffer.  Assume a circular buffer
		self._idx = 0
		self.filled = False		# Has the buffer been filled?

	def record(self, frame, action, reward, is_terminal):
		"""
		Store this state, action and reward.  Flush and start a new batch if necessary
		"""

		self.frames[self._idx,:,:] = frame
		self.actions[self._idx] = action
		self.rewards[self._idx] = reward
		self.terminal[self._idx] = is_terminal

		self._idx += 1

		# Reset the circular buffer
		if self._idx == self.memory_size:
			self._idx = 0
			self.filled = True

	def get_samples(self, size, history_length=4):
		"""
		Return an array of samples
		"""

		state = np.zeros((size, self.height, self.width, 4), np.float32)
		next_state = np.zeros((size, self.height, self.width, 4), np.float32)

		# Figure out to how big the current array is
		if self.filled:
			max_idx = self.memory_size
		else:
			max_idx = self._idx

		# Generate indices to use.  Note that the highest index should be one less than the max
		# so that the next state value can be generated, and should be at least history_length,
		# so enough history is available to fill up the state.
		# Also note that the indices should be shifted by the current index, and modulo'd.  This
		# makes sure that the valid range of indices are mapped propertly to the array
		indices = np.random.randint(3, max_idx-1, (size,)) + self._idx
		indices = indices % max_idx

		# Get the current and next state
		for i in range(4):
			# Sample the prior 4 frames
			n = (indices - 3 + i) % max_idx
			state[:,:,:,i] = self.frames[n,:,:].astype(np.float32)# / 255.0
		next_state[:,:,:,0:3] = state[:,:,:,1:4]
		n = (indices + 1) % max_idx
		next_state[:,:,:,3] = self.frames[n,:,:].astype(np.float32)# / 255.0

		return state, self.actions[indices], self.rewards[indices], next_state, self.terminal[n]

	def save(self, path):
		"""
		"""

		# Make a directory

		if not os.path.exists(path):
			os.makedirs(path)

		filenames = ['frames.npy', 'actions.npy', 'rewards.npy', 'terminal.npy']
		values = [self.frames, self.actions, self.rewards, self.terminal]

		for filename, values in zip(filenames, values):
			f = open(path + '/' + filename, 'wb')
			np.save(f, values)
			f.close()

		f=open(path + '/params.txt', 'w')
		f.write("%d,%d,%d"%(self._idx, self.memory_size, self.filled))
		f.close()

	def load(self, path):
		"""
		"""

		print "Restoring Experience Replay Memory..."

		self.frames = None
		self.actions = None
		self.rewards = None
		self.terminal = None

		f = open(path + '/frames.npy')
		self.frames = np.load(f)
		f.close()

		f = open(path + '/actions.npy')
		self.actions = np.load(f)
		f.close()

		f = open(path + '/rewards.npy')
		self.rewards = np.load(f)
		f.close()

		f = open(path + '/terminal.npy')
		self.terminal = np.load(f)
		f.close()

		f=open(path + '/params.txt')
		info = f.readline()
		f.close()

		values = [int(x) for x in info.split(',')]

		self._idx = values[0]
		self.memory_size = values[1]
		self.filled = bool(values[2])

		print "  Replay Memory Current Index:", self._idx
		print "  Replay Memory Size:", self.memory_size
		print "  Replay Memory is Filled:", self.filled


#! /usr/bin/env python
"""API to use and import library for object recognition"""

import numpy as np
import tensorflow as tf
import os
from core import utils
from PIL import Image

__author__ = "Sayan Paul"
__email__ = "sayanpau@usc.edu"

IMAGE_H, IMAGE_W = 416, 416

OID_CLASS_NAMES_FILE = os.path.join(os.path.dirname(__file__), 'data', 'oid.names')
IMGNET_CLASS_NAMES_FILE = os.path.join(os.path.dirname(__file__), 'data', 'imgnet.names')
MERGED_CLASS_NAMES_FILE = os.path.join(os.path.dirname(__file__), 'data', 'merged.names')

OID_FROZEN_MODEL_DIR = os.path.join(os.path.dirname(__file__), 'frozen_models', 'OID')
IMGNET_FROZEN_MODEL_DIR = os.path.join(os.path.dirname(__file__), 'frozen_models', 'IMGNET')
MERGED_FROZEN_MODEL_DIR = os.path.join(os.path.dirname(__file__), 'frozen_models', 'MERGED')

class ObjectRecognition:
    """Object Recognition API"""

    def __init__(self, session=None, use_gpu=True, score_thresh=0.3, iou_thresh=0.5, dataset_name='OID'):
        """
        Initiate model for API
        :param session: Tensorflow session variable
        :type session: tensorflow session
        :param use_gpu: Use GPU, default True
        :type use_gpu: Boolean
        :param score_thresh: Box score threshold
        :type score_thresh: float
        :param iou_thresh: IOU threshold for non-max suppression
        :type iou_thresh: float
        :param dataset_name: Dataset name for the model (OID, IMGNET, MERGED)
        :type dataset_name: str
        """

        self.img_h = IMAGE_H
        self.img_w = IMAGE_W
        self.use_gpu = use_gpu
        self.score_thresh = score_thresh
        self.iou_thresh = iou_thresh
        self.dataset_name = dataset_name
        self.sess = None
        self.input_tensors = None
        self.output_tensors = None

        def get_tensors(frozen_model_dir):
            if session is not None:
                sess = session
            else:
                if not use_gpu:
                    cpu_nms_graph = tf.Graph()
                    sess = tf.Session(graph=cpu_nms_graph)
                else:
                    gpu_nms_graph = tf.Graph()
                    sess = tf.Session(graph=gpu_nms_graph)
            if not use_gpu:
                input_tensors, output_tensors = utils.read_pb_return_tensors(cpu_nms_graph,
                                                                             os.path.join(frozen_model_dir,
                                                                                "yolov3_cpu_nms.pb"),
                                                                             ["Placeholder:0", "concat_9:0",
                                                                              "mul_6:0", "concat_8:0"])
            else:
                input_tensors, output_tensors = utils.read_pb_return_tensors(gpu_nms_graph,
                                                                             os.path.join(frozen_model_dir,
                                                                                "yolov3_gpu_nms.pb"),
                                                                             ["Placeholder:0", "concat_10:0",
                                                                              "concat_11:0", "concat_8:0",
                                                                              "concat_13:0"])
            return sess, input_tensors, output_tensors

        if dataset_name == 'OID':
            self.classes = utils.read_class_names(OID_CLASS_NAMES_FILE)
            self.sess, self.input_tensors, self.output_tensors = get_tensors(OID_FROZEN_MODEL_DIR)
        elif dataset_name == 'IMGNET':
            self.classes = utils.read_class_names(IMGNET_CLASS_NAMES_FILE)
            self.sess, self.input_tensors, self.output_tensors = get_tensors(IMGNET_FROZEN_MODEL_DIR)
        elif dataset_name == 'MERGED':
            self.classes = utils.read_class_names(MERGED_CLASS_NAMES_FILE)
            self.sess, self.input_tensors, self.output_tensors = get_tensors(MERGED_FROZEN_MODEL_DIR)
        else:
            raise Exception('Unsupported dataset type')

        self.num_classes = len(self.classes)

    def predict_images(self, records):
        """
        Predict boxes, scores and labels for each image records
        :param records:
        :type records:
        :return:
        :rtype:
        """

        result = {'boxes': [], 'probs': [], 'labels': []}

        for record in records:
            preds = self.predict_image(np.expand_dims(record, axis=0))
            result['boxes'].append(preds['boxes'])
            result['probs'].append(preds['probs'])
            result['labels'].append(preds['labels'])

        return result

    def predict_image(self, record):
        """
        Predict boxes, scores and labels for a single image record
        :param record:
        :type record:
        :return:
        :rtype:
        """
        # Pre-process record
        record = self._process_image(record)

        if self.use_gpu:
            boxes, _, labels, probs = self.sess.run(self.output_tensors,
                                                    feed_dict={self.input_tensors: np.expand_dims(record, axis=0)})
        else:
            boxes, scores, probs = self.sess.run(self.output_tensors,
                                                 feed_dict={self.input_tensors: np.expand_dims(record, axis=0)})
            boxes, _, labels, probs = utils.cpu_nms(boxes, scores, probs, self.num_classes,
                                                    score_thresh=self.score_thresh,
                                                    iou_thresh=self.iou_thresh)

        return {'boxes': boxes, 'probs': probs, 'labels': labels}

    def _process_image(self, img):
        """
        Image pre-processing
        :param img: Single image records
        :type img: [HxWx3] numpy array
        :return: Pre-processed image
        :rtype: [self.img_w x self.img_h x 3] numpy array
        """

        img_resized = np.array(np.resize(img, [self.img_w, self.img_h, 3]), dtype=np.float32)
        img_resized = img_resized / 255.
        return img_resized

if __name__ == "__main__":
    model = ObjectRecognition(use_gpu=False, score_thresh=0.4, iou_thresh=0.5)
    image_path = "data/OID/test/Apple/0da61cd490c57814.jpg"
    img = Image.open(image_path)
    predictions = model.predict_image(img)
    image = utils.draw_boxes(img, predictions['boxes'], predictions['labels'], predictions['labels'],
                             model.classes, [IMAGE_H, IMAGE_W], show=True)


from celery import shared_task,current_task
from numpy import random
from scipy.fftpack import fft

@shared_task
def fft_random(n):
    """
    Brainless number crunching just to have a substantial task:
    """
    for i in range(n):
        x = random.normal(0, 0.1, 2000)
        y = fft(x)
        if(i%30 == 0):
            process_percent = int(100 * float(i) / float(n))
            current_task.update_state(state='PROGRESS',
                                      meta={'process_percent': process_percent})
    return random.random()

@shared_task
def add(x,y):
    for i in range(1000000000):
        a = x+y
    return x+y


from keras.models import model_from_json
from keras.models import model_from_json
import pickle
from keras.preprocessing.sequence import pad_sequences
import numpy as np
from keras.optimizers import Adam
from cleaner import Cleaner

maxlen = 500
def main():
    # loading
    with open('../data/neural_network_config/tokenizer.pickle', 'rb') as handle:
        tokenizer = pickle.load(handle)

    # load json and create model
    json_file = open('../data/neural_network_config/model.json', 'r')
    loaded_model_json = json_file.read()
    json_file.close()
    loaded_model = model_from_json(loaded_model_json)

    # load weights into new model
    loaded_model.load_weights("../data/neural_network_config/model.h5")
    print("Loaded model from disk")

    # evaluate loaded model on test data
    loaded_model.compile(optimizer='adam', loss='mean_squared_error',  metrics=['mae','accuracy'])

    clean = Cleaner().clean_text
    sentence_test = ["IMPORTANTE: El restaurante cierra los días de partido o eventos.  El confort y la elegancia son uno de los dúos más apreciados por los sibaritas de todo el mundo. Pero Chamartín ha sabido convertirlo en un muy distinguido trío, puesto que suma la gastronomía asiática a esta lujosa ecuación.  Y es que Zen Market es mucho más que un restaurante, puesto que su calidad raya la perfección. No es casualidad que Ignacio García de Vinuesa y su equipo de profesionales le hayan dado forma a este rincón que también es deportista: se encuentra en una zona exclusiva del estadio Santiago Bernabéu.  Si pensabas que habías probado un pato lacado auténtico, desde eltenedor.es te pedimos que reserves una mesa y disfrutes de un auténtico pato lacado en caviar o quizás un bogavante salteado con sal y pimienta Sechuan. Si ya los probaste, reserva en cualquier caso y para cualquier ocasión. Zen Market es toda una experiencia. ¡No te lo puedes perder!",
    "me gusta mucho, todo está bien",
    "Los platos bien.. Comida divertida. El servicio regular, tardaron en servir el café 20 minutos",
    "decepcionante, la verdad que no me ha gustado demasiado, poco recomendable",
    "Elegimos el hotel por su buena situación, en el centro, en la calle Ancha. Apenas a unos metros de la Catedral y el barrio Húmedo. En ese sentido, perfecto. El problema es que esa localización precisamente es lo que genera que sea imposible dormir. Nos tocó una habitación en el primer piso de la fachada que da a la calle Cervantes, repleta de bares. Hasta las 3 de la mañana con ruidos constantes, golpes, gritos. A partir de esa hora, borrachos gritando. No pudimos pegar ojo, algo que también fue culpa de la estrecha cama de 1,35 cm. Una auténtica pesadilla.  El hotel es un tres estrellas anclado en los finales de los ochenta. El baño y el mobiliario así lo demuestran. Las habitaciones están dispersas por varios edificios, pasillos y más pasillos, escaleras, etc...Caótico.  Otro aspecto negativo fue la televisión. Sólo tenía sintonizadas dos cadenas y no había posibilidad de resintonizar el aparato, ya que estaba bloqueado. Además, estaba contectada la descripción para ciegos, imposible de quitar, por lo que ver una película (lo único que se podía ver, la verdad), era toda una epopeya.  Al menos pudimos disfrutar de un circuito de spa gratuito. Se trata de un spa humilde, pero que no nos vino mal. SI hubiéramos sabido que íbamos a dormir así, nos quedamos en la piscina.  El desayuno no es buffet, es un café con zumo natural y una tostada.  Habíamos reservado aparcamiento, pero viendo las opiniones al respecto acerca de la estrechez e incomodidad, decidimos aparcar en uno público que está a 50 metros...al menos no tengo que volverme loco para entrar en una calle peatonal, maniobrar de locura y arriesgarme.  https://www.tripadvisor.es/ShowUserReviews-g187492-d8670166-r384189174-Hotel_Paris-Leon_Province_of_Leon_Castile_and_Leon.html#"
    "La comida está correcta, buena, pero un poco escasa. El precio adecuado si vas con promoción de El Tenedor. Aunque los camareros son agradables, el servicio es un poco caótico, con intervalos irregulares a la hora de traer la comida. Además, pedimos dos veces la carta para pedir más raciones, pero nunca la trajeron, y la cuenta venía con un error en el descuento aplicado",
    "Meh. Iba con unas expectativas medias y salí decepcionado. Es verdad que la carta pinta bien y que la están renovando (cambio de cocinero), pero no hay nada que sorprenda o deje un buen recuerdo. Sabores que intentan ser originales y acaban con el producto principal. Presentaciones muy similares, abusando de almendras, lechugas... Lo único destacable fueron los postres, especialmente un trampantojo de huevo. ",
    "La entrada en el 2020 ha comportado cambios en la edad de jubilación y en el cálculo de los años cotizados que se tienen en cuenta para determinar la prestación. Las medidas son de carácter automático, ya que forman parte de la reforma de 2011 que hace que la edad para jubilarse se retrase paulatinamente hasta llegar a los 67 años.  ADVERTISING  Todo esto se da mientras resta pendiente saber cuál será la subida de las prestaciones en el 2020, ya que aunque el Gobierno en funciones ha prometido que se subirán el 0,9% y no perderán poder adquisitivo, la medida no se tomará hasta que esté formado un Ejecutivo. En diciembre de 2019 en España se contabilizaban 6.089.294 pensiones de jubilación, con una prestación media de 1.143,55 euros mensuales.   Pensiones en 2020 Los cambios en la edad de jubilación  Respecto a la edad de jubilación, cada año se va retrasando en virtud del régimen establecido en la reforma de 2011 aprobada durante el mandato de José Luis Rodríguez Zapatero. De esta forma, en 2020 la edad legal ordinaria será de 65 años y 10 meses. Esta edad se aplicará a aquellos que han cotizado menos de 37 años.  Si una persona llega a los 65 años en 2020 y ha cotizado 37 años o más, ya podrá jubilarse con 65 años.  En el caso de la jubilación parcial, en la que se combina trabajo y prestación, el mínimo será de 61 años y 10 meses con 35 años o más cotizados; o de 62 años y 8 meses con 33 años cotizados.  Con cada año que pasa es necesaria más edad para acceder a la jubilación, tanto si se ha cotizado por encima o por debajo de los periodos de referencia  Con cada año que pasa es necesaria más edad para acceder a la jubilación, tanto si se ha cotizado por encima o por debajo de los periodos de referencia Pensiones en 2020 Los cambios en el cálculo de la pensión  Por lo que respecta al cálculo de la pensión que se cobrará la momento de jubilarse, en 2020 se tendrán en cuenta los últimos 23 años cotizados. Estos años cotizados conforman la base reguladora, que es la suma de las bases de cotización en dicho periodo. Hay que tener en cuenta que cuantos más años se tengan en cuenta es posible que se recorte más la pensión, ya que en los últimos años de vida laboral es cuando mejores salarios se suelen cobrar.   Esta es otra de las reformas introducidas con los cambios en las pensiones de la década anterior, momento hasta el que se tenían en cuenta los últimos 15 años trabajados. La idea es que para 2022 ya se tengan en cuenta los últimos 25 años cotizados. De esta manera, en 2021 se computarán los últimos 24 años trabajados y en 2022 los últimos 25 años cotizados.  La base reguladora de la pensión se obtiene de dividir los meses de los años exigidos por el divisor correspondiente La base reguladora de la pensión se obtiene de dividir los meses de los años exigidos por el divisor correspondiente (LV) En 2023 El recorte de las pensiones que viene  Otra de las medidas que tendrán un fuerte calado en el sistema es la llegada del factor de sostenibilidad, que se aplicará a partir de 2023 e irá recortando las nuevas pensiones, teniendo en cuenta que los pensionistas vivirán más. Dicha medida en un principio debía aplicarse en 2019.  El conjunto de medidas se puede consultar al detalle en la guía para la jubilación del Ministerio de Trabajo, Migraciones y Seguridad Social.",
    "El Juzgado de Violencia de Género de Elche (Alicante) investiga a un hombre detenido el pasado miércoles por una supuesta agresión a su hija, de 19 años. Tras decretar su libertad provisional, se ha abierto una causa por los supuestos delitos de malos tratos y abuso sexual. La investigación trata de determinar también si, fruto de tal relación incestuosa, la hija de la víctima, de 15 meses, lo es también del investigado.  MÁS INFORMACIÓN El Defensor del Pueblo pide a Interior un plan de seguimiento a menores víctimas de violencia machista Un hombre mata a su expareja y a su hija de 3 años en Esplugues de Llobregat  Según han confirmado a EL PAÍS fuentes del Tribunal Superior de Justicia de la Comunidad Valenciana (TSJCV), tras la detención el juez decretó la libertad provisional del acusado así como una orden de alejamiento con respecto a su hija. Ni la Fiscalía ni la víctima habían solicitado prisión preventiva para el acusado. La causa, que de momento lleva el juzgado de Violencia de Género, se ha iniciado con una investigación previa y el magistrado ha encargado una prueba pericial psicológica de la víctima.  Los   indicios apuntan a que padre e hija mantenían una relación  , afirman las mismas fuentes. Si se confirma, el caso continuará en el juzgado de Violencia de Género mientras que de lo contrario se trasladará a un juzgado de instrucción ordinario al considerarse maltrato de un padre hacia una hija.  Según ha adelantado el periódico Información, la denuncia partió de una vecina de la víctima, que la encontró junto a su casa llorando y con un ojo amoratado. Tras la detención del presunto agresor y su paso a disposición judicial, padre e hija declararon que la bebé es la hija en común de ambos y que mantuvieron relaciones desde que la víctima tenía 16 años. ADVERTISING  Ads by Teads   Larga condena Según el mismo diario, la muchacha estuvo acogida en una vivienda tutelada de la Generalitat valenciana después de que su padre ingresara en prisión y cumpliera una larga condena. Al quedar en libertad, el arrestado contactó de nuevo con ella, la entonces menor abandonó el piso tutelado y se fue a vivir con su padre. Poco después se quedó embarazada y dio a luz cuando todavía era menor de edad. Las mismas fuentes aseguran que ambos declararon en sede judicial que son los padres del bebé.  También apuntan las mismas fuentes que el informe psicológico solicitado por la Fiscalía pretende establecer el estado psicológico de la muchacha, que se ha ido a vivir a casa de una tía, hermana del detenido, por carecer de medios económicos y no poder volver al domicilio familiar.",
    "que alegría que alboroto otro perrito piloto. muy correcto", "lo odio, fatal, que horror", "malo malísimo feo caca, horrible",
    "España ha amanecido este martes con una buena noticia: La Organización Mundial de la Salud (OMS) ha comentado que nuestro país puede estar llegando al pico de incidencia de casos del nuevo coronavirus Covid-19. Sin embargo, no ha sido la única. Entre las tinieblas y la incertidumbre que ha pintado el Covid-19 esta primavera de 2020, cada jornada se conocen informaciones que animan a mantener la esperanza. Un paciente de coronavirus Un paciente de coronavirus 1. El Hospital Gregorio Marañón da el alta de la UCI a su paciente más joven, de 20 años. El Hospital Gregorio Marañón ha dado el alta de la UCI a su paciente de coronavirus más joven, un chico de 20 años que ha pasado a la planta de hospitalización, donde continuará con su recuperación del Covid-19. Según informaron fuentes del centro hospitalario, el Gregorio Marañón ha dado hoy más de 80 altas a pacientes que ya han regresado a su domicilio. El hospital madrileño es uno de los cinco españoles que participan en un ensayo clínico con «sarilumab» para pacientes hospitalizados con Covid-19 en estado grave o crítico con enfermos europeos. El Hospital Vall d'Hebron de Barcelona ha sido el primer centro activado en España y el Gregorio Marañón de Madrid el que ha incluido los primeros pacientes. También participan el Hospital La Paz y el Ramón y Cajal, ambos de Madrid y el Clinic de Barcelona. 2. El emotivo gesto de dos agentes de la Policía Nacional con una anciana por su cumpleaño. La crisis del coronavirus ha desatado un gran estado de alarma y nerviosismo entre la población. En estos tiempos difíciles, la solidaridad se abre paso y sale a la luz el lado más humano de la sociedad. Una muestra de ello es el emotivo gesto que se vivió la semana pasada en un municipio de Aragón. Un equipo de la Policía Nacional estaba realizando una patrulla peatonal por el centro de Jaca (Huesca), y Toñi, una anciana de un edificio cercano, al verlos, salió a aplaudirles. No fue la única. Varios vecinos, que también se asomaron a sus balcones, comentaron a los agentes que aquel día era el cumpleaños de la mujer. Decididos a hacer de aquella una jornada especial para ella, los dos miembros del cuerpo se acercaron a la comisaría para hacerse con una caja de dulces y regresaron al barrio de Toñi. Una vez allí, y sin pensárselo dos veces, Manuel Aldaz, jefe de la Policía Nacional en Jaca, se encaramó a su coche y entregó el presente a la anciana. Amancio Ortega ha hecho grandes esfuerzos para lugar contra el coronavirus Amancio Ortega ha hecho grandes esfuerzos para lugar contra el coronavirus 3. La Fundación Amancio Ortega compra material contra la pandemia de coronavirus por 63 millones de euros. Según ha informado la entidad a través de un comunicado, «las gestiones de compra continúan en marcha con el objetivo de seguir incrementando el volumen de material adquirido». Entre este material destaca, principalmente, la adquisición de 1.450 respiradores para las unidades de cuidados intensivos, tres millones de mascarillas filtrantes para el uso de personal sanitario, un millón de kits de detección del virus y otros equipamientos para centros sanitarios, como 450 camas hospitalarias.",
    "Recuerdo que antes decía, que en mí vivían sentimientos de agonía y tristeza, sentimientos que revivían con cada recuerdo que tenía, mi mente, un lugar abrumado, confundido, desolado y que pensaba que en ningún otro momento eso cambiaría, estaba tan lastimado no sólo por las personas, sino por la vida en general… De un momento a otro llega una pequeña casualidad, aquella que le daría un rumbo diferente a mi vida, que haría despertar aquel yo que se había ocultado por mucho tiempo, aquellos sentimientos que habían desaparecido, aquella pequeña casualidad, cobró mucha importancia, ya no sólo era una casualidad, era la razón de que yo no brincara al fondo de ese hoyo el cual me atraía cada vez más para hundirme dentro de el, era por quién en las mañanas ya no pensaba que todo era repetitivo, sino que todas las mañanas eran días nuevos, para poder vivir, poder disfrutar y sobre todo poder amar",
    "Agricultores afectados por el incendio de junio en la Ribera d'Ebre, Garrigues y Segrià consideran insuficiente el plan de choque anunciado este jueves por la 'consellera' de Agricultura, Teresa Jordà, que propuso ayudas directas por la limpieza de los bosques quemados y créditos blandos por los propietarios de fincas afectadas por el fuego, que arrasó más de 5.000 hectáreas. El presidente de Asaja y portavoz de agricultores afectados por el incendio de Ribera de Ebro, Pere Roqué, dice que los campesinos están desilusionados con la propuesta puesto que necesitan una primera ayuda directa para las explotaciones antes de plantearse un crédito blando. También Unió de Pagesos se ha sumado a la protesta y añaden que las ayudas no tienen en cuenta la pérdida de rentas que supondrán a medio plazo para los afectados, algunos de los cuales se plantean poder continuar con la actividad agrícola o ganadera. Los agricultores anuncian una nueva manifestación por domingo a las 18h en Flix y reclaman una reunión urgente con Jordà la semana que viene. Lamentan que se los emplace en septiembre y le recuerdan que los campesinos afectados no se van de vacaciones. La 'consellera' de Agricultura, Teresa Jordà, anunció una ayuda directa para limpiar los bosques de la madera quemada este agosto y una línea de créditos de 5 millones de euros, cuyos intereses irán a cargo del departamento. También anunció una inversión de 3,5 millones de euros para extender el riego de apoyo a toda la zona afectada por el incendio. Sobre las ayudas a fondo perdido, Jordà se emplazó a seguir hablando en septiembre, dependiendo también de los próximos presupuestos, a pesar de que se comprometió a estudiar caso por caso todas las demandas y necesidades. Los agricultores y ganaderos afectados por el incendio de la Ribera d'Ebre no  tienen bastante y reclaman ayudas directas para recuperar sus explotaciones. La medida del crédito puede ser interesante, siempre y cuando primero haya una ayuda directa para salir adelante, para aquellas explotaciones que los hagan falta más dinero además de la ayuda directa, explica Roqué."]

    sentence_test = [clean(item) for item in sentence_test]

    xnew = tokenizer.texts_to_sequences(sentence_test)
    xnew = pad_sequences(xnew, padding='post', maxlen=maxlen)

    ynew = loaded_model.predict(xnew)
    print(ynew)

    #print(ynew)
    #print(multilabel_binarizer.inverse_transform(np.array([ynew])))

    for i in range(0, len(ynew)):
        print("----")
        print(sentence_test[i])
        print(ynew[i] * 100)

if __name__== "__main__":
  main()


#!/usr/bin/env python
#
# // SPDX-License-Identifier: BSD-3-CLAUSE
#
# (C) Copyright 2018, Xilinx, Inc.
#
from collections import OrderedDict
from copy import deepcopy
from six import string_types as _string_types

import tensorflow as tf
import numpy as np
from tensorflow.python.ops import script_ops as _script_ops

from xfdnn.tools.compile.bin.xfdnn_compiler_tensorflow import TFFrontend
from xfdnn.rt import xdnn_util
from xfdnn.rt import xdnn_util_tf
from xfdnn.rt.xdnn_rt_base import xdnnRT as _xdnnRT
from xfdnn.rt.xdnn_opt import CPUTransform, HWEmuTransform, FPGATransform
from ext.PyTurboJPEG import imread as _imread

## global variables
global_fpga_device    = 'cpu:0'   ## TODO: replace with FPGA:0
global_pyfunc_counter = 0

######################################################
## tensorflow specific utility functions
######################################################
## expanding tf.NodeDef methods
tf.NodeDef.set_name  = xdnn_util_tf.set_name
tf.NodeDef.set_shape = xdnn_util_tf.set_shape
tf.NodeDef.get_shape = xdnn_util_tf.get_shape
tf.NodeDef.get_dtype = xdnn_util_tf.get_dtype

## expanding tf.GraphDef methods
tf.GraphDef.get_node_dict   = xdnn_util_tf.get_node_dict
tf.GraphDef.get_output_dict = xdnn_util_tf.get_output_dict
tf.GraphDef.get_node_index  = xdnn_util_tf.get_node_index
tf.GraphDef.remove_nodes    = xdnn_util_tf.remove_nodes
tf.GraphDef.is_cyclic       = xdnn_util_tf.is_cyclic
tf.GraphDef.all_cycles      = xdnn_util_tf.all_cycles

class TFxdnnRT(_xdnnRT):
    def __init__ (self, args, **kwargs):
        super(TFxdnnRT, self).__init__(TFFrontend, args, **kwargs)

        if not hasattr(self, 'graph_def'):
          self.graph_def = deepcopy(self._graph)

        if not hasattr(self, 'fpga_pynode_dict'):
          self.fpga_pynode_dict = OrderedDict()

    def load_graph(self, args, **kwargs):
        return xdnn_util_tf.load_graph(args, **kwargs)

    def list_inputs_of_graph(self, graph_def):
        return xdnn_util_tf.discover_sourcenodes(graph_def)

    def list_outputs_of_graph(self, graph_def):
        if hasattr(self, 'outputs') and self.outputs is not None:
          return self.outputs
        return xdnn_util_tf.discover_sinknodes(graph_def)

    def extract_subgraph(self, outputs, inputs, inclusive=False, filename=None, session=None):
        destgraph_def = xdnn_util_tf.extract_subgraph(self._graph, outputs, inputs, inclusive)
        destgraph_def, fValidGraph = xdnn_util_tf.freeze_graph(session,
                                                               destgraph_def,
                                                               sinknodes_list=outputs,
                                                               filename=filename,
                                                               freeze_blacklist=[],
                                                               freeze_whitelist=[])
        return destgraph_def

    def device_transforms(self, args):
        subTFs = []
        for partition in self.graph_partitions:
          ## extract supported subgraph
          if partition.supported:
            print('Re-compile partition_index \"{}\"'.format(partition.index))

            name_postfix = '_partition#{:02d}'.format(partition.index)
            filename = self.file_path('.pb', name_postfix=name_postfix)

            subTFs.append((TFxdnnRT(args,
                                    fPartition=False,
                                    startnode=partition.inputMap.values(),
                                    finalnode=partition.outputMap.values(),
                                    inclusive=False,
                                    name_postfix=name_postfix,
                                    weights=filename,
                                    graph_savepath=filename,
                                    cpulayermustgo=True,
                                   ),
                           filename,
                           partition.index
                          )
                         )
          else:
            partition.layers    = list(partition.names)
            partition.variables = {}

        for TF, filename, partition_index in subTFs:
          partition = self.graph_partitions[partition_index]

          print('Transorm partition_index \"{}\"'.format(partition.index))

          time_to_layer_list = []
          for time, layers in TF.compilerSchedule.time_to_layer.items():
            for layer in layers:
              time_to_layer_list.append((time, layer))

          ## NOTE: hack to get rid of input placeholder
          ## FIXME: this might cause problem if TF.compilerJson['inputs'] info is used in
          ##        FPGA. Commented out for now when compiler flag "cpulayermustgo" is used.
          ## ALTERNATIVE?: time_to_layer_list = [tol for tol in time_to_layer_list if not tol[1].startswith('geph__')]
          # time_to_layer_list = [tol for tol in time_to_layer_list if tol[1] not in set(TF.inputs)]

          if args.device.lower() == "cpu":
            opt = CPUTransform(time_to_layer_list, TF.layerparameter_dict, args, TF._graph)
          elif args.device.lower() == "hwemu":
            opt = HWEmuTransform(partition.inputs, TF.pydotGraph, TF.compilerSchedule, args)
          elif args.device.lower() == "fpga":
            if not args.fpga_recipe:
              args.fpga_recipe = {'start': [inp['input_name'] for inp in
                                            TF.compilerJson['inputs']],
                                  'end':   [out['output_name'] for out in
                                            TF.compilerJson['outputs']]}
            if args.xclbin:
              opt = FPGATransform(time_to_layer_list, TF.layerparameter_dict,
                                  TF.compilerJson, args, TF._graph,
                                  filename=filename)
            else:
              raise AttributeError("Must specify path to xclbin when device = FPGA")
          else:
            raise AttributeError("Unsupported device type", args.device)

          #variables hold the inputs/consts of graph
          partition.layers    = opt.getLayers()
          partition.variables = opt.variables
          for l in partition.layers:
            l.setup()

    def rebuild_graph(self):
        self.graph_def = self.rebuild_graph_def()
        # if len(self.graph_partitions) > 0:
        #   self.graph_def = self.rebuild_graph_def()

    def rebuild_graph_def(self):
        #############################
        ## helper function
        #############################
        def discover_consumer_nodes(root_name, partition_node_set, graph_output_dict):
            dst_dict = {}
            consumer_nodes = graph_output_dict[root_name]
            for consumer_name, index in consumer_nodes.items():
              if consumer_name not in partition_node_set:
                dst_dict[consumer_name] = index
            return dst_dict

        #def discover_consumer_nodes((root_name, index), partition_outputs, dst_dict):
        #    ## NOTE: This doesn't work if the supported partition is not maximal
        #    if root_name in self.spt_set:
        #      if index != -1 and root_name in partition_outputs:
        #        return
        #      consumer_nodes = graph_output_dict[root_name]
        #      for consumer_item in consumer_nodes.items():
        #        discover_consumer_nodes(consumer_item, partition_outputs, dst_dict)
        #    else:
        #      dst_dict[root_name] = index

        #############################
        ## helper function
        #############################
        def insert_fpga_pynode(graph_def, partition):
            global global_pyfunc_counter

            # def update_partition_boundries(partition, graph_output_dict):
            #     for extra, layer in partition.spt_set.items():
            #       if layer not in partition.names:
            #         raise RuntimeError('supported layer {} not in partition {}-{}'.format(layer,
            #                                                                               partition.index,
            #                                                                               partition.names))
            #       for output in graph_output_dict[extra].keys():
            #         if output not in partition.spt_set and layer not in partition.outputs:
            #           partition.outputs += [layer]
            #           partition.outputMap.update({layer: extra})

            graph_node_dict, \
            graph_output_dict   = graph_def.get_node_dict(outmap=True)

            input_partitions    = self.reverse_connectivity[partition.index]
            consumer_partitions = self.connectivity[partition.index]

            input_tensors           = []
            placeholder_replace_map = {}

            # update_partition_boundries(partition, graph_output_dict)

            with tf.Graph().as_default() as fpga_subgraph:
              for idx, inp in enumerate(partition.inputs):
                ## NOTE: index of partition in self.graph_partitions must match partition.index
                input_partition = input_partitions[inp]
                if len(set(input_partition)) > 1:
                  raise RuntimeError('input \"{}\" cannot belong to multiple partitions! {}'.format(inp, set(input_partition)))
                if not self.graph_partitions[input_partition[0]].supported:
                  inp = partition.inputs[idx] = partition.inputMap[inp]

                ## NOTE: inp shouldn't have ":" except for multiple outputs of py_func
                inp = inp.split(':')[0]

                ## assuming all py_func inputs are active tensors and FPGA IP data_format is 'NCHW'
                input_nodedef = graph_node_dict[inp]
                input_tensor  = tf.placeholder(input_nodedef.get_dtype(),
                                               shape=input_nodedef.get_shape(), name=inp)
                input_tensors += [input_tensor]
                placeholder_replace_map[input_tensor.op.node_def.name] = inp

              with fpga_subgraph.name_scope(xdnn_util.Trie(partition.names).lcs()):
                with fpga_subgraph.name_scope('fpga_func_{}'.format(global_pyfunc_counter)):

                  # print('_names_in_use: ', fpga_subgraph._names_in_use)

                  output_shapes = [graph_node_dict[output].get_shape() for output in partition.outputs]
                  output_dtypes = [graph_node_dict[output].get_dtype() for output in partition.outputs]

                  if self.data_format != 'NCHW':
                    with fpga_subgraph.name_scope('fpga_preproc'):
                      fpga_input_tensors = [tf.transpose(tensor, [0, 3, 1, 2]) if
                                            len(tensor.get_shape()) == 4 else tensor for
                                            tensor in input_tensors]
                  else:
                    fpga_input_tensors = input_tensors

                  # with tf.device(global_fpga_device):
                  fpga_output_tensors = tf.py_func(partition.forward_exec,
                                                   fpga_input_tensors,
                                                   output_dtypes,
                                                   stateful=False)

                  for fpga_output_tensor, output_shape in zip(fpga_output_tensors, output_shapes):
                    if len(output_shape) == 4:
                      fpga_output_tensor.set_shape([output_shape[i] for i in (0, 3, 1, 2)])
                    else:
                      fpga_output_tensor.set_shape(output_shape)

                  # print('graph py_func tokens: {}'.format(_script_ops._py_funcs._funcs.keys()))

                  if self.data_format != 'NCHW':
                    with fpga_subgraph.name_scope('fpga_postproc'):
                      output_tensors = [tf.transpose(tensor, [0, 2, 3, 1]) if
                                        len(tensor.get_shape()) == 4 else tensor for tensor in
                                        fpga_output_tensors]
                  else:
                    output_tensors = fpga_output_tensors

                  # print('graph py_func tokens: {}'.format(_script_ops._py_funcs._funcs.keys()))

                  global_pyfunc_counter += 1

              ## create identity nodes matching the correct output names
              # /wrk/hdstaff/arminb/Anaconda/envs/deephi_tf/lib/python2.7/site-packages/tensorflow/python/framework/op_def_library.py:394
              # /wrk/hdstaff/arminb/Anaconda/envs/deephi_tf/lib/python2.7/site-packages/tensorflow/python/framework/ops.py:6007
              # /wrk/hdstaff/arminb/Anaconda/envs/deephi_tf/lib/python2.7/site-packages/tensorflow/python/framework/ops.py:4117
              output_tensors = [tf.identity(tensor, name=partition.outputMap[name]) for tensor, name
                                in zip(output_tensors, partition.outputs)]

            ## maintain an ordered dictionary of pynodes for loading the graph
            #self.fpga_pynode_dict[fpga_output_tensors[0].op.node_def.attr['token'].s] = partition
            self.fpga_pynode_dict[fpga_output_tensors[0].op.name] = (partition, [inp.name for inp in
                                                                             fpga_input_tensors],
                                                                     list(output_dtypes))

            fpga_subgraph_def    = fpga_subgraph.as_graph_def(add_shapes=True)
            # fpga_node_map, \
            # fpga_output_node_map = fpga_subgraph_def.get_node_dict(outmap=True)

            for key, value in placeholder_replace_map.items():
              if key != value:
                raise RuntimeError('real and dummy placeholder names mismatched! {}'.format(placeholder_replace_map))

            ##########################################
            ## NOTE: NOT needed anymore (identity node fixes name mismatches)
            ##########################################
            # ## remove dummy placehoders and connect their consumers to the original inputs
            # fpga_nodes = []
            # for fpga_node_name, fpga_node in fpga_node_map.items():
            #   if fpga_node.op == 'Placeholder':
            #     # replace palceholder with original inputs
            #     placeholder_consumers = fpga_output_node_map[fpga_node_name]
            #     for placeholder_consumer_name, input_index in placeholder_consumers.items():
            #       placeholder_consumer_node = fpga_node_map[placeholder_consumer_name]
            #       del placeholder_consumer_node.input[input_index]
            #       placeholder_consumer_node.input.insert(input_index, placeholder_replace_map[fpga_node_name])
            #   else:
            #     # All nodes except dummy placeholders are to be copied to the main graph
            #     fpga_nodes += [fpga_node]

            # ## connect input of fpga_consumer_nodes to output from py_function
            # for i, outp in enumerate(partition.outputs):
            #   #output_consumer_nodes = {}
            #   #discover_consumer_nodes((outp, -1), set(partition.outputs), output_consumer_nodes)
            #   output_consumer_nodes = discover_consumer_nodes(partition.outputMap[outp],
            #                                                   set(partition.names), graph_output_dict)

            #   ## update the input list for the consumers partitions of outp
            #   for consumerPart in consumer_partitions[outp]:
            #     consumerPart = self.graph_partitions[consumerPart]
            #     # print(outp, output_consumer_nodes, consumerPart.index, consumerPart.supported)
            #     if consumerPart.supported:
            #       temp = self.reverse_connectivity[consumerPart.index]
            #       temp[output_tensors[i].name] = temp[outp]
            #       temp.pop(outp)
            #       consumerPart.inputs = [name if name != outp else output_tensors[i].name for name in consumerPart.inputs]
            #       consumerPart.inputMap[output_tensors[i].name] = consumerPart.inputMap[outp]

            #   ## update the input list for the consumer nodes of outp
            #   for output_consumer_node_name, input_index in output_consumer_nodes.items():
            #     output_consumer_node = graph_node_dict[output_consumer_node_name]
            #     del output_consumer_node.input[input_index]
            #     output_consumer_node.input.insert(input_index, output_tensors[i].name)

            ## remove dummy placeholders
            fpga_subgraph_def.remove_nodes([inp.op.name for inp in input_tensors])

            ## remove original output from input graph
            graph_def.remove_nodes([out.op.name for out in output_tensors])

            ## append fpga nodes to original graph
            fpga_nodes = fpga_subgraph_def.node
            graph_def.node.extend(fpga_nodes)

            return graph_def, fpga_nodes

        #############################
        ## function's body
        #############################
        graph_def = deepcopy(self._graph)

        self.fpga_pynode_dict = OrderedDict()

        for t, partition in enumerate(self.graph_partitions):
          if partition.supported:
            graph_def, __ = insert_fpga_pynode(graph_def, partition)

        # check graph integrity
        isCyclic, cycle = graph_def.is_cyclic()
        if isCyclic:
          raise RuntimeError('Graph partitioning resulted in cyclic graph; {}'.format(graph_def.all_cycles()))

        ## To make sure the modified graph_def is valid
        #with tf.Graph().as_default():
        #  tf.import_graph_def(graph_def, name='')

        # cleanup graph (at this point we do not have variables in graph so no initialization (sess) is needed)
        graph_def, fValidGraph = xdnn_util_tf.freeze_graph(None,
                                                           graph_def,
                                                           sinknodes_list=self.outputs,
                                                           remove_training_nodes=False,
                                                           filename=self.file_path('.pb', name_postfix='-fpga'),
                                                          )

        return graph_def

    def load_partitioned_graph(self):
        ## import the base partitioned graph
        tf.reset_default_graph()
        tf.import_graph_def(self.graph_def, name='')
        graph = tf.get_default_graph()

        ## declare py_functions in graph
        py_func_tokens = _script_ops._py_funcs._funcs.keys()
        for token in py_func_tokens:
          _script_ops._py_funcs.remove(token)
        _script_ops._py_funcs._unique_id = 0

        #with tf.device(global_fpga_device):  ## TODO: add FPGA support for TF
        for pyfunc_name, (partition, input_names, output_dtypes) in self.fpga_pynode_dict.items():
          input_tensors = [graph.get_tensor_by_name(name) for name in input_names]

          fpga_output_tensor = tf.py_func(partition.forward_exec, input_tensors, output_dtypes,
                                          stateful=False, name=pyfunc_name)

          # print('graph py_tunctions:   {}'.format(graph._py_funcs_used_in_graph))
          # print('graph py_func tokens: {}'.format(_script_ops._py_funcs._funcs.keys()))

        return graph

    def forward_exec(self, **kwargs):
        def default_forward_exec(**kwargs):
            if 'sess' in kwargs:
              sess  = kwargs['sess']
              graph = sess.graph
            else:
              #config=tf.ConfigProto(log_device_placement=True)
              graph = self.load_partitioned_graph()
              sess  = tf.Session(graph=graph, config=kwargs.get('config', None))

            ## declare input tensors to network graph
            input_names = kwargs.get('input_names', None)
            if not input_names:
              input_names = self.inputs
            else:
              for inp in input_names:
                if not isinstance(inp, _string_types):
                  raise TypeError('input_names should be flattened list of name strings')
            input_tensors  = [graph.get_operation_by_name(name).outputs[0] for name in input_names]

            ## declare output tensors to network graph
            output_names = kwargs.get('output_names', None)
            if not output_names:
              output_names = self.outputs
            else:
              for out in output_names:
                if not isinstance(out, _string_types):
                  raise TypeError('output_names should be flattened list of name strings')
            output_tensors = [graph.get_operation_by_name(name).outputs[0] for name in output_names]

            ## bound the input tensors to input data
            preprocess   = kwargs.get('preprocess', self.preprocess)
            input_values = xdnn_util.make_list(kwargs.get('input_values', None))

            feed_dict = {inp_tensor: inp_val if len(inp_val.shape) == len(inp_tensor.shape) else
                         inp_val[None,...] for inp_tensor, inp_val in zip(input_tensors, preprocess(input_values))}

            output_values = sess.run(output_tensors, feed_dict=feed_dict if feed_dict else None)

            if 'sess' not in kwargs:
              sess.close()

            return {tensor.op.name: val for val, tensor in zip(output_values, output_tensors)}

        if 'forward_exec' in kwargs:
          return kwargs['forward_exec'](*kwargs.get('argv', None))
        else:
          return default_forward_exec(**kwargs)

    def preprocess(self, inputs):
        ## inputs should be a list
        if not isinstance(inputs, list):
          raise TypeError('inputs to preprocessing should be a list')

        res = []
        for inp in inputs:
          if isinstance(inp, np.ndarray):
            res.append(inp)
          elif isinstance(inp, _string_types):
            res.append(_imread(inp))
        return res

    def debug_finalnode(self, input_values, **kwargs):
        input_values = xdnn_util.make_list(input_values)

        input_names  = self.inputs
        output_names = xdnn_util.make_list(self._args.finalnode)

        ## running the original graph
        with tf.Graph().as_default() as graph_org:
          tf.import_graph_def(self._graph, name='')

          node_dict, \
          output_dict    = graph_org.as_graph_def().get_node_dict(outmap=True)
          input_tensors  = [graph_org.get_operation_by_name(name).outputs[0] for name in input_names]
          output_tensors = [graph_org.get_operation_by_name(name).outputs[0] for name in output_names]

          print('org tensor: {}'.format(output_tensors))
          with tf.Session() as sess:
            feed_dict = {tensor: [value] for tensor, value in zip(input_tensors, input_values)}
            ret = {'org': sess.run(output_tensors, feed_dict)}

        ## running the partitioned graph
        graph          = self.load_partitioned_graph()
        input_tensors  = [graph.get_operation_by_name(name).outputs[0] for name in input_names]
        output_tensors = [graph.get_operation_by_name(name).outputs[0] for name in output_names]

        # output_tensors = []
        # for name in output_names:
        #   consumers = output_dict[name].keys()
        #   name = consumers[0] if consumers else name
        #   inp = [node.name for node in graph.get_operation_by_name(name).inputs if 'transpose' in
        #          node.name]
        #   if inp:
        #     ## use this in case output_names are NOT the last operations in global graph
        #     output_tensors.append(inp[0])
        #   else:
        #     ## use this in case output_names are the last operations in global graph
        #     output_tensors.append(graph.get_operation_by_name(name).outputs[0])
        ##########################################

        print('fpga tensor: {}'.format(output_tensors))
        with tf.Session(graph=graph) as sess:
          feed_dict = {tensor: [value] for tensor, value in zip(input_tensors, input_values)}
          ret.update({'fpga': sess.run(output_tensors, feed_dict)})

        for org, fpga in zip(ret['org'], ret['fpga']):
          print('org  max {} min {} mean {} std {}'.format(org.max(), org.min(), org.mean(), org.std()))
          print('FPGA max {} min {} mean {} std {}'.format(fpga.max(), fpga.min(), fpga.mean(), fpga.std()))
          print('DIFF max {} min {} mean {} std {}'.format((fpga-org).max(), (fpga-org).min(),
                                                           (fpga-org).mean(), (fpga-org).std()))

        return ret

#    class GEMXxdnnRT(TFxdnnRT):
#        def __init__(self, compilerFunc, args, **kwargs):
#            args = xdnn_util.dict2attr(args)
#            args.update(kwargs)
#
#            if args.networkfile is None and args.loadpickle is None:
#              raise AttributeError('network argument is missing')
#
#            self.fPartitioned  = args.get('fPartition', True)
#            self.networkfile   = args.networkfile
#            self.data_format   = args.data_format
#            self.save2modeldir = args.save2modeldir   # whether to save to mo directory
#
#            name_postfix       = args.get('name_postfix', '_frozen')
#            self.picklefile    = self.file_path('.pickle', name_postfix=name_postfix)   # save pydotGraph and compilerJson
#
#            ## run compiler
#            compiler = compilerFunc(args,
#                                    savepickle=self.picklefile,
#                                    loadpickle=self.picklefile,
#                                   )
#
#            self._graph, \
#            self.inputs, \
#            self.outputs, \
#            pydotGraph, \
#            compilerSchedule, \
#            ssize, \
#            compilerJson \
#                = compiler.compile()
#
#            if compilerJson is None:
#              raise RuntimeError('Compiler failed to produce valid schedule')
#
#            ## store compiler json
#            with open(self.file_path('.json', name_postfix=name_postfix+'-xdlfCompiler'), "w") as f:
#              json.dump(compilerJson, f, sort_keys=True, indent=4, separators=(',',': '))
#
#            if (self._graph  is None or
#                self.inputs  is None or
#                self.outputs is None):
#              ## in case loading from a pickled file
#              self._graph, \
#              self.inputs, \
#              self.outputs, \
#              self.fValidGraph \
#                  = self.load_graph(args)
#
#            self.unspt_set      = set(compilerJson['unsupported']['list'].keys()) # build unsupported layer set
#            layerparameter_dict = {}
#            layeroutput_dict    = defaultdict(lambda: {})
#            for node in pydotGraph.get_nodes():
#              LayerParameter = node.get('LayerParameter')
#              layer_name = LayerParameter.name
#              layerparameter_dict[layer_name] = LayerParameter
#              if 'blob' not in layer_name and LayerParameter.bottoms:       ## FIXME: hack to remove output blobs from output list
#                for input_index, input_name in enumerate(LayerParameter.bottoms):
#                  layeroutput_dict[input_name][layer_name] = input_index
#
#            self.layerparameter_dict = layerparameter_dict
#            self.pydotGraph = pydotGraph
#            self.compilerSchedule = compilerSchedule
#            self.compilerJson = compilerJson
#
#            graph_partitions = [LayerPartition(0, False),
#                                LayerPartition(1, True),
#                                LayerPartition(2, False),
#                                LayerPartition(1, False)]
#
#            ################## now manually add inputs and outputs for all paritions
#            graph_partitions[0].inputs = ['',]
#            graph_partitions[0].outpus = ['',]
#            graph_partitions[1].inputs = ['',]
#            graph_partitions[1].outpus = ['',]
#            graph_partitions[2].inputs = ['',]
#            graph_partitions[2].outpus = ['',]
#            graph_partitions[3].inputs = ['',]
#            graph_partitions[3].outpus = ['',]
#
#            for partition in graph_partitions:
#              partition.inputMap = {name: name for name in partition.inputs}
#              partition.outputMap = {name: name for name in partition.outputs}
#            ########################################################################
#
#            ## optimize partitions
#            self.graph_partitions = self.refine_graph_partitions(graph_partitions)
#
#            ## device transformation for supported layers
#            ################## now need to continue the device_transforms function below to plug in your
#            ################## own function. your function should support opt.getLayers() and
#            ################## opt.variables.
#            ################## layers from opt.getLayers() should support layer.forward_exec(input)
#            ################## methods. follow xdnn_opt and py files in xfdnn/tools/emu.
#            self.device_transforms(args)
#            ########################################################################
#
#            self.rebuild_graph()
#
#        def device_transforms(self, args):
#            subTFs = []
#            for partition in self.graph_partitions:
#              ## extract supported subgraph
#              print('Transorm partition_index \"{}\"'.format(partition.index))
#              print('inputs:  {}'.format(partition.inputMap.keys()))
#              print('actual:  {}'.format(partition.inputMap.values()))
#              print('outputs: {}'.format(partition.outputMap.keys()))
#              print('actual:  {}'.format(partition.outputMap.values()))
#
#              if partition.supported:
#                opt = GEMXTransform(.....)
#
#                partition.layers    = opt.getLayers()
#                partition.variables = opt.variables


import os

import numpy as np
from scipy.interpolate import RectBivariateSpline

from adf11 import Adf11

argon_data = {
    'ionisation' : 'scd89_ar.dat',
    'recombination' : 'acd89_ar.dat',
    'continuum_power' : 'prb89_ar.dat',
    'line_power' : 'plt89_ar.dat',
    'cx_power' : 'prc89_ar.dat',
}

carbon_data = {
    'ionisation' : 'scd96_c.dat',
    'recombination' : 'acd96_c.dat',
    'continuum_power' : 'prb96_c.dat',
    'line_power' : 'plt96_c.dat',
    'cx_power' : 'prc96_c.dat',
}

neon_data = {
    'ionisation' : 'scd96_ne.dat',
    'recombination' : 'acd96_ne.dat',
    'continuum_power' : 'prb96_ne.dat',
    'line_power' : 'plt96_ne.dat',
}

def _element_data(element):
    e = element.lower()
    if e in ['ar', 'argon']:
        return argon_data
    elif e in ['c', 'carbon']:
        return carbon_data
    elif e in ['ne', 'neon']:
        return neon_data
    else:
        raise NotImplementedError('unknown element: %s' % element)

def _full_path(file_):
    """ Figure out the location of the atomic datafiles. """
    module_path = os.path.dirname(os.path.realpath( __file__ ))
    return os.path.join(module_path, '..', 'adas_data', file_)

class AtomicData(object):
    def __init__(self, coefficients):
        """
        Parameters
        ----------
        element : string
            Name of the element.
        coefficients : dict
            Map of the different rate coefficients.
        """
        self.coeffs = coefficients
        self._check_consistency()
        self._make_element_initial_uppercase()

    def copy(self):
        new_coeffs = {}
        for key, value in self.coeffs.iteritems():
            new_coeffs[key] = value.copy()

        return self.__class__(new_coeffs)

    def _check_consistency(self):
        nuclear_charge = set()
        element = set()
        for coeff in self.coeffs.values():
            nuclear_charge.add(coeff.nuclear_charge)
            element.add(coeff.element.lower())

        assert len(nuclear_charge) == 1, 'inconsistent nuclear charge.'
        assert len(element) == 1, 'inconsistent element name.'

        self.nuclear_charge = nuclear_charge.pop()
        self.element = element.pop()

    @classmethod
    def from_element(cls, element):
        element_data = _element_data(element)

        coefficients = {}
        for key in element_data.keys():
            name = _full_path(element_data[key])
            coefficients[key] = RateCoefficient.from_adf11(name)

        return cls(coefficients)

    def _make_element_initial_uppercase(self):
        e = self.element
        self.element = e[0].upper() + e[1:]

class RateCoefficient(object):
    def __init__(self, nuclear_charge, element, log_temperature, log_density,
            log_coeff, name=None):
        self.nuclear_charge = nuclear_charge
        self.element = element
        self.adf11_file = name

        self.log_temperature = log_temperature
        self.log_density = log_density
        self.log_coeff = log_coeff

        self._compute_interpolating_splines()

    @classmethod
    def from_adf11(cls, name):
        adf11_data = Adf11(name).read()
        nuclear_charge = adf11_data['charge']
        element = adf11_data['element']
        filename = adf11_data['name']

        log_temperature = adf11_data['log_temperature']
        log_density = adf11_data['log_density']
        log_coeff = adf11_data['log_coeff']

        return cls(nuclear_charge, element, log_temperature, log_density,
                log_coeff, name=filename)

    def copy(self):
        log_temperature = self.log_temperature.copy()
        log_density = self.log_density.copy()
        log_coeff = self.log_coeff.copy()
        cls = self.__class__(self.nuclear_charge, self.element, log_temperature,
                log_density, log_coeff, self.adf11_file)
        return cls

    def _compute_interpolating_splines(self):
        self.splines = []
        for k in xrange(self.nuclear_charge):
            x = self.log_temperature
            y = self.log_density
            z = self.log_coeff[k]
            self.splines.append(RectBivariateSpline(x, y, z))

    def __call__(self, k, Te, ne):
        """
        Evaulate the ionisation/recombination coefficients of k'th atomic state
        at a given temperature and density.

        Parameters
        ----------
        k : int
            Ionisation stage. k=0 is the neutral atom, k=Z is  the fully
            stripped ion, where Z is the atomic number.
        Te : array_like
            Temperature in [eV].
        ne : array_like
            Density in [m-3].

        Returns
        -------
        c : array_like
            Rate coefficent in [m3/s].
        """
        c = self.log10(k, Te, ne)
        return np.power(10, c)

    def log10(self, k, Te, ne):
        """
        Evaulate the logarithm of ionisation/recombination coefficients of
        k'th atomic state at a given temperature and density.

        Parameters
        ----------
        k : int
            Ionisation stage. k=0 is the neutral atom, k=Z is  the fully
            stripped ion, where Z is the atomic number.
        Te : array_like
            Temperature in [eV].
        ne : array_like
            Density in [m-3].

        Returns
        -------
        c : array_like
            log10(rate coefficent in [m3/s])
        """

        Te, ne = np.broadcast_arrays(Te, ne)
        log_temperature = np.log10(Te)
        log_density = np.log10(ne)

        c = self.splines[k](log_temperature, log_density)
        return c.diagonal()

    @property
    def temperature_grid(self):
        return 10**(self.log_temperature)

    @property
    def density_grid(self):
        return 10**(self.log_density)

from sys import float_info
class ZeroCoefficient(RateCoefficient):
    def __init__(self):
        pass

    def __call__(self, k, Te, ne):
        Te, ne = np.broadcast_arrays(Te, ne)
        return float_info.min * np.ones_like(Te)


import os,re
import collections
import numpy as np
from scipy.stats import norm
import pyworld as pw
import matplotlib.pyplot as plt
import sys
import h5py

from synth.utils import sig_process, segment, vamp_notes, audio_process, utils, midi_process
from synth.config import config

def write_data(inputs, file_name, feats_dir=config.feats_dir):
    """
    Function to save data. For now, saving each segment separatly.
    Arguments:
        inputs: A dictionary of input values, should be real valued np arrays.
        file_name: The name of the hdf5 file to be saved.
    """
    if not os.path.isdir(feats_dir):
        os.mkdir(feats_dir)
    with h5py.File(os.path.join(feats_dir, file_name), mode='w') as hdf5_file:
        for key in inputs.keys():
            inp = inputs[key]
            if len(inp.shape) == 1:
                inp = np.expand_dims(inp, -1)
            hdf5_file.create_dataset(key, inp.shape, inp.dtype)
            hdf5_file[key][:,:] = inp