from PyQt5 import QtCore, QtGui, QtWidgets

class Ui_Form(object):
    def setupUi(self, Form):
        Form.setObjectName("Form")
        Form.resize(661, 321)
        self.label = QtWidgets.QLabel(Form)
        self.label.setGeometry(QtCore.QRect(50, 40, 301, 21))
        self.label.setObjectName("label")
        self.NameEdit = QtWidgets.QLineEdit(Form)
        self.NameEdit.setGeometry(QtCore.QRect(50, 70, 371, 20))
        self.NameEdit.setObjectName("NameEdit")
        self.Exit = QtWidgets.QPushButton(Form)
        self.Exit.setGeometry(QtCore.QRect(200, 140, 75, 23))
        self.Exit.setObjectName("Exit")
        self.widget = QtWidgets.QWidget(Form)
        self.widget.setGeometry(QtCore.QRect(140, 100, 211, 25))
        self.widget.setObjectName("widget")
        self.horizontalLayout = QtWidgets.QHBoxLayout(self.widget)
        self.horizontalLayout.setContentsMargins(0, 0, 0, 0)
        self.horizontalLayout.setObjectName("horizontalLayout")
        self.Hallo = QtWidgets.QPushButton(self.widget)
        self.Hallo.setObjectName("Hallo")
        self.horizontalLayout.addWidget(self.Hallo)
        self.Clear = QtWidgets.QPushButton(self.widget)
        self.Clear.setObjectName("Clear")
        self.horizontalLayout.addWidget(self.Clear)

        self.retranslateUi(Form)
        self.Exit.clicked.connect(Form.close)
        self.Clear.clicked.connect(self.NameEdit.clear)
        QtCore.QMetaObject.connectSlotsByName(Form)

    def retranslateUi(self, Form):
        _translate = QtCore.QCoreApplication.translate
        Form.setWindowTitle(_translate("Form", "Form"))
        self.label.setText(_translate("Form", "
Masukkan Nama Anda :
"))
        self.Exit.setText(_translate("Form", "Exit"))
        self.Hallo.setText(_translate("Form", "Hallo"))
        self.Clear.setText(_translate("Form", "Clear"))

if __name__ == "__main__":
    import sys
    app = QtWidgets.QApplication(sys.argv)
    Form = QtWidgets.QWidget()
    ui = Ui_Form()
    ui.setupUi(Form)
    Form.show()
    sys.exit(app.exec_())


import argparse
import subprocess
import json
from multiprocessing.pool import Pool

def get_iplist(iplist):
    if '-' in iplist:
        iplist = iplist.split('-')
        new_iplist = [f'{iplist[0][:-1]}{i}' for i in range(int(iplist[0][-1]), int(iplist[1][-1])+1)]
        return new_iplist
    else:
        new_ip = iplist
        return new_ip

def func_ping(ip):
    print(f"ping -c 3 {ip}")
    commond = f"ping -c 3 {ip}"
    subt = subprocess.Popen(commond, shell=True,
                            stdout=subprocess.PIPE, encoding='utf-8')
    flag = 1
    for line in iter(subt.stdout.readline, ''):
        print(line)
        if 'timeout' in line:
            print(f'{ip} 不可用')
            flag = 0
            break
    if flag == 0:
        pass
    else:
        print(f'{ip} 可用')
        ip_used = ip
        return ip_used

    subt.stdout.close()
    subt.wait()

def func_tcp(ip, port):

    print(f'开始扫描端口：{port}')
    print(f"nc -vz -w 2 {ip} {port}")
    commond = f"nc -vz -w 2 {ip} {port}"
    subt = subprocess.Popen(commond, shell=True,
                            stdout=subprocess.PIPE, encoding='utf-8')
    line = subt.stdout.readlines()
    print(line)
    if 'successed' in line:
        print(f'端口：{port} 可用')
        return port
    subt.stdout.close()
    subt.wait()

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-n', '--num', dest='Num', type=int, help='target Num')
    parser.add_argument('-f', '--func', dest='Func', type=str, choices=['ping', 'tcp'], help='target Func')
    parser.add_argument('-ip', '--iplist', dest='Ip', type=str, help='target Ip')
    parser.add_argument('-w', '--write', dest='Write', type=str, help='target FileName')

    cmd_info = parser.parse_args()
    iplist = get_iplist(cmd_info.Ip)
    print(iplist)
    p = Pool(cmd_info.Num)
    if cmd_info.Func == 'ping':
        for ip in iplist:
            p.apply_async(func_ping, args=(ip,))
        p.close()
        p.join()
    else:
        min_port = 1
        max_port = 1024
        print(f'扫描端口的范围为{min_port}--{max_port}')
        for port in range(min_port, max_port+1):
            p.apply_async(func_tcp, args=(iplist, port))
        p.close()
        p.join()


import torch
from torch import nn
import numpy as np
from tqdm import tqdm

from configuration import clstm_config
from preprocess import get_dataloader

# import sranodec as anom

class SRCNN(nn.Module):
    def __init__(self, device):
        super(SRCNN, self).__init__()
        self.device = device
        self.bidirectional = True
        self.n_layers = 2
        self.rnn_hidden = 32
        # less than period
        # self.amp_window_size=24
        # (maybe) as same as period
        # self.series_window_size=24
        # a number enough larger than period
        # self.score_window_size=64
        # self.spec = anom.Silency(self.amp_window_size, self.series_window_size, self.score_window_size)
        # self.sr_layer = self.spec.generate_anomaly_score

        self.dropout = nn.Dropout(0.2).to(device)
        self.cnn1 = nn.Sequential(nn.Conv1d(in_channels=1, out_channels=64, kernel_size=64, stride=1, padding=64),
                                  nn.MaxPool1d(kernel_size=2, stride=2),
                                  ).to(device)
        self.cnn2 = nn.Sequential(nn.Conv1d(in_channels=64, out_channels=128, kernel_size=64, stride=1),
                                  # nn.MaxPool1d(kernel_size=2, stride=2),
                                  ).to(device)

        self.classifier = nn.Sequential(nn.Linear(128, 32), nn.Tanh(), nn.Linear(32, 2)).to(device)
        self.criterion = nn.CrossEntropyLoss()
        self.optimizer = torch.optim.Adam(self.parameters(), lr=1e-4)

    def forward(self, inputs):
        # inputs [batch, len, 1]
        # inputs = inputs.squeeze(dim=-1)
        # print(inputs.shape)
        # inputs = inputs + 0.00001* torch.from_numpy(np.ones((inputs.shape[0],inputs.shape[1]))).cuda().float()
        # sr = []
        # for input in inputs:
        #     sr.append(torch.tensor(self.sr_layer(input.cpu().numpy())).unsqueeze(dim=0).to(self.device))
        # inputs = torch.cat(sr, dim=0)
        # inputs = inputs.unsqueeze(dim=-1).float()
        # inputs [batch, len, 1]
        batch_size, len, _ = inputs.shape
        inputs = inputs.reshape((batch_size, 1, len))
        # print(inputs.shape)
        inputs = self.cnn1(inputs)
        # print(inputs.shape)
        inputs = self.cnn2(inputs)
        # print(inputs.shape)
        inputs = inputs.squeeze()
        # exit()

        pred = self.classifier(inputs)
        # print(pred.shape)
        pred = torch.sigmoid(pred).squeeze(dim=-1)
        return pred

    def run_epoch(self, dataset, train=True, draw=False):
        if train:
            self.train()
        else:
            self.eval()
        l, tp, tn, fp, fn = 0, 0.01, 0.01, 0.01, 0.01
        for x, y in dataset:
            x, y = x.to(self.device), y.to(self.device)
            out = self.forward(x.unsqueeze(dim=-1))
            loss = self.criterion(out, y)
            if train:
                loss.backward()
                self.optimizer.step()
                self.optimizer.zero_grad()

            pred = out.detach().argmax(dim=-1)
            l += loss.item()
            tp += ((pred == y) & (pred == 1)).sum().item()
            tn += ((pred == y) & (pred != 1)).sum().item()
            fp += ((pred != y) & (pred == 1)).sum().item()
            fn += ((pred != y) & (pred != 1)).sum().item()
            fp_ = (pred != y) & (pred == 1)
            if draw:
                for i, ti in enumerate(fp_):
                    if ti == 1:
                        print(x[i])
                        print(y[i])
                        exit()
        cnt = tp + tn + fp + fn
        precision = tp / (tp + fp)
        recall = tp / (tp + fn)
        f1 = 2 * precision * recall / (precision + recall)
        return "loss=%.4f, acc=%.4f, precision=%.4f, recall=%.4f, F1=%.4f" % (
            l / cnt, (tp + tn) / cnt, precision, recall, f1)

if __name__ == '__main__':
    if torch.cuda.is_available():
        device = torch.device('cuda:0')
    else:
        device = torch.device('cpu')
    print(device)

    config = clstm_config()
    train, test = get_dataloader(config.batch_size, rate=0.1, split=0.7, use_sr=False, normalize=True)

    model = SRCNN(device)
    model = model
    print(model)

    for epoch in range(100):
        print('epoch %d:' % epoch, end=' ')
        # train
        model.run_epoch(train, True)
        # test
        print('test: ', model.run_epoch(test, False, epoch==10))


#!/usr/bin/env python

# libraries
import warnings
import numpy as np

## reduction models
from sklearn.decomposition import PCA, FastICA, IncrementalPCA, KernelPCA, FactorAnalysis, TruncatedSVD, SparsePCA, MiniBatchSparsePCA, DictionaryLearning, MiniBatchDictionaryLearning
from sklearn.manifold import TSNE, MDS, SpectralEmbedding, LocallyLinearEmbedding, Isomap

# internal libraries
from ..tools.df2mat import df2mat
from .._shared.helpers import *
from .normalize import normalize as normalizer
from .align import align as aligner

# main function
@memoize
def reduce(x, reduce='IncrementalPCA', ndims=3, normalize=None, align=None,
           model=None, model_params=None, internal=False):
    """
    Reduces dimensionality of an array, or list of arrays

    Parameters
    ----------
    x : Numpy array or list of arrays
        Dimensionality reduction using PCA is performed on this array.

    reduce : str or dict
        Decomposition/manifold learning model to use.  Models supported: PCA,
        IncrementalPCA, SparsePCA, MiniBatchSparsePCA, KernelPCA, FastICA,
        FactorAnalysis, TruncatedSVD, DictionaryLearning, MiniBatchDictionaryLearning,
        TSNE, Isomap, SpectralEmbedding, LocallyLinearEmbedding, and MDS. Can be
        passed as a string, but for finer control of the model parameters, pass
        as a dictionary, e.g. reduce={'model' : 'PCA', 'params' : {'whiten' : True}}.
        See scikit-learn specific model docs for details on parameters supported
        for each model.

    ndims : int
        Number of dimensions to reduce

    model : None
        Deprecated argument.  Please use reduce.

    model_params : None
        Deprecated argument.  Please use reduce.

    align : None
        Deprecated argument.  Please use new analyze function to perform
        combinations of transformations

    normalize : None
        Deprecated argument.  Please use new analyze function to perform
        combinations of transformations

    Returns
    ----------
    x_reduced : Numpy array or list of arrays
        The reduced data with ndims dimensionality is returned.  If the input
        is a list, a list is returned.

    """

    # sub functions
    def reduce_list(x, model):
        split = np.cumsum([len(xi) for xi in x])[:-1]
        x_r = np.vsplit(model.fit_transform(np.vstack(x)), split)
        if len(x)>1:
            return [xi for xi in x_r]
        else:
            return [x_r[0]]

    # dictionary of models
    models = {
        'PCA' : PCA,
        'IncrementalPCA' : IncrementalPCA,
        'SparsePCA' : SparsePCA,
        'MiniBatchSparsePCA' : MiniBatchSparsePCA,
        'KernelPCA' : KernelPCA,
        'FastICA' : FastICA,
        'FactorAnalysis' : FactorAnalysis,
        'TruncatedSVD' : TruncatedSVD,
        'DictionaryLearning' : DictionaryLearning,
        'MiniBatchDictionaryLearning' : MiniBatchDictionaryLearning,
        'TSNE' : TSNE,
        'Isomap' : Isomap,
        'SpectralEmbedding' : SpectralEmbedding,
        'LocallyLinearEmbedding' : LocallyLinearEmbedding,
        'MDS' : MDS
    }

    # deprecated warning
    if (model is not None) or (model_params is not None):
        warnings.warn('Model and model params will be deprecated.  Please use the \
                      reduce keyword.  See API docs for more info: http://hypertools.readthedocs.io/en/latest/hypertools.tools.reduce.html#hypertools.tools.reduce')
        reduce = {}
        reduce['model'] = model
        reduce['params'] = model_params

    # if model is None, just return data
    if reduce is None:
        return x
    else:

        # common format
        x = format_data(x, ppca=True)

        # deprecation warnings
        if normalize is not None:
            warnings.warn('The normalize argument will be deprecated for this function.  Please use the \
                          analyze function to perform combinations of these transformations.  See API docs for more info: http://hypertools.readthedocs.io/en/latest/hypertools.analyze.html#hypertools.analyze')
            x = normalizer(x, normalize=normalize)

        if align is not None:
            warnings.warn('The align argument will be deprecated for this function.  Please use the \
                          analyze function to perform combinations of these transformations.  See API docs for more info: http://hypertools.readthedocs.io/en/latest/hypertools.analyze.html#hypertools.analyze')
            x = aligner(x, align=align)

        # if the shape of the data is already less than ndims, just return it
        if all([i.shape[1]<=ndims for i in x]):
            return x

        # if reduce is a string, find the corresponding model
        if type(reduce) in [str, np.string_]:
            model = models[reduce]
            model_params = {
                'n_components' : ndims
            }
        # if its a dict, use custom params
        elif type(reduce) is dict:
            if type(reduce['model']) is str:
                model = models[reduce['model']]
                if reduce['params'] is None:
                    model_params = {
                        'n_components' : ndims
                    }
                else:
                    model_params = reduce['params']
        if 'n_components' not in model_params:
            model_params['n_components'] = ndims

        # initialize model
        model = model(**model_params)

        # reduce data
        x_reduced = reduce_list(x, model)

        # return data
        if internal or len(x_reduced)>1:
            return x_reduced
        else:
            return x_reduced[0]


#!/usr/bin/python

import argparse

'''
the list of prices show the variance of price during different times of day. list is sorted by time and not by price.

so the basic problem is to find the maximum profit you can make by buying first and then selling later in the day.

what we need to do is to keep track of the min price so far and the max profit so far.

so we need to have some sort of pointer that starts at the beginning of the array and then a second pointer that starts at the second element in the array.

the first element is automatically the current min price so far and the difference between the second element and the first element is the current max profit so far.

if we do a while loop whose conditional is that i and j are both less than the length of the array, we can keep the while loop going without having nested loops I think.

for the first check, we set i to the initial element in array and we set j to the third element in the array since we have already instantiated the the max profit so far is the difference between elemnt 1 and element 0.

let's check to see if j element - i element is > max profit so far -- if it is update max profit

if not, advance j if there is length left in the array -- if not reset i to i + 1 and reset j to the "new" i + 1.

'''

def find_max_profit(prices):

  i = 0
  j = 1
  max_profit = float('-inf')
  while i < len(prices) and j < len(prices):
    difference = prices[j] - prices[i]
    print(difference)
    if difference > max_profit:
      max_profit = difference
      if j + 1 == len(prices):
        #we start over
        if i + 1 == len(prices):
          break
        else:
          i += 1
          j = i + 1 #new i
      else:
        j += 1
    else:
      if j + 1 == len(prices):
        #we start over
        if i + 1 == len(prices):
          break
        else:
          i += 1
          j = i + 1 #new i
      else:
        j += 1

  return max_profit

if __name__ == '__main__':
  # This is just some code to accept inputs from the command line
  parser = argparse.ArgumentParser(description='Find max profit from prices.')
  parser.add_argument('integers', metavar='N', type=int, nargs='+', help='an integer price')
  args = parser.parse_args()

  print("A profit of ${profit} can be made from the stock prices {prices}.".format(profit=find_max_profit(args.integers), prices=args.integers))


import re
import math
import random
import sys
import pprint
import cProfile
from utils import read_text
import time
import argparse

class FCM:

    def __init__(self, text, a=0, k=1):
        self.words = text
        self.fs_cache = {}
        self.a = a
        self.k = k
        self.contexts_seen = None
        self.prob_dic = None

    # Count all subsequences of k+1 words
    def count_subsequences(self, words, k):
        dic = {}

        for i in range(0, len(words)):
            seq = tuple(words[i:i + (k + 1)])
            if len(seq) < k + 1:
                continue
            if seq in dic:
                dic[seq] += 1
            else:
                dic[seq] = 1
        return dic

    # Get probability of a state c in the text
    def frequency_sequence(self, seq, total_len_seq):
        t_seq = tuple(seq)

        if t_seq in self.fs_cache:
            count = self.fs_cache[t_seq]
            result = count / total_len_seq
            return result
        return 0

    # Calculate all probabilities for each context
    def calculate_probabilities(self):
        words_len = len(self.words)
        unique_words = set(self.words)
        seq_count = self.count_subsequences(self.words, self.k)  # { context: num_occors}
        # Sequences Dictionary
        # for key in seq_count:
        #     print(f'{key} -> {seq_count[key]}')
        prob_dic = {}
        contexts_seen = {}
        for i in range(0, words_len):
            c = self.words[i:i + self.k]  # c - context
            if tuple(c) in contexts_seen:
                continue
            # Get the sum of all contexts where c exists
            sum_all_constexts_c = sum(
                [seq_count[tuple(c + [simbol])] for simbol in unique_words if tuple(c + [simbol]) in seq_count])
            all_probs = []
            # Get all possible sequences for c + appending with another simbol
            unique_words_len = len(unique_words)
            for simbol in unique_words:
                seq = tuple(c + [simbol])
                if seq not in seq_count:
                    if self.a:
                        n = 0

                    else:
                        continue
                else:
                    n = seq_count[seq]

                total = (sum_all_constexts_c + self.a * unique_words_len)
                # Prob of event e for context c
                prob_n = (n + self.a) / total

                # print(f'Seq:{seq} Nprob:{prob_n} the total from prob is {total}')
                context = ''.join(c)

                if context in prob_dic:
                    prob_dic[context].append((simbol, prob_n))
                else:
                    prob_dic[context] = [(simbol, prob_n)]
                all_probs.append(prob_n)
            contexts_seen[tuple(c)] = all_probs
        self.contexts_seen = contexts_seen
        self.prob_dic = prob_dic
        return prob_dic

    def calculate_frequencies(self, total_seq):
        for i in total_seq:
            seq_hash = tuple(i)
            if seq_hash in self.contexts_seen:
                if seq_hash in self.fs_cache:
                    self.fs_cache[seq_hash] += 1
                else:
                    self.fs_cache[seq_hash] = 0

    # Calculate entropy taking into account
    def calculate_entropy(self):
        words_len = len(self.words)
        context_entropy_list = []
        if self.contexts_seen == None: return 0

        # Total number of sequences
        total_seq = [self.words[i:i + self.k] for i in range(0, words_len)]
        total_seq_len = len(total_seq)

        # Calculate frequencies before hand
        # Done in order to prevent multiple transversals of the whole text
        self.calculate_frequencies(total_seq)

        for c in self.contexts_seen:
            all_probs = self.contexts_seen[c]
            # Get Hc entropy for each context/row
            # He gets 0 if prob is 0
            entropy_of_context = -1 * sum([prob * math.log(prob, 2) for prob in all_probs])
            # Probability of context/subsequence in total text
            prob_c = self.frequency_sequence(list(c), total_seq_len)
            # print(f'For c={c} entropy={entropy_of_context} Cprob={prob_c}')
            context_entropy_list.append(entropy_of_context * prob_c)
        return sum(context_entropy_list)

def main():
    pp = pprint.PrettyPrinter(indent=4)
    parser = argparse.ArgumentParser(description="Calculate Entropy",
                                     usage="python3 fcm.py -a <smoothing_parameter> -k <order_of_the_model> -path <path_of_the_text_file>")

    parser.add_argument("-a", help="Smoothing parameter", type=float, required=True)
    parser.add_argument("-k", help="Model context size", type=int, required=True)
    parser.add_argument("-path", help="Path to text file", required=True)

    args = parser.parse_args()

    text = read_text(args.path)
    fcm = FCM(text, args.a, args.k)
    prob_dic = fcm.calculate_probabilities()
    entropy = fcm.calculate_entropy()
    # for key in prob_dic:
    #     pp.pprint(f'{key} : {prob_dic[key]}')
    pp.pprint(f'Smoothing: {args.a} and Order: {args.k}')
    pp.pprint(f'Entropy:{entropy}')

if __name__ == "__main__":
    main()


import urllib.request
from bs4 import BeautifulSoup

class Scraper:
    def __init__(self, site):
        self.site = site
    def scrape(self):
        response = urllib.request.urlopen(self.site)
        html = response.read()
        sp = BeautifulSoup(html,"html.parser")
        with open ("output.txt","w") as f:
            for tag in sp.find_all("a"):
                url = tag.get("href")
                if url and 'html' in url:
                    print("\n" + url)
                    f.write("\n" + url)

news = "https://sueddeutsche.de/"
Scraper(news).scrape()


from django import template

from categories.models import Good

register = template.Library()

@register.inclusion_tag('goodtypes_block.html')
def show_goodtypes(category):
    goodtypes = category.goodtype_set.all()
    for goodtype in goodtypes:
    	goodtype.count = Good.objects.filter(goodtype=goodtype).count()
    return {'goodtypes': goodtypes}


from rest_framework import viewsets, routers
from react_app.models import User_Models
from .serializers import UserSerializer

class UserApi(viewsets.ModelViewSet):
    queryset = User_Models.objects.all()
    serializer_class = UserSerializer

    def get_queryset(self):
        queryset = User_Models.objects.all()
        L_id = self.request.query_params.get('id')

        if L_id:
            queryset = queryset.filter(user_id=L_id)

        return queryset

router = routers.DefaultRouter()
router.register(r'user', UserApi)


"""
This file is part of pyS5p

https://github.com/rmvanhees/pys5p.git

Implements a lite interface with the xarray::DataArray

Copyright (c) 2020-2021 SRON - Netherlands Institute for Space Research
   All Rights Reserved

License:  BSD-3-Clause
"""
from pathlib import PurePath

import numpy as np
import xarray as xr

def __get_attrs(dset, field: str) -> dict:
    """
    Return attributes of the HDF5 dataset

    Parameters
    ----------
    dset :  h5py.Dataset
       h5py dataset from which the attributes are read
    field : str
       Name of field in compound dataset

    Returns
    -------
    dict with numpy arrays
    """
    _field = None
    if field is not None:
        try:
            _field = {'name': field,
                      'oneof': len(dset.dtype.names),
                      'index': dset.dtype.names.index(field)}
        except Exception as exc:
            raise RuntimeError(
                f'field {field} not found in dataset {dset.name}') from exc
        # print('_field ', _field)

    attrs = {}
    for key in dset.attrs:
        if key in ('CLASS', 'DIMENSION_LIST', 'NAME', 'REFERENCE_LIST',
                   '_Netcdf4Dimid'):
            continue

        attr_value = dset.attrs[key]
        # print('# ----- ', key, type(attr_value), attr_value)
        if isinstance(attr_value, np.ndarray):
            if len(attr_value) == 1:
                attr_value = attr_value[0]
                # print('# ----- ', key, type(attr_value), attr_value)
            elif _field is not None:
                if len(attr_value) == _field['oneof']:
                    attr_value = attr_value[_field['index']]
                # elif isinstance(attr_value, np.void):
                #    attr_value = attr_value[0]

        attrs[key] = (attr_value.decode('ascii')
                      if isinstance(attr_value, bytes) else attr_value)

    return attrs

def __get_coords(dset, data_sel: tuple) -> list:
    """
    Return coordinates of the HDF5 dataset

    Parameters
    ----------
    dset :  h5py.Dataset
       h5py dataset from which the data is read
    data_sel :  tuple of slice(s)
       A numpy slice generated for example numpy.s_
       Default read while array

    Returns
    -------
    A sequence of tuples [(dims, data), ...]
    """
    coords = []
    if len(dset.dims) == dset.ndim:
        try:
            for ii, dim in enumerate(dset.dims):
                name = PurePath(dim[0].name).name
                if name.startswith('row') or name.startswith('column'):
                    name = name.split(' ')[0]

                buff = dim[0][()]
                if np.all(buff == 0):
                    buff = np.arange(dim[0].size, dtype=dim[0].dtype)
                elif data_sel is not None:
                    buff = buff[data_sel[ii]]

                coords.append((name, buff))
        except RuntimeError:
            coords = []

    return coords

def __set_coords(dset, data_sel: tuple, dims: list) -> list:
    """
    Set coordinates of the HDF5 dataset

    Parameters
    ----------
    dset :  h5py.Dataset or numpy.array
       h5py dataset from which the data is read
    data_sel :  tuple of slice(s)
       A numpy slice generated for example numpy.s_
       Default read while array
    dims : list of strings
       Alternative names for the dataset dimensions if not attached to dataset
       Default coordinate names are ['time', ['row', ['column']]]

    Returns
    -------
    A sequence of tuples [(dims, data), ...]
    """
    if dims is None:
        if dset.ndim > 3:
            raise ValueError('not implemented for ndim > 3')

        dims = ['time', 'row', 'column'][-dset.ndim:]

    coords = []
    for ii in range(dset.ndim):
        buff = np.arange(dset.shape[ii], dtype='u4')
        if data_sel is not None:
            buff = buff[data_sel[ii]]
        coords.append((dims[ii], buff))

    return coords

def __get_data(dset, data_sel: tuple, field: str):
    """
    Return data of the HDF5 dataset

    Parameters
    ----------
    dset :  h5py.Dataset
       h5py dataset from which the data is read
    data_sel :  numpy slice
       A numpy slice generated for example numpy.s_
       Default read while array
    field : str
       Name of field in compound dataset or None

    Returns
    -------
    Numpy array

    Notes
    -----
    Read floats always as doubles
    """
    if data_sel is None:
        data_sel = ()

    if np.issubdtype(dset.dtype, np.floating):
        data = dset.astype(float)[data_sel]
        data[data == float.fromhex('0x1.ep+122')] = np.nan
        return data

    if field is None:
        return dset[data_sel]

    data = dset.fields(field)[data_sel]
    if np.issubdtype(data.dtype, np.floating):
        data = data.astype(float)
        data[data == float.fromhex('0x1.ep+122')] = np.nan
    return data

def h5_to_xr(h5_dset, data_sel=None, *, dims=None, field=None):
    """
    Create xarray::DataArray from a HDF5 dataset (with dimension scales)

    Parameters
    ----------
    h5_dset :  h5py.Dataset
       Data, dimensions, coordinates and attributes are read for this dataset
    data_sel :  slice, optional
       A numpy slice generated for example numpy.s_
    dims :  list of strings, optional
       Alternative names for the dataset dimensions if not attached to dataset
    field : str, optional
       Name of field in compound dataset or None

    Returns
    -------
    xarray.DataArray

    Notes
    -----
    If data_sel is used to select data from a dataset then the number of
    dimensions of data_sel should agree with the HDF5 dataset or one and
    only one Ellipsis has to be used.
    Thus allowed values for data_sel are:
    * [always]: (), np.s_[:], np.s_[...]
    * [1-D dataset]: np.s_[:-1], np.s_[0]
    * [2-D dataset]: np.s_[:-1, :], np.s_[0, :], np.s_[:-1, 0]
    * [3-D dataset]: np.s_[:-1, :, 2:4], np.s_[0, :, :], np.s_[:-1, 0, 2:4]
    * [Ellipsis] np.s_[0, ...], np.s_[..., 4], np.s_[0, ..., 4]

    Examples
    --------
    >>> fid = h5py.File(flname, 'r')        # flname is a HDF5/netCDF4 file
    >>> xdata = h5_to_xr(fid['signal'])
    >>> fid.close()

    Combine Tropomi SWIR data of band7 and band8
    >>> fid = h5py.File(s5p_b7_prod, 'r')   # Tropomi band7 product
    >>> xdata7 = h5_to_xr(fid['signal'])
    >>> fid.close()
    >>> fid = h5py.File(s5p_b8_prod, 'r')   # Tropomi band8 product
    >>> xdata8 = h5_to_xr(fid['signal'])
    >>> fid.close()
    >>> xdata = xr.concat((xdata7, xdata8), dim='spectral_channel')

    Optionally, fix the 'column' dimension
    >>> xdata = xdata.assign_coords(
    >>> ... column=np.arange(xdata.column.size, dtype='u4'))
    """
    if data_sel is not None:
        if data_sel in (np.s_[:], np.s_[...], np.s_[()]):
            data_sel = None
        elif np.isscalar(data_sel):
            data_sel = (np.s_[data_sel:data_sel+1],)
        else:
            buff = ()
            for val in data_sel:
                if val == Ellipsis:
                    for _ in range(h5_dset.ndim - len(data_sel) + 1):
                        buff += np.index_exp[:]
                elif np.isscalar(val):
                    buff += (np.s_[val:val+1],)
                else:
                    buff += (val,)
            data_sel = buff

    # Name of this array
    name = PurePath(h5_dset.name).name

    # Values for this array
    data = __get_data(h5_dset, data_sel, field)

    # Coordinates (tick labels) to use for indexing along each dimension
    coords = []
    if dims is None:
        coords = __get_coords(h5_dset, data_sel)
    if not coords:
        coords = __set_coords(h5_dset, data_sel, dims)

    # Attributes to assign to the array
    attrs = __get_attrs(h5_dset, field)

    # check if dimension of dataset and coordinates agree
    if data.ndim < len(coords):
        for ii in reversed(range(len(coords))):
            if np.isscalar(coords[ii][1]):
                del coords[ii]

    return xr.DataArray(data, name=name, attrs=attrs, coords=coords)

def data_to_xr(data, *, dims=None, name=None, long_name=None, units=None):
    """
    Create xarray::DataArray from a dataset

    Parameters
    ----------
    data :  array-like
       Data to be stored in the xarray
    dims :  list of strings
       Names for the dataset dimensions
    name : str
       A string that names the instance
    units :  str
       Units of the data, default: '1'
    long_name : str
       Long name describing the data, default: empty string

    Returns
    -------
    xarray.DataArray
    """
    coords = __set_coords(data, None, dims)
    attrs = {}
    attrs['units'] = '1' if units is None else units
    attrs['long_name'] = '' if long_name is None else long_name

    return xr.DataArray(data, name=name, attrs=attrs, coords=coords)


"""
Part of this program is based on the implementation by Rico Jonschkowski (rico.jonschkowski@tu-berlin.de):
https://github.com/tu-rbo/learning-state-representations-with-robotic-priors

"""
from __future__ import print_function, division, absolute_import

import argparse
from collections import OrderedDict
import os
import numpy as np
import torch as th

import preprocessing
import models.learner as learner
import plotting.representation_plot as plot_script
from models.learner import SRL4robotics
from pipeline import getLogFolderName, saveConfig, correlationCall
from plotting.losses_plot import plotLosses, plotloss_G_D
from plotting.representation_plot import plotRepresentation
from utils import parseDataFolder, createFolder, getInputBuiltin, loadData, buildConfig, parseLossArguments
from collections import defaultdict

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='State Representation Learning with PyTorch')
    parser.add_argument('--epochs', type=int, default=30, metavar='N',
                        help='number of epochs to train (default: 30)')
    parser.add_argument('--img-shape', type=str, default="(3,64,64)",
                        help='image shape (default "(3,64,64)"')
    parser.add_argument('--seed', type=int, default=1, metavar='S',
                        help='random seed (default: 1)')
    parser.add_argument('--state-dim', type=int, default=2, help='state dimension (default: 2)')
    parser.add_argument('-bs', '--batch-size', type=int, default=32, help='batch_size (default: 32)')
    parser.add_argument('--val-size', type=float, default=0.2, help='Validation set size in percentage (default: 0.2)')
    parser.add_argument('--training-set-size', type=int, default=-1,
                        help='Limit size (number of samples) of the training set (default: -1)')			# for now, we use the default value, there might be an error when use smaller size
    parser.add_argument('-lr', '--learning-rate', type=float, default=0.0005, help='learning rate (default: 0.005)')
    parser.add_argument('-lr_G', '--learning-rate-G', type=float, default=1.0 *
                        1e-5, help='learning rate GAN: Generator (default: None)')
    parser.add_argument('-lr_D', '--learning-rate-D', type=float, default=5.0*1e-5,
                        help='learning rate GAN: Discriminator (default: None)')
    parser.add_argument('--l1-reg', type=float, default=0.0, help='L1 regularization coeff (default: 0.0)')
    parser.add_argument('--l2-reg', type=float, default=0.0, help='L2 regularization coeff (default: 0.0)')
    # parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training')
    parser.add_argument('--gpu-num', type=int, default=0, help='CUDA visible device (use CPU if -1, default: 0)')
    parser.add_argument('--no-display-plots', action='store_true', default=False,
                        help='disables live plots of the representation learned')
    parser.add_argument('--model-type', type=str, default="custom_cnn",
                        choices=['custom_cnn', 'custom_cnn_2', 'resnet', 'mlp', 'linear', 'gan', 'unet','dc'],
                        help='Model architecture (default: "custom_cnn")')
    parser.add_argument('--inverse-model-type', type=str, default="linear",
                        choices=['mlp', 'linear'],
                        help='Inverse model s architecture (default: "linear")')
    parser.add_argument('--data-folder', type=str, default="", help='Dataset folder', required=True)
    parser.add_argument('--log-folder', type=str, default="",
                        help='Folder where the experiment model and plots will be saved. ' +
                             'By default, automatically computing KNN-MSE and saving logs at location ' +
                             'logs/DatasetName/YY-MM-DD_HHhMM_SS_ModelType_ST_DIMN_LOSSES')
    parser.add_argument('--multi-view', action='store_true', default=False,
                        help='Enable use of multiple camera')
    parser.add_argument('--balanced-sampling', action='store_true', default=False,
                        help='Force balanced sampling for episode independent prior instead of uniform')
    parser.add_argument('--losses', nargs='+', default=[], **parseLossArguments(
        choices=["forward", "inverse", "reward", "reward2", "spcls", "priors", "episode-prior", "reward-prior", "triplet",
                 "autoencoder", "vae","cvae", "perceptual", "dae", "random", "gan","cgan", "gan_new", "cgan_new", "cvae_new"],
        help='The wanted losses. One may also want to specify a weight and dimension '
             'that apply as follows: "<name>:<weight>:<dimension>".'))
    parser.add_argument('--beta', type=float, default=1.0,
                        help='(For beta-VAE only) Factor on the KL divergence, higher value means more disentangling.')
    parser.add_argument('--path-to-dae', type=str, default="",
                        help='Path to a pre-trained dae model when using the perceptual loss with VAE')
    parser.add_argument('--state-dim-dae', type=int, default=200,
                        help='state dimension of the pre-trained dae (default: 200)')
    parser.add_argument('--occlusion-percentage', type=float, default=0.5,
                        help='Max percentage of input occlusion for masks when using DAE')
    parser.add_argument('--figdir', type=str, default=None,
                        help="Save figure the 'figdir'.")
    parser.add_argument('--monitor', type=str, default='loss',
                        choices=['pbar', 'loss'],
                        help="Monitor mode: either print the losses ('loss') or show the progressbar ('pbar'). (default 'loss')")
    parser.add_argument('--num-worker', type=int, default=10,
                        help="Number of CPUs to use for dataloader.")
    parser.add_argument('--srl-pre-weights', type=str, default=None,
                        help="Load SRL pretrained weights.")
    parser.add_argument('--debug', action='store_true', default=False,
                        help="Debug mode.")
    parser.add_argument('--Cmax', type=float, default=50,
                        help='(For CCI-VAE or CCI-CVAE)')
    parser.add_argument('--use-cci', action='store_true', default=False, help='if cci is use in VAE')
    parser.add_argument('--ls', action='store_true', default=False, help='used to smooth the real label in dcGAN and CdcGAN')
    parser.add_argument('--add-noise', action='store_true', default=False, help='Add noise to the input of the discriminator of dcGAN and CdcGan ')
    parser.add_argument('--only-action', action='store_true', default=False, help='Conditioned only on action')

    args = parser.parse_args()
    # args.cuda = not args.no_cuda and th.cuda.is_available()
    args.data_folder = parseDataFolder(args.data_folder)
    learner.SAVE_PLOTS = not args.no_display_plots
    learner.N_EPOCHS = args.epochs
    learner.BATCH_SIZE = args.batch_size
    learner.VALIDATION_SIZE = args.val_size
    learner.BALANCED_SAMPLING = args.balanced_sampling
    learner.N_WORKERS = args.num_worker
    th.backends.cudnn.benchmark = True
    # Dealing with losses to use
    has_loss_description = [isinstance(loss, tuple) for loss in args.losses]
    has_consistent_description, has_weight, has_splits = False, False, False
    if all(has_loss_description) and len(has_loss_description) > 0:
        len_description = [len(item_loss) for item_loss in args.losses]
        has_consistent_description = sum(len_description) / len(len_description) == len_description[0]
        has_weight = has_consistent_description
        has_splits = has_consistent_description and len_description[0] == 3

    if any(has_loss_description) and not all(has_loss_description):
        raise ValueError(
            "Either no losses have a defined weight or dimension, or all losses have a defined weight. {}".format(args.losses))

    # If not describing the the losses (weight and or dimension)
    if not has_consistent_description:
        losses = list(set(args.losses))
        losses_weights_dict = None
        split_dimensions = -1

    # otherwise collecting descriptions
    else:
        losses_weights_dict = OrderedDict()
        split_dimensions = OrderedDict()
        for loss, weight, split_dim in args.losses:
            losses_weights_dict[loss] = weight
            split_dimensions[loss] = split_dim
        losses = list(losses_weights_dict.keys())

        if not has_weight:
            split_dimensions = losses_weights_dict
            losses_weights_dict = None
            losses = list(split_dimensions.keys())

        assert not ("triplet" in losses and not args.multi_view), \
            "Triplet loss with single view is not supported, please use the --multi-view option"
    args.losses = losses
    args.split_dimensions = split_dimensions
    if args.img_shape is None:
        img_shape = None  # (3,224,224)
    else:
        img_shape = tuple(map(int, args.img_shape[1:-1].split(",")))
    if args.multi_view is True:
        # Setting variables involved data-loading from multiple cameras,
        # involved also in adapting the input layers of NN to that data
        # PS: those are stacked images - 3 if triplet loss, 2 otherwise
        if "triplet" in losses:
            # preprocessing.preprocess.N_CHANNELS = 9
            img_shape = (9, ) + img_shape[1:]
        else:
            # preprocessing.preprocess.N_CHANNELS = 6
            img_shape = (6, ) + img_shape[1:]

    assert not ("autoencoder" in losses and "vae" in losses), "Model cannot be both an Autoencoder and a VAE (come on!)"
    assert not (("autoencoder" in losses or "vae" in losses)
                and args.model_type == "resnet"), "Model cannot be an Autoencoder or VAE using ResNet Architecture !"
    assert not ("vae" in losses and args.model_type == "linear"), "Model cannot be VAE using Linear Architecture !"
    assert not (args.multi_view and args.model_type == "resnet"), \
        "Default ResNet input layer is not suitable for stacked images!"
    assert not (args.path_to_dae == "" and "vae" in losses and "perceptual" in losses), \
        "To use the perceptual loss with a VAE, please specify a path to a pre-trained DAE model"
    assert not ("dae" in losses and "perceptual" in losses), \
        "Please learn the DAE before learning a VAE with the perceptual loss "
    assert not (args.use_cci and ("vae" not in losses and "cvae" not in losses)), "cci cannot used without vae or cvae"
    assert not ("cvae" in losses and "cvae_new" in losses), "can not cvae and cvae_new at the same time"
    assert not ("cgan" in losses and len(losses)>1), "can not use cgan with other models"
    assert not ("cgan_new" in losses and len(losses)>1), "can not use cgan_new with other models"

    print('Loading data ... ')

    training_data, ground_truth, relative_positions, target_positions = loadData(args.data_folder)
    rewards, episode_starts = training_data['rewards'], training_data['episode_starts']
    actions = training_data['actions']
    # We assume actions are integers
    n_actions = int(np.max(actions) + 1)

    # Try to convert old python 2 format
    try:
        images_path = np.array([path.decode("utf-8") for path in ground_truth['images_path']])
    except AttributeError:
        images_path = ground_truth['images_path']

    # Building the experiment config file
    exp_config = buildConfig(args)

    if args.log_folder == "":
        # Automatically create dated log folder for configs
        createFolder("logs/{}".format(exp_config['data-folder']), "Dataset log folder already exist")
        # Check that the dataset is already preprocessed
        log_folder, experiment_name = getLogFolderName(exp_config)
        args.log_folder = log_folder
    else:
        os.makedirs(args.log_folder, exist_ok=True)
        experiment_name = "{}_{}".format(args.model_type, losses)

    exp_config['log-folder'] = args.log_folder
    exp_config['experiment-name'] = experiment_name
    exp_config['n_actions'] = n_actions
    exp_config['multi-view'] = args.multi_view
    exp_config['img_shape'] = args.img_shape
    exp_config['use_cci'] = args.use_cci
    exp_config['Cmax'] = args.Cmax
    exp_config['learning_rate_D'] = args.learning_rate_D
    exp_config['learning_rate_G'] = args.learning_rate_G
    exp_config['label_smoothing'] = args.ls
    exp_config['add_noise'] = args.add_noise
    exp_config['only_action'] = args.only_action
    exp_config['debug'] = args.debug
    exp_config['pretrained_weights_path'] = args.srl_pre_weights

    if "dae" in losses:
        exp_config['occlusion-percentage'] = args.occlusion_percentage
    print('Log folder: {}'.format(args.log_folder))

    print('Learning a state representation ... ')
    # The dimension of the class (action) in CVAE
    class_dim = 4
    exp_config['class_dim'] = class_dim

    srl = SRL4robotics(args.state_dim, class_dim, img_shape=img_shape, model_type=args.model_type, inverse_model_type=args.inverse_model_type
                        , ls=args.ls,add_noise=args.add_noise,only_action = args.only_action, pretrained_weights_path=args.srl_pre_weights,debug=args.debug,
                       seed=args.seed,log_folder=args.log_folder, learning_rate=args.learning_rate, learning_rate_gan=( args.learning_rate_D,
                        args.learning_rate_G),l1_reg=args.l1_reg, l2_reg=args.l2_reg, cuda=args.gpu_num, multi_view=args.multi_view,losses=losses,
                        losses_weights_dict=losses_weights_dict, n_actions=n_actions, beta=args.beta, use_cci=args.use_cci, Cmax=args.Cmax,
                        split_dimensions=split_dimensions, path_to_dae=args.path_to_dae, state_dim_dae=args.state_dim_dae,
                        occlusion_percentage=args.occlusion_percentage)

    if args.training_set_size > 0:
        limit = args.training_set_size
        actions = actions[:limit]
        images_path = images_path[:limit]
        rewards = rewards[:limit]
        episode_starts = episode_starts[:limit]
        truncate = limit
    else:
        truncate = None
    # Save configs in log folder
    saveConfig(exp_config, print_config=True)
    if args.figdir is not None:
        os.makedirs(args.figdir, exist_ok=True)
    loss_history, learned_states, pairs_name_weights = srl.learn(
        images_path, actions, rewards, episode_starts, figdir=args.figdir, monitor_mode=args.monitor,
        ground_truth=ground_truth, relative_positions=relative_positions, target_positions=target_positions, truncate=truncate)

    # Update config with weights for each losses
    exp_config['losses_weights'] = pairs_name_weights
    saveConfig(exp_config, print_config=True)

    if learned_states is not None:
        print("learned_states.shape ", learned_states.shape)
        srl.saveStates(learned_states, images_path, rewards, args.log_folder)
        name = "Learned State Representation\n {}".format(args.log_folder.split('/')[-1])
        path = "{}/learned_states.png".format(args.log_folder)

        # PLOT REPRESENTATION & CORRELATION
        plotRepresentation(learned_states, rewards, name, add_colorbar=True, path=path)
        correlationCall(exp_config, plot=not args.no_display_plots)

    if "gan" in losses or "cgan" in losses:
        # Save plot
        plotLosses(loss_history[0], path=args.log_folder, name="Autoencoder_losses")
        plotLosses(loss_history[1], path=args.log_folder, name="Dicriminator_losses")
        plotLosses(loss_history[2], path=args.log_folder, name="Generator_losses")
        plotloss_G_D(loss_history[2], loss_history[1], path=args.log_folder, name="G_D_losses")

        loss_history = {**loss_history[0],**loss_history[1],**loss_history[2]}
    elif "gan_new" in losses or "cgan_new" in losses:
        # Save plot
        plotLosses(loss_history[0], path=args.log_folder, name="Dicriminator_losses")
        plotLosses(loss_history[1], path=args.log_folder, name="Generator_losses")
        plotloss_G_D(loss_history[1], loss_history[0], path=args.log_folder, name="G_D_losses")

        loss_history = {**loss_history[0],**loss_history[1]}
    else:
        plotLosses(loss_history, args.log_folder)
    np.savez('{}/loss_history.npz'.format(args.log_folder), **loss_history)


import numpy as np
import torch

def get_val_from_interpolate_disk(interpolate, val, ceil_init_r):
    steps = int(ceil_init_r)
    if interpolate is None or interpolate == "idt" or interpolate == "const":
        get_val = lambda r, init_r: val  # apply the constant value
        steps = 0
    elif interpolate == "lin" or interpolate == "linear":
        get_val = lambda r: r / ceil_init_r
    elif interpolate == "exp" or interpolate == "exponent" or interpolate == "exponential":
        get_val = lambda r: np.exp(r - ceil_init_r)
    elif interpolate == "log" or interpolate == "logarithmic":
        get_val = lambda r: np.log(((r / ceil_init_r) * (np.e - 1)) + 1)
    else:
        raise Exception(f"Unknown interpolation: {interpolate}!")
    return get_val, steps

def get_val_from_interpolate_hyper(interpolate, val, start_r, stop_r):
    """
    :param interpolate: the type of value interpolation
    :param val: the value to be set
    :param stop_r: the starting value of the radius
    :param start_r:  the stoping value of the radius
    :return: the value interpolation function, number of total steps
    """
    steps = (stop_r - start_r) + 1
    delta_r = (stop_r - start_r) + 1
    if interpolate is None or interpolate == "idt" or interpolate == "const":
        get_val = lambda r: val  # apply the constant value
        steps = 0
    elif interpolate == "lin" or interpolate == "linear":
        get_val = lambda r: (stop_r - r) / delta_r
    elif interpolate == "exp" or interpolate == "exponent" or interpolate == "exponential":
        get_val = lambda r: np.exp(start_r - r - 1)
    elif interpolate == "log" or interpolate == "logarithmic":
        get_val = lambda r: np.log(
            (((stop_r - r) / delta_r) * (np.e - 1)) + 1)
    else:
        raise Exception(f"Unknown interpolation: {interpolate}!")
    return get_val, steps

def get_disk_mask(H, W, compress_rate, val=0, interpolate=None, onesided=True):
    # Compute the initial radius of the disk:
    # disk_area / image_area = compress_rate / 100
    # np.pi * r**2 / size_len ^^2 = compress_rate / 100
    # Hence, we compute the initial value of the radius r: init_r in the line
    # below in the following way:
    side_len = H
    # There is only a slight difference in the areas between the onesided and
    # non-onesided cases but we perform the exact computation to limit the
    # errors to minimum.
    # if onesided:
    #     init_r = np.sqrt(2 * (compress_rate / 100) * H * W / np.pi)
    if onesided is False:
        if H != W:
            raise Exception("We only support squared inputs.")

    init_r = np.sqrt((compress_rate / 100) * side_len ** 2 / np.pi)

    y, x = np.ogrid[0:side_len, 0:side_len]
    ctr = side_len // 2  # center
    array_mask = np.ones((side_len, side_len), dtype=np.float32)
    # Start from the slightly higher denominator for the get_val function to
    # incur some decrease in the values ( < 1.0 multiplier) for the coefficients.
    ceil_init_r = np.ceil(init_r)
    get_val, steps = get_val_from_interpolate_disk(interpolate=interpolate,
                                                   val=val,
                                                   ceil_init_r=ceil_init_r)

    r = init_r
    mask = (x - ctr) ** 2 + (y - ctr) ** 2 <= r ** 2
    array_mask[mask] = 0

    # Decrease the disk size and its value of the mask in each iteration.
    for delta in range(steps):
        mask = (x - ctr) ** 2 + (y - ctr) ** 2 <= r ** 2
        array_mask[mask] = get_val(r)
        r -= 1
    # print("array mask:\n", array_mask)
    # Transform the mask to the complex representation, with 2 values for the
    # last dimension being the same.
    tensor_mask = get_tensor_mask(array_mask)
    return tensor_mask, array_mask

def get_inverse_hyper_mask(H, W, compress_rate, val=0, interpolate=None,
                           onesided=True):
    mask, array_mask = get_hyper_mask(H=H, W=W, compress_rate=compress_rate, val=val,
                          interpolate=interpolate, onesided=onesided)
    mask = mask * (-1) + 1
    array_mask = array_mask * (-1) + 1
    return mask, array_mask

def get_hyper_mask(H, W, compress_rate, val=0, interpolate=None,
                   onesided=True):
    # Compute the initial radius of the hyperdisk for onesided=False, the whole
    # FFT map is returned (not-halved).
    # hyper_area / image_area = (1 - compress_rate / 100)
    # np.pi * r**2 / size_len ^^2 = (1 - compress_rate / 100)
    # Hence, we compute the initial value of the radius r: init_r in the line
    # below in the following way:
    side_len = H
    # There is only a slight difference in the areas between the onesided and
    # non-onesided cases but we perform the exact computation to limit the
    # errors to minimum.
    # if onesided:
    #     init_r = np.sqrt(2 * (1 - compress_rate / 100) * H * W / np.pi)
    if onesided is False:
        if H != W:
            raise Exception("We only support squared inputs.")

    if onesided == True:
        multiplier = 2
        WW = W
    else:
        multiplier = 1
        WW = W / 2

    compress_rate = compress_rate / 100

    start_r = np.sqrt(multiplier * (1 - compress_rate) * H * W / np.pi)
    stop_r = np.sqrt((H / 2) ** 2 + WW ** 2)

    start_r = np.floor(start_r)
    stop_r = np.ceil(stop_r)

    y, x = np.ogrid[0:side_len, 0:side_len]
    get_val, steps = get_val_from_interpolate_hyper(interpolate=interpolate,
                                                    val=val,
                                                    start_r=start_r,
                                                    stop_r=stop_r)
    # Decrease the disk size and its value of the mask in each iteration.
    r = start_r
    n = side_len - 1
    # upper-left corner
    mask0 = get_array_mask(x ** 2 + y ** 2 >= r ** 2, side_len)
    # upper-right corner
    mask1 = get_array_mask((x - n) ** 2 + y ** 2 >= r ** 2, side_len)
    # bottom-left corner
    mask2 = get_array_mask(x ** 2 + (y - n) ** 2 >= r ** 2, side_len)
    # bottom-right corner
    mask3 = get_array_mask((x - n) ** 2 + (y - n) ** 2 >= r ** 2, side_len)
    array_mask = mask0 + mask1 + mask2 + mask3
    # If the compression is small, the ones can appear in the same positions for
    # different masks, so limit them to just 1 and does not allow to exceed the
    # bound (otherwise, two-s can emerge).
    array_mask = np.clip(array_mask, a_min=0, a_max=1)

    if steps > 0:
        # Prepare the quadrant masks, so that the mask(x) below can only change values
        # in its quadrant.
        if H % 2 == 0:
            mid = H // 2
        else:
            mid = H // 2 + 1

        # upper-left corner
        quadrant0 = get_array_mask(np.outer(x < mid, y < mid), side_len, val=1,
                                   back=0)
        # upper-right corner
        quadrant1 = get_array_mask(np.outer(x < mid, y >= mid), side_len, val=1,
                                   back=0)
        # bottom-left corner
        quadrant2 = get_array_mask(np.outer(x >= mid, y < mid), side_len, val=1,
                                   back=0)
        # bottom-right corner
        quadrant3 = get_array_mask(np.outer(x >= mid, y >= mid), side_len,
                                   val=1, back=0)

    mask0 = np.ones((side_len, side_len))
    mask1 = np.ones((side_len, side_len))
    mask2 = np.ones((side_len, side_len))
    mask3 = np.ones((side_len, side_len))

    r = start_r
    for delta in range(int(steps)):
        val = get_val(r)

        mask0 = get_array_mask(
            mask=(x ** 2 + y ** 2 >= r ** 2),
            n=side_len,
            val=val,
            back=None,
            array_mask=mask0)

        mask1 = get_array_mask(
            mask=((x - n) ** 2 + y ** 2 >= r ** 2),
            n=side_len,
            val=val,
            back=None,
            array_mask=mask1)

        mask2 = get_array_mask(
            mask=(x ** 2 + (y - n) ** 2 >= r ** 2),
            n=side_len,
            val=val,
            back=None,
            array_mask=mask2)

        mask3 = get_array_mask(
            mask=((x - n) ** 2 + (y - n) ** 2 >= r ** 2),
            n=side_len,
            val=val,
            back=None,
            array_mask=mask3)

        r += 1

    if steps > 0:
        mask0 *= quadrant0
        mask1 *= quadrant1
        mask2 *= quadrant2
        mask3 *= quadrant3

        array_mask = mask0 + mask1 + mask2 + mask3

    # print("array mask:\n", array_mask)
    # Transform the mask to the complex representation, with 2 values for the
    # last dimension being the same.
    tensor_mask = get_tensor_mask(array_mask)
    return tensor_mask, array_mask

def get_tensor_mask(array_mask):
    tensor_mask = torch.from_numpy(array_mask)
    tensor_mask = tensor_mask.unsqueeze(-1)
    # tensor_other = torch.zeros_like(tensor_mask)
    tensor_other = tensor_mask
    tensor_mask = torch.cat((tensor_mask, tensor_other), dim=-1)
    return tensor_mask

def get_array_mask(mask, n=7, val=0, back=1, dtype=np.float32,
                   array_mask=None):
    if back is None:
        if array_mask is None:
            raise Exception("array_mask has to be specified if back is None")
        array = array_mask
    elif back == 1:
        array = np.ones((n, n), dtype=dtype)
    elif back == 0:
        array = np.zeros((n, n), dtype=dtype)
    else:
        array = np.full((n, n), fill_value=back, dtyp=dtype)
    array[mask] = val
    return array

if __name__ == "__main__":
    a, b = 1, 1
    n = 7
    ctr = n // 2  # center
    r = 2

    # y,x = np.ogrid[-a:n-a, -b:n-b]
    # y, x = np.ogrid[a:n - a, b:n - b]
    y, x = np.ogrid[0:n, 0:n]
    # print("x: ", x, " y:", y)
    mask = (x - ctr) ** 2 + (y - ctr) ** 2 <= r ** 2
    # print("mask: ", mask)

    array = np.ones((n, n))
    array[mask] = 0
    print("array: ", array)
    tensor = torch.from_numpy(array)
    tensor = tensor.unsqueeze(-1)
    tensor = torch.cat((tensor, tensor), dim=-1)
    # print("array: ", tensor)

    a, b = 1, 1
    n = 7
    ctr = n // 2  # center
    r = 2

    # y,x = np.ogrid[-a:n-a, -b:n-b]
    # y, x = np.ogrid[a:n - a, b:n - b]
    y, x = np.ogrid[0:n, 0:n]
    # print("x: ", x, " y:", y)
    mask1 = (x ** 2 + y ** 2 >= r ** 2)
    print("mask1: ", mask1)
    mask1 = get_array_mask(mask1)
    print("mask1: ", mask1)

    mask2 = ((x - (
            n - 1)) ** 2 + y ** 2 >= r ** 2)  # and ((x-n-1)**2 + (y-n-1)** 2 >= r ** 2) and (x**2 + (y-n-1)** 2 >= r ** 2)
    print("mask2: ", mask2)
    mask2 = get_array_mask(mask2)
    print("mask2: ", mask2)

    mask3 = ((x - (n - 1)) ** 2 + (y - (
            n - 1)) ** 2 >= r ** 2)  # and ((x-n-1)**2 + (y-n-1)** 2 >= r ** 2) and (x**2 + (y-n-1)** 2 >= r ** 2)
    print("mask3: ", mask3)
    mask3 = get_array_mask(mask3)
    print("mask3: ", mask3)

    mask4 = (x ** 2 + (y - (
            n - 1)) ** 2 >= r ** 2)  # and ((x-n-1)**2 + (y-n-1)** 2 >= r ** 2) and (x**2 + (y-n-1)** 2 >= r ** 2)
    print("mask4: ", mask4)
    mask4 = get_array_mask(mask4)
    print("mask4: ", mask4)

    array = mask1 + mask2 + mask3 + mask4

    print("array: ", array)
    tensor = torch.from_numpy(array)
    tensor = tensor.unsqueeze(-1)
    tensor = torch.cat((tensor, tensor), dim=-1)


import pygame
import random
import math
import csv, os
BLUE = (0,100,255)

class tower(pygame.sprite.Sprite):
    def __init__(self, color, height, width, x ,y):
        super().__init__()
        self.height = height
        self.width = width
        self.color = color
        self.image = pygame.Surface([height, width])
        self.image.fill(color)
        self.rect = self.image.get_rect()
        self.rect.y = y
        self.rect.x = x
        self.placed = 0
        self.radius_n = 0
        self.hit_radius = [50, 100, 150]
        self.in_radius_group = pygame.sprite.Group()
        self.tower_n = 0
        self.closest_enemy = 0
        self.cooldown = 1
        self.timer = pygame.time.get_ticks()

    def update(self):
        self.image.fill(self.color)

        """if self.cooldown == 0:
            now = pygame.time.get_ticks()
            if self.timer - now >= 2000:
                self.cooldown = 1"""

BTD 14 - Problem) all towers share one self.cooldown/Baloons TD Coursework-restart/Tower.py


import glob as gb
import os
import xlwings as xw
import shutil

#os.chdir(r"C:\Users\msweeney\Desktop\MS EM Tools\Python\Python Scripts\Sample SIMs\Sample hsr")

def hsrtocsv():

	hsrfiles = gb.glob('./*.hsr')

	for h in hsrfiles:
		if os.path.isfile(h[:-4] + '.csv'):
			os.remove(h[:-4] + '.csv')

		shutil.copy(h,h[:-4]+".csv")

		wb = xw.Book(h[:-4] + '.csv')
		wb.close()

if __name__ == "__main__":
	hsrtocsv()


from pyramid.view import view_config
from pyramid.httpexceptions import HTTPFound

from .                    import web, method_delete_router
from ..lib.auto_format    import action_ok
from ..templates.helpers  import search_url

#-------------------------------------------------------------------------------
# Faves
#-------------------------------------------------------------------------------

@view_config(route_name='fave', request_method='GET')
@web
def fave_view(request):
    """
    view current faves

    not really needed as faves are also part of the identity dict
    """
    trackids = request.session['faves']

    if request.matchdict['format']=='html':
        raise HTTPFound(location=search_url(trackids=trackids,route='search_list'))

    return action_ok(
        data={'faves':trackids}
    )

@view_config(route_name='fave', request_method='POST')
@web
def fave_add(request):
    """
    Add item to faves in session
    """
    if 'faves' not in request.session:
        request.session['faves'] = []
    request.session['faves'].append(request.params['id'])
    return action_ok(message='added to faves')

@view_config(route_name='fave', custom_predicates=(method_delete_router, lambda info,request: request.params.get('id'))) #request_method='DELETE'
@web
def fave_del(request):
    """
    Remove fave from session
    """
    request.session['faves'].remove(request.params['id'])
    return action_ok(message='removed from faves')


# Modelo preditivo de classificaçãp para prever o valor de uma variável binária (true ou false) a partir de dados numéricos

# Import dos módulos
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
import pandas as pd
import numpy as np

# Dados de treino
n_train = 10
np.random.seed(0)
df_train = pd.DataFrame({"var1":np.random.random(n_train), \
                         "var2": np.random.random(n_train), \
                         "var3": np.random.random(n_train), \
                         "var4":np.random.randint(0,2,n_train).astype(bool),\
                         "target":np.random.randint(0,2,n_train).astype(bool)})

# Dados de treino
n_valid = 3
np.random.seed(1)
df_valid = pd.DataFrame({"var1":np.random.random(n_valid), \
                         "var2": np.random.random(n_valid), \
                         "var3": np.random.random(n_valid), \
                         "var4":np.random.randint(0,2,n_valid).astype(bool),\
                         "target":np.random.randint(0,2,n_valid).astype(bool)})
# Reduzindo a dimensionalidade para 3 componentes
pca = PCA(n_components = 3)

# Aplicando o PCA aos datasets
newdf_train = pca.fit_transform(df_train.drop("target", axis = 1))
newdf_valid = pca.transform(df_valid.drop("target", axis = 1))

# Gerando novos datasets
features_train = pd.DataFrame(newdf_train)
features_valid = pd.DataFrame(newdf_valid)

# Cria o modelo de regressão logística
regr = LogisticRegression()

# Usando o recurso de pipeline do scikit-learn para encadear 2 algoritmos em um mesmo modelo, no caso PCA e Regressão Logística
pipe = Pipeline([('pca', pca), ('logistic', regr)])
pipe.fit(features_train, df_train["target"])
predictions = pipe.predict(features_valid)

# Imprimindo as previsões
print(predictions)


#!/usr/bin/env python3

from Bio import SeqIO
import tompltools

# expect input fq and output fq
parsed_args = tompltools.parse_cli_arguments()

if parsed_args.input_fa:
    print('input_fa:\t%s' % parsed_args.input_fa)
    unsorted_fa = parsed_args.input_fa[0]
else:
    raise TypeError("input_fa not specified (-f)")
if parsed_args.output_fa:
    print('output_fa:\t%s' % parsed_args.output_fa)
    sorted_fasta = parsed_args.output_fa[0]
else:
    raise TypeError("output_fa not specified (--g)")

# make a sorted list of tuples of scaffold name and sequence length
print("Parsing read lengths")
length_id_unsorted = ((len(rec), rec.id) for
                      rec in SeqIO.parse(unsorted_fa, 'fasta'))
length_and_id = sorted(length_id_unsorted)

# get an iterator sorted by read length
print("Sorting reads by length")
longest_to_shortest = reversed([id for (length, id) in length_and_id])

# release scaffolds_file from memory
del(length_and_id)

# build an index of the fasta file
print("Indexing %s by read name" % unsorted_fa)
record_index = SeqIO.index(unsorted_fa, 'fasta')

# write selected records in correct order to disk
print("Ordering records")
ordered_records = (record_index[id] for id in longest_to_shortest)
print("Writing sorted fasta to %s" % sorted_fasta)
SeqIO.write(sequences=ordered_records,
            handle=sorted_fasta,
            format='fasta')


"""
This is a modification to Demo_start_training_proceedure.py where it repeats
the code until it finds a model with a false structure

This file read a `config.yml` file and train a network based on the settings
in that file
"""

import yaml
import shutil
import uuid
from tensorflow.keras import backend as K
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
from nn_tools import read_count
import tensorflow as tf
from Data_loader_stripe import Data_loader_stripe_train, Data_loader_stripe_test
import model_builders as mb
from SaveWeights import MyCallback
import os
from os.path import join
import matplotlib.pyplot as plt;
import numpy as np;

def data_selector(data_name, arguments):
    """ Select a data loader based on `data_name` (str).
Arguments
---------
data_name (str): Name of the data loader
arguments (dict): Dictionary given to the constructor of the data loader

Returns
-------
Data loader with name `data_name`. If not found, an error message is printed
and it returns None.
"""
    if data_name.lower() == "stripe_train":
        return Data_loader_stripe_train(arguments)
    elif data_name.lower() == "stripe_test":
        return Data_loader_stripe_test(arguments)
    else:
        print('Error: Could not find data loader with name %s' % (data_name))
        return None;

def model_selector(model_name, input_shape, output_shape, arguments):
    """ Select a model (network) based on `model_name` (str).
Arguments
---------
model_name (str): Name of the model loader
input_shape (list): List of integers specifying dimensions
output_shape (list): List of integers specifying dimensions
arguments (dict): Arguments to the model function

Returns
-------
Keras model
"""
    if model_name.lower() == "fc3":
        return mb.build_model_fc3(input_shape, output_shape, arguments)
    elif model_name.lower() == "cnndrop":
        return mb.build_model_cnndrop(input_shape, output_shape, arguments)
    elif model_name.lower() == "fc2":
        return mb.build_model_fc2(input_shape, output_shape, arguments)
    elif model_name.lower() == "fc2_cheat":
        return mb.build_model_fc2_cheat(input_shape, output_shape, arguments)
    elif model_name.lower() == "cnn2":
        return mb.build_model_cnn2(input_shape, output_shape, arguments)
    else:
        print('Error: Could not find model with name %s' % (model_name))
        return None;

if __name__ == "__main__":
    """Train a full model based on the settings in `config.yml`"""
    # Turn off unnessesary warnings
    tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.ERROR)

    # Load configuration file
    with open('config.yml') as ymlfile:
        cgf = yaml.load(ymlfile, Loader=yaml.SafeLoader);

    # Set up computational resource
    use_gpu = cgf['COMPUTER_SETUP']['use_gpu']
    print("""\nCOMPUTER SETUP
Use gpu: {}""".format(use_gpu))
    if use_gpu:
        compute_node = cgf['COMPUTER_SETUP']['compute_node']
        os.environ["CUDA_VISIBLE_DEVICES"]= "%d" % (compute_node)
        print('Compute node: {}'.format(compute_node))
    else:
        os.environ["CUDA_VISIBLE_DEVICES"]= "-1"

    # Turn on soft memory allocation
    tf_config = tf.compat.v1.ConfigProto()
    tf_config.gpu_options.allow_growth = True
    tf_config.log_device_placement = False
    sess = tf.compat.v1.Session(config=tf_config)
    #K.v1.set_session(sess)

    # Load train and validatation data
    data_loader_train = data_selector(cgf['DATASET_TRAIN']['name'], cgf['DATASET_TRAIN']['arguments'])
    data_loader_validate = data_selector(cgf['DATASET_VAL']['name'], cgf['DATASET_VAL']['arguments'])
    print('\nDATASET TRAIN')
    print(data_loader_train)
    print('DATASET VALIDATION')
    print(data_loader_validate)

    train_data, train_labels = data_loader_train.load_data();
    val_data, val_labels = data_loader_validate.load_data();

    # Get input and output shape
    input_shape = train_data.shape[1:]
    output_shape = train_labels.shape[1];
    print('input_shape', input_shape)
    # Set the default precision
    model_precision = cgf['MODEL_METADATA']['precision']
    K.set_floatx(model_precision)

    # Print model information
    model_name = cgf['MODEL']['name']
    model_arguments = cgf['MODEL']['arguments']
    print("""MODEL
name: {}
arguments:""".format(model_name))
    for key, value in cgf['MODEL']['arguments'].items():
        print("\t{}: {}".format(key,value))

    # Load model
    model = model_selector(model_name, input_shape, output_shape, model_arguments)

    # Extract training information
    loss_type = cgf['TRAIN']['loss']['type']
    optimizer = cgf['TRAIN']['optim']['type']
    batch_size = cgf['TRAIN']['batch_size']
    metric_list = list(cgf['TRAIN']['metrics'].values())
    shuffle_data = cgf['TRAIN']['shuffle']
    max_epoch = cgf['TRAIN']['max_epoch']
    stpc_type = cgf['TRAIN']['stopping_criteria']['type']
    print("""\nTRAIN
loss: {}
optimizer: {}
batch size: {}
shuffle data between epochs: {}
max epoch: {}
stopping_criteria: {}""".format(loss_type, optimizer, batch_size, shuffle_data,
           max_epoch, stpc_type))

    callbacks = None
    if stpc_type.lower() == 'epoch' or stpc_type.lower() == 'epochs':
        callbacks = None
    elif stpc_type.lower() == "earlystopping" or stpc_type.lower() == "early_stopping":
        arguments = cgf['TRAIN']['stopping_criteria']['arguments']
        es = EarlyStopping(**arguments)
        callbacks = [es]
        for key, value in arguments.items():
            print('\t{}: {}'.format(key,value))
    else:
        print('Unknown stopping criteria')
        callbacks = None
    print('')
    # Compile model
    model.compile(optimizer = optimizer,
                 loss = loss_type,
                 metrics = metric_list)

    # Model metadata
    save_final_model = cgf['MODEL_METADATA']['save_final_model']
    save_best_model = cgf['MODEL_METADATA']['save_best_model']['use_model_checkpoint']
    print("""MODEL METADATA
Precision: {}
Save final model: {}""".format(model_precision, save_final_model))
    if save_final_model or save_best_model: # Get a model id

        dest_model = cgf['MODEL_METADATA']['dest_model']
        model_number_type = cgf['MODEL_METADATA']['model_number_type']
        if model_number_type.lower() == 'fixed':
            model_number = cgf['MODEL_METADATA']['model_number_arguments']['model_id']
        elif model_number_type.lower() == 'count':
            counter_path =\
                cgf['MODEL_METADATA']['model_number_arguments']['counter_path']
            model_number = read_count(counter_path);
        elif model_number_type.lower() == 'uuid':
            model_number = uuid.uuid1().int
        else:
            model_number = -1;

        full_dest_model = join(dest_model, str(model_number))
        if not os.path.isdir(dest_model):
            os.mkdir(dest_model)
        if not os.path.isdir(full_dest_model):
            os.mkdir(full_dest_model)
        else:
            print('Delete all content in the folder: \n{}'.format(full_dest_model))
            shutil.rmtree(full_dest_model);
            os.mkdir(full_dest_model)
        shutil.copyfile('config.yml', join(full_dest_model, 'config.yml'))
        shutil.copyfile('model_builders.py', join(full_dest_model, 'model_builders.py'))
        print("""Model nbr type: {}
Model number: {}
Model dest: {}""".format(model_number_type, model_number, dest_model))

    print('Save best model: {}'.format(save_best_model))

    '''
    if save_best_model:
        arguments = cgf['MODEL_METADATA']['save_best_model']['arguments']
        for key, value in arguments.items():
            print('\t{}: {}'.format(key,value))

        filename = arguments['filepath']
        arguments['filepath'] = join(full_dest_model, filename)
        mc = ModelCheckpoint(arguments['filepath'], monitor=arguments['monitor'], verbose=arguments['verbose'], save_best_only=False, save_weights_only=True, mode=arguments['mode'], period=arguments['period'])
        if callbacks is None:
            callbacks = mc
        elif type(callbacks) is list: # Is a list
            callbacks.append(mc)
        else:
            callbacks = mc
            print('ERROR: Unknown callback type')
    '''
    '''
    # We save weights after each epoch
    if callbacks == None:
        callbacks =[]
    mycallback = MyCallback()
    callbacks.append(mycallback)
    '''

    print('\nStart training the model\n')
    # Train model :)
    print(shuffle_data)
    print(callbacks)

    data_loader_test = data_selector(cgf['DATASET_TEST']['name'], cgf['DATASET_TEST']['arguments'])
    test_data, test_labels = data_loader_test.load_data()
    optimal = 0

    while optimal < 0.9  :
        model = model_selector(model_name, input_shape, output_shape, model_arguments)
        model.compile(optimizer = optimizer,
                      loss = loss_type,
                      metrics = metric_list)
        history = model.fit(train_data, train_labels,
                  epochs=max_epoch,
                  batch_size=batch_size,
                  validation_data=(val_data, val_labels),
                  shuffle=shuffle_data,
                  callbacks = callbacks)

        score = model.evaluate(test_data, test_labels, verbose=0)
        optimal = score[1]

    if save_final_model:
        full_file_name = join(full_dest_model, 'keras_model_files.h5')
        model.save(join(full_dest_model, 'keras_model_files.h5'), save_format='h5')
        print('Saved model as {}'.format(full_file_name))

    # plot the loss history
    plt.figure()
    loss_hist = history.history['loss'][int(max_epoch/2):]
    index = np.linspace(max_epoch/2+1, max_epoch, max_epoch/2);
    plt.plot(index,loss_hist);
    plt.savefig(join(full_dest_model, 'loss_graph.png'));
    # Load and test model on test set

    print('\nDATASET TEST')
    print(data_loader_test)

    score = model.evaluate(test_data, test_labels, verbose=0)
    print('Accuracy on test set: {}%'.format(100*score[1]))


# 16/10/03 12:31:54 INFO ControllerJob: NOMINAL RECORD/S STAGE ID 1 : 3671965.1777167222

import glob

import matplotlib.pyplot as plt
import numpy as np

def mean_nominalrate(fold):
    stageDict = {}
    count = 0
    print(fold)
    for folder in glob.glob(fold + "/*"):
        print(folder)
        for file in glob.glob(folder + "/*.dat"):
            with open(file) as f:
                print(file)
                count += 1
                for line in f:
                    l = line.split(" ")
                    if len(l) > 4 and l[4] == "NOMINAL" and l[5] == "RECORD/S":
                        try:
                            print(l[-3], float(l[-1]))
                            stageDict[l[-3]]["nominalrate"].append(float(l[-1]))
                        except KeyError:
                            stageDict[l[-3]] = {}
                            stageDict[l[-3]]["nominalrate"] = []
                            stageDict[l[-3]]["nominalrate"].append(float(l[-1]))

    print(stageDict)
    with open(fold + "/result.txt", "w") as out:
        for id in sorted(stageDict.keys()):
            out.write(
                "SID " + str(id) + " MEAN " + str(np.mean(stageDict[id]["nominalrate"])) + "\n")
            out.write("SID " + str(id) + " STD " + str(np.std(stageDict[id]["nominalrate"])) + "\n")
            out.write("SID " + str(id) + " MIN " + str(np.min(stageDict[id]["nominalrate"])) + "\n")
            out.write("SID " + str(id) + " MAX " + str(np.max(stageDict[id]["nominalrate"])) + "\n")
            out.write("\n")
            out.write("SID " + str(id) + " % " + str((
                                                     np.std(stageDict[id]["nominalrate"]) / np.mean(
                                                         stageDict[id][
                                                             "nominalrate"])) * 100) + "\n")
            out.write("\n")
            stageDict[id]["nominalrate"] = np.mean(stageDict[id]["nominalrate"])

    print(stageDict)

output_folder = "./results/OK/PageRank/Profiling-NominalRate/"
# output_folder = "./spark-bench/num/app-20161013104553-0000"
import os

for dirpath, dirnames, files in os.walk(output_folder):
    for dir in dirnames:
        mean_nominalrate(dirpath + "/" + dir)
    break

nominalrate_dict = {}
mini_dict = {}
max_dict = {}
mean_dict = {}
for dirpath, dirnames, files in os.walk(output_folder):
    for dir in dirnames:
        nominalrate_dict[dir] = {}
        mini_dict[dir] = {}
        max_dict[dir] = {}
        mean_dict[dir] = {}
        with open(dirpath + "/" + dir + "/result.txt") as f:
            # SID 13 % 19.7066924684
            for line in f:
                l = line.split(" ")
                if len(l) > 1 and l[-2] == "%":
                    nominalrate_dict[dir][int(l[1])] = float(l[-1].replace("\n", ""))
                if len(l) > 1 and l[-2] == "MIN":
                    mini_dict[dir][int(l[1])] = float(l[-1].replace("\n", ""))
                if len(l) > 1 and l[-2] == "MAX":
                    max_dict[dir][int(l[1])] = float(l[-1].replace("\n", ""))
                if len(l) > 1 and l[-2] == "MEAN":
                    mean_dict[dir][int(l[1])] = float(l[-1].replace("\n", ""))
    break

print(nominalrate_dict)

for dirpath, dirnames, files in os.walk(output_folder):
    for dir in dirnames:
        data = []
        for key in nominalrate_dict[dir].keys():
            data.append((key, nominalrate_dict[dir][key]))
        data.sort(key=lambda x: x[0])
        print(data)
        plt.plot([x[0] for x in data], [x[1] for x in data])
    plt.legend(dirnames)
    break
plt.ylabel('% std dev')
plt.xlabel("Stage ID")
plt.savefig(output_folder + "result.png")
plt.close()

for dirpath, dirnames, files in os.walk(output_folder):
    legend = []
    for dir in dirnames:
        if dir == "6EXEC-NOHDFS" or dir == "12EXEC-NOHDFS":
            legend.append(dir)
            data = []
            for key in mini_dict[dir].keys():
                data.append((key, mini_dict[dir][key]))
            data.sort(key=lambda x: x[0])
            print(data)
    plt.legend(legend)
    break
plt.ylabel('MIN DIFF')
plt.xlabel("Stage ID")
plt.savefig(output_folder + "min.png")
plt.close()

A = "6EXEC-6DATANODE"
B = "12EXEC-6DATANODE"
print("\nMIN")
for key in mini_dict[A].keys():
    diff = mini_dict[A][key] - mini_dict[B][key]
    print(str(key) + " " + str(diff / min(mini_dict[A][key], mini_dict[B][key]) * 100))

print("\nMEAN")
for key in mean_dict[A].keys():
    diff = mean_dict[A][key] - mean_dict[B][key]
    print(str(key) + " " + str(diff / min(mean_dict[A][key], mean_dict[B][key]) * 100))

print("\nMAX")
for key in max_dict[A].keys():
    diff = max_dict[A][key] - max_dict[B][key]
    print(str(key) + " " + str(diff / min(max_dict[A][key], max_dict[B][key]) * 100))


# Copyright 2021 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Setup file for the model package passed to ai platform.

  We expect the model package to be placed with tasks/transformer in the same
  parent directory. When calling ai-platform submit, use model.trainer as
  module-name and model as package-path.
"""

import setuptools

_REQUIRED_PACKAGES = [
    'gast>=0.2.2',
    'tensorflow',
    'tensorflow-data-validation',
    'tensorflow-metadata',
    'tensorflow-transform',
]

_MODEL = 'model'
_TRANSFORMER = 'tasks/transformer'

if __name__ == '__main__':
  setuptools.setup(
      name=_MODEL,
      packages=[_MODEL, _TRANSFORMER],
      install_requires=_REQUIRED_PACKAGES)


# A very simple Flask Hello World app for you to get started with...

from flask import Flask, render_template, request

from tinydb import TinyDB, Query

db = TinyDB('my_list.json')
db.insert({'name': 'Fred', 'age': 22})
User = Query()
result=db.search(User.name == 'Fred')

app = Flask(__name__)

@app.route('/')
@app.route('/hello')
@app.route('/hello/')
@app.route('/hello/<name>')
def hello_world(name="friend"):
    return render_template('hello.html', name=name + str(result))

@app.route('/stuff')
def stuff_page():
    stuff = ["Dogs","Kids","Wind","Dust","Towels"]
    return render_template('list.html', name="Stuff", items=stuff, title=False)

@app.route('/form')
def form_page():
    return render_template('form.html')

@app.route('/post_list', methods=['POST'])
def post_list():
    mylist = [request.form['first'],request.form['second'],request.form['third']]
    db.insert({
        "first":request.form['first'],
        "second":request.form['second'],
        "third":request.form['third'],
        })
    return render_template('list.html', name="My Items", items=mylist, title=True)

@app.route('/all_items')
def get_all_items():
    Item=Query()
    items = db.search(Item.first>'')
    return render_template('all_items.html', items=items)

@app.route('/bye')
def bye_world():
    return render_template('bye.html')


from os import system, name
from time import sleep

import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

def texto_pausado(texto,tiempo):
    for i in range(len(texto)):
        print(texto[i], sep='', end='', flush=True); sleep(tiempo)

def clear():
    # windows
    if name == 'nt':
        _ = system('cls')
    # mac and linux
    else:
        _ = system('clear')

clear()

def entrada_real(numero):
    try:
        entrada = float(numero)
    except:
        print("Por favor introduzca una entrada válida.")
        exit()
    return entrada

_run = True
while _run:

    print("----- GRAFICADORA -----\n")

    texto_pausado("(1) Graficar tiempo real\n",0.025)
    texto_pausado("(2) Mover parámetros\n",0.025)
    texto_pausado("(3) Salir\n",0.025)

    opcion = input("> ")

    clear()

    if opcion == '1':

        TIEMPO = []
        THETA = []

        print("----- Graficar en tiempo real -----\n")

        k = entrada_real(input("Constante del resorte de torsión: "))
        A = entrada_real(input("Área de la espira: "))
        B = entrada_real(input("Intensidad de campo magnético: "))
        R = entrada_real(input("Resistencia de la espira: "))
        I = entrada_real(input("Momento de incercia de la espira: "))
        t = entrada_real(input("Tiempo: "))

        def calcular_thetadpt(ang, vel):
            ac_ang = (- k * ang )/I - (((A*B)**2)* ((np.cos(ang)) ** 2) * vel) / (R*I) # Cálculo de la aceleración angular según la ecuación diferencial
            return ac_ang

        def f(tiempo):
            # Valores iniciales
            theta = 1.
            thetapt = 0.
            dt = 0.01 # Paso

            for i in np.arange(0,tiempo,dt):
                TIEMPO.append(i) # Se adjunta el segundo a una lista

                thetadpt = calcular_thetadpt(theta, thetapt) # Se calcula la aceleración angular según las variables
                thetapt += thetadpt * dt # Como thetapt = d/dt (thetapt), theta_f = thetapt_i + thetadpt * dt, aprox
                theta += thetapt * dt # Como thetapt = d/dt (theta), theta_f = thetapt_i + thetadpt * dt, aprox
                THETA.append(theta)

            return theta # Le agregué esto

        # FIGURA 1

        fig1 = plt.figure()
        fig1.canvas.set_window_title('Oscilador')
        ax1 = fig1.add_subplot(111)

        ax1.grid(linestyle = "--")
        ax1.set_facecolor('#000000')

        y_fig1 = f(t)
        cuadros = 1. / 20

        def animacion1(i):
            j = int(i / (1./20))
            if j < 101:
                t = TIEMPO[0:j]
                th = THETA[0:j]
                a = TIEMPO[0]
                b = TIEMPO[200]
            else:
                t = TIEMPO[j-100:j]
                th = THETA[j-100:j]
                a = TIEMPO[j-100]
                b = TIEMPO[j+100]
            ax1.set_xlim(a,b) # Límites de la gráfica
            ax1.set_ylim(-2,2)
            linea, = ax1.plot(t, th, "#05d2ed")
            return linea

        animation1 = FuncAnimation(fig1, func=animacion1, frames=np.arange(0, 100, (1./20)),interval = t) # Altérese el denominador del paso para cambiar los fps

        # FIGURA 2

        # Se definen dos listas que serán las entradas en X y en Y de los extremos del oscilador
        x = [0,0]
        y = [0,0]

        fig2 = plt.figure()
        fig1.canvas.set_window_title('Espira')

        ax2 = fig2.add_subplot(111)
        ax2.set_facecolor('#000000')

        # Límites de la gráfica
        ax2.set_xlim(-2,2)
        ax2.set_ylim(-2,2)

        # Recta que hará de oscilador
        linea, = ax2.plot(0, 0, "#ede505")

        # Recta horizontal
        plt.axhline(0,-2,2,color="gray",linestyle="--")

        def animacion2(i):
            # Se definen las coordenadas de los extremos
            x1 = np.cos(f(i))
            y1 = np.sin(f(i))
            x2 = -np.cos(f(i))
            y2 = -np.sin(f(i))

            # Se alteran las entradas de las listas que definen los extremos de acuerdo a los valores recién calculados

            x[0] = x1
            y[0] = y1

            x[1] = x2
            y[1] = y2

            linea.set_xdata(x) # Se dan los extremos en x del oscilador
            linea.set_ydata(y) # Se dan los extremos en y del oscilador

            return linea

        animation2 = FuncAnimation(fig2, func=animacion2, frames=np.arange(0, 100, (1./60)),interval = t)

        plt.show()

        clear()

    elif opcion == '2':

        from matplotlib.widgets import Slider, Button, RadioButtons

        print("----- Mover parámetros -----\n")

        k_0 = 1.
        A_0 = 1.
        B_0 = 1.
        R_0 = 1.
        I_0 = 1.
        temp = entrada_real(input("Tiempo: "))

        def f(k, A, B, R, I):
            TIEMPO =[]
            THETA = []
            # Valores iniciales
            theta = 1.
            thetapt = 0.
            dt = 0.001 # Paso

            for i in np.arange(0.0, temp, dt):
                TIEMPO.append(i)

                # Se calcula la aceleración angular según las variables
                thetadpt = (- k * theta )/I - (((A*B)**2)* ((np.cos(theta)) ** 2) * thetapt) / (R*I)

                thetapt += thetadpt * dt # Como thetapt = d/dt (thetapt), theta_f = thetapt_i + thetadpt * dt, aprox
                theta += thetapt * dt # Como thetapt = d/dt (theta), theta_f = thetapt_i + thetadpt * dt, aprox
                THETA.append(theta)

            return np.asarray(THETA)

        fig = plt.figure()
        fig.canvas.set_window_title('Oscilador')

        ax = fig.add_subplot(111)
        ax.set_facecolor('#000000')

        fig.subplots_adjust(bottom=0.4)

        t = np.arange(0.0, temp, 0.001)

        # Gráfica inicial
        y = f(k_0, A_0, B_0, R_0, I_0)
        [linea] = ax.plot(t, y, linewidth=2, color='red')
        ax.set_xlim([0, temp])
        ax.set_ylim([-max(y)-1, max(y)+1])

        # k
        k_deslizador_ax  = fig.add_axes([0.1, 0.25, 0.65, 0.03])
        k_deslizador = Slider(k_deslizador_ax, r'$k$', 0.1, 100, valinit=k_0)

        # A
        A_deslizador_ax = fig.add_axes([0.1, 0.20, 0.65, 0.03])
        A_deslizador = Slider(A_deslizador_ax, r'$A$', 0.1, 50, valinit=A_0)

        # B
        B_deslizador_ax = fig.add_axes([0.1, 0.15, 0.65, 0.03])
        B_deslizador = Slider(B_deslizador_ax, r'$B$', 0.1, 50, valinit=B_0)

        # R
        R_deslizador_ax = fig.add_axes([0.1, 0.10, 0.65, 0.03])
        R_deslizador = Slider(R_deslizador_ax, r'$R$', 0.1, 100, valinit=R_0)

        # I
        I_deslizador_ax = fig.add_axes([0.1, 0.05, 0.65, 0.03])
        I_deslizador = Slider(I_deslizador_ax, r'$I$', 0.1, 100, valinit=I_0)

        # Modifica la línea cuando un valor de cualquier deslizador cambia
        def cambio_deslizadores(val):
            linea.set_ydata(f(k_deslizador.val, A_deslizador.val, B_deslizador.val, R_deslizador.val, I_deslizador.val))
            fig.canvas.draw_idle()

        k_deslizador.on_changed(cambio_deslizadores)
        A_deslizador.on_changed(cambio_deslizadores)
        B_deslizador.on_changed(cambio_deslizadores)
        R_deslizador.on_changed(cambio_deslizadores)
        I_deslizador.on_changed(cambio_deslizadores)

        # Botón para reiniciar los parámetros
        boton_reinicio_ax = fig.add_axes([0.85, 0.075, 0.1, 0.04])
        boton_reinicio = Button(boton_reinicio_ax, 'Reiniciar', hovercolor='0.975')
        def reinicio_boton(mouse_event):
            k_deslizador.reset()
            A_deslizador.reset()
            B_deslizador.reset()
            R_deslizador.reset()
            I_deslizador.reset()

        boton_reinicio.on_clicked(reinicio_boton)

        plt.show()

    elif opcion == '3':
        _run = False

    else:
        texto_pausado("Por favor, introduzca una opción válida\n", 0.05)
        texto_pausado(" ",0.8)
        clear()

clear()
exit()


import logging
from django.conf import settings

logger = logging.getLogger(__name__)

class StateDatasetRouter(object):
    """Read/write from speciic State databases"""

    def _db_name(self, model):
        return 'traffic_stops_{}'.format(model._meta.app_label)

    def db_for_read(self, model, **hints):
        """Return state DB if model's app name is a database"""
        state_db = self._db_name(model)
        if state_db in settings.DATABASES:
            name = state_db
        else:
            name = 'default'
        logger.debug('db_for_read({}): {}'.format(state_db, name))
        return name

    def db_for_write(self, model, **hints):
        """Return state DB if model's app name is a database"""
        state_db = self._db_name(model)
        if state_db in settings.DATABASES:
            name = state_db
        else:
            name = 'default'
        logger.debug('db_for_write({}): {}'.format(state_db, name))
        return name

    def allow_syncdb(self, db, model):
        # scenarios:
        #
        # default           traffic_stops_nc    False
        # default           traffic_stops_admin True
        # traffic_stops_nc  traffic_stops_admin False
        # traffic_stops_nc  traffic_stops_nc    True
        #
        name = self._db_name(model)
        if db == 'default':
            if name in settings.DATABASES:
                ret = False
            else:
                ret = True
        else:
            if name in settings.DATABASES:
                ret = True
            else:
                ret = False
        logger.debug('allow_syncdb({}, {}): {}'.format(db, model, ret))
        return ret


import matplotlib
import matplotlib.pylab as plt

from g2p_en import G2p

import tkinter
import argparse
import sys
import numpy as np
import torch

from hparams import create_hparams
from model import Tacotron2
from layers import TacotronSTFT, STFT
from audio_processing import griffin_lim
from train import load_model
from text import text_to_sequence

from hifigan.inference_e2e import inference

def plot_data(data, figsize=(16, 4)):
    matplotlib.use('TkAgg')
    fig, axes = plt.subplots(1, len(data), figsize=figsize)
    for i in range(len(data)):
        axes[i].imshow(data[i], aspect='auto', origin='lower',
                       interpolation='none')
    plt.show()

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-t', '--text', type=str, required=True,
                        help='what the model will say')
    parser.add_argument('-c', '--checkpoint_path', required=True, type=str,
                        help='directory to Tacotron checkpoints used for inference')
    parser.add_argument('-o', '--output_file', default='generated_speech', required=False, type=str,
                        help='filename of output .wav file')
    parser.add_argument('--profiling', action='store_true',
                        required=False, help='enables the profiler')

    args = parser.parse_args()

    hparams = create_hparams()

    checkpoint_path = args.checkpoint_path
    model = load_model(hparams)
    model.load_state_dict(torch.load(checkpoint_path)['state_dict'])
    _ = model.cuda().eval()

    text = args.text
    g2p = G2p()
    sequence = np.array(text_to_sequence(text, g2p))[None, :]
    sequence = torch.autograd.Variable(
        torch.from_numpy(sequence)).cuda().long()

    mel_outputs, mel_outputs_postnet, alignments = model.inference(sequence)

    # Synthesize audio with Hifi-GAN
    checkpoint_file = "hifigan/generator_v1"
    mel = mel_outputs_postnet.float().data
    inference(args.output_file, checkpoint_file, input_mel=mel_outputs_postnet.float().data)

    plot_data((mel_outputs.float().data.cpu().numpy()[0],
            mel_outputs_postnet.float().data.cpu().numpy()[0],
            alignments.float().data.cpu().numpy()[0].T))