import faulthandler; faulthandler.enable()
import numpy as np
import pymeshlab
import polyscope as ps
import polyscope.imgui as psim
import Polymeshlabfusion
import os

class ChooseCC(Polymeshlabfusion.Mesh):
    '''This is the class responsible for choosing the connected components'''

    is_active = False
    connectedcomp = []
    ms = None
    current_cc = 1
    numberofcc = 0

    def __init__(self, Mesh, color=(0.3, 0.5, 1), mode = 'quad'):
        '''Slightly changed constructor method '''
        self.ms = pymeshlab.MeshSet()
        self.ms.add_mesh(Mesh)
        self.id = self.ms.current_mesh().id()
        self.Mesh = Mesh
        self.name = f"CC number {ChooseCC.numberofcc}"
        self.color = color
        self.VisibleMesh = self.register_mesh(mode)
        ChooseCC.connectedcomp.append(self)
        ChooseCC.numberofcc += 1

    def set_current_mesh(self):
        self.ms.set_current_mesh(self.id)

    def __del__(self):
        self.VisibleMesh.remove()
        self.ms.set_current_mesh(self.id)
        self.ms.delete_current_mesh()

    @classmethod
    def callback(cls, PolyGUI):

        if not psim.TreeNode('Choose connected component'):
            cls.remove_all()
            if PolyGUI.current_step>1:
                PolyGUI.layers[PolyGUI.current_step-1].set_enabled(True)
            return

        if psim.Button('Split up'):

            if not PolyGUI.ms.current_mesh().is_point_cloud():
                #ps.remove_all_structures()
                cls.is_active = True
                cls.ms = pymeshlab.MeshSet()
                m = PolyGUI.ms.current_mesh()
                cls.ms.add_mesh(m)
                cls.ms.generate_splitting_by_connected_components(delete_source_mesh=True)

                for cc in cls.ms:
                    ChooseCC(cc)
            else:
                ps.error("The current mesh is not a point cloud!")

        if cls.connectedcomp !=[]:
            cls.connectedcomp[cls.current_cc-1].set_current_mesh()

        current_step_clicked, cls.current_cc = psim.SliderInt('Connected component no', cls.current_cc, v_min=1, v_max=np.max([len(cls.connectedcomp), 1]))

        if psim.Button('This one'):
            Chosen_one = cls.connectedcomp[cls.current_cc-1]
            while len(cls.connectedcomp)>0:
                del cls.connectedcomp[0]
            cls.connectedcomp.append(Chosen_one)
            cls.current_cc = 1

            PolyGUI.ms.add_mesh(cls.connectedcomp[cls.current_cc-1].Mesh)
            PolyGUI.layers.append(Polymeshlabfusion.Mesh(PolyGUI.ms, f"Reconstructed component", (.2,.3,.4)))
            PolyGUI.current_step = len(PolyGUI.layers)
            cls.is_active = False

        cls.mainloop()

        if cls.is_active:
            cls.set_everything_disabled(PolyGUI.layers)

    @classmethod
    def set_everything_disabled(cls,structurelist):
        for structure in structurelist:
            structure.set_enabled(False)

    @classmethod
    def mainloop(cls):
        for i in range(len(cls.connectedcomp)):
            enabled = True if (i==cls.current_cc-1) else False
            cls.connectedcomp[i].set_enabled(enabled)

    @classmethod
    def remove_all(cls):
        while len(cls.connectedcomp)>0:
            del cls.connectedcomp[0]

MS = pymeshlab.MeshSet()

def callback():
    ChooseCC.callback(MS)

def main():
    MS.load_new_mesh(os.getcwd()+'/Experiment-78/Experiment-78+pointcloud.ply')
    #MS.load_new_mesh(os.getcwd()+'/step3.ply')
    ps.init()
    Original = Polymeshlabfusion.Mesh(MS, 'Super cool mesh', (0.5, 1, 0.5))
    ps.set_user_callback(lambda Mesh = MS : ChooseCC.callback(Mesh))
    ps.show()

if __name__ == '__main__':
    main()


import faiss
import json
import numpy as np

class FaissSearch():
    """
The search takes an array of embeddings (can be a single embedding in the array), and returns a list of lists
with the k specified nearest matches for each embedding in the array.
    """

    def __init__(self):
        """
        Loads the embeddings from a JSON file, builds a matrix of each elements UUID, and builds
        an faiss index of that matrix.
        """
        with open("image_embeddings.json", 'r') as f:
            self.embeddings = json.load(f)
        self.element_matrix = self.build_element_matrix()
        self.index = self.build_index() #so in theory I can just return self.index[index]

    def build_element_matrix(self):
        """
        Builds a matrix of each elements UUID.

        Returns:
            A matrix of the UUIDS the elements in the corpus.
        """
        count = 0 # for faster testing
        element_matrix = np.ndarray([0,1000]) # don't have to hard code this; could put embeddings.shape[1]
        for uuid in self.embeddings:
            element = np.array (self.embeddings[uuid])
            element_matrix = np.vstack((element, element_matrix))
            count += 1
            if count == 100: #for faster testing
                break
        return element_matrix.astype('float32')

    def build_index(self):
        """
        Creates a flat index of the element matrix using the faiss module.

        Returns:
            The index is being returned.
        """
        index = faiss.IndexFlatL2(1000)
        index.add(self.element_matrix)
        return index

    def search(self, query_vector, num_results):
        """
        Returns the distances and indices of the closest vectors to the queried vectors.
        The query vector can contain the features of one or several images. A set of results
        will be returned for each set of image features in the quesry vector.

        Args:
            query_vector: a vector containing one or more vectors you want to search for
            num_results: number of results to return

        Returns:
            The index of the closest vector to the query vector.
        """
        d, i = self.index.search(query_vector, num_results)
        return i

    def decode_search_result(self, result):
        decoded_results = [] # a list of lists of UUIDS of near images for each queried element
        for queried_element in result: # result is an array of arrays, each which is the collection of nearest images for each element queried.
            uuids = [] #creates a list of UUIDS for each queried element, to be appended to the total decoded results
            for position in queried_element:#position is a np ndarray with one element in [position]
                result_vector = self.element_matrix[position]
                for uuid, vector in self.embeddings.items(): #vector is a list, result_vector is an array
                    if result_vector.tolist() == vector[0]:
                        uuids.append (uuid)
            decoded_results.append(uuids)
        return decoded_results # a list of lists

if __name__ == '__main__':
    pass


''' Simple lirc setup tool - model part. '''

YAML_MSG = '''
"Cannot import the yaml library. Please install the python3
yaml package, on many distributions known as python3-PyYAML. It is also
available as a pypi package at https://pypi.python.org/pypi/PyYAML.'''

import ast
import configparser
import glob
import os
import os.path
import subprocess
import sys
import urllib.error          # pylint: disable=no-name-in-module,F0401,E0611
import urllib.request        # pylint: disable=no-name-in-module,F0401,E0611

try:
    import yaml
except ImportError:
    print(YAML_MSG)
    sys.exit(1)

REMOTES_LIST = os.path.expanduser('~/.cache/remotes.list')
REMOTES_LIST_URL = "http://lirc-remotes.sourceforge.net/remotes.list"
_MODINIT_PATH = "lirc-modinit.conf"
_OPTIONS_PATH = "/etc/lirc/lirc_options.conf"
_REMOTES_BASE_URI = "http://sf.net/p/lirc-remotes/code/ci/master/tree/remotes"
_RESULTS_DIR = 'lirc-setup.conf.d'
_USAGE = "Usage: lirc-setup [results directory]"
_MANUAL_REMOTE_INSTALL = '(manual install)'

_DEBUG = 'LIRC_DEBUG' in os.environ

README = """
This is some configuration files created by lirc-setup. To install them,
try the following commands:

sudo cp lircd_options.conf /etc/lirc/lircd_options.conf
sudo cp lirc-modinit.conf /etc/modprobe.d
sudo cp lircd.conf /etc/lirc/lircd.conf
sudo cp lircmd.conf /etc/lirc/lircmd.conf

Of course, if you already have a working configuration don't forget to
make backup copies as required! Note that all files are not always present.
"""

def _hasitem(dict_, key_):
    ''' Test if dict contains a non-null value for key. '''
    return key_ in dict_ and dict_[key_]

def _here(path):
    ' Return path added to current dir for __file__. '
    return os.path.join(os.path.dirname(os.path.abspath(__file__)), path)

def parse_options():
    ''' Parse command line optios into a returned dict. '''
    options = {}
    if len(sys.argv) == 1:
        options['results_dir'] = _RESULTS_DIR
    elif len(sys.argv) == 2:
        options['results_dir'] = sys.argv[1]
    else:
        sys.stderr.write(_USAGE)
        sys.exit(1)
    return options

def find_rc(lirc):
    ''' Return the /sys/class/rc/rc* device corresponding to lirc device. '''
    lirc = os.path.basename(lirc)
    for rc in glob.glob('/sys/class/rc/rc*'):
        if os.path.exists(os.path.join(rc, lirc)):
            return rc
    return None

def download_file(view, url, path):
    ''' Download location url to a file. '''
    try:
        urllib.request.urlretrieve(url, path)
    except urllib.error.HTTPError as ex:
        text = "Cannot download %s : %s" % (url, str(ex))
        view.show_warning('Download error', text)

def get_bundled_remotes(config, view):
    ''' Return the bundled remote file for a device config dict,
    a possibly empty list of remote ids.
    '''
    if 'supports' not in config or config['supports'] == 'timing':
        if 'lircd_conf' in config:
            return [config['lircd_conf']]
        else:
            return []
    else:
        found = []
        if 'lircd_conf' in config:
            found = [config['lircd_conf']]
        if _hasitem(view.model.config, 'driver'):
            driver = view.model.config['driver']
            for l in view.model.get_remotes_list(view):
                words = l.split(';')
                if words[7] == driver:
                    found.append(words[0] + "/" + words[1])
        return found

def get_dmesg_help(device):
    ''' Return dmesg lines matching device.'''
    lines = subprocess.check_output('dmesg').decode('utf-8').split('\n')
    rc = find_rc(device)
    if rc:
        rc = os.path.basename(rc)
    else:
        rc = 'fjdsk@$'
    dev = os.path.basename(device)
    return [l for l in lines if dev in l or rc in l]

def list_ttys():
    ''' List all currently used ttys on this host. '''

    def driver_path(s):
        ''' Path to driver directory for given /class/tty/x device,. '''
        return os.path.join(s, 'device', 'driver')

    # (Not tested)
    # def is_inactive_5250(s):
    #     try:
    #         driver = os.path.basename(os.readlink(driver_path(s)))
    #     except (IOError, OSError):
    #         return False
    #     return driver != 'serial5250'

    syslist = glob.glob("/sys/class/tty/*")
    syslist = [s for s in syslist if os.path.exists(driver_path(s))]
    # syslist = [s for s in syslist if not is_inactive_5250(s)]
    devices = ["/dev/" + os.path.basename(s) for s in syslist]
    return devices

def write_results(config, result_dir, view):
    ''' Write the set of new configuration files into results directory, '''
    # pylint: disable=too-many-branches

    def write_modinit(log):
        ''' Possibly write the modprobe.d config file. '''
        if not _hasitem(config, 'modsetup'):
            return log
        modinit = '# Generated by lirc-setup\n'
        modinit += config['modsetup'] + '\n'
        path = os.path.join(result_dir, _MODINIT_PATH)
        with open(path, 'w') as f:
            f.write(modinit)
        log += 'Info: modprobe.d configuration: %s\n' % config['modsetup']
        return log

    def write_blacklist(log):
        ''' Possibly update blacklist in the /etc/modprobe.d file. '''
        if not _hasitem(config, 'blacklist'):
            return log
        blacklist = '# Generated by lirc-setup\n'
        blacklist += config['blacklist'] + '\n'
        path = os.path.join(result_dir, _MODINIT_PATH)
        with open(path, 'a') as f:
            f.write(blacklist)
        log += \
             'Info: modprobe.d blacklist config: %s \n' % config['blacklist']
        return log

    def write_options(options, log):
        ''' Update options in new lirc_options.conf. '''
        inited = False
        if not options.has_section('lircd'):
            options.add_section('lircd')
        if not _hasitem(config, 'lircd_conf'):
            log += 'Warning: No lircd.conf found, requierd by lircd.\n'
        for opt in ['device', 'lircd_conf', 'lircmd_conf']:
            if not _hasitem(config, opt):
                continue
            if not inited:
                log += 'Info: new values in lircd_options.conf\n<tt>'
                inited = True
            value = config[opt]
            if not opt == 'device':
                value = os.path.basename(value)
            options.set('lircd', opt, value)
            log += "%-16s: %s\n" % (opt, value)
        if _hasitem(config, 'modprobe'):
            if not options.has_section('modprobe'):
                options.add_section('modprobe')
            options.set('modprobe', 'modules', config['modprobe'])
            log += "%-16s: %s\n" % ('modules', config['modprobe'])
        log += '</tt>'
        path = os.path.join(result_dir, 'lirc_options.conf')
        with open(path, 'w') as f:
            f.write("# Generated by lirc-setup\n")
            options.write(f)
        return log

    def get_configfiles(log):
        ''' Download lircd.conf and perhaps lircmd.conf, '''

        def error(ex, uri):
            ''' Handle download error. '''
            text = "Cannot download %s : %s" % (uri, str(ex))
            view.show_error("Download error", text)

        if 'lircd_conf' not in config or not config['lircd_conf'] \
            or config['lircd_conf'] == _MANUAL_REMOTE_INSTALL:
                # pylint: disable=bad-indentation
                text = "No lircd.conf defined, skipping"
                view.show_warning("Download error", text)
                return log
        for item in ['lircd_conf', 'lircmd_conf']:
            if item not in config:
                continue
            uri = os.path.join(_REMOTES_BASE_URI, config[item])
            path = os.path.join(result_dir, os.path.basename(config[item]))
            try:
                urllib.request.urlretrieve(uri + '?format=raw', path)
                log += 'Info: Downloaded %s to %s\n' % (str(uri), str(path))
            except urllib.error.HTTPError as ex:
                error(ex, uri)
        return log

    options = configparser.RawConfigParser()
    options.read(_OPTIONS_PATH)
    log = 'Writing installation files in %s\n' % result_dir
    log = write_options(options, log)
    log = write_modinit(log)
    log = write_blacklist(log)
    log = get_configfiles(log)
    path = os.path.join(result_dir, 'README')
    with open(path, 'w') as f:
        f.write(README)
    return log

def check_resultsdir(dirpath):
    ''' Check that dirpath is ok, return possibly empty error message.'''
    if not os.path.exists(dirpath):
        try:
            os.makedirs(dirpath)
            return ''
        except os.error as err:
            return "Cannot create directory %s:" % dirpath + str(err)
    elif not os.listdir(dirpath):
        if os.access(dirpath, os.W_OK):
            return ''
        else:
            return "Directory %s is not writeable" % dirpath
    else:
        return "Directory %s is in the way. Please remove it or use" \
            " another directory." % dirpath

def check_modules(config):
    ''' Check modules options, return results as a string. '''
    if 'modules' not in config or not config['modules']:
        return ''
    with open('/proc/modules') as f:
        all_modules = f.readlines()
    all_modules = [m.split()[0] for m in all_modules]

    modules = ast.literal_eval(config['modules'])
    if not isinstance(modules, list):
        modules = []

    s = ''
    for m in modules:
        cmd = ['sh -c "find /lib/modules/$(uname -r) -name %s.ko"' % m]
        try:
            found = subprocess.check_output(cmd, shell=True).decode('utf-8')
        except (OSError, subprocess.CalledProcessError):
            found = []
        else:
            found = found.strip()
        if m in found:
            s += 'modules: %s: OK, module exists\n' % m
            if m in all_modules:
                s += 'modules: %s: OK, module is loaded\n' % m
            else:
                s += 'modules: %s: Info, module is not loaded.\n' % m
                if 'modprobe' not in config:
                    config['modprobe'] = list()
                config['modprobe'].extend([m])
        else:
            s += 'modules: %s: Error, module does not exist\n' % m
    return s

class DeviceListModel(object):
    ''' The list of devices corresponding to the device: wildcard in config.'''
    AUTO_CONFIG = "atilibusb"   # A driver configuration with device: auto.

    def __init__(self, config):
        self.driver_id = config['id']
        self.config = config
        self.device_pattern = config['device']
        self.label_by_device = {}
        self.list_devices()
        if not self.label_by_device:
            self.label_by_device = {}

    def list_devices(self):
        ''' List all available devices. '''
        assert self is True, 'Invalid call to abstract list_devices()'

    def is_direct_installable(self):
        ''' Return True  if this can be installed without a dialog. '''
        return len(self.label_by_device) == 1

    def is_empty(self):
        ''' Return true if there is no matching device.'''
        return len(self.label_by_device) == 0

class EventDeviceListModel(DeviceListModel):
    ''' List of devinput /dev/input/ devices . '''

    def __init__(self, model):
        config = model.find_config('id', 'devinput')
        DeviceListModel.__init__(self, config)

    def list_devices(self):
        ''' Return a dict label_by_device, labels from /input/by-id. '''
        self.label_by_device = {}
        oldcwd = os.getcwd()
        try:
            os.chdir("/dev/input/by-id/")
            for l in glob.glob("/dev/input/by-id/*"):
                try:
                    device = os.path.realpath(os.readlink(l))
                except OSError:
                    device = l
                self.label_by_device[device] = os.path.basename(l)
        except FileNotFoundError:    # pylint: disable=undefined-variable
            pass
        finally:
            os.chdir(oldcwd)

class LircDeviceListModel(DeviceListModel):
    '''  /dev/lirc? device list. '''

    def __init__(self, model):
        config = model.find_config('id', 'default')
        DeviceListModel.__init__(self, config)

    def list_devices(self):
        ''' Return a dict label_by_device, labels for /dev/lirc devices. '''
        self.label_by_device = {}
        for dev in glob.glob('/dev/lirc?'):
            rc = find_rc(dev)
            self.label_by_device[dev] = "%s (%s)" % (dev, rc)

class GenericDeviceListModel(DeviceListModel):
    ''' Generic /dev/xxx* device list. '''

    def list_devices(self):
        ''' Return a dict label_by_device, labels for matching devices. '''
        self.label_by_device = {}
        for match in glob.glob(self.config['device']):
            self.label_by_device[match] = match

class SerialDeviceListModel(DeviceListModel):
    ''' Let user select a device for a userspace serial driver. '''

    def list_devices(self):
        ''' Return a dict label_by_device, labels for matching devices. '''
        self.label_by_device = {}
        for dev in list_ttys():
            rc = find_rc(dev)
            if rc:
                self.label_by_device[dev] = "%s (%s)" % (dev, rc)
            else:
                self.label_by_device[dev] = dev

class UdpPortDeviceList(DeviceListModel):
    ''' Dummy list for the udp driver port. '''

    def __init__(self, model):
        config = model.find_config('id', 'udp')
        DeviceListModel.__init__(self, config)

    def list_devices(self):
        self.label_by_device = {'8765': 'default port 8765'}

    def is_direct_installable(self):
        return False

    def is_empty(self):
        ''' Return true if there is no matching device.'''
        return False

class AutoDeviceList(DeviceListModel):
    ''' Dummy list for the driver: 'auto'  entries. '''

    def __init__(self, model):
        config = model.find_config('id', self.AUTO_CONFIG)
        DeviceListModel.__init__(self, config)

    def list_devices(self):
        self.label_by_device = \
            {'auto': 'automatically probed device'}

    def is_direct_installable(self):
        return True

    def is_empty(self):
        return False

def device_list_factory(config, model):
    ''' Given a device: wildcard from config, return a DeviceList. '''

    device_wildcard = config['device']
    if device_wildcard.startswith('/dev/input'):
        return EventDeviceListModel(model)
    elif device_wildcard.startswith('/dev/lirc'):
        return LircDeviceListModel(model)
    elif device_wildcard.startswith('/dev/tty'):
        return SerialDeviceListModel(config)
    elif device_wildcard == 'auto':
        return AutoDeviceList(model)
    elif device_wildcard == 'udp_port':
        return UdpPortDeviceList(model)
    else:
        return GenericDeviceListModel(config)

class Model(object):
    ''' The basic model is the selected remote, driver and device. '''

    NO_SUCH_REMOTE = None
    NO_SUCH_DRIVER = 0
    DEVICE_ATTR = ['modinit', 'lircmd_conf', 'driver', 'device', 'note',
                   'label', 'modprobe', 'conflicts', 'modules', 'supports']

    def __init__(self):
        self.config = {}
        self.configs = {}
        self.device_list = None
        self.remotes_index = None
        self._listeners = []

    def reset(self):
        ''' Reset to pristine state... '''
        Model.__init__(self)

    def _call_listeners(self):
        ''' Call all listeners. '''
        for listener in self._listeners:
            listener()

    def add_listener(self, listener):
        ''' Add function to be called when model changes.'''
        self._listeners.append(listener)

    def set_remote(self, remote):
        ''' Update the remote configuration file. '''
        self.config['lircd_conf'] = remote
        self._call_listeners()

    def clear_remote(self):
        ''' Unset the lircd_conf status. '''
        self.config['lircd_conf'] = ''
        self._call_listeners()

    def set_manual_remote(self):
        ''' Set the remote to 'no remote', i.e., manual install. '''
        self.set_remote(_MANUAL_REMOTE_INSTALL)

    def is_remote_manual(self):
        ''' return True if lircd.conf should be installed manually. '''
        return self.config['lircd_conf'] == _MANUAL_REMOTE_INSTALL

    def set_capture_device(self, config):
        ''' Given a device dict update selected capture device. '''
        for key in self.DEVICE_ATTR:
            if key in config:
                self.config[key] = str(config[key])
        if 'lircd_conf' in config and 'lircd_conf' not in self.config:
            self.config['lircd_conf'] = config['lircd_conf']
        if 'supports' not in self.config:
            self.config['supports'] = 'timing'
        self._call_listeners()

    def clear_capture_device(self):
        ''' Indeed: unset the capture device. '''
        for key in self.DEVICE_ATTR:
            if key in self.config:
                del self.config[key]
        self._call_listeners()

    def set_device(self, device):
        ''' Update the device  + label part in config.'''
        self.config['device'] = device
        self.config['label'] += ' on ' + device
        self._call_listeners()

    def get_remotes_list(self, view):
        ''' Download and return the directory file as a list of lines. '''
        if not self.remotes_index:
            if not os.path.exists(REMOTES_LIST):
                download_file(view, REMOTES_LIST_URL, REMOTES_LIST)
            with open(REMOTES_LIST) as f:
                list_ = f.read()
            self.remotes_index = [l for l in list_.split('\n') if l]
        return self.remotes_index

    def is_installable(self):
        ''' Do we have a configuration which can be installed? '''
        for item in ['lircd_conf', 'driver']:
            if item not in self.config or not self.config[item]:
                return False
        conf = self.config['lircd_conf']
        conf = conf.replace('(', '').replace(')', '').lower()
        return conf != 'any' and conf != 'none'

    def get_bundled_driver(self, remote, view):
        ''' Return the bundled capture device file for remote,
        possibly None (no such remote) or 0 (no such driver)
        '''
        for l in self.get_remotes_list(view):
            words = l.split(';')
            key = words[0] + "/" + words[1]
            if key == remote:
                if words[7] == 'no_driver':
                    return None
                config = self.find_config('id', words[7])
                if not config:
                    return self.NO_SUCH_DRIVER
                return config
        return self.NO_SUCH_REMOTE

    def has_lircd_conf(self):
        ''' Return if there is a valid lircd_conf in config. '''
        if 'lircd_conf' not in self.config:
            return False
        cf = self.config['lircd_conf']
        if not cf:
            return False
        cf = cf.replace('(', '').replace(')', '').replace(' ', '').lower()
        if not cf or cf == 'any' or cf == 'none':
            return False
        return True

    def has_label(self):
        ''' Test if there is a valid driver label in config. '''
        if 'label' not in self.config:
            return False
        lbl = self.config['label']
        if not lbl or lbl.lower().endswith('(none)'):
            return False
        return True

    def load_configs(self):
        ''' Load config files into self.configs. '''
        if self.configs:
            return
        self.configs = {}
        if os.path.exists(_here('configs')):
            configs = _here('configs')
        elif os.path.exists(_here('../configs')):
            configs = _here('../configs')
        else:
            return None
        for path in glob.glob(configs + '/*.conf'):
            with open(path) as f:
                cf = yaml.load(f.read())
            self.configs[cf['config']['id']] = cf['config']

    def get_configs(self):
        ''' Return all configurations. '''
        self.load_configs()
        return self.configs

    def find_config(self, key, value):
        ''' Return item (a dict) in configs where config[key] == value. '''
        self.load_configs()
        found = \
            [c for c in self.configs.values() if key in c and c[key] == value]
        if len(found) > 1:
            print("find_config: not properly found %s, %s): " % (key, value)
                  + ', '.join([c['id'] for c in found]))
            return None
        elif not found:
            print("find_config: Nothing  found for %s, %s): " % (key, value))
            return None
        return dict(found[0])

# vim: set expandtab ts=4 sw=4:


# -*- coding: utf-8 -*-
"""
Created on Tue May 29 10:43:31 2018

@author: m.meliani
"""

import numpy as np
import matplotlib.pyplot as plt
from smt.applications.mfk import MFK, NestedLHS
from aerostructures import MFEGO
from modal_LF import ModalLF
from modal_HF import ModalHF
from openmdao.api import Problem

dim = 2
n = 7 # number of low and high-fidelity points
nlevel=2
ub0 = 30.
lb0 = 0.
ub1 = 70.
lb1 = 40.

costs = np.array([1,100]) # low and high-fidelity costs
initial_cost = np.sum(np.array(n)*costs)

def modal_HF(X):
    pb = Problem()
    pb.root = ModalHF()

    pb.setup()
    print('###########')
    print('X',X)

    pb['sweep'] = X[0]
    pb['b'] = X[1]
    pb.run()
    res = 1/pb['mode_1_2_dist']

    return res

def modal_LF(X):
    pb = Problem()
    pb.root = ModalLF()

    pb.setup()
    print('###########')
    print('X',X)

    pb['sweep'] = X[0]
    pb['b'] = X[1]
    pb.run()
    res = 1/pb['mode_1_2_dist']

    return res

#np.random.seed(1)
xlimits = np.array([[0.0, 1.0],[0.0, 1.0]])
X = NestedLHS(nlevel=nlevel, xlimits=xlimits)
Xc, Xe = X(n)

Xe[:,0] = Xe[:,0]*(ub0-lb0)+lb0
Xc[:,0] = Xc[:,0]*(ub0-lb0)+lb0
Xe[:,1] = Xe[:,1]*(ub1-lb1)+lb1
Xc[:,1] = Xc[:,1]*(ub1-lb1)+lb1

yc = np.zeros(2*n)
ye = np.zeros(n)

for i in range(n):
    ye[i] = modal_HF(Xe[i])
for i in range(2*n):
    yc[i] = modal_LF(Xc[i])

print(Xe)
print(ye)

#time.sleep(5)

sm = MFK(theta0=np.array(Xe.shape[1]*[1.]), print_global = False)
sm.set_training_values(Xc, yc, name = 0)
sm.set_training_values(Xe, ye)
sm.train()

bounds=np.array([[lb0,ub0],[lb1,ub1]])
algo_EGO = MFEGO([modal_LF, modal_HF],sm,bounds, costs = costs)
#Running the EGO algorithm for n_iter
budget = 15
Y_min, sm = algo_EGO.run(budget)

target_level = algo_EGO.levels[-1]

print( "######################################")
print( "Optimization summary Modal CRM")
print( "###################################### \n\n\n")

#
#x = np.linspace(0.,1., 50)
#y = np.copy(x)
#xv, yv = np.meshgrid(x, y)
#
#x_test= np.array([np.ravel(xv), np.ravel(yv)]).T
#y_test = sm.predict_values(x_test)
#y_mse = sm.predict_variances
#
#fig = plt.figure()
#ax = fig.add_subplot(111, projection='3d')
#ax.plot_surface(xv, yv, y_test.reshape((50, 50)), label ='surrogate')
##ax.plot_surface(xv, yv, currin_LF(x_test).reshape((50,50)),label ='LF')
#ax.plot_surface(xv, yv, currin_HF(x_test).reshape((50,50)),label ='HF reference')
#
#plt.show()
#
print( "######################################")
print( "minimum found by ego x=",algo_EGO.doe_x[target_level][algo_EGO.doe_y[target_level].argmin()]," y=",algo_EGO.doe_y[target_level].min())

print( "######################################")
print( "Initial DOE \n\n\n")
fig1 = plt.figure()
ax1 = fig1.add_subplot(111)
ax1.scatter(Xc[:,0],Xc[:,1],\
            marker = 'd', c = "green",s=20, label ='initial cheap DOE')
ax1.scatter(Xe[:,0],Xe[:,1],\
            marker = 'o', c = "blue", s =40, label ='initial expensive DOE')
ax1.legend(loc=0)
plt.show()

print( "######################################")
print( "Updated DOE : End of Opitmization \n\n\n")
fig2 = plt.figure()
ax2 = fig2.add_subplot(111)
ax2.scatter(algo_EGO.doe_x[0][:,0],algo_EGO.doe_x[0][:,1],\
            marker = 'd', c = "green",s=20, label ='cheap DOE')
ax2.scatter(algo_EGO.doe_x[None][:,0],algo_EGO.doe_x[None][:,1],\
            marker = 'o', c = "blue", s =40, label ='expensive DOE')
ax2.scatter(algo_EGO.doe_x[target_level][algo_EGO.doe_y[target_level].argmin()][0],\
                           algo_EGO.doe_x[target_level][algo_EGO.doe_y[target_level].argmin()][1],\
                           c='red',marker = '*',s = 80, label ='opt')
ax2.legend(loc=0)

ax1.set_xlim(lb0-0.1,ub0+.1)
ax1.set_ylim(lb1-0.1,ub1+.1)
ax2.set_xlim(lb0-0.1,ub0+.1)
ax2.set_ylim(lb1-0.1,ub1+.1)
plt.show()

print("######################################")
print( "Gain figure \n\n\n")
fig3 = plt.figure()
ax3 = fig3.add_subplot(111)
ax3.plot(np.log(np.array(algo_EGO.cost_hist) + initial_cost), algo_EGO.gain, c = "black", marker = "o")
ax3.set_title("Gain vs cost graph")
ax3.set_xlabel("log(Total Cost)")
ax3.set_ylabel("Gain")
plt.show()


from datetime import datetime, timedelta
import pandas as pd

def oneHourIntervalBrushing(data_hour):
    if len(data_hour['event_time']) == 1:
        # not brushing for sure
        return ""
    else:
#         calculate the concentrate rate
        unique_buyer_list = []
        suspicious_buyer = ""
        highest_order = 0
        unique_buyer_list = data_hour.userid.unique()
        num_orders = len(data_hour['shopid'])
        num_unique_buyers = len(unique_buyer_list)
        concentrate_rate = num_orders / num_unique_buyers
        if float(concentrate_rate) >= 3:
            # need to calculate propotion
            for each_buyer in unique_buyer_list:
                if len(data_hour[data_hour.userid == each_buyer]) > highest_order:
                    highest_order = len(data_hour[data_hour.userid == each_buyer])
                    suspicious_buyer = each_buyer
        return suspicious_buyer

def uniqueShopBrushing(data_shop):
    starting_time = data_shop['event_time'].iloc[0]
    # print(starting_time)
    buyerid_str_list = "0"
    if len(data_shop['event_time']) == 1 :
        buyerid = oneHourIntervalBrushing(data_shop)
        if buyerid == "":
            pass
    for i in range(0, len(data_shop['event_time'])):
        date_str = data_shop['event_time'].iloc[i]
        starting_time_obj = datetime(int(date_str[0:4]), int(date_str[5:7]), int(date_str[8:10]), int(date_str[11:13]),
                                     int(date_str[14:16]), int(date_str[17:19]))

        for j in range(i+1, len(data_shop['event_time'])):

            date_str = data_shop['event_time'].iloc[j]
            ending_time_obj = datetime(int(date_str[0:4]), int(date_str[5:7]), int(date_str[8:10]),
                                         int(date_str[11:13]),
                                         int(date_str[14:16]), int(date_str[17:19]))
            if ending_time_obj > (starting_time_obj + timedelta(hours=1)):
                buyerid = oneHourIntervalBrushing(data_shop[i:j])
                if buyerid == "" :
                    pass
                else:
                    if buyerid_str_list == "0":
                        buyerid_str_list = str(buyerid)
                    else:
                        if str(buyerid) in str(buyerid_str_list):
                            pass
                        else:

                            buyerid_str_list = str(buyerid_str_list) + ("&")
                            buyerid_str_list = str(buyerid_str_list) + (str(buyerid))
                break

    return buyerid_str_list

def brushing():
    data = pd.read_csv("./order_brush_order.csv")

    data = data.sort_values(by=['shopid', 'event_time'])
    # data = data.head(n=10000)
    shopid_list = data.shopid.unique()
    final_shopid = []
    final_buyerid_str_list = []
    for indiv_shop in shopid_list:
        # print(data.loc[data['shopid'] == indiv_shop])

        buyerid_str_list = uniqueShopBrushing(data.loc[data['shopid'] == indiv_shop])
        final_shopid.append(indiv_shop)
        final_buyerid_str_list.append(buyerid_str_list)

    final_dataframe = pd.DataFrame({'shopid':   final_shopid,
                   'userid': final_buyerid_str_list})
    # print(final_shopid)
    # print(final_buyerid_str_list)
    final_dataframe.to_csv("./output.csv", index=False)
if __name__ =="__main__":

    brushing()


import time
import copy
import numpy as np
import math

from options import Options
opt = Options().parse()  # set CUDA_VISIBLE_DEVICES before import torch

import torch
import torchvision
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import random
import numpy as np

from models.autoencoder import Model
from data.modelnet_shrec_loader import ModelNet_Shrec_Loader
from data.shapenet_loader import ShapeNetLoader
from util.visualizer import Visualizer

if __name__=='__main__':
    if opt.dataset=='modelnet' or opt.dataset=='shrec':
        trainset = ModelNet_Shrec_Loader(opt.dataroot, 'train', opt)
        dataset_size = len(trainset)
        trainloader = torch.utils.data.DataLoader(trainset, batch_size=opt.batch_size, shuffle=True, num_workers=opt.nThreads)
        print('#training point clouds = %d' % len(trainset))

        testset = ModelNet_Shrec_Loader(opt.dataroot, 'test', opt)
        testloader = torch.utils.data.DataLoader(testset, batch_size=opt.batch_size, shuffle=False, num_workers=opt.nThreads)
    elif opt.dataset=='shapenet':
        trainset = ShapeNetLoader(opt.dataroot, 'train', opt)
        dataset_size = len(trainset)
        trainloader = torch.utils.data.DataLoader(trainset, batch_size=opt.batch_size, shuffle=True, num_workers=opt.nThreads)
        print('#training point clouds = %d' % len(trainset))

        tesetset = ShapeNetLoader(opt.dataroot, 'test', opt)
        testloader = torch.utils.data.DataLoader(tesetset, batch_size=opt.batch_size, shuffle=False, num_workers=opt.nThreads)
    else:
        raise Exception('Dataset error.')

    model = Model(opt)

    visualizer = Visualizer(opt)

    best_loss = 99
    for epoch in range(601):

        epoch_iter = 0
        for i, data in enumerate(trainloader):
            iter_start_time = time.time()
            epoch_iter += opt.batch_size

            if opt.dataset=='modelnet' or opt.dataset=='shrec':
                input_pc, input_sn, input_label, input_node, input_node_knn_I = data
                model.set_input(input_pc, input_sn, input_label, input_node, input_node_knn_I)
            elif opt.dataset=='shapenet':
                input_pc, input_sn, input_label, input_seg, input_node, input_node_knn_I = data
                model.set_input(input_pc, input_sn, input_label, input_node, input_node_knn_I)

            model.optimize()

            if i % 100 == 0:
                # print/plot errors
                t = (time.time() - iter_start_time) / opt.batch_size

                errors = model.get_current_errors()

                visualizer.print_current_errors(epoch, epoch_iter, errors, t)
                visualizer.plot_current_errors(epoch, float(epoch_iter) / dataset_size, opt, errors)

                # print(model.autoencoder.encoder.feature)
                visuals = model.get_current_visuals()
                visualizer.display_current_results(visuals, epoch, i)

        # test network
        if epoch >= 0 and epoch%1==0:
            batch_amount = 0
            model.test_loss.data.zero_()
            for i, data in enumerate(testloader):
                if opt.dataset == 'modelnet' or opt.dataset=='shrec':
                    input_pc, input_sn, input_label, input_node, input_node_knn_I = data
                    model.set_input(input_pc, input_sn, input_label, input_node, input_node_knn_I)
                elif opt.dataset == 'shapenet':
                    input_pc, input_sn, input_label, input_seg, input_node, input_node_knn_I = data
                    model.set_input(input_pc, input_sn, input_label, input_node, input_node_knn_I)
                model.test_model()

                batch_amount += input_label.size()[0]

                # # accumulate loss
                model.test_loss += model.loss_chamfer.detach() * input_label.size()[0]

            model.test_loss /= batch_amount
            if model.test_loss.item() < best_loss:
                best_loss = model.test_loss.item()
            print('Tested network. So far lowest loss: %f' % best_loss )

        # learning rate decay
        if epoch%20==0 and epoch>0:
            model.update_learning_rate(0.5)

        # save network
        if epoch%1==0 and epoch>0:
            print("Saving network...")
            model.save_network(model.encoder, 'encoder', '%d_%f' % (epoch, model.test_loss.item()), opt.gpu_id)
            model.save_network(model.decoder, 'decoder', '%d_%f' % (epoch, model.test_loss.item()), opt.gpu_id)


import sys
import os
import logging
from sentence_transformers import SentenceTransformer, util

dir_path = os.path.dirname(os.path.realpath(__file__))
print(dir_path)

sys.path.append("{}/../src/".format(dir_path))
print(sys.path)

root = logging.getLogger()
root.setLevel(logging.DEBUG)
handler = logging.StreamHandler(sys.stdout)
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
root.addHandler(handler)

def test_st():
    image_model = SentenceTransformer('clip-ViT-B-32-multilingual-v1')
    model = SentenceTransformer("all-MiniLM-L6-v2")

    # Our sentences we like to encode
    sentences = [
        "This framework generates embeddings for each input sentence",
        "Sentences are passed as a list of string.",
        "The quick brown fox jumps over the lazy dog.",
    ]

    # Sentences are encoded by calling model.encode()
    sentence_embeddings = model.encode(sentences)

    # Print the embeddings
    for sentence, embedding in zip(sentences, sentence_embeddings):
        print("Sentence:", sentence)
        print("Embedding:", embedding)
        print("")

    # Single list of sentences

    """
    sentences = [
        "The cat sits outside",
        "A man is playing guitar",
        "I love pasta",
        "The new movie is awesome",
        "The cat plays in the garden",
        "A woman watches TV",
        "The new movie is so great",
        "Do you like pizza?",
    ]
    """
    sentences = [
        "猫坐在外面",
        "狗坐在上面",
        "狗坐在里面",
        "一个男人在弹吉他",
        "我爱意大利面",
        "新电影太精彩了",
        "猫在花园里玩耍",
        "一个女人在看电视",
        "新电影太棒了",
        "你喜欢披萨吗？",
    ]

    # Compute embeddings
    #embeddings = model.encode(sentences, convert_to_tensor=True)
    embeddings = model.encode(sentences)
    print("embeddings as follows: ")
    print(embeddings)

    # Compute cosine-similarities for each sentence with each other sentence
    cosine_scores = util.cos_sim(embeddings, embeddings)

    # Find the pairs with the highest cosine similarity scores
    pairs = []
    for i in range(len(cosine_scores) - 1):
        for j in range(i + 1, len(cosine_scores)):
            pairs.append({"index": [i, j], "score": cosine_scores[i][j]})

    # Sort scores in decreasing order
    pairs = sorted(pairs, key=lambda x: x["score"], reverse=True)

    for pair in pairs[0:10]:
        i, j = pair["index"]
        print(
            "{} \t\t {} \t\t Score: {:.4f}".format(
                sentences[i], sentences[j], pair["score"]
            )
        )

if __name__ == "__main__":
    test_st()


from celery import shared_task
from django.core.mail import EmailMessage
from otpmicroservice.settings import EMAIL_HOST, EMAIL_HOST_USER
import logging
logging.basicConfig(level='DEBUG')

@shared_task()
def  send_mail_func(mail_body,toMail):
    #operations
    try:
        email = EmailMessage(
                    "OTP for into PocketHR",
                    mail_body,
                    EMAIL_HOST_USER,
                    [toMail],
                    [],
        )
        email.send()
        return "Done"
    except Exception as e:
        logging.debug(e+" : send_mail_func()")
        return str(e)


# -*- coding: utf-8 -*-
"""
Created on Sun Feb 12 10:07:49 2023

@author: anush
"""
import cv2
import numpy as np

img1=cv2.imread("E:\data\dog1.jpg")
img2=cv2.imread("E:\data\cat1.jpg")

img1 = cv2.resize(img1,(300,400))
img2 = cv2.resize(img2,(300,400))
#cv2.imshow("dog",img1)
#cv2.imshow("cat",img2)

def blend(x):
    pass

img =np.zeros((400,400,3),np.uint8)
cv2.namedWindow("win")#create a track bar
cv2.createTrackbar("alpha","win",1,100,blend)

switch = "0 :OFF \n 1:ON"
cv2.createTrackbar(switch,"win",0,1,blend)

while True:
    s =cv2.getTrackbarPos(switch,"win")
    a =cv2.getTrackbarPos("alpha","win")
    n=float(a/100)
    print(n)

    if s==0:
        dst =img[:]
    else:
        dst =cv2.addWeighted(img1,1-n,img2,n,0)
        cv2.putText(dst,str(a),(20,60),cv2.FONT_ITALIC,
                    2,(0,125,255),2)
    cv2.imshow("dst",dst)
    k=cv2.waitKey(1)
    if k==27:
        break

cv2.waitKey(0)
cv2.destroyAllWindows()


"""empty message

Revision ID: 4b4bc246171
Revises: 5930870a26b
Create Date: 2015-03-18 10:08:55.404885

"""

# revision identifiers, used by Alembic.
revision = '4b4bc246171'
down_revision = '5930870a26b'

from alembic import op
import sqlalchemy as sa

def upgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('payment',
    sa.Column('id', sa.Integer(), nullable=False),
    sa.Column('date', sa.String(length=60), nullable=False),
    sa.Column('token_id', sa.String(length=60), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.drop_table('customer')
    ### end Alembic commands ###

def downgrade():
    ### commands auto generated by Alembic - please adjust! ###
    op.create_table('customer',
    sa.Column('id', sa.INTEGER(), nullable=False),
    sa.Column('name', sa.VARCHAR(length=60), nullable=False),
    sa.Column('token_id', sa.VARCHAR(length=60), nullable=True),
    sa.PrimaryKeyConstraint('id')
    )
    op.drop_table('payment')
    ### end Alembic commands ###


from rest_framework import serializers
from .models import (
    Rent,
    RentFeature
)
from cars.serializers import (
    CarListSearializer,
    CarDetailSearializer,
)
from companies.serializers import (
    CompanyDetailSearializer,
)

class RentFeatureSerializer(serializers.ModelSerializer):
    class Meta:
        model = RentFeature
        fields = (
            'id',
            'title',
        )

class RentListSearializer(serializers.ModelSerializer):
    car = CarListSearializer()
    class Meta:
        model = Rent
        fields = (
            'id',
            'car',
            'price',
            'total_cost',
            'deposit',
        )

class RentDetailSearializer(serializers.ModelSerializer):
    car = CarDetailSearializer()
    features = RentFeatureSerializer(many=True)
    company = CompanyDetailSearializer()
    class Meta:
        model = Rent
        fields = (
            'id',
            'car',
            'features',
            'price',
            'deposit',
            'total_cost',
            'total_cost_discount',
            'company',
        )

class RentFilterSerializer(serializers.Serializer):
    start_date = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S", required = False)
    end_date = serializers.DateTimeField(format="%Y-%m-%d %H:%M:%S", required = False)

class RentByLocationSerializer(serializers.Serializer):
    pick_up = serializers.IntegerField()
    drop_off = serializers.IntegerField()


import threading
import pymongo
import logging
from scrapy import signals

from douban.items.movie_item import MovieItem

logger = logging.getLogger(__name__)

class MongoPipeline(object):
    lock = threading.Lock()

    def __init__(self, mongo_uri, mongo_db):
        self.mongo_uri = mongo_uri
        self.mongo_db = mongo_db
        self.client = pymongo.MongoClient(self.mongo_uri)
        logger.info('Mongodb connected.')
        self.db = self.client[self.mongo_db]

    @classmethod
    def from_crawler(cls, crawler):
        pipeline = cls(
            mongo_uri=crawler.settings.get('MONGO_URI'),
            mongo_db=crawler.settings.get('MONGO_DATABASE', 'items')
        )
        crawler.signals.connect(pipeline.disconnect, signals.engine_stopped)
        return pipeline

    def disconnect(self):
        self.client.close()
        logger.info('Mongodb disconnected.')

    def process_item(self, item, spider):
        with self.lock:
            self.db[item.Meta.db_collection_name]\
                .find_one_and_update({'id': item['id']}, {'$set': dict(item)}, upsert=True)
        return item


#! /usr/bin/env python3

from datetime import datetime

class CarSale():
    def __init__(self,
                    model_year: int,
                    mileage: float,
                    price: float,
                    posted_datetime: datetime,
                    model: str,
                    source: str,
                    location: str,
                    description: str):
        self.model_year = model_year
        self.mileage = mileage
        self.price = price
        self.posted_datetime = posted_datetime
        self.model = model
        self.source = source
        self.location = location
        self.description = description


from django.shortcuts import render
from .models import Contact

# Create your views here.
def index(request):
    try:
        contact = Contact.objects.get(name='Yuriy', last_name='Lyashko')
    except:
        name, last_name, date_of_birth, bio, email, jabber, skype, other_contacts = Contact.get_contact_data()
        contact = Contact(name=name, last_name=last_name, date_of_birth=date_of_birth, bio=bio, email=email,
                          jabber=jabber, skype=skype, other_contacts=other_contacts
                          )
        contact.save()
    finally:
        contact.bio, contact.other_contacts = eval(contact.bio), eval(contact.other_contacts)
        return render(request, 'index.html', {'contact': contact})


from models.graph_model import GraphModel
from typing import List, Tuple

class GraphMapHelper(object):
    @staticmethod
    def db_entity_to_graph_model(db_entity: tuple) -> GraphModel:
        graph_model = GraphModel()
        graph_model.graph_id = db_entity[0]
        graph_model.graph_name = db_entity[1]
        return graph_model

    @staticmethod
    def db_entities_to_graph_models(db_entities: List[Tuple]) -> List[GraphModel]:
        list_graph_models: List[GraphModel] = []
        for list_of_graph_values in db_entities:
            node_model: GraphModel = GraphMapHelper.db_entity_to_graph_model(list_of_graph_values)
            list_graph_models.append(node_model)
        return list_graph_models


'''
CAD.GetLayerNames
Goal: Get all layer names of an import instance.
diegojsanchez@gmail.com #masalladedynamo
v: 1.0.1
'''
#bibliotecas
import clr
clr.AddReference('RevitAPI')
import Autodesk
from Autodesk.Revit.DB import FilteredElementCollector, ImportInstance
clr.AddReference('RevitServices')
from RevitServices.Persistence import DocumentManager
doc = DocumentManager.Instance.CurrentDBDocument
#definiciones
def tolist(x):
    if isinstance(x,list) == True: return x
    else: return [x]
#inputs
cad = UnwrapElement(tolist(IN[0]))
#mainbody
capas = [x.Category.SubCategories for x in cad]
nombres = [x.Name for x in capas[0]]
#output
OUT = sorted(nombres)


# -*- coding: utf-8 -*-

# Define here the models for your scraped items
#
# See documentation in:
# http://doc.scrapy.org/en/latest/topics/items.html

import scrapy
import requests
import time
import logging
import configparser

# Config parser
cfg = configparser.ConfigParser()
cfg.read("general.cfg")

class ProxyfetcherItem(scrapy.Item):
    # Build of the item
    ip = scrapy.Field()
    port = scrapy.Field()
    country = scrapy.Field()
    con_type = scrapy.Field()
    response_time = scrapy.Field()
    full_address = scrapy.Field()

    def status_check(self, item):
        # Save timestamp for later calculation of elapsed time
        timestamp = time.time()
        # Call the judge for the proxy
        try:
            judge = requests.get(cfg.get('judge', "url", fallback="http://azenv.net/"),
                                 proxies = {"http" : "http://{}:{}".format(item["ip"], item["port"])},
                                 timeout = 5)
            # Only store the item if the judge makes a correct answer
            if (judge.status_code == 200) and ("HTTP_HOST" in judge.text):
                item["response_time"] = round(time.time() - timestamp, 3)
                return item
        # Exception handling
        except requests.exceptions.Timeout as e:
            logging.info("Timeout: "+str(e))
        except requests.exceptions.ProxyError as e:
            logging.info("Proxy error: "+str(e))
        except requests.exceptions.ConnectionError as e:
            logging.info("Connection Error: "+str(e))
        except requests.exceptions.ChunkedEncodingError as e:
            logging.info("Connection probably broken.")
        except Exception as e:
            logging.info("Uncaught exception: " +str(e))


import os
import yaml
import pickle
import logging
import numpy as np
import pandas as pd
from typing import Tuple

from ml.gridsearches.gridsearch import GridSearch

import bokeh
from bokeh.plotting import figure, output_file, show, save
from bokeh.models import ColumnDataSource, HoverTool
from bokeh.models import ColumnDataSource, Whisker
from bokeh.sampledata.autompg import autompg as df
from bokeh.layouts import gridplot

from sklearn.metrics import confusion_matrix
from collections import Counter
from matplotlib import pyplot as plt
import seaborn as sns

from labels.concatenators.concatenator import Concatenator

class LabelPlotter:

    def __init__(self, settings, concatenator: Concatenator):
        """Plotting the results of the concatenated labels

        Args:
            settings ([type]): [description]
        """
        self._name = 'label_plotter'
        self._notation = 'lblpltr'
        self._settings = settings
        self._legend_addition = concatenator.get_legend_addition()
        self._order = concatenator.get_order()
        self._combinations = concatenator.get_combinations()

        self._load_palettes()
        self._load_id_dictionary()
        self._load_demographics()

    def _load_palettes(self):
        raise NotImplementedError

    def _load_id_dictionary(self):
        """Loads the summaries of sequences and information about the students

        => Implement it in the settings
        """
        raise NotImplementedError

    def _load_demographics(self):
        """Returns the demographics
        """
        raise NotImplementedError

    def _get_percentage(self, df, grouping:list, stratification: str):
        """Used to retrieve the percentage of students in each categories, potentially stratified by "stratification"

        Args:
            df ([type]): dataframe with the information
            grouping (list): what to sort the students in
            stratification (str): how to stratify the students

        Returns:
            [type]: dataframe with the percentage of each of the students in each of the groups in grouping, potentially stratified by stratification
        """
        print('You need to edit the demographics columns to the available demographics')
        raise NotImplementedError
        # d = df[['language']].groupby(grouping).nunique()['lid'].reset_index()

        # strat = df[[stratification, 'lid']]
        # strat = strat.groupby(stratification).nunique()['lid'].reset_index()
        # strat.columns = [stratification, 'total_counts']

        # new = d.merge(strat, how='inner', on=stratification)
        # new['height'] = new['lid'] / new['total_counts']
        # return new

    def _plot_label_distribution(self, new_predictions: dict):
        """Plot the label distribution in absolute numbers, without stratification
        Args:
            new_predictions (dict):
                key [index from the id_dictionary]: {
                    class 1 [the first class to concatenate with the others]:
                        raw proba: the vector of probability
                        pred: the actual prediction (integer of the label)
                        proba: the probability of being pred
                    ...
                    class n [the nth class to concatenate with the others]:
                        raw proba: the vector of probability
                        pred: the actual prediction (integer of the label)
                        proba: the probability of being pred
                }
        """

        raise NotImplementedError

    def _plot_label_distribution_stratified(self, new_predictions: dict, stratifier: str):
        """Plot the percentage of students in each of the labels, stratified by the stratified

        Args:
            new_predictions (dict):
                key [index from the id_dictionary]: {
                    class 1 [the first class to concatenate with the others]:
                        raw proba: the vector of probability
                        pred: the actual prediction (integer of the label)
                        proba: the probability of being pred
                    ...
                    class n [the nth class to concatenate with the others]:
                        raw proba: the vector of probability
                        pred: the actual prediction (integer of the label)
                        proba: the probability of being pred
                }
            stratifier (str): what to stratify the students with
        """

        raise NotImplementedError

    def _plot_confusion_matrix(self, new_predictions:dict, stratifier='no_strat'):
        """Plot the confusion matrix for the new labels, possibly stratified by a factor

        Args:
            new_predictions (dict):
            key [index from the id_dictionary]: {
                    class 1 [the first class to concatenate with the others]:
                        raw proba: the vector of probability
                        pred: the actual prediction (integer of the label)
                        proba: the probability of being pred
                    ...
                    class n [the nth class to concatenate with the others]:
                        raw proba: the vector of probability
                        pred: the actual prediction (integer of the label)
                        proba: the probability of being pred
                }
        """
        cm = {}
        for iid in new_predictions:
            cm[iid] = {
                'preds': new_predictions[iid]['predicted_label'],
                'truth': new_predictions[iid]['ground_truth'],
                'gender': self._demographics_map[iid]['gender'],
                'year': self._demographics_map[iid]['year'],
                'field': self._demographics_map[iid]['field'],
                'no_strat': 'nothing'
        }

        cm_df = pd.DataFrame(cm).transpose()

        for strat in cm_df[stratifier].unique():
            df = cm_df[cm_df[stratifier] == strat]
            df = confusion_matrix(df['truth'], df['preds'], labels=self._combinations['labels'])
            df = pd.DataFrame(df, columns=self._combinations['labels'], index=self._combinations['labels'])
            df.set_index(self._combinations['labels'])
            sns.heatmap(df, cmap="YlGnBu")
            plt.title('Confusion Matrix stratified with ' + stratifier + '=' + str(strat))
            plt.ylabel('truth')
            plt.xlabel('prediction')

            if self._settings['save']:
                path = '../reports/' + self._settings['concatenation']['report_folder'] + '/figures/confusion_matrix_n' + self._notation
                path += '_strat' + strat
                path += 'exp' + self._settings['experiment']['name'] + '_algorithm' + self._settings['experiment']['algorithm']
                path += '_feat' + self._settings['experiment']['features'] + '_seq' + self._settings['experiment']['sequencer']
                path += '_class' + self._settings['experiment']['classname']
                path +='.svg'
                plt.savefig(path, format='svg')
            if self._settings['show']:
                plt.show()
            else:
                plt.close()
        raise NotImplementedError

    def _measure_comparison(self, results: pd.DataFrame):
        raise NotImplementedError

    def plot(self, new_predictions={}, results=pd.DataFrame()):
        raise NotImplementedError


import numpy as np
from PIL import Image, ImageDraw, ImageFont

class PillowMisc:
    def pil2cv(imgPIL):
        imgCV_RGB = np.array(imgPIL, dtype=np.uint8)
        imgCV_BGR = np.array(imgPIL)[:, :, ::-1]
        return imgCV_BGR

    def cv2pil(imgCV):
        imgCV_RGB = imgCV[:, :, ::-1]
        imgPIL = Image.fromarray(imgCV_RGB)
        return imgPIL

    def put_text(img, text, org, fontFace, fontScale, color):
        x, y = org
        b, g, r = color
        colorRGB = (r, g, b)
        imgPIL = PillowMisc.cv2pil(img)
        draw = ImageDraw.Draw(imgPIL)  # ImageDraw.text() の xy は右上基準
        fontPIL = ImageFont.truetype(font=fontFace, size=fontScale)
        draw.text(xy=(x, y), text=text, fill=colorRGB, font=fontPIL)
        imgCV = PillowMisc.pil2cv(imgPIL)
        return imgCV


# -*- coding: utf-8 -*-
"""
Created on Thu Jun  3 09:57:19 2021

@author: juandc
"""

# %%
### INITIALIZATION

# reset variables etc.
from IPython import get_ipython
get_ipython().magic('reset -sf')

# libraries
import community_simulator
import community_simulator.usertools
import community_simulator.visualization
import numpy as np
#import pandas as pd
# import matplotlib
#import matplotlib.pyplot as plt
# import seaborn as sns
# colors = sns.color_palette()
# %matplotlib inline

import random
# import math
import copy
import os

# check if running on Windows
if os.name == 'nt':
    par = False
else:
    par = True

import time
start_time = time.time()

np.random.seed(1)
random.seed(1)

# %%
### USER-DEFINED FUNCTIONS

# stabilize communities through serial passaging
def stabilizeCommunities(plate):

    # parameters
    dilution_factor = 1/100
    n_transfers = 20
    time_per_cycle = 1
    # extinction_threshold = 1e-4 # we don't use this because we don't use StadyState()

    # run stabilization protocol
    Ntraj, Rtraj = plate.RunExperiment(f = np.diag(np.ones(plate.n_wells))*dilution_factor,
                                       T = time_per_cycle,
                                       npass = n_transfers,
                                       compress_resources=False,
                                       compress_species=True)

    return([plate.N,plate.R])

# Bray-Curtis similarity
# p and q are lists (of absolute abundances, sum need not be 1)
def bray_curtis(p,q):

    # C_pq: sum of lesser values of common species
    c = sum([min(p[i],q[i]) for i in range(len(p)) if p[i]>0 and q[i]>0])

    # Bray-Curtis similarity
    bc = 2*c/(sum(p) + sum(q))
    return bc

# similarity index
# p: invasive
# q: resident
# pq: coalesced
def mysim(p,q,pq):

    # if any of the vectors is empty, return nan (this could happen e.g. if maintenance costs are too high)
    if sum(p)==0 or sum(q)==0 or sum(pq)==0:
        return np.nan

    # otherwise...
    else:

        # normalization
        p = [p_i/sum(p) for p_i in p]
        q = [q_i/sum(q) for q_i in q]
        pq = [pq_i/sum(pq) for pq_i in pq]

        # relative similarity
        return bray_curtis(p,pq)/(bray_curtis(p,pq) + bray_curtis(q, pq))

# %%
### GENERATE RANDOM PARAMETERS, RUN MODEL, GET R^2, REPEAT

# list of R^2
r_squared = ['NA' for i in range(100)]

# repeat simulation 100 times
for n_sim in range(100):

    # print progress
    print('Running simulation %s out of 100' % (n_sim+1))
    time.sleep(0.1)

    # general assumptions (common to all simulations)
    assumptions = community_simulator.usertools.a_default.copy()
    assumptions['n_wells'] = 50 # number of communities

    assumptions['response'] = 'type I'
    assumptions['regulation'] = 'independent' # 'independent' is standard, 'energy' or 'mass' enable preferential consumption of resources
    assumptions['sampling'] = 'Gamma' # 'Binary', 'Gaussian', 'Uniform' or 'Gamma' (sampling of the matrix c)
    assumptions['supply'] = 'off'
    assumptions['R0_food'] = 1000

    assumptions['c0'] = 0.0 # background consumption rate in binary model
    assumptions['c1'] = 1.0 # specific consumption rate in binary model

    assumptions['rs'] = 0.0 # control parameter for resource secretion: 0 means random secretions, 1 means species only secrete resources they can consume (relevant only if 'metabolism' is 'specific')

    # randomly sample variable parameters
    assumptions['S'] = random.randint(10,90) # number of species sampled at initialization -- uniformly distributed between 10 and 90

    n_families = random.randint(1,5) # number of species families -- uniformly distributed between 1 and 5
    n_species_per_family = random.randint(400,1200) # number of species per family -- uniformly distributed between 400 and 1200
    assumptions['SA'] = [n_species_per_family] * n_families
    assumptions['Sgen'] = random.randint(100,380) # number of generalists -- uniformly distributed between 100 and 380

    n_resource_classes = random.randint(3,8) # number of resource classes -- uniformly distributed between 1 and 5
    n_resources_per_class = random.randint(3,17) # number of resources per class -- uniformly distributed between 3 and 17
    assumptions['MA'] = [n_resources_per_class] * n_resource_classes

    #assumptions['l'] = [random.uniform(0.45,0.95) for i in range(sum(assumptions['MA']))] # leakage fraction -- uniformly distributed between 0.45 and 0.95, resource-specific
    assumptions['l'] = random.uniform(0.45,0.95) # leakage fraction -- uniformly distributed between 0.45 and 0.95, resource-unspecific

    #assumptions['m'] = [random.uniform(0,0.2) for i in range(sum(assumptions['SA']) + assumptions['Sgen'])] # species maintenance costs -- uniformly distributed between 0 and 0.2, species-specific
    assumptions['m'] = random.uniform(0,0.2) # species maintenance costs -- uniformly distributed between 0 and 0.2, species-unspecific

    assumptions['q'] = random.uniform(0.5,1) # preference strength (0 for generalists, 1 for specialists) -- uniformly distributed between 0.5 and 0.9
    assumptions['sigc'] = random.uniform(2,4) # standard deviation of sum of consumption rates for Gaussian and Gamma models -- uniformly distributed between 2 and 4
    assumptions['sparsity'] = random.uniform(0.05,0.95) # variability in secretion fluxes among resources (must be less than 1) -- uniformly distributed between 0.05 and 0.95
    assumptions['fs'] = random.uniform(0.05,0.45) # fraction of secretion flux to resources of the same type as the consumed one -- uniformly distributed between 0.05 and 0.45
    assumptions['fw'] = random.uniform(0.05,0.45) # fraction of secretion flux to waste resources -- uniformly distributed between 0.05 and 0.45
    assumptions['metabolism'] = ['common','specific'][random.randint(0,1)] # common or species-specific D matrix -- chosen arbitrarily with equal probabilities

    # parameters (make new matrices, etc)
    params = community_simulator.usertools.MakeParams(assumptions)

    # dynamics
    def dNdt(N,R,params):
        return community_simulator.usertools.MakeConsumerDynamics(assumptions)(N,R,params)
    def dRdt(N,R,params):
        return community_simulator.usertools.MakeResourceDynamics(assumptions)(N,R,params)

    dynamics = [dNdt,dRdt]

    # %%
    ### SEED AND STABILIZE COMMUNITIES

    # make species pools
    pool_overlap = 0.0 # 0: no overlap, 1: full overlap (same pool)
    n_species = sum(assumptions['SA']) + assumptions['Sgen']
    n_pool = int(int(n_species/2) + pool_overlap*int(n_species/2))

    n = list(range(n_species))
    random.shuffle(n)

    pools = [n[:n_pool],
             n[-n_pool:]]
    pools[0].sort()
    pools[1].sort()

    # function to sample from pool
    def sampleFromPool(pool_number):
        N = np.zeros((n_species,assumptions['n_wells']))
        for i in range(assumptions['n_wells']):
            Ni = random.sample(pools[pool_number],assumptions['S'])
            N[Ni,i] = 1
        return N

    # generic initial state (will modify this later for specific plate settings)
    init_state = community_simulator.usertools.MakeInitialState(assumptions)

    # initialize resident plate
    N0_resident = init_state[0]
    N0_resident.loc[:,:] = sampleFromPool(0)
    init_state_resident = (N0_resident,
                           init_state[1])
    resident_plate = community_simulator.Community(init_state_resident,
                                                   dynamics,
                                                   params,
                                                   parallel=par,
                                                   scale=1e6)

    # initialize invasive plate
    N0_invasive = init_state[0]
    N0_invasive.loc[:,:] = sampleFromPool(1)
    init_state_invasive = (N0_invasive,
                           init_state[1])
    invasive_plate = community_simulator.Community(init_state_invasive,
                                                   dynamics,
                                                   params,
                                                   parallel=par,
                                                   scale=1e6)

    # stabilize plates
    N_resident, R_resident = stabilizeCommunities(resident_plate)
    N_invasive, R_invasive = stabilizeCommunities(invasive_plate)

    # make and stabilize coalescence plate
    N0_coalescence = (N_resident + N_invasive)/2*(1/100) # added dilution factor
    init_state_coalescence = (N0_coalescence,
                              init_state[1]) # resource init_state same as before
    coalescence_plate = community_simulator.Community(init_state_coalescence,
                                                      dynamics,
                                                      params,
                                                      parallel=par,
                                                      scale=1e6)
    N_coalescence, R_coalescence = stabilizeCommunities(coalescence_plate)

    # %%
    ### MONOCULTURES

    # find most abundant species in each resident/invasive community
    dominants_resident = list(list(zip(*N_resident.idxmax()))[1])
    dominants_invasive = list(list(zip(*N_invasive.idxmax()))[1])
    all_species = list(list(zip(*N_resident.index))[1])

    # initialize resident monoculture plate
    init_state_monoculture = community_simulator.usertools.MakeInitialState(assumptions)
    N0_monoculture_resident = init_state_monoculture[0]
    N0_monoculture_resident.iloc[:,:] = 0
    N0_monoculture_invasive = copy.deepcopy(N0_monoculture_resident)

    # set dominant species' abundance to 1 for each well
    for i in range(len(dominants_resident)):
        n = np.where(np.isin(all_species,dominants_resident[i]))[0][0]
        N0_monoculture_resident.iloc[n,i] = 1

    for i in range(len(dominants_invasive)):
        n = np.where(np.isin(all_species,dominants_invasive[i]))[0][0]
        N0_monoculture_invasive.iloc[n,i] = 1

    # make plates
    init_state_monoculture_resident = (N0_monoculture_resident,
                                       init_state_monoculture[1])
    monoculture_resident_plate = community_simulator.Community(init_state_monoculture_resident,
                                                               dynamics,
                                                               params,
                                                               parallel=par,
                                                               scale=1e6)

    init_state_monoculture_invasive = (N0_monoculture_invasive,
                                       init_state_monoculture[1])
    monoculture_invasive_plate = community_simulator.Community(init_state_monoculture_invasive,
                                                               dynamics,
                                                               params,
                                                               parallel=par,
                                                               scale=1e6)

    # stabilize monocultures (we just allow to grow for a single cycle)
    monoculture_resident_plate.Propagate(1,compress_resources=False,compress_species=True)
    monoculture_invasive_plate.Propagate(1,compress_resources=False,compress_species=True)

    # %%
    ### PAIRWISE COMPETITION

    # make plate (rescale so most abundant species at t=0 has abundance 1 (times the scaling factor of the plate))
    N0_pairwise = (monoculture_resident_plate.N + monoculture_invasive_plate.N)
    N0_pairwise = N0_pairwise/N0_pairwise.max()
    init_state_pairwise = (N0_pairwise,
                           init_state[1])
    pairwise_plate = community_simulator.Community(init_state_pairwise,
                                                   dynamics,
                                                   params,
                                                   parallel=par,
                                                   scale=1e6)

    # stabilize
    N_pairwise, R_pairwise = stabilizeCommunities(pairwise_plate)

    # %%
    ### STATISTICS

    # get relative similarity between invasive and coalesced communities
    q = ['NA' for i in range(assumptions['n_wells'])]

    for i in range(assumptions['n_wells']):

        # community compositions as lists
        R = N_resident.iloc[:,i].tolist()
        I = N_invasive.iloc[:,i].tolist()
        C = N_coalescence.iloc[:,i].tolist()

        q[i] = mysim(I,R,C)

    # get fraction in pairwise competition
    f_pairwise = ['NA' for i in range(assumptions['n_wells'])]
    f_pairwise_null = ['NA' for i in range(assumptions['n_wells'])]
    F = N_pairwise/N_pairwise.sum()
    F_null = N0_pairwise/N0_pairwise.sum()
    F = F.droplevel(0)
    F_null = F_null.droplevel(0)
    for i in range(assumptions['n_wells']):
        f_pairwise[i] = F.loc[dominants_invasive[i]][i]
        f_pairwise_null[i] = F_null.loc[dominants_invasive[i]][i]

    # %%
    ### FIT AND SAVE R^2

    r_squared[n_sim] = np.corrcoef(f_pairwise,q)[0,1]**2


from logging import getLogger

import reversion

from datahub.dbmaintenance.management.base import CSVBaseCommand
from datahub.dbmaintenance.utils import parse_bool, parse_uuid, parse_uuid_list
from datahub.investment.project.models import InvestmentProject

logger = getLogger(__name__)

class Command(CSVBaseCommand):
    """
    Command to update investment_project.uk_region_locations and
    investment_project.allow_blank_possible_uk_regions.
    """

    def add_arguments(self, parser):
        """Define extra arguments."""
        super().add_arguments(parser)

        parser.add_argument(
            '--ignore-old-regions',
            action='store_true',
            default=False,
            help='If True it does not check if the regions match the old_uk_region_locations '
                 'column.',
        )

    def _process_row(self, row, simulate=False, ignore_old_regions=False, **options):
        """Process one single row."""
        pk = parse_uuid(row['id'])
        investment_project = InvestmentProject.objects.get(pk=pk)
        allow_blank_possible_uk_regions = parse_bool(row['allow_blank_possible_uk_regions'])
        uk_region_locations = parse_uuid_list(row['uk_region_locations'])

        current_regions = investment_project.uk_region_locations.all()
        current_region_ids = set(region.pk for region in current_regions)
        if (investment_project.allow_blank_possible_uk_regions == allow_blank_possible_uk_regions
                and current_region_ids == set(uk_region_locations)):
            return

        if not ignore_old_regions:
            old_uk_region_locations = parse_uuid_list(row['old_uk_region_locations'])

            if current_region_ids != set(old_uk_region_locations):
                return

        investment_project.allow_blank_possible_uk_regions = allow_blank_possible_uk_regions

        if simulate:
            return

        with reversion.create_revision():
            investment_project.save(
                update_fields=('allow_blank_possible_uk_regions',),
            )
            investment_project.uk_region_locations.set(uk_region_locations)
            reversion.set_comment('Possible UK regions data migration correction.')


# -*- coding: utf-8 -*-
from json import dumps
import requests

BASE = "http://localhost:8080/"
HEADER = {'content-type': 'application/json'}

def alta_habitacion():
    print("Introduce el número de plazas de la habitación:")
    while True:
        try:
            plazas = int(input("> "))
            if plazas >= 1:
                break
            else:
                print("El numero de plazas tiene que ser mayor que 0")
        except ValueError:
            print("Debe introducir números.")
    equipamiento = []
    print("Introduce el quipamiento de la habitación ('Salir' para seguir con el procedimiento):")
    while True:
        nuevo_equipamiento = input("> ")
        if nuevo_equipamiento == "Salir":
            break;
        else:
            equipamiento.append(nuevo_equipamiento)
    print("Introduce el precio por noche de la habitacion:")
    while True:
        try:
            precio = int(input("> "))
            if precio >= 0:
                break
            else:
                print("El precio no puede ser negativo.")
        except ValueError:
            print("Debe introducir números.")
    payload = {'plazas': plazas, 'equipamiento': equipamiento, 'precio': precio}
    r = requests.post(BASE, data=dumps(payload), headers=HEADER)
    if r.status_code == 400:
        print(r.json()['error_description'])
    else:
        print("Habitación creada correctamente.")
    iniciar_seleccion()

def borrar_habitacion():
    print("Introduce la ID de la habitación que desea borrar:")
    while True:
        try:
            id_habitacion = int(input("> "))
            if id_habitacion >= 1:
                break
            else:
                print("La ID tiene que ser mayor que 0")
        except ValueError:
            print("Debe introducir números.")
    r = requests.delete(BASE + id_habitacion.__str__(), headers=HEADER)
    if r.status_code == 409 or r.status_code == 404:
        print(r.json()['error_description'])
    else:
        print("Habitación eliminada correctamente.")
    iniciar_seleccion()

def modificar_habitacion():
    print("Introduce la ID de la habitación que desea modificar:")
    while True:
        try:
            id_habitacion = int(input("> "))
            if id_habitacion >= 1:
                break
            else:
                print("La ID tiene que ser mayor que 0")
        except ValueError:
            print("Debe introducir números.")
    r = requests.get(BASE + id_habitacion.__str__(), headers=HEADER)
    if r.status_code == 404:
        print(r.json()['error_description'])
    else:
        print("Elige que opción deseas realizar: ")
        print("     1. Modificar plazas.")
        print("     2. Modificar equipamiento.")
        print("     3. Modificar precio.")
        print("     4. Modificar disponibilidad.")
        print("     5. Salir.")
        while True:
            try:
                opcionseleccionada = int(input("> "))
                if opcionseleccionada >= 1 and opcionseleccionada <= 5:
                    if opcionseleccionada == 1:
                        print("Introduce el nuevo número de plazas de la habitación:")
                        while True:
                            try:
                                plazas = int(input("> "))
                                if plazas >= 1:
                                    break
                                else:
                                    print("El numero de plazas tiene que ser mayor que 0")
                            except ValueError:
                                print("Debe introducir números.")
                        y = requests.put(BASE + id_habitacion.__str__() + '/plazas', params={'plazas': plazas}, headers=HEADER)
                        if y.status_code == 409:
                            print(y.json()['error_description'])
                        else:
                            print("El valor plazas se cambio correctamente.")
                    if opcionseleccionada == 2:
                        print("Elige que opción deseas realizar: ")
                        print("     1. Reemplazar listado completo de equipamiento.")
                        print("     2. Añadir equipamiento.")
                        print("     3. Eliminar equipamiento.")
                        print("     4. Salir.")
                        while True:
                            try:
                                opcionmodificar = int(input("> "))
                                if opcionmodificar >= 1 and opcionmodificar <= 3:
                                    equipamiento = []
                                    print("Introduce el quipamiento de la habitación ('Salir' para seguir con el procedimiento):")
                                    while True:
                                        nuevo_equipamiento = input("> ")
                                        if nuevo_equipamiento == "Salir":
                                            break;
                                        else:
                                            equipamiento.append(nuevo_equipamiento)
                                    datos = {'equipamiento': equipamiento}
                                    if opcionmodificar == 1:
                                        y = requests.put(BASE + id_habitacion.__str__() + '/equipamiento', data=dumps(datos), headers=HEADER)
                                        if y.status_code == 409:
                                            print(y.json()['error_description'])
                                        else:
                                            print("El equipamiento se cambio correctamente")
                                    if opcionmodificar == 2:
                                        y = requests.put(BASE + id_habitacion.__str__() + '/equipamiento/add', data=dumps(datos), headers=HEADER)
                                        if y.status_code == 409:
                                            print(y.json()['error_description'])
                                        else:
                                            print("El nuevo equipamiento se añadio correctamente")
                                    if opcionmodificar == 3:
                                        y = requests.put(BASE + id_habitacion.__str__() + '/equipamiento/eliminar', data=dumps(datos), headers=HEADER)
                                        if y.status_code == 409:
                                            print(y.json()['error_description'])
                                        else:
                                            print("El equipamiento se elimino correctamente")
                                    break
                                else:
                                    if opcionmodificar == 4:
                                        print("Opción seleccionada: Salir.")
                                        break
                                    else:
                                        print("Opción no valida")
                                    break
                                break
                            except ValueError:
                                print("Debe introducir números.")
                    if opcionseleccionada == 3:
                        print("Introduce el nuevo precio por noche de la habitacion:")
                        while True:
                            try:
                                precio = int(input("> "))
                                if precio >= 0:
                                    break
                                else:
                                    print("El precio no puede ser negativo.")
                            except ValueError:
                                print("Debe introducir números.")
                        y = requests.put(BASE + id_habitacion.__str__() + '/precio', params={'precio': precio}, headers=HEADER)
                        if y.status_code == 409:
                            print(y.json()['error_description'])
                        else:
                            print("El precio se cambio correctamente.")
                    if opcionseleccionada == 4:
                        if r.json()['disponible']:
                            y = requests.put(BASE + id_habitacion.__str__() + '/disponibilidad', params={'disponible': False},headers=HEADER)
                        else:
                            y = requests.put(BASE + id_habitacion.__str__() + '/disponibilidad', params={'disponible': True},headers=HEADER)
                        print("La disponibilidad de la habitación se cambio correctamente.")
                    if opcionseleccionada == 5:
                        print("Opción seleccionada: Salir.")
                        break
                    break
                else:
                    print("Opción no valida")
            except ValueError:
                print("Debe introducir números.")
    iniciar_seleccion()

def consultar_habitaciones():
    r = requests.get(BASE, headers=HEADER)
    if r.status_code == 204:
        print("No existen habitaciones registradas en el sistema.")
    else:
        for i in r.json():
            print("ID de la habitación: ", r.json()[i]['id'])
            print("     Plazas: ", r.json()[i]['plazas'])
            print("     Precio: ", r.json()[i]['precio'])
            print("     Equipamiento: ")
            for j in r.json()[i]['equipamiento']:
                print("         ", j)
            print("     Disponible: ", r.json()[i]['disponible'])
    iniciar_seleccion()

def consultar_habitacion():
    print("Introduce la ID de la habitación que desea consultar:")
    while True:
        try:
            id_habitacion = int(input("> "))
            if id_habitacion >= 1:
                break
            else:
                print("La ID tiene que ser mayor que 0")
        except ValueError:
            print("Debe introducir números.")
    r = requests.get(BASE + id_habitacion.__str__(), headers=HEADER)
    if r.status_code == 404:
        print(r.json()['error_description'])
    else:
        print("ID de la habitación: ", r.json()['id'])
        print("     Plazas: ", r.json()['plazas'])
        print("     Precio: ", r.json()['precio'])
        print("     Equipamiento: ")
        for i in r.json()['equipamiento']:
            print("         ", i)
        print("     Disponible: ", r.json()['disponible'])
    iniciar_seleccion()

def consultar_habitaciones_ocupadas():
    r = requests.get(BASE+'ocupadas', headers=HEADER)
    for i in r.json():
        print("ID de la habitación: ", r.json()[i]['id'])
        print("     Plazas: ", r.json()[i]['plazas'])
        print("     Precio: ", r.json()[i]['precio'])
        print("     Equipamiento: ")
        for j in r.json()[i]['equipamiento']:
            print("         ", j)
        print("     Disponible: ", r.json()[i]['disponible'])
    iniciar_seleccion()

def consultar_habitaciones_desocupadas():
    r = requests.get(BASE+'disponibles', headers=HEADER)
    for i in r.json():
        print("ID de la habitación: ", r.json()[i]['id'])
        print("     Plazas: ", r.json()[i]['plazas'])
        print("     Precio: ", r.json()[i]['precio'])
        print("     Equipamiento: ")
        for j in r.json()[i]['equipamiento']:
            print("         ", j)
        print("     Disponible: ", r.json()[i]['disponible'])
    iniciar_seleccion()

def iniciar_seleccion():
    print("Elige que opción deseas realizar: ")
    print("     1. Dar de alta una habitación.")
    print("     2. Eliminar una habitación.")
    print("     3. Modificar los datos de una habitación.")
    print("     4. Consultar la lista completa de habitaciones.")
    print("     5. Consultar una habitación mediante identificador.")
    print("     6. Consultar la lista de habitaciones ocupadas.")
    print("     7. Consultar la lista de habitaciones desocupadas.")
    print("     8. Salir.")
    while True:
        try:
            opcionseleccionada = int(input("> "))
            if opcionseleccionada >=1 and opcionseleccionada <=8:
                if opcionseleccionada == 1:
                    alta_habitacion()
                if opcionseleccionada == 2:
                    borrar_habitacion()
                if opcionseleccionada == 3:
                    modificar_habitacion()
                if opcionseleccionada == 4:
                    consultar_habitaciones()
                if opcionseleccionada == 5:
                    consultar_habitacion()
                if opcionseleccionada == 6:
                    consultar_habitaciones_ocupadas()
                if opcionseleccionada == 7:
                    consultar_habitaciones_desocupadas()
                if opcionseleccionada == 8:
                    print("Opción seleccionada: Salir.")
                    break
                break
            else:
                print("Opción no valida")
        except ValueError:
            print("Debe introducir números.")

iniciar_seleccion()


import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.models as models
from models.metrics import AdaCos
#from models.osnet import osnet_ibn_x1_0
from models.osnet import osnet_x1_0
from models.metrics import ArcMarginProduct
import pytorch_pfn_extras as ppe
import numpy as np

class Model(nn.Module):
    def __init__(self, n_classes, feature_extractor=None, metric_learning=False, emb_dims=512):
        super().__init__()
        self.ef = True if 'efficient' in feature_extractor else False
        self.ef_rb = True if 'rfb_ef' in feature_extractor else False
        self.ef_as = True if 'as_ef' in feature_extractor else False
        self.osnet = True if 'osnet' in feature_extractor else False
        self.metric_learning = metric_learning
        ef_last_ch = {'efficientnet-b0':1280, 'efficientnet-b1':1280,'efficientnet-b2':1408,
                      'efficientnet-b3':1536, 'efficientnet-b4':1792, 'efficientnet-b5':2048}
        if self.ef:
            from efficientnet_pytorch import EfficientNet
            self.features = EfficientNet.from_pretrained(feature_extractor)
            #last_ch = 1280
            last_ch = ef_last_ch[feature_extractor]
        elif self.osnet:
            self.features = osnet_x1_0(pretrained=True)
            last_ch = 1000
        elif self.ef_rb:
            feature_extractor = 'efficientnet-{}'.format(feature_extractor.split('-')[-1])
            from models.efficientnet import EfficientNetRFB
            #self.features = EfficientNetRFB(feature_extractor, output_blocks_ind=[2, 4, 10, 15])
            self.features = EfficientNetRFB(feature_extractor, output_blocks_ind=[5, 9, 15,21,26,31])
            last_ch = ef_last_ch[feature_extractor]
        elif self.ef_as:
            feature_extractor = 'efficientnet-{}'.format(feature_extractor.split('-')[-1])
            from models.aspp import ASPP
            from models.efficientnet import EfficientNetASPP
            last_ch = ef_last_ch[feature_extractor]
            aspp = ASPP(output_stride=32, BatchNorm=nn.BatchNorm2d, inplanes=last_ch)
            self.features = EfficientNetASPP(feature_extractor, aspp)
        else:
            self.features = models.resnet34(pretrained=True)
            last_ch = 1000
        if not metric_learning:
            fc_ch = 1280
            if last_ch == 2048:
                fc_ch = 1680
            self.fc0 = nn.Linear(last_ch, fc_ch)
            self.fc1 = nn.Linear(fc_ch, n_classes)
        else:
            self.bn0 = nn.BatchNorm2d(last_ch)
            self.drop_out = nn.Dropout()
            self.fc = nn.Linear(last_ch * 16 * 16, 512)
            self.bn1 = nn.BatchNorm1d(512)
            self.emb = AdaCos(512, 4)
            #self.emb = ArcMarginProduct(512, 4)
        self.last_ch = last_ch

    def forward(self, x, t=None):
        if self.ef or self.ef_rb or self.ef_as:
            if self.ef:
                h = self.features.extract_features(x)
            else:
                h = self.features(x)
            if not self.metric_learning:
                h = F.avg_pool2d(h, h.size()[2:]).reshape((-1, self.last_ch))
        else:
            h = self.features(x)

        if not self.metric_learning:
            h = self.fc0(h)
            h = self.fc1(h)
        else:
            h = self.bn0(h)
            h = self.drop_out(h)
            h = h.view(h.size(0), -1)
            h = self.fc(h)
            h = self.bn1(h)
            #if t is not None:
            h = self.emb(h, t)
        return h


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

"""
Prepare dataset for bayesian variable selection.
"""

import argparse
import logging

import pandas as pd
from sklearn.pipeline import Pipeline

from src.common.df_csv_writing import write_df_to_csv
from src.common.json_logging import setup_logging
from src.data.imputations import impute
from src.data.imputations.impute import DataImputer
from src.data.na_thresholding.transformer import ThresholdingMissingDataColumnDropper, ValuePresenceRowFilter

setup_logging("src/common/logging.json")  # setup logger
logger = logging.getLogger(__name__)

if __name__ == "__main__":

    # configure parser and parse arguments
    parser = argparse.ArgumentParser(description='Prepare dataset for bayesian variable selection.')
    parser.add_argument('--dataset', type=str, help='The path to the dataset file', required=True)
    parser.add_argument('--csv_separator', type=str, default=',', help='The separator of the data columns')
    parser.add_argument('--diagnosis_indicator_column', action='append', help='Typical measurements done when evaluating the plausibility of a certain diagnosis. Repeat flag to provide multiple columns')
    parser.add_argument('--skip_na_drop_threshold', action='store_true', help='Skip drop thresholding')
    parser.add_argument('--na_drop_threshold', type=float, default=1.0, help='Minimum amount of NA in column for it to be dropped [0;1]')
    parser.add_argument('--skip_imputation', action='store_true', help='Skip imputation')
    parser.add_argument('--imputation_type', type=str, default='ITERATIVE', help='The type of imputation to perform')
    parser.add_argument('--niter', type=int, default=1000, help='The number of iterations to perform, in case an interative method is chosen')

    args = parser.parse_args()
    dataset_path = args.dataset
    csv_separator = args.csv_separator
    indicator_columns = args.diagnosis_indicator_column
    do_na_drop_threshold = not args.skip_na_drop_threshold
    na_drop_threshold = args.na_drop_threshold
    do_imputation = not args.skip_imputation
    imputation_type = impute.ImputationType[args.imputation_type]
    iteration_qty = args.niter

    logger.info(str({"message": "PREPARE NEW DATASET",
                     "path": dataset_path,
                     "imputation_type": imputation_type,
                     "iteration_qty": iteration_qty})
                )

    file_appendix = ''  # prepare file appendix telling us what has been done

    logger.info(str({"message": "Load dataset",
                     "path": dataset_path}))
    df = pd.read_csv(dataset_path, header=0, sep=csv_separator)  # read data from csv

    # prepare pipeline and run it
    components = []
    if indicator_columns is not [] and indicator_columns is not None:
        components.append(('filter_by_indicator_columns', ValuePresenceRowFilter(required_columns=indicator_columns)))
    file_appendix += '_indicatorCols_' + str(indicator_columns)

    if do_na_drop_threshold:
        components.append(('drop_above_threshold_na_columns', ThresholdingMissingDataColumnDropper(na_drop_threshold=na_drop_threshold)))
        file_appendix += '_naLimit_' + str(na_drop_threshold)

    if do_imputation:
        components.append(('impute_missing_values', DataImputer(imputation_type=imputation_type, iteration_qty=iteration_qty)))
        file_appendix += '_impType_' + imputation_type.name + '_nIter_' + str(iteration_qty)

    pipeline = Pipeline(components)
    df = pipeline.fit_transform(df)  # apply pipeline

    logger.info({"message": "Results",
                 "dataset dimensions":
                 df.shape})

    # write dataset to file
    write_df_to_csv(df=df, store_path='data/interim/', initial_path=dataset_path, file_appendix=file_appendix)


#!/usr/bin/env python3
import argparse
import collections
import re
import sys

parser = argparse.ArgumentParser()
parser.add_argument("train_data", type=str, help="Training data file")
args = parser.parse_args()

lemma_strip_re = re.compile(r"(?<=.)(?:`|_|-[^0-9]).*$")
def lemma_tag_key(lemma, tag):
    return lemma_strip_re.sub("", lemma) + "\t" + tag

train = collections.defaultdict(lambda: dict())
with open(args.train_data, mode="r", encoding="utf-8") as train_file:
    for line in train_file:
        line = line.rstrip("\n")
        if not line: continue

        form, lemma, tag, *_ = line.split("\t")
        train[form][lemma_tag_key(lemma, tag)] = (lemma, tag)

for line in sys.stdin:
    line = line.rstrip("\n")

    if not line:
        print()
    else:
        form, lemma, tag, *analyses = line.split("\t")
        assert len(analyses) % 2 == 0

        if len(analyses):
            train_analyses = list(train.get(form, {}).items()) \
                + list(train.get(form[0] + form[1:].lower(), {}).items()) \
                + list(train.get(form.lower(), {}).items())
            if train_analyses:
                analyses_set = set(lemma_tag_key(analyses[i], analyses[i + 1]) for i in range(0, len(analyses), 2))
                for key, lemma_tag in train_analyses:
                    if key not in analyses_set:
                        analyses.extend(lemma_tag)
                        analyses_set.add(key)

        print("\t".join([form, lemma, tag] + analyses))