# Main container for all decks
#
import os
import threading
import logging
import pickle
from queue import Queue
import pkg_resources

from PIL import Image, ImageFont

from cockpitdecks import __version__, LOGFILE, FORMAT
from cockpitdecks import ID_SEP, SPAM, SPAM_LEVEL, ROOT_DEBUG
from cockpitdecks import CONFIG_FOLDER, CONFIG_FILE, SECRET_FILE, EXCLUDE_DECKS, ICONS_FOLDER, FONTS_FOLDER, RESOURCES_FOLDER
from cockpitdecks import DEFAULT_SYSTEM_FONT, DEFAULT_LABEL_FONT, DEFAULT_LABEL_SIZE, DEFAULT_LABEL_COLOR, DEFAULT_LABEL_POSITION
from cockpitdecks import DEFAULT_ICON_NAME, DEFAULT_ICON_COLOR, DEFAULT_ICON_TEXTURE, DEFAULT_LOGO, DEFAULT_WALLPAPER
from cockpitdecks import DEFAULT_ANNUNCIATOR_COLOR, ANNUNCIATOR_STYLES, DEFAULT_ANNUNCIATOR_STYLE, HOME_PAGE
from cockpitdecks import COCKPIT_COLOR, COCKPIT_TEXTURE
from cockpitdecks import Config
from cockpitdecks.resources.color import convert_color, has_ext
from cockpitdecks.simulator import DatarefListener
from cockpitdecks.decks import DECK_TYPES

logging.addLevelName(SPAM_LEVEL, SPAM)
logger = logging.getLogger(__name__)
# logger.setLevel(logging.DEBUG)

if LOGFILE is not None:
    formatter = logging.Formatter(FORMAT)
    handler = logging.FileHandler(
        LOGFILE, mode="a"
    )
    handler.setFormatter(formatter)
    logger.addHandler(handler)

class CockpitBase:
    """As used in Simulator

    [description]
    """
    def __init__(self):
        pass

    def set_logging_level(self, name):
        if name in self._debug:
            l = logging.getLogger(name)
            if l is not None:
                l.setLevel(logging.DEBUG)
                l.info(f"set_logging_level: {name} set to debug")
            else:
                logger.warning(f"logger {name} not found")

    def reload_pages(self):
        pass

class Cockpit(DatarefListener, CockpitBase):
    """
    Contains all deck configurations for a given aircraft.
    Is started when aicraft is loaded and aircraft contains CONFIG_FOLDER folder.
    """
    def __init__(self, simulator):
        CockpitBase.__init__(self)
        DatarefListener.__init__(self)
        self._debug = ROOT_DEBUG.split(",")   # comma separated list of module names like cockpitdecks.page or cockpitdeck.button_ext

        self._config = None          # content of aircraft/deckconfig/config.yaml
        self.default_config = None   # content of resources/config.yaml

        self.name = "Cockpitdecks"
        self.icao = "ZZZZ"

        self.sim = simulator(self)

        self.disabled = False
        self.default_pages = None      # for debugging

        self.has_reload = False
        self.reload_loop_run = False
        self.reload_loop_thread = None
        self.reload_queue = Queue()

        self.devices = []

        self.acpath = None
        self.cockpit = {}  # all decks: { deckname: deck }

        self.default_logo = DEFAULT_LOGO
        self.default_wallpaper = DEFAULT_WALLPAPER

        self.fonts = {}
        self.default_font = None
        self.default_label_font = DEFAULT_LABEL_FONT
        self.default_label_size = DEFAULT_LABEL_SIZE
        self.default_label_color = convert_color(DEFAULT_LABEL_COLOR)
        self.default_label_position = DEFAULT_LABEL_POSITION

        self.icon_folder = None
        self.icons = {}
        self.cache_icon = False
        self.default_icon_name = DEFAULT_ICON_NAME
        self.default_icon_color = convert_color(DEFAULT_ICON_COLOR)
        self.default_icon_texture = DEFAULT_ICON_TEXTURE
        self.default_annun_texture = None
        self.default_annun_color = convert_color(DEFAULT_ANNUNCIATOR_COLOR)

        self.fill_empty_keys = True
        self.annunciator_style = DEFAULT_ANNUNCIATOR_STYLE
        self.cockpit_color = convert_color(COCKPIT_COLOR)
        self.cockpit_texture = COCKPIT_TEXTURE

        self.default_home_page_name = HOME_PAGE

        self.init()

    def init(self):
        """
        Loads all devices connected to this computer.
        """
        self.scan_devices()

    def get_id(self):
        return self.name

    def get_button_value(self, name):
        a = name.split(ID_SEP)
        if len(a) > 0:
            if a[0] == self.name:
                if a[1] in self.cockpit.keys():
                    return self.cockpit[a[1]].get_button_value(ID_SEP.join(a[1:]))
                else:
                    logger.warning(f"so such deck {a[1]}")
            else:
                logger.warning(f"no such cockpit {a[0]}")
        else:
            logger.warning(f"invalid name {name}")
        return None

    def inspect(self, what: str = None):
        """
        This function is called on all instances of Deck.
        """
        logger.info(f"Cockpitdecks Rel. {__version__} -- {what}")

        if "thread" in what:
            logger.info(f"{[(t.name,t.isDaemon(),t.is_alive()) for t in threading.enumerate()]}")
        elif what.startswith("datarefs"):
            self.inspect_datarefs(what)
        elif what == "monitored":
            self.inspect_monitored(what)
        else:
            for v in self.cockpit.values():
                v.inspect(what)

    def inspect_datarefs(self, what: str = None):
        if what.startswith("datarefs"):
            for dref in self.sim.all_datarefs.values():
                logger.info(f"{dref.path} = {dref.value()} ({len(dref.listeners)})")
                if what.endswith("listener"):
                    for l in dref.listeners:
                        logger.info(f"  {l.name}")
        else:
            logger.info(f"to do")

    def inspect_monitored(self, what: str = None):
        for dref in self.sim.datarefs.values():
            logger.info(f"{dref}")

    def scan_devices(self):
        if len(DECK_TYPES) == 0:
            logger.error(f"no driver")
            return
        logger.info(f"drivers installed for {', '.join([f'{decktype} {pkg_resources.get_distribution(decktype).version}' for decktype in DECK_TYPES.keys()])}; scanning..")
        for decktype, builder in DECK_TYPES.items():
            decks = builder[1]().enumerate()
            logger.info(f"found {len(decks)} {decktype}") # " ({decktype} {pkg_resources.get_distribution(decktype).version})")
            for name, device in enumerate(decks):
                device.open()
                serial = device.get_serial_number()
                device.close()
                if serial in EXCLUDE_DECKS:
                    logger.warning(f"deck {serial} excluded")
                    del decks[name]
                self.devices.append({
                    "type": decktype,
                    "device": device,
                    "serial_number": serial
                })
            logger.debug(f"using {len(decks)} {decktype}")
        logger.debug(f"..scanned")

    def get_device(self, req_serial: str, req_type: str):
        """
        Get a HIDAPI device for the supplied serial number.
        If found, the device is opened and reset and returned open.

        :param    req_serial:  The request serial
        :type      req_serial:  str
        """
        # No serial, return deck if only one deck of that type
        if req_serial is None:
            i = 0
            good = None
            for deck in self.devices:
                if deck["type"] == req_type:
                    good = deck
                    i = i + 1
            if i == 1 and good is not None:
                logger.debug(f"only one deck of type {req_type}, returning it")
                device = good["device"]
                device.open()
                if device.is_visual():
                    image_format = device.key_image_format()
                    logger.debug(f"key images: {image_format['size'][0]}x{image_format['size'][1]} pixels, {image_format['format']} format, rotated {image_format['rotation']} degrees")
                else:
                    logger.debug(f"no visual")
                device.reset()
                return device
            else:
                if i > 1:
                    logger.warning(f"more than one deck of type {req_type}, no serial to disambiguate")
            return None
        ## Got serial, search for it
        for deck in self.devices:
            if deck["serial_number"] == req_serial:
                device = deck["device"]
                device.open()
                if device.is_visual():
                    image_format = device.key_image_format()
                    logger.debug(f"key images: {image_format['size'][0]}x{image_format['size'][1]} pixels, {image_format['format']} format, rotated {image_format['rotation']} degrees")
                else:
                    logger.debug(f"no visual")
                device.reset()
                return device
        logger.warning(f"deck {req_serial} not found")
        return None

    def start_aircraft(self, acpath: str):
        """
        Loads decks for aircraft in supplied path and start listening for key presses.
        """
        self.load_aircraft(acpath)
        self.run()

    def load_aircraft(self, acpath: str):
        """
        Loads decks for aircraft in supplied path.
        """
        if self.disabled:
            logger.warning(f"Cockpitdecks is disabled")
            return
        # Reset, if new aircraft
        if len(self.cockpit) > 0:
            self.terminate_aircraft()
            self.sim.clean_datarefs_to_monitor()
            logger.warning(f"{os.path.basename(self.acpath)} unloaded")

        if len(self.devices) == 0:
            logger.warning(f"no device")
            return

        self.cockpit = {}
        self.icons = {}
        # self.fonts = {}
        self.acpath = None

        self.load_defaults()

        if acpath is not None and os.path.exists(os.path.join(acpath, CONFIG_FOLDER)):
            self.acpath = acpath
            self.load_icons()
            self.load_fonts()
            self.create_decks()
            self.load_pages()
        else:
            if acpath is None:
                logger.error(f"no aircraft folder")
            elif not os.path.exists(acpath):
                logger.error(f"no aircraft folder {acpath}")
            else:
                logger.error(f"no Cockpitdecks folder '{CONFIG_FOLDER}' in aircraft folder {acpath}")
            self.create_default_decks()

    def load_pages(self):
        if self.default_pages is not None:
            logger.debug(f"default_pages {self.default_pages.keys()}")
            for name, deck in self.cockpit.items():
                if name in self.default_pages.keys():
                    if self.default_pages[name] in deck.pages.keys() and deck.home_page is not None:  # do not refresh if no home page loaded...
                        deck.change_page(self.default_pages[name])
                    else:
                        deck.change_page()
            self.default_pages = None
        else:
            for deck in self.cockpit.values():
                deck.change_page()

    def reload_pages(self):
        for name, deck in self.cockpit.items():
            deck.reload_page()

    def load_defaults(self):
        """
        Loads default values for font, icon, etc. They will be used if no layout is found.
        """
        def locate_font(fontname: str) -> str:
            if fontname in self.fonts.keys():
                logger.debug(f"font {fontname} already loaded")
                return fontname

            # 1. Try "system" font
            try:
                test = ImageFont.truetype(fontname, self.default_label_size)
                logger.debug(f"font {fontname} found in computer system fonts")
                return fontname
            except:
                logger.debug(f"font {fontname} not found in computer system fonts")

            # 2. Try font in resources folder
            fn = None
            try:
                fn = os.path.join(os.path.dirname(__file__), RESOURCES_FOLDER, fontname)
                test = ImageFont.truetype(fn, self.default_label_size)
                logger.debug(f"font {fontname} found locally ({RESOURCES_FOLDER} folder)")
                return fn
            except:
                logger.debug(f"font {fontname} not found locally ({RESOURCES_FOLDER} folder)")

            # 3. Try font in resources/fonts folder
            fn = None
            try:
                fn = os.path.join(os.path.dirname(__file__), RESOURCES_FOLDER, FONTS_FOLDER, fontname)
                test = ImageFont.truetype(fn, self.default_label_size)
                logger.debug(f"font {fontname} found locally ({FONTS_FOLDER} folder)")
                return fn
            except:
                logger.debug(f"font {fontname} not found locally ({FONTS_FOLDER} folder)")

            logger.debug(f"font {fontname} not found")
            return None

        # 0. Some variables defaults
        fn = os.path.join(os.path.dirname(__file__), RESOURCES_FOLDER, CONFIG_FILE)
        self.default_config = Config(fn)

        # Load global defaults from resources/config.yaml file or use application default
        self._debug = self.default_config.get("debug", ",".join(self._debug)).split(",")
        self.set_logging_level(__name__)

        self.sim.set_roundings(self.default_config.get("roundings", {}))
        self.sim.set_slow_datarefs(self.default_config.get("slow-datarefs", {}))

        self.default_logo = self.default_config.get("default-logo", DEFAULT_LOGO)
        self.default_wallpaper = self.default_config.get("default-wallpaper", DEFAULT_WALLPAPER)
        self.default_label_font = self.default_config.get("default-label-font", DEFAULT_LABEL_FONT)
        self.default_label_size = self.default_config.get("default-label-size", DEFAULT_LABEL_SIZE)
        self.default_label_color = self.default_config.get("default-label-color", convert_color(DEFAULT_LABEL_COLOR))
        self.default_label_position = self.default_config.get("default-label-position", DEFAULT_LABEL_POSITION)
        self.cache_icon = self.default_config.get("cache-icon", self.cache_icon)
        self.default_icon_name = DEFAULT_ICON_NAME
        self.default_icon_texture = self.default_config.get("default-icon-texture", DEFAULT_ICON_TEXTURE)
        self.default_icon_color = self.default_config.get("default-icon-color", DEFAULT_ICON_COLOR)
        self.default_icon_color = convert_color(self.default_icon_color)
        self.default_annun_texture = self.default_config.get("default-annunciator-texture")
        self.default_annun_color = self.default_config.get("default-annunciator-color", DEFAULT_ANNUNCIATOR_COLOR)
        self.default_annun_color = convert_color(self.default_annun_color)
        self.annunciator_style = self.default_config.get("annunciator-style", DEFAULT_ANNUNCIATOR_STYLE.value)
        self.annunciator_style = ANNUNCIATOR_STYLES(self.annunciator_style)
        self.cockpit_texture = self.default_config.get("cockpit-texture", COCKPIT_TEXTURE)
        self.cockpit_color = self.default_config.get("cockpit-color", COCKPIT_COLOR)
        self.cockpit_color = convert_color(self.cockpit_color)
        self.default_home_page_name = self.default_config.get("default-homepage-name", HOME_PAGE)
        #
        #
        # DO NOT FORGET SET DEFAULTS IN create_decks()
        #
        #

        # 1. Load global icons
        #   (They are never cached when loaded without aircraft.)
        rf = os.path.join(os.path.dirname(__file__), RESOURCES_FOLDER, ICONS_FOLDER)
        if os.path.exists(rf):
            icons = os.listdir(rf)
            for i in icons:
                if has_ext(i, "png"):  # later, might load JPG as well.
                    fn = os.path.join(rf, i)
                    image = Image.open(fn)
                    self.icons[i] = image

        # 2. Finding a default font for Pillow
        #   WE MUST find a default, system font at least

        # 2.1 We try the requested "default label font"
        tryed_our_default = self.default_label_font == DEFAULT_LABEL_FONT
        if self.default_label_font not in self.fonts.keys():
            f = locate_font(self.default_label_font)
            if f is not None:  # found one, perfect
                self.fonts[self.default_label_font] = f
                self.default_font = f
                logger.debug(f"default label font set to {self.default_label_font}")
            else:  # if we are here, self.default_label_font is not found, we must find another one
                self.default_label_font = None

        # 2.2 We try our "default provided label font"
        if self.default_label_font is None and not tryed_our_default:
            f = locate_font(DEFAULT_LABEL_FONT)
            if f is not None:  # found it, perfect
                self.default_label_font = DEFAULT_LABEL_FONT
                self.fonts[DEFAULT_LABEL_FONT] = f
                self.default_font = f
                logger.debug(f"default label font set to {DEFAULT_LABEL_FONT}")
            # if we are here, self.default_label_font is not found, we must find another one

        # 2.3 We try the "default system font"
        f = locate_font(DEFAULT_SYSTEM_FONT)
        if f is not None:  # found it, perfect, keep it as default font for all purposes
            self.fonts[DEFAULT_SYSTEM_FONT] = f
            self.default_font = f
            logger.debug(f"default font set to {DEFAULT_SYSTEM_FONT}")
            if self.default_label_font is None:  # additionnally, if we don't have a default label font, use it
                self.default_label_font = DEFAULT_SYSTEM_FONT
                logger.debug(f"default label font set to {DEFAULT_SYSTEM_FONT}")

        if self.default_label_font is None and len(self.fonts) > 0:
            first_one = list(self.fonts.keys())[0]
            self.default_label_font = first_one
            self.default_font = first_one
            logger.debug(f"no default font found, using first available font")

        if self.default_label_font is None:
            logger.error(f"no default font")

        # 4. report summary if debugging
        logger.debug(f"default fonts {self.fonts.keys()}, default={self.default_font}, default label={self.default_label_font}")

    def create_decks(self):
        sn = os.path.join(self.acpath, CONFIG_FOLDER, SECRET_FILE)
        serial_numbers = Config(sn)
        fn = os.path.join(self.acpath, CONFIG_FOLDER, CONFIG_FILE)
        self._config = Config(fn)
        if self._config.is_valid():
            #
            #
            # DO NOT FORGET SET DEFAULTS IN load_defaults()
            #
            #
            self.default_logo = self._config.get("default-logo", self.default_logo)
            self.default_wallpaper = self._config.get("default-wallpaper", self.default_wallpaper)
            self.default_label_font = self._config.get("default-label-font", self.default_label_font)
            self.default_label_size = self._config.get("default-label-size", self.default_label_size)
            self.default_label_color = self._config.get("default-label-color", self.default_label_color)
            self.default_label_color = convert_color(self.default_label_color)
            self.default_label_position = self._config.get("default-label-position", self.default_label_position)
            self.default_icon_texture = self._config.get("default-icon-texture", self.default_icon_texture)
            self.default_icon_color = self._config.get("default-icon-color", self.default_icon_color)
            self.default_icon_color = convert_color(self.default_icon_color)
            self.default_annun_texture = self._config.get("default-annunciator-texture", self.default_annun_texture)
            self.default_annun_color = self._config.get("default-annunciator-color", self.default_annun_color)
            self.default_annun_color = convert_color(self.default_annun_color)
            self.annunciator_style = self._config.get("annunciator-style", self.annunciator_style)
            self.annunciator_style = ANNUNCIATOR_STYLES(self.annunciator_style)
            self.cockpit_texture = self._config.get("cockpit-texture", self.cockpit_texture)
            self.cockpit_color = self._config.get("cockpit-color", self.cockpit_color)
            self.cockpit_color = convert_color(self.cockpit_color)
            self.default_home_page_name = self.default_config.get("default-homepage-name", self.default_home_page_name)

            logger.debug(f"label font={self.default_label_font}, logo={self.default_logo}, wallpaper={self.default_wallpaper}, texture={self.cockpit_texture}")
            decks = self._config.get("decks")
            if decks is not None:
                deck_type_count = {}
                for deck_config in decks:
                    ty = deck_config.get("type")
                    if ty is not None:
                        if ty not in deck_type_count:
                            deck_type_count[ty] = 0
                        deck_type_count[ty] = deck_type_count[ty] + 1

                cnt = 0
                for deck_config in decks:
                    name = deck_config.get("name", f"Deck {cnt}")

                    disabled = deck_config.get("disabled")
                    if type(disabled) != bool:
                        if type(disabled) == str:
                            disabled = disabled.upper() in ["YES", "TRUE"]
                        elif type(disabled) in [int, float]:
                            disabled = int(disabled) != 0
                    if disabled:
                        logger.info(f"deck {name} disabled, ignoring")
                        continue

                    decktype = deck_config.get("type")
                    if decktype not in DECK_TYPES.keys():
                        logger.warning(f"invalid deck type {decktype}, ignoring")
                        continue

                    serial = deck_config.get("serial")
                    if serial is None:  # get it from the secret file
                        serial = serial_numbers[name] if name in serial_numbers.keys() else None

                    # if serial is not None:
                    device = self.get_device(req_serial=serial, req_type=decktype)
                    if device is not None:
                        #
                        deck_config["model"] = device.deck_type()
                        if serial is None:
                            if deck_type_count[decktype] > 1:
                                logger.warning(f"only one deck of that type but more than one configuration in config.yaml for decks of that type, ignoring")
                                continue
                            deck_config["serial"] = device.get_serial_number()
                            logger.info(f"deck {decktype} {name} has serial {deck_config['serial']}")
                        else:
                            deck_config["serial"] = serial
                        if name not in self.cockpit.keys():
                            self.cockpit[name] = DECK_TYPES[decktype][0](name=name, config=deck_config, cockpit=self, device=device)
                            cnt = cnt + 1
                            logger.info(f"deck {decktype} {name} added")
                        else:
                            logger.warning(f"deck {name} already exist, ignoring")
                    # else:
                    #    logger.error(f"deck {decktype} {name} has no serial number, ignoring")
            else:
                logger.warning(f"no deck in config file {fn}")
        else:
            logger.warning(f"no config file {fn}")

    def create_default_decks(self):
        """
        When no deck definition is found in the aicraft folder, Cockpit loads
        a default X-Plane logo on all deck devices. The only active button is index 0,
        which toggle X-Plane map on/off.
        """
        self.acpath = None

        # {
        #    "type": decktype,
        #    "device": device,
        #    "serial_number": serial
        # }
        for deck in self.devices:
            decktype = deck.get("type")
            if decktype not in DECK_TYPES.keys():
                logger.warning(f"invalid deck type {decktype}, ignoring")
                continue
            device = deck["device"]
            device.open()
            device.reset()
            name = device.id()
            config = {
                "name": name,
                "model": device.deck_type(),
                "serial": device.get_serial_number(),
                "layout": None,   # Streamdeck will detect None layout and present default deck
                "brightness": 75  # Note: layout=None is not the same as no layout attribute (attribute missing)
            }
            self.cockpit[name] = DECK_TYPES[decktype][0](name, config, self, device)

    # #########################################################
    # Cockpit data caches
    #
    def load_icons(self):
        # Loading icons
        #
        dn = os.path.join(self.acpath, CONFIG_FOLDER, ICONS_FOLDER)
        if os.path.exists(dn):
            self.icon_folder = dn
            cache = os.path.join(dn, "_icon_cache.pickle")
            if os.path.exists(cache) and self.cache_icon:
                with open(cache, "rb") as fp:
                    self.icons = pickle.load(fp)
                logger.info(f"{len(self.icons)} icons loaded from cache")
            else:
                # # Global, resource folder icons
                # #                                   # #
                # # THEY ARE NOW LOADED IN LOAD_DEFAULTS # #
                # #                                   # #
                # rf = os.path.join(os.path.dirname(__file__), RESOURCES_FOLDER, ICONS_FOLDER)
                # if os.path.exists(rf):
                #    icons = os.listdir(rf)
                #    for i in icons:
                #        if has_ext(i, "png"):  # later, might load JPG as well.
                #            fn = os.path.join(rf, i)
                #            image = Image.open(fn)
                #            self.icons[i] = image

                # Aircraft specific folder icons
                icons = os.listdir(dn)
                for i in icons:
                    if has_ext(i, "png"):  # later, might load JPG as well.
                        fn = os.path.join(dn, i)
                        image = Image.open(fn)
                        self.icons[i] = image

                if self.cache_icon:  # we cache both folders of icons
                    with open(cache, "wb") as fp:
                        pickle.dump(self.icons, fp)
                    logger.info(f"{len(self.icons)} icons cached")
                else:
                    logger.info(f"{len(self.icons)} icons loaded")

    def load_fonts(self):
        # Loading fonts.
        # For custom fonts (fonts found in the fonts config folder),
        # we supply the full path for font definition to ImageFont.
        # For other fonts, we assume ImageFont will search at OS dependent folders or directories.
        # If the font is not found by ImageFont, we ignore it.
        # So self.icons is a list of properly located usable fonts.
        #
        # 1. Load fonts supplied by Cockpitdeck in its resource folder
        rn = os.path.join(os.path.dirname(__file__), RESOURCES_FOLDER, FONTS_FOLDER)
        if os.path.exists(rn):
            fonts = os.listdir(rn)
            for i in fonts:
                if has_ext(i, ".ttf") or has_ext(i, ".otf"):
                    if i not in self.fonts.keys():
                        fn = os.path.join(rn, i)
                        try:
                            test = ImageFont.truetype(fn, self.default_label_size)
                            self.fonts[i] = fn
                        except:
                            logger.warning(f"default font file {fn} not loaded")
                    else:
                        logger.debug(f"font {i} already loaded")

        # 2. Load fonts supplied by the user in the configuration
        dn = os.path.join(self.acpath, CONFIG_FOLDER, FONTS_FOLDER)
        if os.path.exists(dn):
            fonts = os.listdir(dn)
            for i in fonts:
                if has_ext(i, ".ttf") or has_ext(i, ".otf"):
                    if i not in self.fonts.keys():
                        fn = os.path.join(dn, i)
                        try:
                            test = ImageFont.truetype(fn, self.default_label_size)
                            self.fonts[i] = fn
                        except:
                            logger.warning(f"custom font file {fn} not loaded")
                    else:
                        logger.debug(f"font {i} already loaded")

        # 3. DEFAULT_LABEL_FONT and DEFAULT_SYSTEM_FONT loaded in load_defaults()

        logger.info(f"{len(self.fonts)} fonts loaded, default label font={self.default_label_font}")

    # #########################################################
    # Cockpit start/stop/reload procedures
    #
    # Note: Reloading the deck is done from a separate (dedicated) thread through a queue.
    #
    # The reason is that the reload is provoked by a keypress which is handled in a callback
    # from a deck thread. On reload, the deck will be stopped, initialized, and restarted
    # leading somehow in the destruction of the objects that had created the thread.
    # When pressing a new button, callback would terminate Cockpitdeck.
    # To prevent that, the callback just enqueue a request to perform a reload and exits right away.
    # We do the reload from another thread, external to the callback,
    # that cleanly stops, initializes, and restarts the deck.
    #
    # Note: Only started if has_reload is True. has_reload is set if a Reload button activation is configured.
    #      (Otherwise there is no need to start the reload loop since nothing can provoke it.)
    #
    def start_reload_loop(self):
        if not self.has_reload:
            logger.warning(f"no reload button detected, not starting")
            return
        if not self.reload_loop_run:
            self.reload_loop_thread = threading.Thread(target=self.reload_loop)
            self.reload_loop_thread.name = f"Cockpit::reloader"
            self.reload_loop_run = True
            self.reload_loop_thread.start()
            logger.debug(f"started")
        else:
            logger.warning(f"already running")

    def reload_loop(self):
        while self.reload_loop_run:
            e = self.reload_queue.get()  # blocks infinitely here
            if e == "reload":
                self.reload_decks(just_do_it=True)
            elif e == "stop":
                self.stop_decks(just_do_it=True)
        logger.debug(f"ended")

    def end_reload_loop(self):
        if self.reload_loop_run:
            self.reload_loop_run = False
            self.reload_queue.put("wake up!")  # to unblock the Queue.get()
            # self.reload_loop_thread.join()
            logger.debug(f"stopped")
        else:
            logger.warning(f"not running")

    def reload_decks(self, just_do_it: bool = False):
        """
        Development function to reload page yaml without leaving the page
        Should not be used in production...
        """
        # A security... if we get called we must ensure reloader is running...
        if not self.reload_loop_run:
            logger.warning(f"reload loop not running. Starting..")
            self.has_reload = True
            self.start_reload_loop()
        if just_do_it:
            logger.info(f"reloading decks..")
            self.default_pages = {}  # {deck_name: currently_loaded_page_name}
            for name, deck in self.cockpit.items():
                self.default_pages[name] = deck.current_page.name
            self.load_aircraft(self.acpath)  # will terminate it before loading again
            logger.info(f"..done")
        else:
            self.reload_queue.put("reload")
            logger.debug(f"enqueued")

    def stop_decks(self, just_do_it: bool = False):
        """
        Stop decks gracefully. Since it also terminates self.reload_loop_thread we cannot wait for it
        since we are called from it ... So we just tell it to terminate.
        """
        if just_do_it:
            logger.info(f"stopping decks..")
            self.stop()
        else:
            self.reload_queue.put("stop")
            logger.debug(f"enqueued")

    def dataref_changed(self, dataref):
        """
        This gets called when dataref AIRCRAFT_DATAREF is changed, hence a new aircraft has been loaded.
        """
        v = dataref.value()
        if v is not None and v == 1:
            logger.info(f"current aircraft loaded {dataref.path}={dataref.value()}")
        else:
            logger.info(f"new aircraft loaded {dataref.path}={dataref.value()}")

    def terminate_aircraft(self):
        logger.info(f"terminating..")
        for deck in self.cockpit.values():
            deck.terminate()
        self.cockpit = {}
        nt = len(threading.enumerate())
        if not self.sim.use_flight_loop and nt > 1:
            logger.info(f"{nt} threads")
            logger.info(f"{[t.name for t in threading.enumerate()]}")
        logger.info(f"..done")

    def terminate_devices(self):
        for deck in self.devices:
            decktype = deck.get("type")
            if decktype not in DECK_TYPES.keys():
                logger.warning(f"invalid deck type {decktype}, ignoring")
                continue
            device = deck["device"]
            DECK_TYPES[decktype][0].terminate_device(device, deck["serial_number"])

    def terminate_all(self, threads:int = 1):
        logger.info(f"terminating..")
        # Terminate decks
        self.terminate_aircraft()
        # Terminate dataref collection
        if self.sim is not None:
            logger.info("..terminating connection to simulator..")
            self.sim.terminate()
            logger.debug("..deleting connection to simulator..")
            del self.sim
            self.sim = None
            logger.debug("..connection to simulator deleted..")
        # Terminate reload loop
        if self.reload_loop_run:
            self.end_reload_loop()
        logger.info(f"..terminating devices..")
        self.terminate_devices()
        logger.info(f"..done")
        left = len(threading.enumerate())
        if left > threads:  # [MainThread and spinner]
            logger.error(f"{left} threads remaining")
            logger.error(f"{[t.name for t in threading.enumerate()]}")

    def run(self):
        if len(self.cockpit) > 0:
            # Each deck should have been started
            # Start reload loop
            logger.info(f"starting..")
            self.sim.connect()
            logger.info(f"..connect to simulator loop started..")
            self.start_reload_loop()
            logger.info(f"..reload loop started..")
            if not self.sim.use_flight_loop:
                logger.info(f"{len(threading.enumerate())} threads")
                logger.info(f"{[t.name for t in threading.enumerate()]}")
                logger.info(f"..started")
                logger.info(f"serving {self.name}")
                for t in threading.enumerate():
                    try:
                        t.join()
                    except RuntimeError:
                        pass
                logger.info(f"terminated")
        else:
            logger.warning(f"no deck")
            if self.acpath is not None:
                self.terminate_all()

    # #########################################################
    # XPPython Plugin Hooks
    #
    def start(self):
        logger.info(f"starting..")
        # do nothing, started in when enabled
        logger.info(f"done")

    def stop(self):
        logger.info(f"stopping..")
        self.terminate_all()
        logger.info(f"done")

    def enable(self):
        self.load(self.acpath)
        self.disabled = False
        logger.info(f"enabled")

    def disable(self):
        self.terminate_aircraft()
        self.disabled = True
        logger.info(f"disabled")


import argparse
import pathlib
import csv
import json

parser = argparse.ArgumentParser(
    description="Converts a CSV file to JSON. Output will be in the same directory as input."
)

parser.add_argument('file', help='Path to csv file')
parser.add_argument('--output', '-o', help='Output location')

args = parser.parse_args()

path = pathlib.Path(args.file)

if not path.exists():
    print("Path does not exist")
    exit(1)

if path.is_dir():
    print("Path is a directory")
    exit(1)

if path.suffix != '.csv':
    print("File extension is not .csv")

if args.output is None:

    out_dir = path.parent  # type: pathlib.Path
    out_path = out_dir.joinpath(path.stem + '.json')

else:
    out_path = pathlib.Path(args.output)

items = []

with open(path, 'r') as f:
    reader = csv.DictReader(f)
    for row in reader:
        items.append(row)

with out_path.open('w') as f:
    json.dump(items, f)


import logging

import requests
from flask import current_app as app
from structlog import wrap_logger

from response_operations_ui.exceptions.exceptions import ApiError

logger = wrap_logger(logging.getLogger(__name__))

def get_iac_details(iac):
    logger.info("Retrieving iac")
    if not iac:
        logger.warning("No iac provided")
        return None

    url = f'{app.config["IAC_URL"]}/iacs/{iac}'
    response = requests.get(url, auth=app.config["BASIC_AUTH"])

    try:
        response.raise_for_status()
    except requests.exceptions.HTTPError:
        if response.status_code == 404:
            logger.warning("IAC code not found")
            return
        logger.error("Error retrieving iac")
        raise ApiError(response)

    logger.info("Successfully retrieved iac")
    return response.json()

def is_iac_active(iac):
    iac_response = get_iac_details(iac)
    return iac_response.get("active") if iac_response else None


import os
import xml.dom.minidom as minidom
import xml.etree.ElementTree as ET
import pandas as pd

def Unique(filename):

    _FileName = ''
    _ToolID = ''
    _Plugin = ''
    _FieldName = ''
    _Rename_or_Destination = ''
    _CreateFieldName = ''
    _UpdateFieldName = ''
    _LeftJoinField = ''
    _RightJoinField = ''
    _Expression_1 = ''
    _Expression_2 = ''
    _Expression_3 = ''
    _GroupBy = ''
    _SortOrder = ''
    _UpdateField = ''
    _MultifileValue = ''
    _HeaderField = ''
    _DataField = ''
    _IsSelected = ''
    _NoOfRows = ''
    _NumFields = ''
    _ErrorHandlingText = ''

    Plugin_value = "Unique"
    file = os.path.splitext(os.path.basename(filename))[0]
    field_name=""
    list_row=[]

    DOMTree = minidom.parse(filename)
    Node = DOMTree.getElementsByTagName("Node")
    for node in Node:
        GuiSettings=node.getElementsByTagName("GuiSettings")
        child=node.childNodes.item(1)
        full_plugin_name=child.getAttribute("Plugin")
        tool_id=node.getAttribute("ToolID")
        if Plugin_value in full_plugin_name:
            unique_flds = node.getElementsByTagName("UniqueFields")
            for u in unique_flds:
                fields = u.getElementsByTagName("Field")
                for f in fields:
                    field_name=f.getAttribute("field")
                    list_row.append([str(file),str(tool_id),str(full_plugin_name),str(field_name),
                    str(_Rename_or_Destination),str(_CreateFieldName),str(_UpdateFieldName),str(_LeftJoinField),
                    str(_RightJoinField),str(_Expression_1),str(_Expression_2),str(_Expression_3),
                    str(_GroupBy),str(_SortOrder),str(_UpdateField),str(_MultifileValue),str(_HeaderField),
                    str(_DataField),str(_IsSelected),str(_NoOfRows),str(_NumFields),str(_ErrorHandlingText)])

    unique_df=pd.DataFrame(data=list_row,columns=["FileName","ToolID","Plugin","FieldName",
            "Rename_or_Destination","CreateFieldName","UpdateFieldName","LeftJoinField",
            "RightJoinField","Expression_1","Expression_2","Expression_3","GroupBy","SortOrder",
            "UpdateField","MultifileValue","HeaderField","DataField","IsSelected","NoOfRows","NumFields",
            "ErrorHandlingText"])
    return unique_df.drop_duplicates()

# if __name__ == "__main__":
#     file_input = "08_Belvidere_Data_Normalization V15 Weekly.yxmd"
#     filename = os.path.abspath(os.path.join(file_input))
#     Unique_Tx(filename).to_csv("unique_new.csv",index=False)
#


#-*- coding: utf-8 -*-

########  本文件呈现 卡方检验 相关知识点，包括：

# Part1.卡方检验scipy.stats.chisquare
#   卡方值chi、显著性p
# Part2.交叉表-可视化

######################################################################

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from scipy import stats

######################################################################
########  Part1.卡方检验
######################################################################
#直接对计数和期望值的计算
# from scipy import stats
# obs = [102, 102, 96, 105, 95, 100]
# exp = [100, 100, 100, 100, 100, 100]
# chi2, p = stats.chisquare(f_obs = obs, f_exp = exp)

# 1、读取数据集
filename = 'Telephone.csv'
df = pd.read_csv(filename, encoding='gbk')
# print(df.columns.tolist())

eduLevel = ['初中','高中','大专','本科','研究生']
df['教育水平'] = df['教育水平'].astype('category').cat.set_categories(eduLevel)

# 判断学历与套餐是否有关系
catCol1 = '套餐类型'
catCol2 = '流失'

def chi2Test(df, catCol1, catCol2, threshold=0.05):
    # 计数值
    grouped = df.groupby(by=[catCol1,catCol2])
    obs = grouped[catCol2].count()
    obs.name = '计数值'
    # print(obs)

    # 期望值
    total = obs.sum()
    exps = pd.Series(index=obs.index)
    for tc, lv in obs.index:
        val = obs[tc].sum() * obs[:,lv].sum() /total
        # exps.append(val)
        exps[(tc, lv)] = val
    exps.name = '期望值'
    # print(exps)

    # 卡方检验
    chi2, p = stats.chisquare(f_obs=obs, f_exp = exps)
    print('\nchi2={:.4},p={:.4}'.format(chi2, p))

    if p < threshold:
        print('因子[{}]与[{}]有显著相关性。'.format(catCol1, catCol2))
    else:
        print('因子[{}]与[{}]没有显著相关性!'.format(catCol1, catCol2))

    return p < threshold

ret = chi2Test(df, catCol1, catCol2)

# 遍历：卡方检验
catCols = ['居住地', '婚姻状况', '教育水平', '性别', '电子支付', '套餐类型']
target = '流失'

cols = []
for col in catCols:
    ret = chi2Test(df, col, target)
    if ret:
        cols.append(col)
print('\n有显著影响的因子有：\n', cols)

######################################################################
########  Part2.交叉表-可视化
######################################################################

# 交叉表
df2 = pd.pivot_table(df, index='套餐类型',
            columns='流失', values='UID',
            aggfunc='count')

# 复式柱状图
df2.plot(kind='bar')

# 堆积柱状图
df2.plot(kind='bar', stacked=True)

# 堆积百分比柱状图
total = df2.sum(axis=1)
for col in df2.columns:
    df2[col] = df2[col]/total   #转化为行百分比
df2.plot(kind='bar', stacked=True)

# 对影响因素作百分比堆积柱状图
for col in cols:
    df2 = pd.pivot_table(df,
                index=col,
                columns=target,
                values='UID',
                aggfunc='count')

    #转化为行百分比
    total = df2.sum(axis=1)
    for cl in df2.columns:
        df2[cl] = df2[cl]/total

    df2.plot(kind='bar', stacked=True)


import streamlit as st
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go

st.set_page_config(layout="wide")

path = "./data/kaggle_survey_2021_responses.csv"

@st.cache
def load_data():
    data = pd.read_csv(path, low_memory=False)
    return data

def profile_plot(df, profile_metric, position):

    if profile_metric != "Country":

        grouped_df = df.groupby(["Position", profile_metric]).size().unstack().fillna(0)
        vals = grouped_df.loc[position].values

        fig = go.Figure(
            go.Pie(
                labels=grouped_df.columns,
                values=vals,
                hole=0.3,
                pull=(vals == max(vals)) * 0.2,
            )
        )
        fig.update_layout(width=600, height=500)
        if profile_metric == "Age":
            fig.update_layout(title=dict(text=f"Age distribution of {position}", x=0.5))
        if profile_metric == "Education":
            fig.update_layout(title=dict(text=f"Education level of {position}", x=0.5))
        if profile_metric == "Programming experience":
            fig.update_layout(
                title=dict(text=f"Programming experience of {position}", x=0.5)
            )
        st.plotly_chart(fig)
    else:
        grouped_df = (
            df[df["Position"] == position]["Country"].value_counts().reset_index()
        )
        grouped_df["Country"] = (grouped_df["Country"] * 100) / (
            grouped_df["Country"].sum()
        )
        grouped_df.rename(
            columns={"index": "country", "Country": "percentage"}, inplace=True
        )
        map = px.choropleth(
            grouped_df,
            locations=grouped_df.country,
            color=grouped_df.percentage,
            locationmode="country names",
            color_continuous_scale="viridis",
            title=f"Country residence of {position}",
            range_color=[0, 10],
        )
        map.update(layout=dict(title=dict(x=0.5)))
        st.plotly_chart(map)

def profile_desc(df, profile_metric, position):
    """
    Return some descriptions of the plot given a profile metric and the position title
    """

    data = df[df["Position"] == position][profile_metric].value_counts()
    if profile_metric == "Age":
        st.markdown(f"Most {position} aged **{data.idxmax()}**.")
    if profile_metric == "Country":
        st.markdown(f"Most {position} are from **{data.idxmax()}**.")
    if profile_metric == "Education":
        st.markdown(
            f"Most {position} have an education of or plan to attain an education of **{data.idxmax()}**."
        )
    if profile_metric == "Programming experience":
        st.markdown(
            f"Most {position} have programming experience of **{data.idxmax()}**."
        )

def job_bar_plot(df, position, industry):
    """
    Display bar plot to show the job responsibilities of given position title and industry
    """
    data = df.loc[position].sort_values()
    df = pd.DataFrame({"Job description": data.index, "Percentage": data.values * 100})
    fig = px.bar(
        df,
        x="Percentage",
        y="Job description",
        text=[str(round(val)) + "%" for val in df["Percentage"]],
        orientation="h",
        color="Percentage",
        width=650,
        height=450,
    )
    fig.update_layout(plot_bgcolor="rgba(0,0,0,0)")
    fig.update_layout(
        title=dict(text=f"Job responsibilities of {position} in {str.lower(industry)}", x=0.5)
    )
    st.plotly_chart(fig)

def main():
    data = load_data()
    response = data[1:]
    questions = list(data.iloc[0])
    N_response = response.shape[0]
    N_questions = len(questions) - 1
    N_countries = len(response["Q3"].unique())
    job_title = [
        "Business Analyst",
        "Data Scientist",
        "Data Analyst",
        "Machine Learning Engineer",
        "DBA/Database Engineer",
        "Data Engineer",
        "Statistician",
        "Software Engineer",
        "Research Scientist",
    ]
    job_desc = [
        "Analyze data for business decisions",
        "Build data infrastructure",
        "Build ML Prototypes",
        "Build ML service for workflows",
        "Experiment/improve exisiting models",
        "Research to advance ML",
        "Other",
    ]
    metrics = ["Age", "Country", "Education", "Programming experience"]
    # Rename cols
    response.rename(
        columns={
            "Time from Start to Finish (seconds)": "Duration",
            "Q1": "Age",
            "Q2": "Gender",
            "Q3": "Country",
            "Q4": "Education",
            "Q5": "Position",
            "Q6": "Programming experience",
            "Q20": "Industry",
        },
        inplace=True,
    )
    # Remove outliers in duration (>1931 upper fence val)
    response["Duration"] = response["Duration"].astype(int)
    response = response[response["Duration"] <= 1931]

    # Replace the text to make it's clearer to display
    response.replace(
        "Some college/university study without earning a bachelorâ€™s degree",
        "Some college study",
        inplace=True,
    )  # Replace the text to make it's clearer to display

    # Q24 asks people about their job duty
    Q24_question = [question for question in response.columns if "Q24" in question]
    Q24_question.remove("Q24_Part_7")  # remove none option in the answers
    job_activ = response.groupby("Position")[Q24_question].agg(
        lambda x: x.notnull().mean()
    )
    job_activ.columns = job_desc

    industries = [
        i for i in response.Industry.unique() if str(i) != "nan" and str(i) != "Other"
    ]
    industries.insert(0,"All industries")

    st.title("Data People Explorer")
    st.markdown(
        "Ever wondered about what the job titles are in data science and what the people working with data are like? \
        This dashboard takes answers from people working with data all over the world using the anual survey of Kaggle,\
        one of the largest data science community, and allows you to discover everything you are curious about data people."
    )
    st.header("Survey overview:")
    st.markdown(":clock1: Time window of the survey: 09/01/2021 - 10/04/2021")

    col1, spacer2, col2, spacer3, col3, spacer4 = st.columns([3.5, 1, 3.5, 1, 3.5, 1])
    with col1:
        st.markdown(":busts_in_silhouette: " + str(N_response) + " respondents")
    with col2:
        st.markdown(":question:" + "42 questions")
    with col3:
        st.markdown(":earth_americas: " + str(N_countries) + " countries/territories")
    with st.expander("Click here to view the raw data ðŸ‘‰"):
        st.dataframe(response)
    st.markdown("___")

    st.subheader("Select a job position you are interested to begin")
    spacer1, col1, col2, spacer2 = st.columns([0.2, 2.5, 4.4, 0.2])
    with col1:
        position = st.selectbox("I'm interested to know", job_title)

    spacer1, col1, col2, spacer2 = st.columns([0.2, 2.5, 4.4, 0.2])
    with col1:
        st.markdown(
            f"What do you want to know about {position}? Their age, educational level, country of residence or programming experience? Select below to find out!"
        )

        profile_metric = st.selectbox("Select a profile metric", metrics)

        st.text("")
        profile_desc(response, profile_metric, position)

    with col2:
        profile_plot(response, profile_metric, position)

    st.header("What do data people do?")

    spacer1, col1, spacer2, col2, spacer3 = st.columns((0.2, 2.5, 0.4, 4.4, 0.2))
    with col1:
        st.markdown(
            f"What do {position} do in their daily job? Select below to see their job responsibility in all industries or a certain industry."
        )

        industry = st.selectbox("", industries)

    with col2:

        if industry == "All industries":

            job_bar_plot(job_activ, position, industry)

        else:
            job_activ_indus = (
                response[response["Industry"] == industry]
                .groupby("Position")[Q24_question]
                .agg(lambda x: x.notnull().mean())
            )
            job_activ_indus.columns = job_desc
            job_bar_plot(job_activ_indus, position, industry)

if __name__ == "__main__":
    main()


import openai as ai
import tempfile
import os
import sys

from utils import tts, stt, play_audio, record_audio, kaldi

from dotenv import load_dotenv

load_dotenv()

def chat(question,chat_log = None) -> str:
    if(chat_log == None):
        chat_log = start_chat_log
    prompt = f"{chat_log}\nHuman: {question}\nAI:"
    response = completion.create(prompt = prompt, engine =  "ada", temperature = 0.85,top_p=1, frequency_penalty=0,
    presence_penalty=0.7, best_of=2,max_tokens=100,stop = "\nHuman: ")
    return prompt, response.choices[0].text

def modify_start_message(chat_log,question,answer) -> str:
    if chat_log == None:
        chat_log = start_chat_log
    chat_log += f"Human: {question}\nAI: {answer}\n"
    return chat_log

ai.api_key = os.getenv('OPENAI_KEY')

completion = ai.Completion()

start_chat_log = """Human: Hello, I am Human.
AI: Hello, human I am openai gpt3.
Human: How are you?
AI: I am fine, thanks for asking.
"""

train = input("\nDo you want to train the openai chatbot (True/False): ")
if(train == "True"):
    print("\n(To stop the training enter stop in the question)\n")
    while(True):
        question = input("Question: ")
        if question == "stop":
            break
        answer = input("Answer: ")
        start_chat_log = modify_start_message(start_chat_log,question,answer)
        print("\n")

duration = 0
chat_log = start_chat_log
tmp_file = tempfile.NamedTemporaryFile().name

print("\nAsk openai questions (to quit type duration=0)")
while True:
    print(' '*20)
    sys.stdout.write("\033[F")
    duration = int(input("Question duration (seconds): "))
    print(' '*20)
    sys.stdout.write("\033[F")
    if duration == 0:
        break

    record_audio(tmp_file, seconds=duration)

    question = kaldi(tmp_file)
    print(f'Human: {question}')
    chat_log, answer = chat(question, chat_log)
    chat_log += answer

    tts(answer, tmp_file)
    play_audio(tmp_file)
    print("AI: ",answer)


import pandas as pd
import numpy as np
import matplotlib as mpl
import seaborn as sns
from sklearn.svm import SVC
from sklearn.ensemble import BaggingClassifier
from sklearn.datasets import make_classification
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import KFold
import matplotlib.pyplot as plt
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.ensemble import VotingClassifier
import yfinance as yf
import math
import os.path
import time
from sklearn.model_selection import train_test_split
from sklearn import metrics, tree
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import train_test_split, RandomizedSearchCV
from math import sqrt

# BITCOIN #

cryptocurrencies = ['BTC-USD']
bitcoin_df = yf.download(cryptocurrencies, start='2019-01-01',
                end='2022-03-01', parse_dates = ['Month'])

bitcoin_df["Volume"] = bitcoin_df["Volume"].astype("float")
bitcoin_df.info()

bitcoin_df = bitcoin_df.drop(columns = ['Adj Close'])

bitcoin_features = bitcoin_df
features = ['Open', 'High', 'Low', 'Volume']
X_bitcoin = bitcoin_features[features]
y_bitcoin = bitcoin_features['Close']

X_train, X_test, y_train, y_test = train_test_split(X_bitcoin, y_bitcoin, test_size = 0.30, random_state = 42)

def rmse(x, y): return math.sqrt(((x-y)**2).mean())
def print_score(m):
    res = [rmse(m.predict(X_train), y_train),
           rmse(m.predict(X_test), y_test),
                m.score(X_train, y_train),
                m.score(X_test, y_test)]
    print(f"RMSE Training Set: {res[0]} \nRMSE validation set: {res[1]} \nR2 validation set: {res[3]}")

from sklearn.linear_model import LinearRegression
lm = LinearRegression()
lm.fit(X_train, y_train)

predictions = lm.predict(X_test)
predictions

print('MAE:', metrics.mean_absolute_error(y_test, predictions))
print('MSE:', metrics.mean_squared_error(y_test, predictions))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, predictions)))

def reg_accuracy(y_true, y_pre):
    return_var = []
    from math import sqrt
    rmse = sqrt(metrics.mean_squared_error(y_true, y_pre))
    return_var.append(rmse)
    print ("RMSE: ", rmse)
    r2 = metrics.r2_score(y_true, y_pre)
    return_var.append(r2)
    print("R2: ", r2)
    mae = metrics.mean_absolute_error(y_true, y_pre)
    return_var.append(mae)
    print("MAE: " , mae)

    if 0 in y_true:
        print("MAPE erróneo")
        return_var.append(0)

    else:
        mape = round(np.mean(np.abs((y_true - y_pre) / y_true))*100,4)
        print('MAPE :', mape)
        print('======================')
        print('Model Accuracy(%) :', 100 - mape)
        print('======================')
        return_var.append(mape)
        return_var.append(100-mape)
    return return_var

# CARDANO #

cryptocurrencies = ['BTC-USD']
cardano_df = yf.download(cryptocurrencies, start='2019-01-01',
                end='2022-03-01', parse_dates = ['Month'])

cardano_df["Volume"] = cardano_df["Volume"].astype("float")
cardano_df.info()

cardano_df = cardano_df.drop(columns = ['Adj Close'])

cardano_features = cardano_df
features = ['Open', 'High', 'Low', 'Volume']
X_cardano = cardano_features[features]
y_cardano = cardano_features['Close']

X_train, X_test, y_train, y_test = train_test_split(X_cardano, y_cardano, test_size = 0.30, random_state = 42)

from sklearn.linear_model import LinearRegression
lm = LinearRegression()
lm.fit(X_train, y_train)

predictions = lm.predict(X_test)
predictions

print('MAE:', metrics.mean_absolute_error(y_test, predictions))
print('MSE:', metrics.mean_squared_error(y_test, predictions))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, predictions)))

# ETHEREUM #

cryptocurrencies = ['BTC-USD']
ethereum_df = yf.download(cryptocurrencies, start='2019-01-01',
                end='2022-03-01', parse_dates = ['Month'])

ethereum_df["Volume"] = ethereum_df["Volume"].astype("float")
ethereum_df.info()

ethereum_df = ethereum_df.drop(columns = ['Adj Close'])

ethereum_features = ethereum_df
features = ['Open', 'High', 'Low', 'Volume']
X_ethereum = ethereum_features[features]
y_ethereum = ethereum_features['Close']

X_train, X_test, y_train, y_test = train_test_split(X_ethereum, y_ethereum, test_size = 0.30, random_state = 42)

from sklearn.linear_model import LinearRegression
lm = LinearRegression()
lm.fit(X_train, y_train)

predictions = lm.predict(X_test)
predictions

print('MAE:', metrics.mean_absolute_error(y_test, predictions))
print('MSE:', metrics.mean_squared_error(y_test, predictions))
print('RMSE:', np.sqrt(metrics.mean_squared_error(y_test, predictions)))


from django import forms
from .models import Gene

class GeneForm(forms.ModelForm):
    class Meta:
        model = Gene
        fields = ("chromosome", "start", "end", "symbol")
        labels = {
            'chromosome':'Cromosoma',
            'start':'Comienzo',
            'end':'Fin',
            'symbol':'Notacion (symbol)',
        }

    def __init__(self, *args, **kwargs):
        super(GeneForm,self).__init__(*args, **kwargs)
class GeneFormList(forms.Form):
    class Meta:
        fields = ("chromosome", "start", "end", "symbol", "rs")
        labels = {
            'chromosome':'Cromosoma',
            'start':'Comienzo',
            'end':'Fin',
            'symbol':'Notacion (symbol)',
            'rs':'Rs.',
        }
    def __init__(self, *args, **kwargs):
        super(GeneFormList,self).__init__(*args, **kwargs)

class DiseaseByGeneForm(forms.Form):
    content = forms.CharField()
    class Meta:
        # model =
        fields = ("chromosome",)
        labels = {
            'chromosome':'Gene (Chormosome) : ',
        }
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)


from collections import deque

def find_player_and_bunnies(curr_matrix):
    player_pos, bunnies_positions = (), []
    for c_row in range(rows):
        for c_col in range(cols):

            if curr_matrix[c_row][c_col] == "P":
                player_pos = (c_row, c_col)
            elif curr_matrix[c_row][c_col] == "B":
                bunnies_positions.append((c_row, c_col))

    return player_pos, bunnies_positions

def check_player_win(current_row, current_col):
    return 0 <= current_row < rows and 0 <= current_col < cols

def check_if_position_is_bunny(current_row, current_col, curr_matrix):
    return curr_matrix[current_row][current_col] == "B"

def bunnies_spreading(found_bunnies, curr_matrix, commands_dict, player_indices):
    found_player = False
    for bunny in found_bunnies:
        for command in commands_dict:
            c_row = bunny[0] + commands_dict[command][0]
            c_col = bunny[1] + commands_dict[command][1]

            if 0 <= c_row < rows and 0 <= c_col < cols:
                if curr_matrix[c_row][c_col] == "P":
                    found_player = True

                curr_matrix[c_row][c_col] = "B"

    return found_player

rows, cols = [int(n) for n in input().split()]
matrix = []

for row in range(rows):
    matrix.append(list(input()))

positions = deque(list(input()))

all_commands = {
    'L': (0, -1),
    'R': (0, +1),
    'U': (-1, 0),
    'D': (+1, 0)
}

player_idx, bunnies = find_player_and_bunnies(matrix)
dead, dead_string = False, ""
won = False

while positions:
    position = positions.popleft()
    c_row = all_commands[position][0] + player_idx[0]
    c_col = all_commands[position][1] + player_idx[1]

    if not check_player_win(c_row, c_col):
        matrix[player_idx[0]][player_idx[1]] = "."
        bunnies_spreading(bunnies, matrix, all_commands, player_idx)
        won = True
        break

    if check_if_position_is_bunny(c_row, c_col, matrix):
        dead_string = f"dead: {c_row} {c_col}"
        matrix[player_idx[0]][player_idx[1]] = "B"
        bunnies_spreading(bunnies, matrix, all_commands, player_idx)
        dead = True
        break

    else:
        matrix[c_row][c_col] = "P"
        matrix[player_idx[0]][player_idx[1]] = "."
        player_idx = (c_row, c_col)

    if bunnies_spreading(bunnies, matrix, all_commands, player_idx):
        dead_string = f"dead: {player_idx[0]} {player_idx[1]}"
        matrix[player_idx[0]][player_idx[1]] = "B"
        dead = True
        break

    else:
        bunnies = find_player_and_bunnies(matrix)[1]

for nested_lst in matrix:
    print(*nested_lst, sep="")

if won:
    print(f"won: {player_idx[0]} {player_idx[1]}")
if dead:
    print(dead_string)

3. Python Advanced/Python Advanced - Exercises/4. Multidimensional Lists - Exercise 1/10_radioactive_mutant_vampire_bunnies.py


# General Packages
import enum
import typing

# Torch Packages
from torch import optim

# ENUMS as an auxiliary support to avoid spelling issues
class CustomOptimizer(enum.Enum):
    SGD = ['sgd', 'SGD']
    ADAM = ['adam', 'Adam', 'ADAM']
    Adagrad = ['adagrad', 'Adagrad', 'ADAGRAD']
    RMSprop = ['rmsprop', 'RMSprop', 'RMSPROP']

def get_default_optimizer(model_params):
    """
    Creates an Adam optimizer object with predefined default settings.

    :param model_params: Model parameter.
    :return: Returns default optimizer - Adam -
    object with default parameter (learning_rate=1e-3, weight_decay=5e-5).
    """
    return optim.Adam(model_params, lr=1e-3, weight_decay=5e-2)

def get_optimizer(optimizer: typing.Union[str, 'CustomOptimizer'], optimizer_params,
                  optimizer_args):
    """
    Creates a custom optimizer with custom optimizer arguments.

    :param optimizer: Name of the optimizer, e.g. 'Adam', 'SGD', ...
    :param optimizer_params: Model (Network) parameters.
    :param optimizer_args: Custom parameters like learning_rate, weight_decay, etc.
    :return: Selected optimizer with custom parameters.
    """
    if (optimizer == CustomOptimizer.SGD) or (optimizer in CustomOptimizer.SGD.value):
        return optim.SGD(params=optimizer_params,
                         lr=optimizer_args['learning_rate'],
                         momentum=optimizer_args['momentum'],
                         weight_decay=optimizer_args['weight_decay'],
                         )
    elif (optimizer == CustomOptimizer.Adagrad) or (optimizer in CustomOptimizer.Adagrad.value):
        return optim.Adagrad(params=optimizer_params,
                             lr=optimizer_args['learning_rate'],
                             weight_decay=optimizer_args['weight_decay'],
                             )
    elif (optimizer == CustomOptimizer.ADAM) or (optimizer in CustomOptimizer.ADAM.value):
        return optim.Adam(params=optimizer_params,
                          lr=optimizer_args['learning_rate'],
                          betas=optimizer_args['betas'],
                          weight_decay=optimizer_args['weight_decay']
                          )
    elif (optimizer == CustomOptimizer.RMSprop) or (optimizer in CustomOptimizer.RMSprop.value):
        return optim.RMSprop(params=optimizer_params,
                             lr=optimizer_args['learning_rate'],
                             momentum=optimizer_args['momentum'],
                             alpha=optimizer_args['alpha'],
                             weight_decay=optimizer_args['weight_decay'],
                             )


import argparse
import copy
import os

from parallelize import Parallel

parser = argparse.ArgumentParser()
parser.add_argument("--max_cpu")
args = parser.parse_args()

max_cpu = 1
try:
    max_cpu = int(args.max_cpu)
except:
    pass
sleep_interval = 5

param_combs = {
                "unconstrained" : {"ENFORCE_HOP_OPTIMALITY"       : False,\
                                   "ENFORCE_FANIN"                : False,\
                                   "ENFORCE_FANOUT"               : False,\
                                   "ENFORCE_EXTERNAL_SYMMETRY"    : False,\
                                   "ENFORCE_INTERNAL_SYMMETRY"    : False,\
                                   "ENFORCE_CONTINUATION"         : False,\
                                   "ENFORCE_RELAXED_CONTINUATION" : False,\
                                   "ENFORCE_TWO_TURNS"            : False,\
                                   "ENFORCE_ONE_TURN"             : False,\
                                   "LIMIT_MUX_SIZE_NUMBER"        : -1,\
                                   "LIMIT_FANOUT_SIZE_NUMBER"     : -1,\
                                   "ENFORCE_MUX_PAIR_INPUT_SHARE" : -1,\
                                   "WL_TRADEOFF"                  : 0.0,\
                                   "FLOORPLAN_SEED"               : 19225,\
                                  }
              }

#Limiting mux size number:
for name, cnt in (("one", 1), ("two", 2), ("four", 4)):
    param_combs.update({"%s_mux_size" % name : copy.deepcopy(param_combs["unconstrained"])})
    param_combs["%s_mux_size" % name].update({"LIMIT_MUX_SIZE_NUMBER" : cnt})
########################

#Limiting mux and fanout size number:
for name, cnt in (("one", 1), ("two", 2), ("four", 4)):
    param_combs.update({"%s_fanout_size" % name : copy.deepcopy(param_combs["%s_mux_size" % name])})
    param_combs["%s_fanout_size" % name].update({"LIMIT_FANOUT_SIZE_NUMBER" : cnt})
########################

#Enforcing mux pair input sharing:
for name, cnt in (("one", 1), ("two", 2), ("three", 3), ("four", 4), ("five", 5)):
    param_combs.update({"%s_input_shared" % name : copy.deepcopy(param_combs["one_fanout_size"])})
    param_combs["%s_input_shared" % name].update({"ENFORCE_MUX_PAIR_INPUT_SHARE" : cnt})
    param_combs["%s_input_shared" % name].update({"ENFORCE_FANIN" : True})
    param_combs["%s_input_shared" % name].update({"ENFORCE_FANOUT" : True})
########################

#Optimizing wirelength:
for wl in (0.1, 0.3, 0.5, 0.7, 0.9):
    param_combs.update({"wl_%.1f" % wl : copy.deepcopy(param_combs["unconstrained"])})
    param_combs["wl_%.1f" % wl].update({"WL_TRADEOFF" : wl})
    param_combs["wl_%.1f" % wl].update({"ENFORCE_FANIN" : True})
    param_combs["wl_%.1f" % wl].update({"ENFORCE_FANOUT" : True})
########################

#Enforcing turns and symmetries:
param_combs.update({"all_turns" : copy.deepcopy(param_combs["wl_0.5"])})
param_combs["all_turns"].update({"ENFORCE_TWO_TURNS" : True, "ENFORCE_RELAXED_CONTINUATION" : True})
param_combs["all_turns"].update({"ENFORCE_FANIN" : True})
param_combs["all_turns"].update({"ENFORCE_FANOUT" : True})

param_combs.update({"internal_symmetry" : copy.deepcopy(param_combs["wl_0.5"])})
param_combs["internal_symmetry"].update({"ENFORCE_INTERNAL_SYMMETRY" : True})
param_combs["internal_symmetry"].update({"ENFORCE_FANIN" : True})
param_combs["internal_symmetry"].update({"ENFORCE_FANOUT" : True})

param_combs.update({"external_symmetry" : copy.deepcopy(param_combs["wl_0.5"])})
param_combs["external_symmetry"].update({"ENFORCE_EXTERNAL_SYMMETRY" : True})
param_combs["external_symmetry"].update({"ENFORCE_FANIN" : True})
param_combs["external_symmetry"].update({"ENFORCE_FANOUT" : True})

param_combs.update({"full_symmetry" : copy.deepcopy(param_combs["wl_0.5"])})
param_combs["full_symmetry"].update({"ENFORCE_INTERNAL_SYMMETRY" : True, "ENFORCE_EXTERNAL_SYMMETRY" : True})
param_combs["full_symmetry"].update({"ENFORCE_FANIN" : True})
param_combs["full_symmetry"].update({"ENFORCE_FANOUT" : True})
########################

#Enforcing hop-optimality:
param_combs.update({"hop_optimality" : copy.deepcopy(param_combs["one_fanout_size"])})
param_combs["hop_optimality"].update({"ENFORCE_HOP_OPTIMALITY" : True})
param_combs["hop_optimality"].update({"ENFORCE_FANIN" : True})
param_combs["hop_optimality"].update({"ENFORCE_FANOUT" : True})
########################

with open("src/generate_architecture/config.py", "r") as inf:
    default_txt = inf.read()

##########################################################################
def fill_in(comb):
    """Fills in a configuration file.

    Parameters
    ----------
    comb : str
        Name of the combination as stored in the param_combs dictionary.

    Returns
    -------
    str
        Filled-in configuration.
    """

    txt = ""
    for line in default_txt.splitlines():
        replaced = False
        for key, val in param_combs[comb].items():
            if line.startswith("%s " % key):
                new_line = line.replace("= %s" % line.split('=', 1)[-1].split()[0], "= %s" % str(val))
                txt += new_line + "\n"
                replaced = True
                break
        if not replaced:
            txt += line + "\n"

    return txt
##########################################################################

with open("src/setenv.py", "r") as inf:
    setenv_txt = inf.read()

# WL = 1.0 does not require running avalanche routing. Hence just modify the floorplan seed and solve the ILPs:

run = lambda d : "cd run_exploration/%s/src/generate_architecture/ && python -u ilp_setup.py --config config --vpr_log sample_vpr.log && mkdir ../test_%s && cp sample_ilp.log ../test_%s/ilp_iter_1.log && cp sample_ilp.log ../test_%s/ilp_iter_2.log && rm sample_ilp.log" % (d, d, d, d)

for seed in (19225, 25124, 43033, 50936, 5300):
    key = "wl_1.0_%d" % seed
    comb = copy.deepcopy(param_combs["wl_0.1"])
    comb["WL_TRADEOFF"] = 1.0
    comb["FLOORPLAN_SEED"] = seed
    param_combs.update({key : comb})
    txt = fill_in(key)
    del param_combs[key]
    os.system("mkdir run_exploration/wl_1.0_shuffle_seed_%d" % seed)
    os.system("cp -r src run_exploration/wl_1.0_shuffle_seed_%d/" % seed)
    with open("run_exploration/wl_1.0_shuffle_seed_%d/src/generate_architecture/config.py" % seed, "w") as outf:
       outf.write(txt)
    with open("run_exploration/wl_1.0_shuffle_seed_%d/src/setenv.py" % seed, "w") as outf:
       outf.write(setenv_txt.replace("%SHUFFLE_SEED%", str(seed)))
    os.system(run("wl_1.0_shuffle_seed_%d" % seed))

for comb in param_combs:
    txt = fill_in(comb)
    for seed in (19225, 25124, 43033, 50936, 5300):
        os.system("mkdir run_exploration/%s_shuffle_seed_%d" % (comb, seed))
        os.system("cp -r src run_exploration/%s_shuffle_seed_%d/" % (comb, seed))
        with open("run_exploration/%s_shuffle_seed_%d/src/generate_architecture/config.py" % (comb, seed), "w") as outf:
            outf.write(txt)
        with open("run_exploration/%s_shuffle_seed_%d/src/setenv.py" % (comb, seed), "w") as outf:
            outf.write(setenv_txt.replace("%SHUFFLE_SEED%", str(seed)))

run = lambda d : "cd run_exploration/%s/src/generate_architecture/ && python -u explore_avalanche.py --base_cost 1.0 --scaling_factor 9 --avalanche_iter 25 --adoption_threshold 1.1 --wd test_%s" % (d, d)

all_dirs = sorted(os.listdir("run_exploration"))
calls = []
for d in all_dirs:
    if "wl_1.0" in d:
        continue
    calls.append(run(d))

runner = Parallel(max_cpu, sleep_interval)
runner.init_cmd_pool(calls)
runner.run()

del runner

#Run the reference avalanche-search-only experiments:

comb = "avalanche_only"
calls = []
for seed in (19225, 25124, 43033, 50936, 5300):
    os.system("mkdir run_exploration/%s_shuffle_seed_%d" % (comb, seed))
    os.system("cp -r src run_exploration/%s_shuffle_seed_%d/" % (comb, seed))
    with open("run_exploration/%s_shuffle_seed_%d/src/setenv.py" % (comb, seed), "w") as outf:
        outf.write(setenv_txt.replace("%SHUFFLE_SEED%", str(seed)))
    calls.append(run("%s_shuffle_seed_%d" % (comb, seed)).replace("explore_avalanche.py", "explore_avalanche_no_ilp.py"))

runner = Parallel(max_cpu, sleep_interval)
runner.init_cmd_pool(calls)
runner.run()


import numpy as np
import collections
import pandas as pd
import ConfigParameters as Cp
import statsmodels.api as sm
# np.seterr(invalid='ignore')     # removes RuntimeWarning: invalid value encountered in greater return sum(S > tol)

def metrics(portfolio, simulation_name):
    dfb = portfolio.df
    dft = portfolio.trades_df

    risk_free_rate = 0.0
    pm = collections.OrderedDict()

    pm['simulation_name'] = simulation_name
    pm['Simulation_start_date'] = dfb.head(1).index[0].strftime("%d-%b-%Y")
    pm['Simulation_end_date'] = dfb.tail(1).index[0].strftime("%d-%b-%Y")
    sim_days = (dfb.tail(1).index[0] - dfb.head(1).index[0]).days
    pm['Simulation_duration_days'] = sim_days

    pm['equity_start'] = "{0:,.2f}".format(dfb['Equity'][0])
    pm['equity_end'] = "{0:,.2f}".format(dfb['Equity'][-1])
    pm['total_net_profit'] = "{0:,.2f}".format(dfb['Equity'][-1] - dfb['Equity'][0])

    total_return_factor = dfb['Equity'][-1] / dfb['Equity'][0]
    pm['Total_return'] = "{0:,.2f}%".format(100.0 * (total_return_factor - 1.0))

    average_annual_return_pct = 100.0 * (pow(total_return_factor, (365.0 / sim_days)) - 1)
    pm['Average_Annual_Return'] = "{0:,.2f}%".format(average_annual_return_pct)

    pm['equity_daily_returns_standard_deviation'] = dfb['equity_daily_returns'].std()

    pm['total_number_of_trades'] = "{:,}".format(len(dft.index))
    pm['average_profit_loss_per_trade'] = "{0:,.2f}".format(dft['trade net profit'].mean())
    pm['average_duration_of_trade_days'] = "{0:,.2f}".format(dft['holding period days'].mean())

    pm['total_number_of_winning_trades'] = "{:,}".format(dft[dft['trade net profit'] >= 0.0].count()[0])

    pm['total_number_of_winning_trades_longs'] = "{:,}".format(dft[(dft['trade net profit'] >= 0.0) &
                                                                   (dft['quantity'] > 0)].count()[0])

    pm['total_number_of_winning_trades_shorts'] = "{:,}".format(dft[(dft['trade net profit'] >= 0.0) &
                                                                    (dft['quantity'] < 0)].count()[0])

    pm['net_profit_on_winning_trades'] = "{0:,.2f}".format(dft[dft['trade net profit'] >= 0.0].sum()
                                                           ['trade net profit'])

    pm['average_profit_on_winning_trades'] = "{0:,.2f}".format(dft[dft['trade net profit'] >= 0.0].mean()
                                                               ['trade net profit'])
    pm['average_duration_of_winning_trades_days'] = "{:,.0f}".format(dft[dft['trade net profit'] >= 0.0].mean()
                                                                     ['holding period days'])

    pm['total_number_of_losing_trades'] = "{:,}".format(dft[dft['trade net profit'] < 0.0].count()[0])

    pm['total_number_of_losing_trades_longs'] = "{:,}".format(dft[(dft['trade net profit'] < 0.0) &
                                                                  (dft['quantity'] > 0)].count()[0])

    pm['total_number_of_losing_trades_shorts'] = "{:,}".format(dft[(dft['trade net profit'] < 0.0) &
                                                                   (dft['quantity'] < 0)].count()[0])

    pm['net_profit_on_losing_trades'] = "{0:,.2f}".format(dft[dft['trade net profit'] < 0.0].sum()
                                                          ['trade net profit'])

    pm['average_profit_on_losing_trades'] = "{0:,.2f}".format(dft[dft['trade net profit'] < 0.0].mean()
                                                              ['trade net profit'])
    pm['average_duration_of_losing_trades_days'] = "{:,.0f}".format(dft[dft['trade net profit'] < 0.0].mean()
                                                                    ['holding period days'])

    losing_trades_that_hit_max_stop = dft[dft['trade net profit'] < -abs(Cp.amount_to_risk_per_trade)].count()[0]
    pm['losing_trades_that_hit_max_stop'] = "{:,}".format(losing_trades_that_hit_max_stop)

    win_loss_ratio = abs(dft[dft['trade net profit'] >= 0.0].sum()['trade net profit'] /
                         dft[dft['trade net profit'] < 0.0].sum()['trade net profit'])
    pm['Win/Loss Ratio'] = "{0:,.2f}".format(win_loss_ratio)

    pm['worst_drawdown'] = "{0:,.2f}".format(dfb['Drawdown'].min())
    pm['worst_drawdown_pct'] = "{0:,.2f}%".format(100.0 * dfb['Drawdown_pct'].min())
    pm['longest_drawdown_duration_days'] = "{:,.0f}".format(dfb['DrawdownDuration'].max())

    # sharpe ratio: aim for SR > 1
    sharpe_ratio = np.sqrt(252) * (dfb['equity_daily_returns'].mean() - (risk_free_rate / 252)) / dfb['equity_daily_returns'].std()
    pm['Sharpe_ratio'] = "{0:.2f}".format(sharpe_ratio)

    # Beta and Correlation to FTSE100
    beta, rsquared = calc_stats(dfb, 'equity_daily_returns', 'Change_pct_' + Cp.ticker_benchmark)
    pm['beta_to_benchmark'] = beta
    pm['rsquared_to_benchmark'] = rsquared

    pm['Correlation_to_benchmark'] = dfb['equity_daily_returns'].corr(dfb['Change_pct_' + Cp.ticker_benchmark])

    # Exposure
    pm['max_gross_exposure'] = "{0:,.2f}".format(dfb['GrossExposure'].max())

    # Commission
    pm['total_commission'] = "{0:,.2f}".format(dft['commission open'].sum() + dft['commission close'].sum())

    portfolio.metrics = pm
    portfolio.metrics_df = pd.DataFrame.from_dict(pm, orient='index')
    # print(portfolio.metrics)

def calc_stats(df, y_column, x_column):
    df = df.tail(-(Cp.lookback * 4))    # remove the first 180 days
    df = df.head(-1)                    # remove last line
    X = sm.add_constant(df[x_column].tolist(), prepend=False)
    y = df[y_column].tolist()
    mod = sm.OLS(endog=y, exog=X, missing='drop', hasconst=True)
    res = mod.fit()
    beta = res.params[1]
    r2 = res.rsquared
    return beta, r2


import streamlit as st
import tempfile
import os
from os.path import join
import random

import torch
import numpy as np
import cv2

from model import all_models, get_model
from vector_cv_tools.utils import VideoReader

if torch.cuda.is_available():
    device = torch.device('cuda')
else:
    device = torch.device('cpu')

DOCUMENTS_ROOT = os.getenv("CV_DEMO_DOC_ROOT", default="./documents")
MEDIA_FILE_ROOT = join(DOCUMENTS_ROOT, "covid_classification")

CKPT_ROOT = join(MEDIA_FILE_ROOT, "checkpoints")
IMG_ROOT = join(MEDIA_FILE_ROOT, "imgs")
MARKDOWN_ROOT = join(MEDIA_FILE_ROOT, "markdowns")
SAMPLE_ROOT = join(MEDIA_FILE_ROOT, "sample_videos")
TEST_ROOT = join(MEDIA_FILE_ROOT, "test_videos")

sample_videos = [
    "norm1_crop.mp4",
    "norm2_crop.mp4",
    "covid1_crop.mp4",
    "covid2_crop.mp4",
    "pnue1_crop.mp4",
    "pnue2_crop.mp4",
]

test_videos = [
    "normal.mp4",
    "covid.mp4",
    "pneumonia.mp4",
]

sample_video_bytes = [open(join(SAMPLE_ROOT, vid), 'rb').read() for vid in sample_videos]
test_video_bytes = [open(join(TEST_ROOT, vid), 'rb').read() for vid in test_videos]

classes = [
    "Normal/Other",
    "COVID",
    "Pneumonia",
]

loaded_models = {
    "MC3": get_model("MC3_18")(),
    "R3D": get_model("R3D18")(),
}

for name, model in loaded_models.items():
    checkpoint = join(CKPT_ROOT, name + ".ckpt")
    state = torch.load(checkpoint)["model"]
    model.load_state_dict(state)
    model = model.to(device)
    model.eval()

model_card_mc3 = open(join(MARKDOWN_ROOT, "MC3.md")).read()
model_card_r3d = open(join(MARKDOWN_ROOT, "R3D.md")).read()

datacard = open(join(MARKDOWN_ROOT, "datacard.md")).read()
datasource = open(join(MARKDOWN_ROOT, "datasource.md")).read()

desc = open(join(MARKDOWN_ROOT, "covid_intro.md")).read()

def get_video_tensors(path):
    vid = list(VideoReader(path).to_iter())
    dim = (128, 128)
    vid = [ cv2.resize(vid[i], dim, interpolation = cv2.INTER_AREA) \
                for i in range(0, len(vid), 2)]
    vid = np.array(vid)
    vid = torch.from_numpy(vid)
    vid = vid.float() / 255.0
    vid = vid.permute(3, 0, 1, 2)
    vid = vid.unsqueeze(0)
    return vid

test_video_tensors = [get_video_tensors(join(TEST_ROOT, p)) for p in test_videos]
rand_shuffle = random.randint(0, 3)

inference_text = [("", []), ("", []), ("", [])]
ground_truth_res = [False, False, False]

def reset():
    global inference_text
    global ground_truth_res
    inference_text = [("", []), ("", []), ("", [])]
    ground_truth_res = [False, False, False]

def video_classification_page(state):

    st.title("Classification of COVID-19 Based on Lung Ultra-sound")

    # INTRO
    col1, col2 = st.beta_columns(2)
    col1.markdown(desc)
    col2.image(join(IMG_ROOT, "vector_logo.jpg"))

    # Data
    st.markdown(datacard)
    col1, col2 = st.beta_columns([1, 1])
    col1.markdown(datasource)
    col2.markdown("## Conceptual flow of the data collection and processing")
    col2.image(join(IMG_ROOT, "conceptual_flow.png"))

    # Data samples
    example_expander = st.beta_expander("Data Samples")
    cols = [2, 4, 4]
    for i in range(0, len(sample_video_bytes), 2):
        col0, col1, col2 = example_expander.beta_columns(cols)
        col0.markdown("**{}**".format(classes[i // 2]))
        col1.video(sample_video_bytes[i])
        col2.video(sample_video_bytes[i + 1])

    # Model
    st.markdown("# Let's start with selecting a model!")
    models = ("None", "Resnet 3D Model (R3D)",
                      "Mixed Convolutional Network (MC3)")
    selected_model = st.selectbox("", models)

    if len(selected_model) == 0 or selected_model == "None":
        return

    col1, col2 = st.beta_columns([2, 1])
    if "MC3" in selected_model:
        model_card = model_card_mc3
        img_path = join(IMG_ROOT, "mc3.png")
        model_key = "MC3"
    else:
        model_card = model_card_r3d
        img_path = join(IMG_ROOT, "r3d.png")
        model_key = "R3D"

    col1.markdown(model_card)
    col2.image(img_path, width=200, caption="Model Architecture")

    # Live Demo
    demo_expander = st.markdown("# Test the model on real (unseen) videos")
    model_for_inference = loaded_models[model_key]
    demo_expander = st.beta_expander("Test Samples")
    if demo_expander.button("Reset", key="reset"):
        reset()

    cols = [4, 2, 2, 2]
    for i in range(len(test_video_bytes)):
        i = (i + rand_shuffle) % len(test_video_bytes)
        col0, col1, col2, col3 = demo_expander.beta_columns(cols)

        col0.video(test_video_bytes[i])
        col1.markdown("__Take a guess below__")
        user_pred = col1.selectbox("", ["I Don't Know"] + classes,
                                   key="select{}".format(i))
        model_pred = None

        col2.markdown("---")
        if col2.button("Test Video Against Model", key="pred{}".format(i)):
            pred = model_for_inference(test_video_tensors[i].to(device))
            pred_idx = torch.argmax(pred).item()
            beta = 0.5
            pred = pred * beta
            pred = torch.nn.Softmax(dim=0)(pred.flatten()).tolist()

            model_pred = classes[pred_idx]

            prediction_text = ["{:<15}: {:.2f}%".format(cls, prob * 100) \
                                    for cls, prob in zip(classes, pred)]

            inference_text[i] = model_pred, prediction_text

        model_pred, prediction_text = inference_text[i]

        for t in prediction_text:
            col2.write(t)

        if model_pred:
            col2.markdown("\n*__Prediction: {}__*\n".format(model_pred))

        col3.markdown("---")
        if col3.button("Show Ground Truth", key="gt{}".format(i)):
            ground_truth_res[i] = True

        if ground_truth_res[i]:
            ground_truth = classes[i]
            col3.write("Ground Truth:")
            col3.write("__{}__".format(ground_truth))

            col3.markdown("---")
            if model_pred == ground_truth:
                col3.write("Model is correct!!")
            else:
                col3.write("Model is wrong...")

            col3.markdown("---")
            if user_pred == ground_truth:
                col3.write("You are correct!!")
            else:
                col3.write("You are wrong...")


import logging
import os
from datetime import datetime
from logging.handlers import RotatingFileHandler
from typing import Dict

import json

# environ params
LOG_LEVEL = 'LOG_LEVEL'
LOG_CONSOLE_FORMAT = 'LOG_CONSOLE_FORMAT'
LOG_FILE_FORMAT = 'LOG_FILE_FORMAT'
LOG_ENABLE_FILE = 'LOG_ENABLE_FILE'
LOG_FILE_PATH = 'LOG_FILE_PATH'
LOG_MAX_BYTES = 'LOG_MAX_BYTES'
LOG_BACKUP_COUNT = 'LOG_BACKUP_COUNT'

# constant
LOG_NAME = 'modelscope-agent'
INFO_LOG_FILE_NAME = 'info.log'
ERROR_LOG_FILE_NAME = 'error.log'

def get_formatter(log_format_env):
    if log_format_env == 'json':
        formatter = JsonFormatter()
    else:
        formatter = TextFormatter()
    return formatter

class JsonFormatter(logging.Formatter):
    """
    Custom formatter to output logs in JSON format.
    """

    def format(self, record):
        log_record = {
            'timestamp': datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
            'level': record.levelname,
            'message': record.getMessage(),
            # Extract additional fields if they are in the 'extra' dict
            'uuid': getattr(record, 'uuid', None),
            'request_id': getattr(record, 'request_id', None),
            'content': getattr(record, 'content', None),
            'error': getattr(record, 'error', None),
            'step': getattr(record, 'step', None)
        }
        # Clean up any extra fields that are None (not provided)
        log_record = {k: v for k, v in log_record.items() if v is not None}
        if record.exc_info:
            log_record['exc_info'] = self.formatException(record.exc_info)
        return json.dumps(log_record, ensure_ascii=False)

class TextFormatter(logging.Formatter):
    """
    Custom formatter to output logs in text format.
    """

    def format(self, record):
        timestamp = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')[:-3]
        level = record.levelname
        message = record.getMessage()

        # Collect additional fields if they are in the 'extra' dict
        uuid = getattr(record, 'uuid', '-')
        request_id = getattr(record, 'request_id', '-')
        content = getattr(record, 'content', '-')
        step = getattr(record, 'step', '-')
        error = getattr(record, 'error', '-')

        # Format the log record as text
        log_message = f'{timestamp} - {LOG_NAME} - {level} - '
        log_message = log_message + f' | message: {message}'
        if uuid != '-':
            log_message += f' | uuid: {uuid}'
        if request_id != '-':
            log_message += f' | request_id: {request_id}'
        if content != '-':
            log_message += f' | content: {content}'
        if step != '-':
            log_message += f' | step: {step}'
        if error != '-':
            log_message += f' | error: {error}'

        if record.exc_info:
            log_message += f' | Exception: {self.formatException(record.exc_info)}'

        return log_message

class AgentLogger:

    def __init__(self):
        self.logger = logging.getLogger(LOG_NAME)
        self.logger.propagate = False
        log_level = os.getenv(LOG_LEVEL, 'INFO').upper()
        self.logger.setLevel(getattr(logging, log_level))

        # Create console handler with TextFormatter
        console_log_formatter = get_formatter(
            os.getenv(LOG_CONSOLE_FORMAT, 'normal').lower())
        console_handler = logging.StreamHandler()
        console_handler.setFormatter(
            get_formatter(log_format_env=console_log_formatter))
        self.logger.addHandler(console_handler)

        if os.environ.get(LOG_ENABLE_FILE, 'on').lower() == 'on':
            _log_dir = os.getenv(LOG_FILE_PATH, f'{os.getcwd()}/logs')
            os.makedirs(_log_dir, exist_ok=True)
            self.set_file_handle(
                log_dir=_log_dir,
                max_bytes=int(os.environ.get(LOG_MAX_BYTES, 50 * 1024 * 1024)),
                backup_count=int(os.environ.get(LOG_BACKUP_COUNT, 7)))

    def set_file_handle(self,
                        log_dir,
                        max_bytes=50 * 1024 * 1024,
                        backup_count=7):
        """
        Args:
            log_dir (str): Directory to save the log files.
            max_bytes (int): Maximum size in bytes for a single log file, default 50MB.
            backup_count (int): The number of log files to keep, default is 7.
        """
        # Create file handlers with JsonFormatter
        file_log_formatter = get_formatter(
            os.getenv(LOG_FILE_FORMAT, 'json').lower())
        info_file_path = os.path.join(log_dir, INFO_LOG_FILE_NAME)
        info_file_handler = RotatingFileHandler(
            info_file_path,
            mode='a',
            maxBytes=max_bytes,
            backupCount=backup_count)
        info_file_handler.setFormatter(file_log_formatter)
        info_file_handler.setLevel(logging.INFO)

        # Create error file handler
        error_file_path = os.path.join(log_dir, ERROR_LOG_FILE_NAME)
        error_file_handler = RotatingFileHandler(
            error_file_path,
            mode='a',
            maxBytes=max_bytes,
            backupCount=backup_count)
        error_file_handler.setFormatter(file_log_formatter)
        error_file_handler.setLevel(logging.ERROR)

        # Add handlers to the logger
        self.logger.addHandler(info_file_handler)
        self.logger.addHandler(error_file_handler)

    def info(self,
             uuid: str = 'default_user',
             request_id: str = 'default_request_id',
             content: Dict = None,
             step: str = '',
             message: str = '',
             error: str = ''):
        if content is None:
            content = {}

        self.logger.info(
            message,
            extra={
                'uuid': uuid,
                'request_id': request_id,
                'content': content,
                'step': step,
                'error': error
            })

    def error(self,
              uuid: str = 'default_user',
              request_id: str = 'default_request_id',
              content: Dict = None,
              step: str = '',
              message: str = '',
              error: str = ''):
        if content is None:
            content = {}

        self.logger.error(
            message,
            extra={
                'uuid': uuid,
                'request_id': request_id,
                'content': content,
                'step': step,
                'error': error
            })

agent_logger = AgentLogger()


from datetime import timedelta
from celery import Celery

# from celery.schedules import crontab

from backend.config import settings

beat_schedule = {
    'send_email': {
        'task': 'send_email',
        # 'schedule': crontab(minute=1),
        'schedule': timedelta(seconds=5),
    }
}

app_celery = Celery("backend")
app_celery.config_from_object(obj=settings.celery)
app_celery.conf.beat_schedule = beat_schedule


import pyodbc
from PyQt5 import QtWidgets
from matplotlib.backends.backend_qt5agg import FigureCanvasQTAgg as FigureCanvas
from matplotlib.figure import Figure
import locale

# Defina o locale para formatar os valores como reais (R$)
locale.setlocale(locale.LC_ALL, 'pt_BR.utf8')

server = '<server>.database.windows.net'
database = '<database>'
username = '<username>'
password = '<password>'
driver = '{ODBC Driver 17 for SQL Server}' # or 'SQL Server'

cnxn = pyodbc.connect(
    f'DRIVER={driver};SERVER={server};PORT=1433;DATABASE={database};UID={username};PWD={password}'
)

cursor = cnxn.cursor()

# Consulta para obter os dados para a primeira visualização
query1 = """
SELECT TOP 10 NomeFavorecido, ValorTotalRecebido
FROM ResumoFavorecido
ORDER BY ValorTotalRecebido DESC
"""

cursor.execute(query1)
results1 = cursor.fetchall()

# Consulta para obter os dados para a segunda visualização
query2 = """
SELECT SiglaUF, COUNT(DISTINCT CodigoFavorecido) as NumFavorecidos
FROM Recebimento
GROUP BY SiglaUF
ORDER BY NumFavorecidos DESC
"""

cursor.execute(query2)
results2 = cursor.fetchall()

results2_sorted = sorted(results2, key=lambda x: x[1], reverse=True)
top_results2 = results2_sorted[:len(results2_sorted)-6]
other_results2 = results2_sorted[len(results2_sorted)-6:]
other_sum = sum(row[1] for row in other_results2)
results2 = top_results2 + [('Outros', other_sum)]

# Consulta para obter os dados para a terceira visualização
query3 = """
SELECT TOP 10 NomeOrgaoSuperior, SUM(ValorRecebido) as TotalRecebido
FROM Recebimento JOIN OrgaoSuperior ON Recebimento.CodigoOrgaoSuperior = OrgaoSuperior.CodigoOrgaoSuperior
GROUP BY NomeOrgaoSuperior
ORDER BY TotalRecebido DESC
"""

cursor.execute(query3)
results3 = cursor.fetchall()

cursor.close()
cnxn.close()

class MainWindow(QtWidgets.QMainWindow):
    def __init__(self):
        super().__init__()

        central_widget = QtWidgets.QWidget()
        self.setCentralWidget(central_widget)

        main_layout = QtWidgets.QVBoxLayout()
        central_widget.setLayout(main_layout)

        h_layout = QtWidgets.QHBoxLayout()
        main_layout.addLayout(h_layout)

        fig1 = Figure()
        ax1 = fig1.add_subplot(111)
        ax1.bar([row[0] for row in results1], [row[1]/1000000000 for row in results1])
        ax1.set_title('Top 10 Favorecidos por Valor Total Recebido')
        ax1.set_xlabel('Nome Favorecido')
        ax1.set_ylabel('Valor Total Recebido (bilhões de R$)')

        ax1.yaxis.set_major_formatter(lambda x, pos: locale.currency(x, grouping=True))

        for label in ax1.get_xticklabels():
            label.set_rotation(10)
            label.set_ha('right')

        fig1.subplots_adjust(bottom=0.3, left=0.2)

        canvas1 = FigureCanvas(fig1)
        h_layout.addWidget(canvas1, stretch=2)

        fig2 = Figure()
        ax2 = fig2.add_subplot(111)
        ax2.pie([row[1] for row in results2], labels=[row[0] for row in results2])
        ax2.set_title('Número de Favorecidos por UF')
        canvas2 = FigureCanvas(fig2)
        h_layout.addWidget(canvas2, stretch=1)

        fig3 = Figure()
        ax3 = fig3.add_subplot(111)
        ax3.plot([row[0] for row in results3], [row[1]/1000000000 for row in results3])
        ax3.set_title('Top 10 Órgãos Superiores por Total Recebido')
        ax3.set_xlabel('Nome do Órgão Superior')
        ax3.set_ylabel('Total Recebido (bilhões de R$)')

        ax3.yaxis.set_major_formatter(lambda x, pos: locale.currency(x, grouping=True))

        for label in ax3.get_xticklabels():
            label.set_rotation(10)
            label.set_ha('right')

        fig3.subplots_adjust(bottom=0.3)

        canvas3 = FigureCanvas(fig3)
        main_layout.addWidget(canvas3)

if __name__ == '__main__':
    app = QtWidgets.QApplication([])
    window = MainWindow()
    window.show()
    app.exec_()