"""

Extracting root sum of squares from edge-wise timeseries.
=========================================================
This example uses a ``RSSETSMarker`` to compute root sum of squares
of the edge-wise timeseries using the Schaefer parcellation
(100 rois and 200 rois, 17 Yeo networks) for a 4D nifti BOLD file.

Authors: Leonard Sasse, Sami Hamdan, Nicolas Nieto, Synchon Mandal

License: BSD 3 clause

"""

import tempfile

import junifer.testing.registry  # noqa: F401
from junifer.api import collect, run
from junifer.storage import SQLiteFeatureStorage
from junifer.utils import configure_logging

###############################################################################
# Set the logging level to info to see extra information:
configure_logging(level="INFO")

##############################################################################
# Define the DataGrabber interface
datagrabber = {
    "kind": "SPMAuditoryTestingDataGrabber",
}

###############################################################################
# Define the markers interface
markers = [
    {
        "name": "Schaefer100x17_RSSETS",
        "kind": "RSSETSMarker",
        "parcellation": "Schaefer100x17",
    },
    {
        "name": "Schaefer200x17_RSSETS",
        "kind": "RSSETSMarker",
        "parcellation": "Schaefer200x17",
    },
]

###############################################################################
# Create a temporary directory for junifer feature extraction:
# At the end you can read the extracted data into a ``pandas.DataFrame``.
with tempfile.TemporaryDirectory() as tmpdir:
    # Define the storage interface
    storage = {
        "kind": "SQLiteFeatureStorage",
        "uri": f"{tmpdir}/test.sqlite",
    }
    # Run the defined junifer feature extraction pipeline
    run(
        workdir=tmpdir,
        datagrabber=datagrabber,
        markers=markers,
        storage=storage,
        elements=["sub001"],  # we calculate for one subject only
    )
    # Collect extracted features data
    collect(storage=storage)
    # Create storage object to read in extracted features
    db = SQLiteFeatureStorage(uri=storage["uri"])

    # List all the features
    print(db.list_features())
    # Read extracted features
    df_vbm = db.read_df(feature_name="BOLD_Schaefer100x17_RSSETS")

###############################################################################
# Now we take a look at the dataframe
df_vbm.head()


import os
from uuid import uuid4
import undetected_chromedriver as uc
from selenium import webdriver
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
import polars as pl
from selenium.webdriver.common.by import By
from bot.constants import IMPLEMENTED
from selenium.common.exceptions import NoSuchElementException,StaleElementReferenceException,ElementNotInteractableException
import logging
from selenium.webdriver.support import expected_conditions
import time

class UcChromeDriver(uc.Chrome):
    """"
    Description:
    This class is a wrapper for the undetected_chromedriver package but with basic options

    """

    def __init__(self)->None:

        options = webdriver.ChromeOptions()

        options.add_argument('--log-level=1')
        # Adding argument to disable the AutomationControlled flag
        options.add_argument("--disable-blink-features=AutomationControlled")

        # Exclude the collection of enable-automation switches
        options.add_experimental_option("excludeSwitches", ["enable-automation"])

        # Turn-off userAutomationExtension
        options.add_experimental_option("useAutomationExtension", False)

        self.driver = uc.Chrome(executable_path="chromedriver",chrome_options=options)

        # driver.set_window_size(1920,1080)

    def get_driver(self):
        return self.driver

class Bot:
    """
    Description:
    """
    def __init__(self,
                 website:str,
                 job_title:str="Data Scientist",
                 search_location:str="Paris",
                 ) -> None:

        if website not in list(IMPLEMENTED.keys()):
            raise NotImplementedError(f"{website} not implemented yet")

        self.job_title = job_title
        self.search_location = search_location

        self.website = website

        self.driver = UcChromeDriver().get_driver()
        self.driver.implicitly_wait(5)

        self.CONSTANTS = IMPLEMENTED[website]
        self.wait = WebDriverWait(self.driver, 5,poll_frequency=0.2)

    def find_element_try(self,element=None,verbose=False,*args,**kwargs):
        """
        Description:
        This method will try to find an element and return None if it fails"""
        try:
           if element:
               return element.find_element(*args,**kwargs)
           return self.driver.find_element(*args,**kwargs)
        except NoSuchElementException as e:
            if verbose:
                print(e)
            return None

    def save_offers(self,offers,schema=None):
        """"
        Description:
        This method will save the offers in parquet format"""
        if offers:
            if offers[0]["job_title"]:
                try:
                    pl.DataFrame(
                        offers,
                        schema=schema
                    ).write_parquet(f"{self.CONSTANTS['DATA_PATH']}/{self.website}_{uuid4()}.parquet")
                except Exception as e:
                    print(offers)
                    print(e)

    def get_offers_off_page(self):
        """
        Description:
        This method will get the offers on the page and return a list of offers"""

        # self.wait.until(EC.visibility_of_all_elements_located(self.CONSTANTS["LIST_PAGE_OFFERS"]))

        return self.wait.until(EC.visibility_of_all_elements_located(self.CONSTANTS["LIST_PAGE_OFFERS"]))

    def search_job(self):
        """

        Description:
        This method will search the job title in the search bar and click on the search button"""

        self.driver.get(self.CONSTANTS["START_URL"])

        search_bar = self.wait.until(EC.visibility_of_element_located(tuple(self.CONSTANTS["SEARCH_BAR"].values()) ))

        webdriver.ActionChains(self.driver).move_to_element(search_bar).click().perform()

        webdriver.ActionChains(self.driver).send_keys(self.job_title).perform()
        # self.driver.find_element(**self.CONSTANTS["SEARCH_BAR"])
        location  = self.driver.find_element(**self.CONSTANTS["SEARCH_LOCATION"])

        location.send_keys(Keys.CONTROL + "a")
        location.send_keys(Keys.DELETE)
        location.clear()

        webdriver.ActionChains(self.driver).move_to_element(location).click().perform()

        # self.search_location=""
        location.send_keys(self.search_location)
        location.send_keys(Keys.ENTER)

        self.page = 1

        self.offers = []

class IndeedBot(Bot):
    """
    Description:
    This class is a bot that will scrap the indeed website and save the data in parquet forma
    """

    def __init__(self,
                 *args,
                 **kwargs
                 ) -> None:
        """
        Description:
        This method is the constructor of the class
        Parameters:
        job_title: str
        This parameter is the job title that will be searched
        @TODO: Add a parameter to choose the location
        """
        super().__init__("indeed",
                          *args,
                        **kwargs)

    def run(self):
        """
        Description:
        This method is the main method of the class, it will run the bot and save the data in parquet format
        @TODO : Add a method to save the data in a database
        @TODO : Implement logging
        """
        self.search_job()

        while self.page:

            if self.page != 1:
                self.driver.get(self.page.get_attribute('href'))
                webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()

            result_list = self.get_offers_off_page()
            self.go_trough_offers(result_list)
            self.next_page()
            self.save_offers(self.offers)
            # page = None
        self.driver.quit()
        print("Done")

    def next_page(self):
        """
        Description:
        This method will find the next page button"""
        self.page = self.find_element_try(**self.CONSTANTS["NEXT_PAGE"])

    def go_trough_offers(self,result_list):
        """
        Description:
        This method will go trough the offers and save the data in a list of dict"""

        for offer in result_list:

                offer_dict= dict()
                for i in ("job_title","id","company","text","salary_range"):
                    offer_dict[i]=None
                try:
                    os.listdir(self.CONSTANTS["DATA_PATH"])
                    list_fetch_ids = pl.scan_parquet(f'{self.CONSTANTS["DATA_PATH"]}/*.parquet').filter(pl.col("text").is_not_null()).select(pl.col("id").unique()).collect().to_series()

                except FileNotFoundError as e :
                    list_fetch_ids = []
                try:

                    offer_dict['id'] = offer.find_element(**self.CONSTANTS["OFFER_ID"]).get_attribute('id')
                    if offer_dict['id']  not in list_fetch_ids:

                        offer_dict["job_title"] =offer.find_element(**self.CONSTANTS["OFFER_TITLE"]).text
                        offer_dict["company"] = offer.find_element(**self.CONSTANTS["OFFER_COMPANY"]).text
                        webdriver.ActionChains(self.driver).move_to_element(offer).click(offer ).perform()
                        text_offer =self.driver.find_element(*self.CONSTANTS["OFFER_TEXT"])
                        offer_dict["text"] = text_offer.text
                        offer_dict["salary_range"] = offer.find_element(**self.CONSTANTS["OFFER_SALARY"]).text
                        self.offers.append(offer_dict)

                except Exception as e:
                    print(offer_dict.get("id",None))
                    print(e)

class GlassdoorBot(Bot):
    """
    Description:
    This class is a bot that will scrap the glassdoor website and save the data in parquet forma
    """

    def __init__(self,
                 *args,
                 **kwargs
                 ) -> None:
        """
        Description:
        This method is the constructor of the class
        Parameters:
        job_title: str
        This parameter is the job title that will be searched
        """
        super().__init__("glassdoor",
                          *args,
                        **kwargs)

    def run(self):
        self.search_job()
        while self.page:

            if self.page != 1:
                # self.driver.get(self.page.get_attribute('href'))
                webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()

            result_list = self.get_offers_off_page_glassdoor()
            self.go_trough_offers(result_list)
            self.next_page()
            # s
            # self.save_offers(self.offers,schema=schema)
            # page = None
        schema = dict(
                job_title=pl.Utf8(),
                id=pl.Utf8(),
                company=pl.Utf8(),
                text=pl.Utf8(),
                salary_range=pl.Utf8(),
                location=pl.Utf8(),
                age=pl.Utf8(),
                company_rating=pl.Utf8(),
                company_info=pl.Utf8(),
                is_easy_apply=pl.Utf8(),
                salary_range_provenance=pl.Utf8()

            )
        self.save_offers(self.offers,schema=schema)
        self.driver.quit()
        print("Done")
    def get_offers_off_page_glassdoor(self):

        """
        Description:
        This method will get the offers on the page and return a list of offers"""

        # self.wait.until(EC.visibility_of_all_elements_located(self.CONSTANTS["LIST_PAGE_OFFERS"]))
        offers= self.wait.until(EC.visibility_of_all_elements_located(self.CONSTANTS["LIST_PAGE_OFFERS"]))
        offer_list = []
        for offer in offers:
            ignored_exceptions=(NoSuchElementException,StaleElementReferenceException,)
            o = WebDriverWait(self.driver, 10,ignored_exceptions=ignored_exceptions).until(expected_conditions.visibility_of(offer
                                                                                )).get_attribute('data-id')

            offer_list.append(o)

        return offer_list
    def next_page(self):
        """
        Description:
        This method will find the next page button"""
        self.page = self.driver.find_element(**self.CONSTANTS["NEXT_PAGE"])
        if self.page:
            webdriver.ActionChains(self.driver).move_to_element(self.page ).click(self.page ).perform()
        else:
            print("No more pages")

    def go_trough_offers(self,result_list):
        """
        Description:
        This method will go trough the offers and save the data in a list of dict"""
        self.count = 0
        for c,offer in enumerate(result_list):

            self.count = c + 1

            offer_dict= dict()
            for i in ("job_title","id","company","text","salary_range","location","age","company_rating","company_info","is_easy_apply","salary_range_provenance"):
                offer_dict[i]=None
            try:
                os.listdir(self.CONSTANTS["DATA_PATH"])
                list_fetch_ids = pl.scan_parquet(f'{self.CONSTANTS["DATA_PATH"]}/*.parquet').filter(pl.col("text").is_not_null()).select(pl.col("id").unique()).collect().to_series()

            except FileNotFoundError as e :
                list_fetch_ids = []

            offer_dict['id'] = offer

            if offer_dict['id']  not in list_fetch_ids:

                # offer_dict["job_title"] =offer.find_element(**self.CONSTANTS["OFFER_TITLE"]).text
                offer_xpath = f"//li[@data-id='{offer_dict['id']}']"
                location_xpath = f"//li[@data-id='{offer_dict['id']}']" + self.CONSTANTS["OFFER_LOCATION"]["value"]
                age_xpath = f"//li[@data-id='{offer_dict['id']}']" + self.CONSTANTS["OFFER_AGE"]["value"]
                company_rating_xpath = f"//li[@data-id='{offer_dict['id']}']" + self.CONSTANTS["COMPANY_RATING"]["value"]
                company_info_xpath = f"//li[@data-id='{offer_dict['id']}']" + self.CONSTANTS["COMPANY_INFO"]["value"]
                is_easy_apply_xpath = f"//li[@data-id='{offer_dict['id']}']" + self.CONSTANTS["OFFER_IS_EASY_APPLY"]["value"]

                salary_range_provenance_xpath = f"//li[@data-id='{offer_dict['id']}']" + self.CONSTANTS["OFFER_SALARY_PROVENANCE"]["value"]
                salary_range_xpath = f"//li[@data-id='{offer_dict['id']}']" + self.CONSTANTS["OFFER_SALARY"]["value"]
                job_title_xpath = f"//li[@data-id='{offer_dict['id']}']" + self.CONSTANTS["OFFER_JOB_TITLE"]["value"]

                ignored_exceptions=(NoSuchElementException,StaleElementReferenceException,)
                offer_element = WebDriverWait(self.driver, 10,ignored_exceptions=ignored_exceptions).until(expected_conditions.visibility_of_element_located((By.XPATH,offer_xpath)
                                                                                ))

                webdriver.ActionChains(self.driver).move_to_element(offer_element).click(offer_element ).perform()
                webdriver.ActionChains(self.driver).send_keys(Keys.ESCAPE).perform()

                offer_dict["location"] = self.find_element_try(by=By.XPATH,value=location_xpath)
                offer_dict["age"] = self.find_element_try(by=By.XPATH,value=age_xpath)

                offer_dict["salary_range_provenance"] = self.find_element_try(by=By.XPATH,value=salary_range_provenance_xpath)

                offer_dict["job_title"] = self.find_element_try(**self.CONSTANTS["OFFER_JOB_TITLE"])
                #takes too much time
                offer_dict["company_rating"] = self.find_element_try(**self.CONSTANTS['COMPANY_RATING'])
                #takes too much time
                offer_dict["company_info"] = self.find_element_try(**self.CONSTANTS["COMPANY_INFO"])
                offer_dict["is_easy_apply"] = self.find_element_try(**self.CONSTANTS["OFFER_IS_EASY_APPLY"])
                offer_dict["text"] = self.find_element_try(**self.CONSTANTS["OFFER_TEXT"])
                offer_dict["company"] = self.find_element_try(**self.CONSTANTS["OFFER_COMPANY"])
                offer_dict["salary_range"] = self.find_element_try(by=By.XPATH,value=salary_range_xpath)

                for i in offer_dict:
                    if type(offer_dict[i])==uc.webelement.WebElement:
                        offer_dict[i] = offer_dict[i].text

                self.offers.append(offer_dict)

                if c % 5 == 0 :
                    schema = dict(
                        job_title=pl.Utf8(),
                        id=pl.Utf8(),
                        company=pl.Utf8(),
                        text=pl.Utf8(),
                        salary_range=pl.Utf8(),
                        location=pl.Utf8(),
                        age=pl.Utf8(),
                        company_rating=pl.Utf8(),
                        company_info=pl.Utf8(),
                        is_easy_apply=pl.Utf8(),
                        salary_range_provenance=pl.Utf8()

                    )
                    self.save_offers(self.offers,schema=schema)

class LinkedinBot(Bot):
    """
    Description:
    This class is a bot that will scrap the glassdoor website and save the data in parquet forma
    """

    def __init__(self,
                 *args,
                 **kwargs
                 ) -> None:
        """
        Description:
        This method is the constructor of the class
        Parameters:
        job_title: str
        This parameter is the job title that will be searched
        """
        super().__init__("linkedin",
                          *args,
                        **kwargs)

    def get_offers_off_page_linkedin(self):

        """
        Description:
        This method will get the offers on the page and return a list of offers"""

        # self.wait.until(EC.visibility_of_all_elements_located(self.CONSTANTS["LIST_PAGE_OFFERS"]))
        offers= self.wait.until(EC.visibility_of_all_elements_located(self.CONSTANTS["LIST_PAGE_OFFERS"]))
        offer_list = []
        for offer in offers:
            ignored_exceptions=(NoSuchElementException,StaleElementReferenceException,)
            o = WebDriverWait(self.driver, 10,ignored_exceptions=ignored_exceptions).until(expected_conditions.visibility_of(offer
                                                                                ))

            offer_list.append(o)

        return offer_list

    def run(self):
        self.search_job()
        schema = dict(
                        job_title=pl.Utf8(),
                        id=pl.Utf8(),
                        company=pl.Utf8(),
                        location=pl.Utf8(),
                        age=pl.Utf8(),
                        date=pl.Utf8(),
                        date_parsed=pl.Float64(),
                        benefits=pl.Utf8(),
                        text=pl.Utf8(),
                        candidates=pl.Utf8(),
                    )
        while True:

            if not self.find_element_try(by=By.CLASS_NAME,value="infinite-scroller__show-more-button"):
                # Comment: if there is no more offers to load, stop
                break

            result_list = self.get_offers_off_page_linkedin()

            ids_to_pass = pl.read_parquet('linkedin_data/linkedin*.parquet').select("id").unique().to_series().to_list()

            result_list_filtered = list(filter(
                lambda x: x.get_attribute("data-entity-urn") not in ids_to_pass,result_list
            ))

            if not result_list_filtered :

                webdriver.ActionChains(self.driver).move_to_element(result_list[-1]).scroll_to_element(result_list[-1]).scroll_by_amount(delta_x=0,delta_y=100).perform()
                next_button =   self.find_element_try(by=By.CLASS_NAME,value="infinite-scroller__show-more-button")
                time.sleep(2)

                if next_button:
                    try:
                        webdriver.ActionChains(self.driver).move_to_element(next_button).click(next_button).perform()
                    except  ElementNotInteractableException as e:
                        continue

                continue

            for offer in result_list_filtered:
                offer_dict = {}
                offer_dict["job_title"] = offer.find_element(**self.CONSTANTS["OFFER_JOB_TITLE"]).text
                offer_dict["id"] = offer.get_attribute("data-entity-urn")
                offer_dict["company"] = offer.find_element(**self.CONSTANTS["OFFER_COMPANY"]).text
                offer_dict["location"] = offer.find_element(**self.CONSTANTS["OFFER_LOCATION"]).text

                offer_dict["age"] = self.find_element_try(offer,**self.CONSTANTS["OFFER_AGE"])
                if not offer_dict["age"]:
                    offer_dict["age"] = self.find_element_try(offer,**self.CONSTANTS["OFFER_AGE_NEW"])

                if offer_dict["age"]:
                    offer_dict["age"],offer_dict["date"]= offer_dict["age"].text,offer_dict["age"].get_attribute("datetime")

                offer_dict["date_parsed"] = time.time()

                offer_dict["benefits"] = self.find_element_try(offer,**self.CONSTANTS["OFFER_BENEFITS"])
                if offer_dict["benefits"]:
                    offer_dict["benefits"]= offer_dict["benefits"].text

                webdriver.ActionChains(self.driver).move_to_element(offer).click(offer).perform()
                time.sleep(2)
                offer_dict["text"] = self.driver.find_element(*self.CONSTANTS["OFFER_TEXT"]).text
                offer_dict["candidates"] = self.driver.find_element(**self.CONSTANTS["OFFER_CANDIDATES"]).text
                self.offers.append(offer_dict)

                print(offer_dict)
                # s
                # self.save_offers(self.offers,schema=schema)
                # page = None

            self.save_offers(self.offers,schema=schema)

        print("Done")
        time.sleep(60)
        self.driver.quit()


from PIL import Image

def decode_message(file_path):
    image = Image.open(file_path)
    image.show()
    width, height = image.size
    message = ''
    for x in range(30,width):
        y = 0
        while y < height and image.getpixel((x,y)) != 0:
            y += 1
        if y < height:
            message += chr(y)
            print('ok')
    return message

def main():
    print(decode_message('code.png'))

if __name__ == "__main__":
    main()


import error
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import curve_fit, fmin
from uncertainties import ufloat
from uncertainties import unumpy

vapor_pressure = [23.88, 23.75, 25.80, 27.90, 28.70, 29.90, 33.40, 33.70]
inv_temp = np.array([0.003354, 0.003332, 0.003299, 0.003277, 0.003245, 0.003224, 0.003193, 0.003173])
temp = 1/inv_temp
temperature = unumpy.uarray([temp], [0.6, 0.6, 0.6, 0.6,0.6, 0.6, 0.6, 0.6,])
inv_temperature = 1/temperature
print(inv_temperature)

def exponential(x, a, b):
    return a*np.exp(-b*x)

(a, b), cov = curve_fit(exponential, inv_temp, vapor_pressure)

print((a, b))
print(cov)

x_data = np.linspace(0.0031, 0.0034, 100)
y_data = exponential(x_data, a, b)

# Plotting
plt.grid(True)

fig, axes = plt.subplots()
axes.set_ylabel('Vapor Pressure [Bar]', fontsize=24)
axes.set_xlabel('1/Temperature [1/K]', fontsize=24)
axes.set_title('Arrhenius Graph', fontsize=24)
plt.yscale("log")

axes.errorbar(inv_temp, vapor_pressure,fmt=" ", xerr=0.0000065, yerr=error.err_vapour_pressure_bar, capsize=2, capthick=1)

axes.plot(x_data, y_data, color="green")

axes.legend(["Best fit curve", "Data points"], fontsize=24)

plt.show()


import json
import logging
from typing import Any, Optional, Type, Union

import gluonts
from gluonts.core import fqname_for
from gluonts.core.serde import dump_code
from gluonts.dataset.common import Dataset
from gluonts.evaluation import Evaluator, backtest
from gluonts.model.estimator import Estimator
from gluonts.model.forecast import Quantile
from gluonts.model.forecast_generator import QuantileForecastGenerator
from gluonts.model.predictor import Predictor
from gluonts.mx.model.predictor import RepresentableBlockPredictor
from gluonts.mx.model.estimator import GluonEstimator
from gluonts.support.util import maybe_len
from gluonts.transform import FilterTransformation, TransformedDataset

from .env import TrainEnv

logger = logging.getLogger(__name__)

def log_metric(metric: str, value: Any) -> None:
    logger.info(f"gluonts[{metric}]: {dump_code(value)}")

def log_version(forecaster_type):
    name = fqname_for(forecaster_type)
    version = forecaster_type.__version__

    logger.info(f"Using gluonts v{gluonts.__version__}")
    logger.info(f"Using forecaster {name} v{version}")

def run_train_and_test(
    env: TrainEnv, forecaster_type: Type[Union[Estimator, Predictor]]
) -> None:
    log_version(forecaster_type)

    logger.info(
        "Using the following data channels: %s", ", ".join(env.datasets)
    )

    forecaster = forecaster_type.from_inputs(
        env.datasets["train"], **env.hyperparameters
    )
    logger.info(
        f"The forecaster can be reconstructed with the following expression: "
        f"{dump_code(forecaster)}"
    )

    if isinstance(forecaster, Predictor):
        predictor = forecaster
    else:
        predictor = run_train(
            forecaster=forecaster,
            train_dataset=env.datasets["train"],
            validation_dataset=env.datasets.get("validation"),
            hyperparameters=env.hyperparameters,
        )

    predictor.serialize(env.path.model)

    if "test" in env.datasets:
        run_test(env, predictor, env.datasets["test"], env.hyperparameters)

def run_train(
    forecaster: Estimator,
    train_dataset: Dataset,
    hyperparameters: dict,
    validation_dataset: Optional[Dataset],
) -> Predictor:
    num_workers = (
        int(hyperparameters["num_workers"])
        if "num_workers" in hyperparameters.keys()
        else None
    )
    shuffle_buffer_length = (
        int(hyperparameters["shuffle_buffer_length"])
        if "shuffle_buffer_length" in hyperparameters.keys()
        else None
    )
    num_prefetch = (
        int(hyperparameters["num_prefetch"])
        if "num_prefetch" in hyperparameters.keys()
        else None
    )
    if isinstance(forecaster, GluonEstimator):
        return forecaster.train(
            training_data=train_dataset,
            validation_data=validation_dataset,
            num_workers=num_workers,
            num_prefetch=num_prefetch,
            shuffle_buffer_length=shuffle_buffer_length,
        )
    else:
        return forecaster.train(
            training_data=train_dataset, validation_data=validation_dataset
        )

def run_test(
    env: TrainEnv,
    predictor: Predictor,
    test_dataset: Dataset,
    hyperparameters: dict,
) -> None:
    len_original = maybe_len(test_dataset)

    test_dataset = TransformedDataset(
        test_dataset,
        FilterTransformation(
            lambda x: x["target"].shape[-1] > predictor.prediction_length
        ),
    )

    len_filtered = len(test_dataset)

    if len_original is not None and len_original > len_filtered:
        logger.warning(
            f"Not all time-series in the test-channel have "
            f"enough data to be used for evaluation. Proceeding with "
            f"{len_filtered}/{len_original} "
            f"(~{int(len_filtered / len_original * 100)}%) items."
        )

    forecast_it, ts_it = backtest.make_evaluation_predictions(
        dataset=test_dataset, predictor=predictor, num_samples=100
    )

    test_quantiles = (
        [
            Quantile.parse(quantile).name
            for quantile in hyperparameters["test_quantiles"]
        ]
        if "test_quantiles" in hyperparameters.keys()
        else None
    )

    if isinstance(predictor, RepresentableBlockPredictor) and isinstance(
        predictor.forecast_generator, QuantileForecastGenerator
    ):
        predictor_quantiles = predictor.forecast_generator.quantiles
        if test_quantiles is None:
            test_quantiles = predictor_quantiles
        elif not set(test_quantiles).issubset(set(predictor_quantiles)):
            logger.warning(
                f"Some of the evaluation quantiles `{test_quantiles}` are "
                f"not in the computed quantile forecasts `{predictor_quantiles}`."
            )
            test_quantiles = predictor_quantiles

    if test_quantiles is not None:
        logger.info(f"Using quantiles `{test_quantiles}` for evaluation.")
        evaluator = Evaluator(quantiles=test_quantiles)
    else:
        evaluator = Evaluator()

    agg_metrics, item_metrics = evaluator(
        ts_iterator=ts_it,
        fcst_iterator=forecast_it,
        num_series=len(test_dataset),
    )

    # we only log aggregate metrics for now as item metrics may be very large
    for name, score in agg_metrics.items():
        logger.info(f"#test_score ({env.current_host}, {name}): {score}")

    # store metrics
    with open(env.path.model / "agg_metrics.json", "w") as agg_metric_file:
        json.dump(agg_metrics, agg_metric_file)
    with open(env.path.model / "item_metrics.csv", "w") as item_metrics_file:
        item_metrics.to_csv(item_metrics_file, index=False)


import json

def infntIN_ir_seed0(args):
    args.load_step = 2215000
    args.load_port = 26001
    args.load_dbname = 'combinet-test'
    args.load_colname = 'combinet'
    args.load_expId = 'inst_infant_18_fx'

    args.identifier = 'infntIN-ir-seed0'
    cfg_kwargs = {
            'network_func': 'get_resnet_18',
            'network_func_kwargs': '{"num_cat": 128}',
            }
    args.cfg_kwargs = json.dumps(cfg_kwargs)
    return args


#!/usr/bin/env python
"""
A semi-complicated example of sizer use
"""

import wx

class MyFrame(wx.Frame):
    def __init__(self, parent, id, title):
        wx.Frame.__init__(
            self,
            parent,
            id,
            title,
            size=wx.DefaultSize,
            style=wx.DEFAULT_DIALOG_STYLE | wx.MAXIMIZE_BOX |
            # wx.THICK_FRAME |
            wx.RESIZE_BORDER)
        self.status = self.CreateStatusBar()

        btn1above = wx.Button(self, -1, "Create")
        btn2above = wx.Button(self, -1, "Ok")
        btn1right = wx.Button(self, -1, "Cancel")
        btnbrowser = wx.Button(self, -1, "Browse")
        btnmenu = wx.Button(self, -1, "Menu")
        lst = wx.ListCtrl(self, -1)
        self.chkMatchCase = wx.CheckBox(self, -1, "Match Case")
        self.chkRegularExpression = wx.CheckBox(self, -1, "RegularExpression")
        self.chkSubDirectories = wx.CheckBox(self, -1, "Subdirectories")

        helpsizer = wx.BoxSizer(wx.VERTICAL)
        helpsizer.Add((0, 5), 0, wx.ALL, 0)
        helpsizer.Add(btnbrowser, 0, wx.ALL, 0)
        helpsizer.Add((0, 30), 0, wx.ALL, 0)
        helpsizer.Add(btnmenu, 0, wx.ALL, 0)
        helpsizer.Add((0, 40), 0, wx.ALL, 0)
        helpsizer.Add(btn1above, 0, wx.ALL, 0)
        helpsizer.Add((0, 5), 0, wx.ALL, 0)
        helpsizer.Add(self.chkRegularExpression, 0, wx.ALL, 0)
        helpsizer.Add((0, 25), 0, wx.ALL, 0)
        helpsizer.Add(self.chkMatchCase, 0, wx.ALL, 0)
        helpsizer.Add((0, 5), 0, wx.ALL, 0)
        helpsizer.Add(self.chkSubDirectories, 0, wx.ALL, 0)
        helpsizer.Add((0, 30), 0, wx.ALL, 0)
        helpsizer.Add(btn2above, 0, wx.ALL, 0)
        helpsizer.Add((0, 10), 0, wx.ALL, 0)
        helpsizer.Add(btn1right, 0, wx.ALL, 0)

        stat1 = wx.StaticText(self, -1, "Directory:")
        stat2 = wx.StaticText(self, -1, "File Pattern:")
        stat3 = wx.StaticText(self, -1, "Search For:")
        text1 = wx.ComboBox(self, -1, "", wx.DefaultPosition, (300, -1))
        text2 = wx.ComboBox(self, -1, "")
        text3 = wx.ComboBox(self, -1, "")

        topsizer = wx.FlexGridSizer(3, 2, 5, 5)
        topsizer.AddGrowableCol(1)
        topsizer.Add(stat1, 0)
        topsizer.Add(text1, 1, wx.GROW)
        topsizer.Add(stat2, 0)
        topsizer.Add(text2, 1, wx.GROW)
        topsizer.Add(stat3, 0)
        topsizer.Add(text3, 1, wx.GROW)

        leftSizer = wx.BoxSizer(wx.VERTICAL)
        leftSizer.Add(topsizer, 0, wx.EXPAND | wx.ALL, 5)
        leftSizer.Add(lst, 1, wx.EXPAND | wx.ALL, 5)

        newSizer = wx.BoxSizer(wx.HORIZONTAL)
        newSizer.Add(leftSizer, 1, wx.EXPAND | wx.ALL, 10)
        newSizer.Add(helpsizer, 0, wx.ALIGN_TOP | wx.ALL, 10)
        lastSizer = wx.BoxSizer(wx.VERTICAL)
        self.SetSizer(newSizer)

        self.Fit()
        self.Show()

if __name__ == "__main__":
    app = wx.App(False)
    Panel = MyFrame(None, wx.ID_ANY, title="Find Files")
    app.MainLoop()


#Evaluation

#accuracy
# PA
# TP(PP)
# FP(PN)
# FN(NP)
# TN(NN)

import torch
import torch.nn as nn
import torch.optim as optim
def Evaluation(Y_hat,Y):
    print(Y_hat)
    print(Y)
    from sklearn.metrics import f1_score
    TP,FP,FN,TN = 0,0,0,0
    for i in range(len(Y_hat)):
        if int(Y_hat[i]) == 1 and int(Y[i]) ==1:
            TP+=1
        elif int(Y_hat[i]) == 1 and int(Y[i]) ==0:
            FP +=1
        elif int(Y_hat[i]) == 0 and int(Y[i]) ==1:
            FN +=1
        elif int(Y_hat[i]) == 0 and int(Y[i]) ==0:
            TN +=1
        else:
            print('[ERROR]')
    Accuracy = (TP+TN)/(TP+FP+FN+TN)
    Precision = (TP)/(TP+FP)
    Recall = (TP)/(TP+FN)
    F1 = f1_score(Y, Y_hat)

    print('Accuracy:',Accuracy)
    print('Precision:',Precision)
    print('Recall:',Recall)
    print('F1:',F1)

def Train_Eval_Process_Layer(train_X,train_Y,test_X,test_Y):
    # RetaGNN + Self Attention
    import pyprind
    import pickle
    epoch_num =10
    input_dim = 8
    hidden_dim = 8
    model = double_LSTM_model().cuda()
    optimizer = optim.Adam(model.parameters())
    criterion = nn.BCELoss()
    for epoch_  in range(epoch_num):
        model.train()
        for i in pyprind.prog_bar(range(len(train_X))):
            batch_X,batch_Y = train_X[i],train_Y[i] #(b,l,d) ,(b,)
            batch_Y_hat = model(batch_X).squeeze(dim=-1)
            loss = criterion(batch_Y_hat, batch_Y.float())
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
            #print('loss:',loss)
        model.eval()
        pred_Y = list()
        for i in range(len(test_X)):
            pred_Y.append(model(test_X[i]).view(1,-1))
        test_Y_hat = torch.cat(pred_Y,0).cpu().data.numpy()
        test_Y_hat_list = list()
        for i in range(test_Y_hat.shape[0]):
            if test_Y_hat[i,0] >= 0.5:
                test_Y_hat_list.append(1)
            else:
                test_Y_hat_list.append(0)
        Evaluation(test_Y_hat_list,test_Y)

#single2batch layer for RetaGNN + Self Attention
import numpy as np
def Single2Batch_Layer_v2(X,Y,X_text):
    train_rate = 0.8
    batch_size = 32
    train_size = int(train_rate * len(X))
    import random
    data_index = [i for i in range(len(X))]
    shuffle_data_index = random.sample(data_index,len(data_index))
    train_data_index = shuffle_data_index[:train_size]
    test_data_index = shuffle_data_index[train_size:]
    if train_size % batch_size != 0 :
        batch_num = int(train_size / batch_size) + 1
    else:
        batch_num = int(train_size / batch_size)
    train_X,train_Y = list(),list()
    X_virtual,Y_virtual = list(),list()
    for i in range(len(train_data_index)):
        X_i,Y_i = X[train_data_index[i]],Y[train_data_index[i]]
        Y_i = np.array([Y_i])
        X_virtual.append(X_i)
        Y_virtual.append(Y_i.reshape(1,1))
        if len(X_virtual) >= batch_size :
            Y_virtual = np.concatenate(Y_virtual)
            Y_virtual = torch.tensor(Y_virtual)
            train_X.append(X_virtual)
            train_Y.append(Y_virtual)
            X_virtual,Y_virtual = list(),list()
        elif len(train_X) == batch_num - 1 and i == len(train_data_index)-1:
            Y_virtual =  np.concatenate(Y_virtual)
            Y_virtual = torch.tensor(Y_virtual)
            train_X.append(X_virtual)
            train_Y.append(Y_virtual)
    test_X,test_Y,test_X_text = list(),list(),list()
    for i in range(len(test_data_index)):
        X_i,Y_i = X[test_data_index[i]],Y[test_data_index[i]]
        X_i_text = X_text[test_data_index[i]]
        test_X.append(X_i)
        test_Y.append(Y_i)
        test_X_text.append(X_i_text)
    return train_X,train_Y,test_X,test_Y,test_X_text

import torch.nn as nn
class LSTM_model(nn.Module):
    def __init__(self, input_dim, hidden_dim):
        super(LSTM_model,self).__init__()
        self.input_dim,self.hidden_dim = input_dim,hidden_dim
        self.lstm = nn.LSTM(input_dim, hidden_dim)
        self.attention_weight = nn.Linear(hidden_dim, 1)
        self.fc = nn.Linear(hidden_dim, hidden_dim)
        self.sigmoid_layer = nn.Sigmoid()

    def Attention_Layer(self,hidden_vec):
        self.attn = self.attention_weight(hidden_vec)
        out = torch.sum(hidden_vec * self.attn,1)
        return out

    def forward(self, X,sent_type=False):
        hidden_vec, (h_n, c_n) = self.lstm(X)
        attn_layer_out = self.Attention_Layer(hidden_vec)
        out = self.fc(attn_layer_out)
        #out = self.sigmoid_layer(out)
        return out

class double_LSTM_model(nn.Module):
    def __init__(self):
        super(double_LSTM_model,self).__init__()
        self.lstm_layer = LSTM_model(1024,128).cuda()
        self.fc = nn.Linear(128, 1)
        self.sigmoid_layer = nn.Sigmoid()
    def forward(self,X):
        sent_emd_list = list()
        for i in range(len(X)):
            X_i = X[i]
            sent_embbeding = self.lstm_layer(X_i.view(1,-1,1024),sent_type=True).cuda()
            sent_emd_list.append(sent_embbeding.view(1,-1))
        sent_emd = torch.cat(sent_emd_list,0)
        sent_emd = torch.sum(sent_emd,0).cuda()
        pred_Y = self.fc(sent_emd)
        pred_Y = self.sigmoid_layer(pred_Y)
        return pred_Y


import pandas as pd
import numpy as np
import graphdatascience
import itertools

def strfdelta(tdelta, fmt):
    d = {'days': str(tdelta.days).zfill(2)}
    h, rem = divmod(tdelta.seconds, 3600)
    m, s = divmod(rem, 60)
    d['hours'] = str(h).zfill(2)
    d['minutes'] = str(m).zfill(2)
    d['seconds'] = str(s).zfill(2)
    return fmt.format(**d)

def minutes_to_duration(x):
    return pd.to_timedelta(x, unit='m') \
        .apply(lambda x: strfdelta(x, '{days} days, {hours} hrs, {minutes} min'))

def minutes_to_duration_arr(x):
    tds = pd.to_timedelta(x, unit='m')
    return [strfdelta(td, '{days}days-{hours}hrs-{minutes}min') for td in tds]

def minutes_to_duration_arr_col(x):
    return x.apply(minutes_to_duration_arr)

def clear_all_graphs(gds):
    g_names = gds.graph.list().graphName.tolist()
    for g_name in g_names:
        g = gds.graph.get(g_name)
        gds.graph.drop(g)

def historic_path_counts(gds, sid, tid):
    df = gds.run_cypher('''
        //match incoming legIds (incoming leg numbers are > 0)
        MATCH(:EntryPoint {airportId: $sourceAirportId})-[r:RECEPTION]->() WHERE r.legNumber > 0
        WITH r.legId AS incomingLegId, r.shipmentId AS shipmentId

        //filter to only shipmentIds that go to target airport
        MATCH()-[:DELIVERY {shipmentId:shipmentId}]->(:Destination {airportId: $targetAirportId})
        WITH incomingLegId, shipmentId

        //get transfer point airport ids to optimize path search
        MATCH()-[:DELIVERY {legId:incomingLegId}]->(n:TransferPoint)
        WITH incomingLegId, shipmentId, n.airportId AS transferAirportId

        //incoming path
        MATCH p=(:EntryPoint {airportId: $sourceAirportId})-[*1..12 {legId: incomingLegId}]->(:TransferPoint {airportId: transferAirportId})
        WITH incomingLegId, shipmentId, transferAirportId, nodes(p) AS n, relationships(p) AS r
        WITH incomingLegId, shipmentId, transferAirportId,
           apoc.text.join([i IN range(1,size(n)) WHERE (i=1) OR (n[i].airportId <> n[i-1].airportId )| n[i].name], ' -> ') AS incomingAirports,
           reduce(s=0, i in r | s + i.effectiveMinutes ) AS incomingCost

        //outgoing path (leg numbers are -1)
        MATCH p=(:TransferPoint {airportId: transferAirportId})-[*1..12 {shipmentId: shipmentId, legNumber:-1}]->(:Destination {airportId: $targetAirportId})
        WITH incomingLegId, incomingAirports, incomingCost, nodes(p) AS n, relationships(p) AS r
        WITH incomingLegId, incomingAirports,
            incomingAirports + ' -> ' + apoc.text.join([i IN range(1,size(n)) WHERE n[i].airportId <> n[i-1].airportId | n[i].name], ' -> ')  AS airportPath,
            incomingCost + reduce(s=0, i in r | s + i.effectiveMinutes  ) AS totalCost

        //aggregate and return
        RETURN DISTINCT airportPath, size(collect(incomingLegId)) AS historicPathCount, avg(totalCost) AS historicAvgCostMin, stDev(totalCost) AS historicCostStdMin, collect(totalCost) AS historicCostsMin
        ORDER BY historicPathCount DESC
    ''', params={'sourceAirportId': sid, 'targetAirportId': tid})

    df['historicAvgCost'] = minutes_to_duration(df.historicAvgCostMin)
    df['historicCostStd'] = minutes_to_duration(df.historicCostStdMin)
    df['historicCosts'] = minutes_to_duration_arr_col(df.historicCostsMin)
    df.drop(columns=['historicAvgCostMin', 'historicCostStdMin', 'historicCostsMin'], inplace=True)
    return df

def get_yen_dfs(gds, g, source_node_ids, target_node_ids, k, max_avg_time=np.Inf):
    df_list = []
    for source_node_id in source_node_ids:
        for target_node_id in target_node_ids:
            tdf = gds.shortestPath.yens.stream(g, sourceNode=source_node_id, targetNode=target_node_id,
                                               k=k, relationshipWeightProperty='averageEffectiveMinutes')
            if max_avg_time < np.Inf:
                tdf = tdf[tdf.totalCost <= max_avg_time]
            df_list.append(tdf)
    return df_list

def get_solution_costs(row, path_dfs):
    solution = row.solutionIndex
    rels = {}
    path_costs = []
    for n in range(len(solution)):
        path_costs.append(path_dfs[n].totalCost[solution[n]])
        for r in path_dfs[n].path[solution[n]].relationships:
            rels[f'{r.start_node.id}-{r.end_node.id}'] = r
    return sum([r.get('cost') for r in rels.values()]), path_costs, list(rels.values())

def top_k_solutions(gds, g, source_node_ids, target_node_ids, top_k=10, yen_candidates=10, max_avg_time=np.Inf):
    # get paths
    path_dfs = get_yen_dfs(gds, g, source_node_ids, target_node_ids, yen_candidates, max_avg_time)
    # get solutions
    solution_df = pd.DataFrame(itertools.product(*[range(path_df.shape[0]) for path_df in path_dfs])) \
        .apply(tuple, axis=1).to_frame(name='solutionIndex')
    solution_df[['totalCost', 'pathCosts', 'relationships']] = solution_df.apply(get_solution_costs,
                                                                                args=([path_dfs]), axis=1,
                                                                                result_type='expand')
    # return top solutions
    return solution_df.sort_values('totalCost')[:top_k].reset_index(drop=True)

def top_k_solutions_from_names(gds, g, source_airport_names, target_airport_name, top_k=10, yen_candidates=10,
                               max_avg_time=np.Inf):
    source_node_ids = [gds.find_node_id(['EntryPoint'], {'name': i}) for i in source_airport_names]
    target_node_ids = [gds.find_node_id(['Destination'], {'name': target_airport_name})]
    return top_k_solutions(gds, g, source_node_ids, target_node_ids, top_k=top_k, yen_candidates=yen_candidates,
                           max_avg_time=max_avg_time)

def top_k_solutions_from_airport_ids(gds, g, source_airport_ids, target_airport_id, top_k=10, yen_candidates=10,
                                     max_avg_time=np.Inf):
    source_node_ids = [gds.find_node_id(['EntryPoint'], {'airportId': i}) for i in source_airport_ids]
    target_node_ids = [gds.find_node_id(['Destination'], {'airportId': target_airport_id})]
    return top_k_solutions(gds, g, source_node_ids, target_node_ids, top_k=top_k, yen_candidates=yen_candidates,
                           max_avg_time=max_avg_time)

def format_nodes_and_rels(relationships):
    records = []
    for r in relationships:
        records.append({'sourceNodeId': r.start_node.id, 'targetNodeId': r.end_node.id, 'cost': r.get('cost')})
    rel_df = pd.DataFrame.from_records(records)
    return pd.DataFrame(set(rel_df.sourceNodeId).union(rel_df.targetNodeId), columns=['nodeId']), rel_df

def remove_solution_from_db(gds, solution_tag, solution_tag_name='solutionTag', rel_type='SOLUTION_PATH'):
    gds.run_cypher(f'MATCH()-[r:{rel_type}]->() WHERE {solution_tag_name} = "{solution_tag}" DELETE r')

def remove_solution_type_from_db(gds, solution_tag_name='solutionTag', rel_type='SOLUTION_PATH'):
    gds.run_cypher(f'MATCH()-[r:{rel_type}]->() DELETE r')
    gds.run_cypher(f'DROP INDEX {rel_type.lower()}_{solution_tag_name}_index IF EXISTS')

def write_solution_to_db(gds, relationships, solution_tag, solution_tag_name='solutionTag', rel_type='SOLUTION_PATH'):
    _, rels_df = format_nodes_and_rels(relationships)

    gds.run_cypher(f'''
        CREATE INDEX {rel_type.lower()}_{solution_tag_name}_index
        IF NOT EXISTS FOR ()-[r:{rel_type}]-()
        ON (r.{solution_tag_name})
    ''')

    gds.run_cypher(f'''
        UNWIND $rels AS rels
        WITH toInteger(rels.sourceNodeId) AS sourceNodeId,
            toInteger(rels.targetNodeId) AS targetNodeId,
            rels.cost AS cost
        MATCH(n1) WHERE id(n1) = sourceNodeId
        MATCH(n2) WHERE id(n2) = targetNodeId
        MERGE(n1)-[r:{rel_type} {{{solution_tag_name}:"{solution_tag}"}}]->(n2)
        SET r.cost=cost
    ''', params={'rels': rels_df.to_dict('records')})


import os
from datetime import datetime

from lxml import etree

from lib.model.track import GpsTrack, GpsTrackpoint

_XML_NAMESPACE = {
    "tcd": "http://www.garmin.com/xmlschemas/TrainingCenterDatabase/v2",
    "xsi": "http://www.w3.org/2001/XMLSchema-instance"
}

def _get_trackpoints_from_track(
        trackpoints: list[etree._Element]
    ) -> tuple[list[GpsTrackpoint], set[datetime]]:
    """
    Extract the points and time of the read track
    """
    gps_trackpoints: list[GpsTrackpoint] = []
    track_times: set[datetime] = set()
    for trackpoint in trackpoints:

        try:
            trackpoint_time = trackpoint.find(
                './/tcd:Time',
                namespaces=_XML_NAMESPACE
            ).text
            trackpoint_lat_deg = trackpoint.find(
                './/tcd:Position/tcd:LatitudeDegrees',
                namespaces=_XML_NAMESPACE
            ).text
            trackpoint_lon_deg = trackpoint.find(
                './/tcd:Position/tcd:LongitudeDegrees',
                namespaces=_XML_NAMESPACE
            ).text
        except AttributeError:
            continue

        # Time comes with timezone included
        trackpoint_time = datetime.strptime(trackpoint_time, '%Y-%m-%dT%H:%M:%S%z')
        gps_trackpoints.append(
            GpsTrackpoint(
                longitude=trackpoint_lon_deg,
                latitude=trackpoint_lat_deg,
                time=trackpoint_time,
            )
        )
        track_times.add(trackpoint_time)
    return gps_trackpoints, track_times

def read_tcx_track(folder: str, file: bytes) -> GpsTrack|None:
    """
    Read a single GPS track
    """
    filename = os.fsdecode(file)
    if not filename.endswith('.tcx'):
        return None

    file_path = os.path.join(folder, filename)

    # Get all trackpoints
    root: etree._ElementTree = etree.parse(file_path, parser=None)
    trackpoints: list[etree._Element] = root.xpath(
        ".//tcd:Track/tcd:Trackpoint",
        namespaces=_XML_NAMESPACE)

    # Extract lat, lon and time from each trackpoint
    try:
        gps_trackpoints, track_times = _get_trackpoints_from_track(trackpoints)
    except AttributeError:
        return None

    # If no valid point is found ignore the file
    if len(gps_trackpoints) == 0:
        return None

    return GpsTrack(
        min_time=min(track_times),
        max_time=max(track_times),
        trackpoints=gps_trackpoints,
        filename=filename
    )


import functools
import os.path
import subprocess
import sys

from pyperf._utils import shell_quote, popen_communicate
from pyperf._worker import WorkerTask

def bench_command(command, task, loops):
    path = os.path.dirname(__file__)
    script = os.path.join(path, '_process_time.py')
    run_script = [sys.executable, script]

    args = run_script + [str(loops)] + command
    proc = subprocess.Popen(args,
                            stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT,
                            universal_newlines=True)
    output = popen_communicate(proc)[0]
    if proc.returncode:
        raise Exception("Command failed with exit code %s"
                        % proc.returncode)

    rss = None
    try:
        lines = output.splitlines()
        timing = float(lines[0])
        if len(lines) >= 2:
            rss = int(lines[1])
    except ValueError:
        raise ValueError("failed to parse script output: %r" % output)

    if rss:
        # store the maximum
        max_rss = task.metadata.get('command_max_rss', 0)
        task.metadata['command_max_rss'] = max(max_rss, rss)
    return timing

class BenchCommandTask(WorkerTask):
    def __init__(self, runner, name, command):
        command_str = ' '.join(map(shell_quote, command))
        metadata = {'command': command_str}
        task_func = functools.partial(bench_command, command)
        WorkerTask.__init__(self, runner, name, task_func, metadata)

    def compute(self):
        WorkerTask.compute(self)
        if self.args.track_memory:
            value = self.metadata.pop('command_max_rss', None)
            if not value:
                raise RuntimeError("failed to get the process RSS")

            self._set_memory_value(value)


from flask import Flask, render_template
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
SQLALCHEMY_TRACK_MODIFICATIONS = False
app.config.from_object('config')
db = SQLAlchemy(app)

# Sample HTTP error handling
@app.errorhandler(404)
def not_found(error):
    return render_template('404.html'), 404

from app.mod_auth.controllers import mod_auth as auth_module

# Register blueprint(s)
app.register_blueprint(auth_module)
# app.register_blueprint(xyz_module)
# ..

# Build the database:
db.create_all()


import serial

class Light:

    def __init__(self, comport, baudrate=9600):
        self.ser = serial.Serial(comport, baudrate, timeout=1)

    def on(self, terminal):
        assert terminal in [0, 1, 2, 3]
        # Turn on light 1
        if terminal == 0:
            self.ser.write('$310FF16'.encode('utf-8'))
        # Turn on light 2
        elif terminal == 1:
            self.ser.write('$320FF15'.encode('utf-8'))
        # Turn on light 3
        elif terminal == 2:
            self.ser.write('$330FF14'.encode('utf-8'))
        # Turn on light 4
        elif terminal == 3:
            self.ser.write('$340FF13'.encode('utf-8'))

    def off(self, terminal):
        assert terminal in [0, 1, 2, 3]
        # Turn off light 1
        if terminal == 0:
            self.ser.write('$3100016'.encode('utf-8'))
        # Turn off light 2
        elif terminal == 1:
            self.ser.write('$3200015'.encode('utf-8'))
        # Turn off light 3
        elif terminal == 2:
            self.ser.write('$3300014'.encode('utf-8'))
        # Turn off light 4
        elif terminal == 3:
            self.ser.write('$3400013'.encode('utf-8'))


import random
import string
from Summoner import Summoner, create_default_summoner, Class
from Items.Armor import Weapon
from utils import list_from_file

def get_name_list(type):
    file = "fantasy_names.txt"
    if type == "summoner":
        file = "fantasy_names.txt"
    elif type == "weapon":
        file = "weapon_names.txt"
    return list_from_file("fantasy_names/" + file)

# type = summoner, weapon, armor
def generate_name(type):
    name = ""
    nameList = get_name_list(type)

    random.seed()
    rand = random.randrange(len(nameList))
    enemyName = nameList[rand]
    if type == "summoner":
        rand = random.randrange(len(nameList))
        enemyName += " " + nameList[rand]
    return enemyName

# If type is 0, returns a random summoner as enemy
def generate_random_enemy(p, type):
    name = ""
    if type == 0:  # summoner
        name = generate_name("summoner")
        summ = create_default_summoner(name)

        # random class
        random.seed()
        rand = random.randrange(0, 4)
        summ.cl = rand

        # setting level conditions
        if (
            p.level <= 4
        ):  # if player is level 4 or below, enemy is not above player level
            lwrBound = p.level - 3
            uprBound = p.level
        else:  # otherwise level is random in range:
            lwrBound = p.level - 5  # at least 5 levels lower than player
            uprBound = p.level + 2  # at most 2 levels above player

        if lwrBound < 1:  # ensuring no negative or 0 level
            lwrBound = 1

        # create level of enemy
        rand = random.randrange(lwrBound, uprBound + 1)
        summ.level = rand

        # level up stats acccording to class
        for i in range(summ.level):
            summ.level_up_stats()
        summ.full_heal()
        summ.reset_combat_stats()
    return summ

# generate a weapon based on class of summoner p
def generate_random_weapon(p):
    # generate random weapon name
    name = generate_name("weapon")
    # generate some flavor text // TODO //
    flavor = "A cool weapon!"
    allStatStrings = [
        "health",
        "mana",
        "healthr",
        "manar",
        "ad",
        "ap",
        "armor",
        "mr",
        "prio",
        "crit",
    ]
    allStatNums = []
    # for i in range(len(allStatStrings)):  # populate stat increases with all 0 for now
    #    allStatNums.append(0)

    # depending on class, increase likelihood of certain stats increasing
    biasStats = ["ad"]
    if p.clStr == Class.MAGE:
        biasStats = ["ap", "mana", "manar"]
    elif p.clStr == Class.TANK:
        biasStats = ["health", "healthr", "armor", "mr"]
    elif p.clStr == Class.FIGHTER:
        biasStats = ["health", "ad", "armor", "mr"]
    elif p.clStr == Class.MARKSMAN:
        biasStats = ["ad", "prio", "crit"]

    allStatsZero = True
    while allStatsZero:
        for stat in allStatStrings:
            coef = 0  # coefficient for current stat being iterated on
            if stat in biasStats:  # if current stat gets 50% chance of increase
                rand = random.randint(0, 1)
            else:  # if current stat gets 10% chance of increase
                rand = random.randint(0, 9)
            if rand == 0:  # if chance of increase succeeds
                # change this to change stat increases, currently 0-5x player level
                coef = random.randint(0, 50)
                coef = coef / 10  # coef is between 0 and 5 in increments of 0.1
            # add stat change to stat array to be passed to weapon
            allStatNums.append(round(p.level * coef))
            # constructor
        for (
            stat
        ) in (
            allStatNums
        ):  # check if no stat buffs. if that's the case, run the while loop once again
            if stat != 0:
                allStatsZero = False
        if allStatsZero:
            allStatNums = []

    # determine stat scaling
    if (
        "healthr" in biasStats
    ):  # ensuring damage only scales off of hp, mana, ad, ap, armor, mr
        biasStats.remove("healthr")
    if "manar" in biasStats:
        biasStats.remove("manar")
    if "prio" in biasStats:
        biasStats.remove("prio")
    if "crit" in biasStats:
        biasStats.remove("crit")
    allStatStrings.remove("healthr")
    allStatStrings.remove("manar")
    allStatStrings.remove("prio")
    allStatStrings.remove("crit")
    rand = random.randint(1, 10)
    stat_scale = "ad"  # default ad just in case random breaks
    if p.level == int(1):  # make it guaranteed to scale off bias stat for level 1
        rand = 8
    # 60% chance it'll scale off of a random stat (could be bias stat still)
    if rand <= 6:
        # subtract 3 to prevent scaling from prio and crit
        x = random.randint(0, len(allStatStrings) - 1)
        stat_scale = allStatStrings[x]
    else:  # 40% chance it'll scale off of bias stat
        x = random.randint(0, len(biasStats) - 1)
        stat_scale = biasStats[x]
    # determine pct stat scaling
    x = random.randint(0, 100)
    x = x / 10  # x is between 0 and 5 by increments of 0.1
    y = pow(1.6, x) + 10  # y is between 0 and ~110
    pct_scale = y / 100

    # determine physical or magic damage
    rand = random.randint(0, 1)
    dmg_type = ""
    if rand == 0:
        dmg_type = "physical"
    else:
        dmg_type = "magic"

    # determine effects
    effects = ["None"]

    # create weapon
    weapon = Weapon(
        name,
        allStatNums[0],
        allStatNums[1],
        allStatNums[2],
        allStatNums[3],
        allStatNums[4],
        allStatNums[5],
        allStatNums[6],
        allStatNums[7],
        allStatNums[8],
        allStatNums[9],
        flavor,
        stat_scale,
        pct_scale,
        dmg_type,
        effects,
    )

    return weapon

def battle(p):
    # random chance of encountering a monster or enemy summoner
    random.seed()
    rand = random.randint(1, 100)  # generate number 0-99

    # if rand < 40:  # 40% chance to encounter enemy summoner
    #    p = generate_random_enemy(p, 0)
    return p


import appdaemon.plugins.hass.hassapi as hass
import appdaemon.plugins.mqtt.mqttapi as mqtt
import globals
import json
import random

"""
Monitor events and output changes to the verbose_log. Nice for debugging purposes.
Arguments:
 - events: List of events to monitor
"""

class RassToHappyBubbles(mqtt.Mqtt):
    base_topic = "home/presence"
    def initialize(self) -> None:
        self.log('Initialize RassToHappyBubbles')
        self.mqtt_subscribe('{}/#'.format(self.base_topic), namespace='mqtt')
        self.listen_event(self.event_listener, 'MQTT_MESSAGE', namespace='mqtt')

    def event_listener(self, event_name, data, kwargs):
        topic = data.get('topic', '')
        if self.base_topic	not in topic:
        	return
        #array = ['hall', 'balcony']
        #random.shuffle(array)
        #device=array[0]

        device = topic.replace('{}/'.format(self.base_topic), "")
        payload = json.loads(data['payload']) if data.get('payload') else {}
        id = payload.get('id', None)
        hb_topic="happy-bubbles/ble/{}/raw/{}".format(device,id)
        payload['hostname'] = device
        payload['mac'] = payload.get('id', '')

        payload_data = json.dumps(payload)
        self.mqtt_publish(hb_topic, payload_data, qos = 0, retain = False, namespace='mqtt')
        # self.log('publishing to: {}: {}'.format(hb_topic, payload_data))


from fastapi import (
    APIRouter,
    HTTPException,
    Depends,
    status
)
from typing import List
from schemas.user import (
    UserInResp,
    UserInDB,
    UserInDbUpdate,
    UsersInResp
)
from crud.user import (
    find_user_by_id,
    create_new_user,
    get_all_users,
    update_user_in_db,
    delete_user_in_db,
    get_all_users_count
)
from utils.dependencies import (
    get_current_user
)
from uuid import UUID
from application import ps
from permissions_system.constants import (
    InternalTables,
    PermissionTypesEnum
)
from schemas.common import (
    DeleteResp
)

router = APIRouter()

@router.get("/{user_id}", response_model=UserInResp)
async def get_user_by_id(
    user_id: UUID,
    current_user=Depends(get_current_user)
):
    """
    Get a specific user by id
    """

    user = await find_user_by_id(user_id)
    if not user:
        raise HTTPException(
            status_code=404,
            detail="User not found.")
    return UserInResp(**user)

@router.get("/", response_model=UsersInResp)
async def get_users(
    offset: int = 0,
    limit: int = 10,
    current_user=Depends(get_current_user)
):
    """
    Get list of all users and
    total count of items for this resource in DB.
    """
    users = await get_all_users(offset, limit)
    count = await get_all_users_count()

    return UsersInResp(users=users, total_count=count)

@router.post("/", response_model=UserInResp)
async def create_user(user_in: UserInDB, current_user=Depends(get_current_user)):
    """
    Create new user.
    """
    user_id = await create_new_user(user_in)
    user = await find_user_by_id(user_id)
    return user

@router.put("/{user_id}", response_model=UserInResp)
async def update_user(
        user_id: UUID,
        user_in: UserInDbUpdate,
        current_user=Depends(get_current_user)):
    """
    Update specific user details.
    """
    user = await update_user_in_db(user_id, user_in)
    return user

@router.delete("/{user_id}", response_model=DeleteResp)
async def delete_user(
    user_id: UUID,
    current_user=Depends(get_current_user)
):
    """
    Delete specific user.
    """
    resp = {
        "deleted": None
    }
    if current_user["id"] != user_id and \
            current_user["group"] != "super_admin":
        raise HTTPException(status.HTTP_403_FORBIDDEN,
                            "You are not authorised for this operation.")

    deleted = await delete_user_in_db(user_id)
    resp.update(deleted=deleted)
    return resp


import re
import datetime
import threading
from queue import Queue
from pathlib import Path
from collections import defaultdict
from user_agents import parse

# 分析日志
def log_analysis(log_line: str):
    func_dict = {'datetime': lambda time: datetime.datetime.strptime(time, '%d/%b/%Y:%H:%M:%S %z'),  # 转换时间的对应函数
                 'request': lambda mode: dict(zip(('method', 'url', 'protocol'), mode.split())),  # 请求分解的对应函数
                 'useragent': lambda x: parse(x)
                 }

    # 用正则分组名有：remote, datetime, request, status, useragent
    # 日志格式：10.211.55.2 - - [14/Jul/2020:18:54:58 +0800] "GET / HTTP/1.1" 403 4897 "-" "curl/7.64.1"
    pattern = '''(?P<remote>^[\d.]{7,}) - - \[(?P<datetime>[^\[\]]+)\] "(?P<request>[^"]+)" (?P<status>\d{3,}) \d{3,} "[^"]+" "(?P<useragent>[^"]+)"$'''
    matcher = re.match(pattern, log_line)

    if matcher:  # 判断是否匹配上了，如未匹配，函数默认会返回 None
        return {k: func_dict.get(k, lambda x: x)(v) for k, v in
                matcher.groupdict().items()}  # matcher.groupdict() 会返回一个带分组名的字典，例如 {'remote': xxxxxxxx}

# 加载日志文件
def open_file(file):
    with open(file, encoding='utf8') as f:
        for line in f:
            d = log_analysis(line)
            if d:
                yield d
            else:
                # TODO 不合格的数据有多少
                # print('########################')
                continue

# 批量读取文件，并判断是否是文件
def load(*path):
    for file in path:
        f = Path(file)
        if not f.exists():  # 判断文件是否存在
            continue
        elif f.is_dir():  # 如果是目录，就迭代目录内的所有文件
            for x in f.iterdir():
                if x.is_file():  # 如果是文件就读取，否则就忽略了
                    yield from open_file(x)
        elif f.is_file():  # 如果是文件，就直接读取
            yield from open_file(f)

# 时间滑动窗口
def windows(src: Queue, handler, interval: int, width: int):
    """
    :param src: 数据源
    :param handler: 负责处理数据的函数
    :param interval: 时间间隔
    :param width: 时间宽度
    """
    start = datetime.datetime.strptime('1970/01/01 00:00:00 +0800', '%Y/%m/%d %H:%M:%S %z')
    current = datetime.datetime.strptime('1970/01/01 00:00:01 +0800', '%Y/%m/%d %H:%M:%S %z')
    delta = datetime.timedelta(seconds=(width - interval))  # 重复的部分 = 宽度 - 间隔

    buffer = []

    while True:
        data_dict = src.get()
        if data_dict:
            buffer.append(data_dict)
            current = data_dict['datetime']

        if (current - start).total_seconds() >= interval:
            ret = handler(buffer)

            buffer = [x for x in buffer if x['datetime'] > current - delta]
            start = current

# 消息分发器
def dispatcher(src):
    # 队列列表
    queues = []
    threads = []

    def reg(handler, interval, width):
        q = Queue()
        queues.append(q)

        t = threading.Thread(target=windows, args=(q, handler, interval, width))
        threads.append(t)

    def run():
        for t in threads:
            t.start()

        for x in src:
            for q in queues:
                q.put(x)

    return reg, run

# 状态码分析的 handler
def handler_status(buffer):
    d = {}
    start = buffer[0]['datetime']  # 因为日志是按时间顺序记录的，那么在同一个 buffer 内，第一条日志的时间就是这个 buffer 的起始时间
    end = buffer[-1]['datetime']  # 最后一条日志的时间就是这个 buffer 的结束时间

    for i in buffer:
        status = i['status']
        if status not in d.keys():
            d[status] = 0
        d[status] += 1

    # 总数
    total = sum(d.values())

    for k, v in d.items():
        p = (v / total) * 100  # 计算平均值
        d[k] = [d[k], p]  # 把平均值塞回d中

    # 输出到控制台
    print(f'在 {start.date()}【 {start.time()} - {end.time()} 】内')
    for status in d.items():
        print(' ' * 40, f'{status[0]} 出现 {status[1][0]} 次，占比 {int(status[1][1])}%')
    print('\n')

    return start, end, d

# 统计用户的浏览器
ua_dict = defaultdict(int)  # 这是记录 agent 出现次数的字典，对于 agent 的版本，看一小段时间是没有意义的，应该放在函数外，可记录下完整的次数

def user_agent(buffer):
    for i in buffer:
        ua = i['useragent']
        ua_dict[(ua.browser.family, ua.browser.version_string)] += 1

    # 输出到控制台
    print(f'截止：{buffer[-1]["datetime"]}')
    sort_ua = sorted(ua_dict.items(), key=lambda x: x[1], reverse=True)  # 按出现次数排序
    for agent in sort_ua:
        print(' ' * 40, f'客户端：{agent[0][0]:<20} 版本：{agent[0][1]:<15} 出现次数：{agent[1]}')
    print('\n')

    return ua_dict

if __name__ == '__main__':
    # 从命令行参数获取路径的方式
    # import sys
    # path = sys.argv[1]

    # 直接指定路径，测试时使用
    path = './test_file/access_1.log'

    reg, run = dispatcher(load(path))

    # 注册 handler，时间(s)：时间窗口的间隔、宽度
    # 日志会以一个时间窗口为单位，分发到注册的 handler
    # 日志已经被格式化完毕，handler 只要按自己的需求提取并选择如何处理即可
    reg(handler_status, 5, 5)  # 负责统计状态码
    reg(user_agent, 5, 5)  # 负责统计 agent 信息

    # 启动
    run()


from flask import Flask, request, jsonify
from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy.orm import sessionmaker
from sqlalchemy.ext.declarative import declarative_base
import psycopg2
from flask_sqlalchemy import SQLAlchemy

app = Flask(__name__)
engine = create_engine("sqlite:///citizen.db")
app.config['SQLALCHEMY_DATABASE_URI'] = 'sqlite:///citizen.db'
db = SQLAlchemy(app)

#create model
Base = declarative_base()
class Citizen( db.Model):
    __tablename__ = 'citizen'
    id = db.Column(Integer, primary_key=True)
    prenom = db.Column(String(20))
    nom = db.Column(String(20))
    cni = db.Column(String(50))

#db.create_all()
#print('okffffff')
#Base.metadata.create_all(engine)
#print ("Table citizen crée ...")
Session = sessionmaker(bind= engine)

# definition des endpoints
@app.route("/api/citizen", methods=["POST"])
def create():
    try:
        #session = Session()
        new_citizen = Citizen(
            prenom = request.json['prenom'],
            nom = request.json['nom'],
            cni = request.json['cni'])
        db.session.add(new_citizen)
        db.session.commit()
        return new_citizen.cni
    except Exception as e:
        print(f" erreur '{e}' ")
        return {"error": "erreur lors de la creation du citoyen."}, 500

@app.route("/api/citizen", methods=['GET'])
def get():
    try:
        session = Session()
        citizen = db.session.query(Citizen).all()
        if citizen:
            result =[]
            for c in citizen:
                result.append({"id": c.id, "name": c.prenom, "nom":c.nom, "cni":c.cni})
            return jsonify(result)
        else:
            return jsonify({"error": f"Citizen non trouvé"}), 404
    except Exception as e:
        print(f"erreur '{e}'")
        return {"une erreur est survenue"}, 500

if __name__ == '__main__':
    app.run(debug=True, host="0.0.0.0")


from importlib import metadata

import typer

from oterm.app.oterm import app

cli = typer.Typer()

@cli.command()
def oterm(version: bool = typer.Option(None, "--version", "-v")):
    if version:
        typer.echo(f"oterm v{metadata.version('oterm')}")
        exit(0)
    app.run()

if __name__ == "__main__":
    cli()


from typing import Optional, Tuple

from posggym import core

import posggym.envs.grid_world.render as render_lib

import posggym.envs.grid_world.driving.model as dmodel
from posggym.envs.grid_world.driving.grid import DrivingGrid

class DrivingEnv(core.DefaultEnv):
    """The Driving Grid World Environment.

    A general-sum 2D grid world problem involving multiple agents. Each agent
    controls a vehicle and is tasked with driving the vehicle from it's start
    location to a destination location while avoiding crashing into obstacles
    or other vehicles.

    This environment requires each agent to navigate in the world while also
    taking care to avoid crashing into other players. The dynamics and
    observations of the environment are such that avoiding collisions requires
    some planning in order for the vehicle to brake in time or maintain a good
    speed. Depending on the grid layout, the environment will require agents to
    reason about and possibly coordinate with the other vehicles.

    Agents
    ------
    Varied number

    State
    -----
    Each state is made up of the state of each vehicle, which in turn is
    defined by:

    - the (x, y) (x=column, y=row, with origin at the top-left square of the
      grid) of the vehicle,
    - the direction the vehicle is facing NORTH=0, SOUTH=1, EAST=2, WEST=3),
    - the speed of the vehicle (REVERSE=-1, STOPPED=0, FORWARD_SLOW=1,
      FORWARD_FAST=2),
    - the (x, y) of the vehicles destination
    - whether the vehicle has reached it's destination or not
    - whether the vehicle has crashed or not

    Actions
    -------
    Each agent has 5 actions: DO_NOTHING=0, ACCELERATE=1, DECELERATE=2,
    TURN_RIGHT=3, TURN_LEFT=4

    Observation
    -----------
    Each agent observes their current speed along with the cells in their local
    area. The size of the local area observed is controlled by the `obs_dims`
    parameter. For each cell in the observed are the agent observes whether
    they are one of four things: VEHICLE=0, WALL=1, EMPTY=2, DESTINATION=3.
    Each agent also observes the (x, y) coordinates of their destination,
    whether they have reached the destination, and whether they have crashed.

    Each observation is represented as a tuple:
        ((local obs), speed, destination coord, destination reached, crashed)

    Reward
    ------
    All agents receive a penalty of 0.00 for each step. They also recieve a
    penalty of -0.5 for hitting an obstacle (if ``obstacle_collision=True``),
    and -1.0 for hitting another vehicle. A reward of 1.0 is given if the agent
    reaches it's destination.

    Transition Dynamics
    -------------------
    Actions are deterministic and movement is determined by direction the
    vehicle is facing and it's speed:

    - Speed=-1 (REVERSE) - vehicle moves one cell in the opposite direction
    - Speed=0 (STOPPED) - vehicle remains in same cell
    - Speed=1 (FORWARD_SLOW) - vehicle move one cell in facing direction
    - Speed=1 (FORWARD_FAST) - vehicle moves two cells in facing direction

    Accelerating increases speed by 1, while deceleration decreased speed by 1.
    If the vehicle will hit a wall or an other vehicle when moving from one
    cell to another then it remains in it's current cell and its crashed state
    variable is updated. Once a vehicle reaches it's destination it is stuck.

    Episodes ends when all agents have either reached their destination or
    crashed, or the episode step limit is reached.

    """

    metadata = {"render.modes": ["human", "ansi", "rgb_array"]}

    def __init__(
        self,
        grid: DrivingGrid,
        num_agents: int,
        obs_dim: Tuple[int, int, int],
        obstacle_collisions: bool,
        **kwargs,
    ):
        self._model = dmodel.DrivingModel(
            grid, num_agents, obs_dim, obstacle_collisions, **kwargs
        )
        self._obs_dim = obs_dim
        self._viewer = None
        self._renderer: Optional[render_lib.GWRenderer] = None
        super().__init__()

    def render(self, mode: str = "human"):
        if mode == "ansi":
            grid_str = self._model.grid.get_ascii_repr(
                [vs.coord for vs in self._state],
                [vs.facing_dir for vs in self._state],
                [vs.dest_coord for vs in self._state],
            )

            output = [
                f"Step: {self._step_num}",
                grid_str,
            ]
            if self._last_actions is not None:
                action_str = ", ".join(
                    [dmodel.ACTIONS_STR[a] for a in self._last_actions]
                )
                output.insert(1, f"Actions: <{action_str}>")
                output.append(f"Rewards: <{self._last_rewards}>")

            return "\n".join(output) + "\n"
        elif mode in ("human", "rgb_array"):
            grid = self.model.grid
            if mode == "human" and self._viewer is None:
                # pylint: disable=[import-outside-toplevel]
                from posggym.envs.grid_world import viewer

                self._viewer = viewer.GWViewer(  # type: ignore
                    "Driving Env",
                    (min(grid.width, 9), min(grid.height, 9)),
                    num_agent_displays=self.n_agents,
                )
                self._viewer.show(block=False)  # type: ignore

            if self._renderer is None:
                self._renderer = render_lib.GWRenderer(
                    self.n_agents, grid, [], render_blocks=True
                )

            agent_obs_coords = tuple(
                self._model.get_obs_coords(vs.coord, vs.facing_dir)
                for vs in self._state
            )
            agent_coords = tuple(vs.coord for vs in self._state)
            agent_dirs = tuple(vs.facing_dir for vs in self._state)

            # Add agent destination locations
            other_objs = [
                render_lib.GWObject(
                    vs.dest_coord,
                    render_lib.get_agent_color(i),
                    render_lib.Shape.RECTANGLE,
                    # make dest squares slightly different to vehicle color
                    alpha=0.2,
                )
                for i, vs in enumerate(self._state)
            ]
            # Add visualization for crashed agents
            for i, vs in enumerate(self._state):
                if vs.crashed:
                    other_objs.append(
                        render_lib.GWObject(vs.coord, "yellow", render_lib.Shape.CIRCLE)
                    )

            env_img = self._renderer.render(
                agent_coords,
                agent_obs_coords,
                agent_dirs=agent_dirs,
                other_objs=other_objs,
                agent_colors=None,
            )
            agent_obs_imgs = self._renderer.render_all_agent_obs(
                env_img,
                agent_coords,
                agent_dirs,
                agent_obs_dims=self._obs_dim,
                out_of_bounds_obj=render_lib.GWObject(
                    (0, 0), "grey", render_lib.Shape.RECTANGLE
                ),
            )

            if mode == "human":
                self._viewer.display_img(env_img, agent_idx=None)  # type: ignore
                for i, obs_img in enumerate(agent_obs_imgs):
                    self._viewer.display_img(obs_img, agent_idx=i)  # type: ignore
            else:
                return (env_img, agent_obs_imgs)
        else:
            super().render(mode)

    @property
    def model(self) -> dmodel.DrivingModel:
        return self._model

    def close(self) -> None:
        if self._viewer is not None:
            self._viewer.close()  # type: ignore
            self._viewer = None


import logging
import warnings
from typing import Dict, List, Union, Any

import torch
from datasets import Dataset, DatasetDict
from datasets.arrow_dataset import Batch

from oslo.transformers.tasks.data_base import BaseProcessor

try:
    from transformers import (
        AutoTokenizer,
        PreTrainedTokenizerBase,
        RobertaTokenizer,
    )
    from transformers.file_utils import PaddingStrategy
except ImportError:
    print("You have to install `transformers` to use `oslo.transformers` modules")

logger = logging.getLogger(__name__)
logging.captureWarnings(True)

class ProcessorForTokenClassification(BaseProcessor):
    def __init__(
        self,
        tokenizer: PreTrainedTokenizerBase,
        max_seq_length: int,
        dataset: Union[Dataset, DatasetDict] = None,
    ) -> None:
        super().__init__(tokenizer, max_seq_length)
        if dataset is None:
            raise ValueError(
                "dataset argument must be set. (dataset: Union[Dataset, DatasetDict])"
            )

        if self._tokenizer.pad_token is None:
            warnings.warn(
                "If pad token doesn't exist in tokenizer, it can be a problem when applying padding."
            )

        self.label_names = self.get_label_names(dataset)
        self.label_map = self.get_label_map(self.label_names)

    def __call__(self, examples: Batch) -> Dict[str, List[int]]:
        column_names = [k for k, v in examples.items()]
        assert (
            "tokens" in column_names
        ), "The name of dataset column that you want to tokenize must be 'tokens' (not 'text')"

        dict_of_training_examples: Dict[str, List[int]] = self._tokenizer(
            examples["tokens"],
            truncation=True,
            max_length=self._max_seq_length,
            is_split_into_words=True,
            verbose=False,
        )
        all_labels = examples["labels"]
        new_labels = []
        self.make_B_to_I_label(self.label_names)

        for i, labels in enumerate(all_labels):
            word_ids = dict_of_training_examples.word_ids(batch_index=i)
            new_labels.append(self.align_labels_with_tokens(labels, word_ids))

        dict_of_training_examples["labels"] = new_labels
        return dict_of_training_examples

    def get_label_names(self, dataset: Union[Dataset, DatasetDict]) -> List[str]:
        if isinstance(dataset, Dataset):
            assert (
                "labels" in dataset.features
            ), "The name of dataset column that you want to use as a label must be 'labels'"

            features = dataset.features["labels"]
            label_names = features.feature.names
        else:
            assert (
                "train" in dataset.keys()
            ), "The key name of train dataset must be 'train'"
            assert (
                "labels" in dataset["train"].features
            ), "The name of dataset column that you want to use as a label must be 'labels'"

            features = dataset["train"].features["labels"]
            label_names = features.feature.names

        self.label_names = label_names

        return label_names

    def get_label_map(
        self, label_names: Union[List[str], Dataset, DatasetDict]
    ) -> Dict[str, Dict[str, str]]:
        if isinstance(label_names, Dataset) or isinstance(label_names, DatasetDict):
            label_names = self.get_label_names(label_names)

        id2label = {str(i): label for i, label in enumerate(label_names)}
        label2id = {v: k for k, v in id2label.items()}

        self.id2label = id2label
        self.label2id = label2id

        return {"label2id": label2id, "id2label": id2label}

    def make_B_to_I_label(self, label_names: List[str]) -> List[int]:
        # Map that sends B-Xxx label to its I-Xxx counterpart
        b_to_i_label = []
        for idx, label in enumerate(label_names):
            if label.startswith("B-") and label.replace("B-", "I-") in label_names:
                b_to_i_label.append(label_names.index(label.replace("B-", "I-")))
            else:
                b_to_i_label.append(idx)

        self.b_to_i_label = b_to_i_label

        return b_to_i_label

    def align_labels_with_tokens(self, labels, word_ids) -> List[int]:
        previous_word_idx = None
        label_ids = []
        for word_idx in word_ids:
            # Special tokens have a word id that is None. We set the label to pad token id
            if word_idx is None:
                label_ids.append(self._tokenizer.pad_token_id)
            # We set the label for the first token of each word.
            elif word_idx != previous_word_idx:
                label_ids.append(labels[word_idx])
            else:
                label_ids.append(self.b_to_i_label[labels[word_idx]])
            previous_word_idx = word_idx

        return label_ids

class DataCollatorForTokenClassification(object):
    """
    Data collator that will dynamically pad the inputs received, as well as the labels.
    """

    def __init__(
        self,
        processor: ProcessorForTokenClassification,
        padding_side: str = "right",
        label_pad_token_id: int = -100,
    ):
        if not isinstance(processor, ProcessorForTokenClassification):
            warnings.warn(
                "DataCollatorForTokenClassification is suitable for ProcessorForTokenClassification."
            )

        if processor._tokenizer.pad_token is None:
            warnings.warn(
                "If pad token doesn't exist in tokenizer, it can be a problem when applying padding."
            )

        self.tokenizer = processor._tokenizer
        self.padding_side = padding_side
        self.label_pad_token_id = label_pad_token_id

    def __call__(self, features: List[Dict[str, Any]]):
        label_name = "labels"
        labels = [feature[label_name] for feature in features]
        return self.pad(features, labels, label_name)

    def pad(self, features, labels, label_name):
        batch = self.tokenizer.pad(
            features,
            return_attention_mask=True,
            return_tensors=None,
        )

        sequence_length = torch.tensor(batch["input_ids"]).shape[1]
        padding_side = self.tokenizer.padding_side
        if padding_side == "right":
            batch[label_name] = [
                list(label) + [self.label_pad_token_id] * (sequence_length - len(label))
                for label in labels
            ]
        else:
            batch[label_name] = [
                [self.label_pad_token_id] * (sequence_length - len(label)) + list(label)
                for label in labels
            ]

        batch = {k: torch.tensor(v, dtype=torch.int64) for k, v in batch.items()}
        batch[label_name].masked_fill_(
            batch[label_name] == self.tokenizer.pad_token_id, self.label_pad_token_id
        )
        return batch


import matplotlib.pyplot as plt
import numpy as np
file1="changed_ipcp"
file2="ipcp"

with open(file1, 'r') as f1:
    data1 = [line.strip().split() for line in f1.readlines()]

with open(file2, 'r') as f2:
    data2 = [line.strip().split() for line in f2.readlines()]

fig, ax = plt.subplots(figsize=(14, 10.5))
combined_data = []
k=0
mul1=1
mul2=1
for row in data1:
    name=row[0].split('.trace.gz')[0]
    name=name.split('.champsimtrace')[0]
    value1=row[1]
    mul1*=float(value1)
    value2 = data2[k][1]
    mul2*=float(value2)
    k+=1
    combined_data.append((name, float(value1), float(value2)))

# Sort the data by name
combined_data.sort(key=lambda x: x[0])

# Extract the data into separate lists for plotting
names = [x[0] for x in combined_data]
values1 = [x[1] for x in combined_data]
values2 = [x[2] for x in combined_data]

N = len(names)
ind = np.arange(N)
width = 0.35

bar1 = plt.bar(ind, values1, width, color='r')
bar2 = plt.bar(ind+width, values2, width, color='b')

plt.axhline(y=mul1**(1/len(names)), color='r', linestyle='--')  # Add a horizontal line at y
plt.axhline(y=mul2**(1/len(names)), color='b', linestyle='--')
file1=file1.replace(".txt","")
file2=file2.replace(".txt","")
print(f"Cummulative ipc of {file1}:",mul1**(1/len(names)))
print(f"Cummulative ipc of {file2}:",mul2**(1/len(names)))

plt.xlabel("Traces")
plt.ylabel('IPC') # or IPC or MISS RATE
plt.title("IPC comparison")
plt.xticks(ind+width/2, names)
plt.xticks(rotation=90)
plt.legend((bar1, bar2), (file1, file2))
#plt.ylim(ymin=100000)
#plt.ylim(0, 1.15)
# Adjust top margin
fig.subplots_adjust(top=0.95)
plt.savefig(f"{file1}_{file2}.png")


from django.urls import path

from .views import (
    home,
    QuizListView,
    QuestionListView,
    QuizDetailView,
    QuizCreateView,
    QuestionCreateView
)

urlpatterns = [
    path("", home, name="home"),
    path("quiz/", QuizListView.as_view(), name="quiz_list"),
    path("quiz/<int:pk>", QuizDetailView.as_view(), name="quiz_detail"),
    path("quiz/create", QuizCreateView.as_view(), name="quiz_create"),
    path("quiz/<int:quiz_id>/questions/", QuestionListView.as_view(), name="question_list"),
    path("quiz/<int:quiz_id>/questions/create", QuestionCreateView.as_view(), name="question_create")
]


from flask import Flask, render_template, request, redirect
from main import Palette
from werkzeug.utils import secure_filename
from forms import UploadForm
import os
import datetime

app = Flask(__name__)

SECRET_KEY = os.urandom(32)
app.config['SECRET_KEY'] = SECRET_KEY
PEOPLE_FOLDER = os.path.join('static', 'photo')
app.config['UPLOAD_FOLDER'] = PEOPLE_FOLDER

current_year = datetime.datetime.now().year
print(current_year)

@app.route("/", methods=["GET", "POST"])
def home():
    form = UploadForm()
    if request.method == "GET":
        image = None
        for file in os.listdir(os.path.join(PEOPLE_FOLDER)):
            if os.path.isfile(os.path.join(PEOPLE_FOLDER, file)):
                image = os.path.join(PEOPLE_FOLDER, file)
                break
        return render_template('home.html', form=form, image=image, year=current_year);
    elif request.method == "POST":
        for file in os.listdir(PEOPLE_FOLDER):
            os.remove(os.path.join(PEOPLE_FOLDER, file))
            # if os.path.isfile(file):
            #     os.remove(file)
        if form.validate_on_submit():
            folder = 'images/'
            if not os.path.exists(folder):
                os.mkdir(folder)
            number = int(request.form['color_count'])
            if request.files['file']:
                f = request.files['file']
                filename = secure_filename(f.filename)
                print (number)
                new_path = os.path.join(folder, filename)
                print(new_path)
                f.save(new_path)
                palette = Palette(filename, 1200, folder, number)
                palette.get_image()
                colors = palette.get_colors()
                palette.save_to_static(folder, PEOPLE_FOLDER, 1080, filename)
                file_to_send = os.path.join(PEOPLE_FOLDER, filename)
                palette.delete_from_images()
                print(colors)
                return render_template('home.html', form=form, colors=enumerate(colors), image=file_to_send, year=current_year)

@app.route("/features")
def features():
    return render_template('features.html', year=current_year)

if __name__ == '__main__':
    app.run(debug=True, port=5000);


import requests
from bs4 import BeautifulSoup

# 기상청(도시별 현재날씨) 주소
address = 'https://www.weather.go.kr/weather/observation/currentweather.jsp'

source = requests.get(address)
soup = BeautifulSoup(source.content, "html.parser")

table = soup.find('table', {'class':'table_develop3'})

print("Today's Weather")
print('지점\t 기온\t 습도')

for tr in table.find_all('tr'):
    tds = list(tr.find_all('td'))
    for td in tds:
        if td.find('a'):
            point = td.find('a').text
            temp = tds[5].text
            humidity = tds[9].text
            print("{0:<7} {1:<7} {2:<7}".format(point,temp,humidity))

# 하나의 항목만 선택하고 싶을 때, Selector를 가져와서 넣으면 됨
weather = soup.select_one('#digital-forecast > div.cmp-dfs-slider.hr1-fct')
weather = soup.select('#digital-forecast > div.cmp-dfs-slider.hr1-fct')


'''
16/08/2019 @queenjobo @ksamocha

Script to run enrichment test part of the DeNovoWEST testing framework

For usage see 'python DNE_test.py -h'

make a change
'''

__version__ = 1.2
__date__ = '2019-01-09'

#IMPORTS--------------

import logging
import random
import argparse
import pandas as pd
import numpy as np
import itertools
import datetime
from scipy import stats
from probabilities import *

#set seed
#random.seed(31)
#np.random.seed(31)
random.seed(2014)
np.random.seed(2014)

#get todays date
now = datetime.datetime.now()
today = str(now).split(' ')[0]

#default paths
wdicpath = "input/loess_weight_dic_v4.tab"
denovopath = "input/de_novos.txt.gz"
outpath = "DeNovoWEST_enrichment_results_" + today + ".tab"

#FUNCTIONS---------------

def get_options():
    '''
    parse options
    '''
    parser = argparse.ArgumentParser()
    #parser.add_argument('--weightdic', default=wdicpath,
    #    help='path to dictionary of weights to assign variants')
    parser.add_argument('--rates', help='path to all possible exome SNVs with specific rates and weights')
    parser.add_argument('--denovos', default=denovopath,  help='path to annotated denovo variants')
    parser.add_argument('--output', default= outpath, help = "output file to save to")
    parser.add_argument('--nmales',type = int, help = 'number of males in study')
    parser.add_argument('--nfemales', type = int, help = 'number of females in study')
    parser.add_argument('--hgbuild', default = "hg19", help = 'genome assembly build')
    parser.add_argument('--nsim',default = 1000000000, type = int,help = 'minimum number of simulations for each gene (default 1 billion)')
    parser.add_argument('--pvalcap',type = float, default = 1.0, help = "stop simulations if cumulative p-value > pvalcap (default 1)")
    return parser.parse_args()

def load_denovos(path):
    '''
    get a DataFrame of DDD de novos
    '''
    logging.info("Loading de novos")
    denovos = pd.read_table(path, sep='\t', na_values='.', keep_default_na=True) # added sep
    denovos['Chrom'] = denovos['Chrom'].astype(str)
    denovos['Position'] = denovos['Position'].astype(int)
    # denovos['score'] = denovos['score'].astype(float)
    # denovos = fix_consequences(denovos)
    return(denovos)

def load_rates(ratespath,genes):
    '''
    load rates file
    '''
    #rates = pd.read_table(ratespath,names = ['symbol','chrom','pos','ref','alt','cq','prob','raw','score','maf','constrained'])
    logging.info("Loading rates")
    rates = pd.read_table(ratespath, sep='\t', na_values='.', keep_default_na=True)
    #subset to only genes observed in de novo file
    rates = rates[rates['GeneID'].isin(genes)]
    rates['Chrom'] = rates['Chrom'].astype(str)
    #rates['Position'] = rates['Position'].astype(int)
    return(rates)

def correct_for_x_chrom(rates, male_n, female_n, hgbuild):
    '''
    correct for X chromosome
    '''
    autosomal = 2 * (male_n + female_n)
    female_transmit = male_n + female_n
    male_transmit = female_n

    # get scaling factors using the alpha from the most recent SFHS (Scottish
    # Family Health Study) phased de novo data.
    alpha = 3.4
    male_k = 2 / (1 + (1 / alpha))
    female_k = 2 / (1 + alpha)

    # correct the non-PAR chrX genes for fewer transmissions and lower rate
    # (dependent on alpha)
    # need to account for PAR part of chrX! <-- TBD
    if hgbuild=="hg19" or hgbuild=="b37":
        hapX = rates['Chrom'].isin(['X', 'chrX']) & rates['Position'].between(2699520,154931044)
    elif hgbuild=="hg38" or hgbuild=="b38":
        hapX = rates['Chrom'].isin(['X', 'chrX']) & rates['Position'].between(2781479,155701382)
    else:
        hapX = rates['Chrom'].isin(['X', 'chrX'])

    x_factor = ((male_transmit * male_k) + (female_transmit * female_k)) / autosomal
    x_factor = pd.Series([x_factor] * len(hapX), index=rates.index)
    x_factor[~hapX] = 1

    rates['prob'] *= x_factor

    return rates

def get_expected_rates(rates, male, female, hgbuild):
    '''
    get expected mutation rates
    '''
    logging.info("get expected")
    autosomal = 2 * (male + female)
    rates['prob'] = rates['prob'] * autosomal

    return correct_for_x_chrom(rates, male, female, hgbuild)

#def test_gene(rates,denovos,gene,nsim,indel_weights,pvalcap):
def test_gene(rates,denovos,gene,nsim,pvalcap):
    '''
    test single gene given rates object and de novos
    '''
    generates = rates[rates.GeneID == gene]
    # Indel rates has been added to the rates df
    # generates = get_indel_rates(generates,indel_weights)
    # here add inframe/frameshift rates
    s_obs = denovos[denovos.GeneID == gene].weight.sum()
    pval,info,s_exp = get_pvalue(generates,s_obs,nsim,pvalcap)
    return(s_exp,s_obs,pval,info)

#def test_all_genes(rates,denovos,nsim,indel_weights,pvalcap):
def test_all_genes(rates,denovos,nsim,pvalcap):
    '''
    test all genes for enrichment
    '''
    logging.info("Starting tests: ")
    #genes = rates.symbol.unique()
    genes = denovos.GeneID.unique()
    results = []
    for gene in genes:
        #skip gene if no observed de novos in our dataset
        #if denovos[denovos.symbol == gene].weight.sum() == 0:
        #    continue
        #skip gene if no rates in our dataset
        if gene not in rates.GeneID.unique():
            logging.info("could not find " + str(gene))
            continue
        logging.info("testing " + str(gene))
        #s_exp,s_obs,pval,info = test_gene(rates,denovos,gene,nsim,indel_weights,pvalcap)
        s_exp,s_obs,pval,info = test_gene(rates,denovos,gene,nsim,pvalcap)
        # extract hgnc_id
        # hgnc = denovos.loc[denovos['GeneID'] == gene, 'hgnc_id'].iloc[0]
        # results.append((gene,hgnc,s_exp,s_obs,pval,info))
        results.append((gene,s_exp,s_obs,pval,info))
    return(results)

def main():
    args = get_options()

    #initialise log file
    logging.basicConfig(filename=args.output.split(".")[0]+".log",level=logging.DEBUG)

    #load denovos
    denovos = load_denovos(args.denovos)
    #get genes in de novo file so only run test on these
    genes = denovos.GeneID.unique()
    logging.info("genes:" + ",".join(genes))

    #use de novo mutation weights
    #denovos,_ = get_weights(denovos,args.weightdic,False)
    denovos = denovos[denovos.weight.apply(np.isreal)]
    denovos = denovos.dropna(subset=['weight'])

    #load rates and get expected
    rates = load_rates(args.rates,genes)
    rates = rates.dropna(subset=['weight','prob'])

    logging.info("rates loaded")
    if len(rates.index) == 0:
        logging.info("rates are empty")
        return

    if args.hgbuild != "hg19" and args.hgbuild != "b37" and args.hgbuild != "hg38" and args.hgbuild != "b38":
        logging.info("cannot recognize genome assembly build : " + args.hgbuild)
        return

    rates = get_expected_rates(rates, args.nmales, args.nfemales, args.hgbuild)
    #get weights
    #logging.info("get rates weights")
    #rates,indel_weights = get_weights(rates,args.weightdic,False)
    rates = rates[rates.weight.apply(np.isreal)]

    # run gene tests
    #results = test_all_genes(rates,denovos,args.nsim,indel_weights,args.pvalcap)
    results = test_all_genes(rates,denovos,args.nsim,args.pvalcap)

    #save results to file
    # df = pd.DataFrame.from_records(results,columns = ["symbol","hgnc_id","expected","observed","p-value","info"])
    df = pd.DataFrame.from_records(results,columns = ["GeneID","Expected","Observed","P-value","Info"])
    df.to_csv(args.output, sep = "\t", index = False)

#------------------SCRIPT--------------------------------------

if __name__=='__main__':
    main()


import numpy as np
from collections import Counter

def read_gsea_file(input_file):
    output = []
    with open(input_file, 'r') as f:
        f.readline()
        for line in f:
            line = line.strip().split(',')
            output.append([float(i) for i in line])
    return np.array(output)

def read_drug_id(input_file):
    with open(input_file, 'r') as f:
        drug_id = f.readline().strip().split(',')
    return drug_id

drug_id = read_drug_id('drugbank_drug_id.csv')

pos_score = read_gsea_file('enrichment_score_up.csv')
neg_score = read_gsea_file('enrichment_score_down.csv')
idx_score = pos_score * neg_score
idx_score = np.where(idx_score < 0, 1, 0)
combined_score = (pos_score - neg_score) / 2 * idx_score

top_sorted_idx = np.argsort(combined_score, axis=0)[-10:, :]
top_drug_idx = Counter(top_sorted_idx.flatten())
output = [['DrugBank ID', 'Number of cell lines of which drug appears in top-10 candidate']]
c1 = 'DrugBank ID'
c2 = 'Number of cell lines of which drug appears in top-10 candidate list'
print("{:<15} {:<15}".format(c1, c2))
for k, v in sorted(top_drug_idx.items(), key=lambda item: item[1], reverse=True):
    print("{:<15} {:<15}".format(drug_id[k], str(v)))