from sklearn.semi_supervised import LabelSpreading, LabelPropagation
import datasets as ds
import numpy as np
import adversarial
from sklearn.metrics import pairwise
from label_propagation_cached import LabelPropagationCached
from sklearn.exceptions import ConvergenceWarning
import warnings
import time
from config import Config
from greedy_adv_attack import (
    multiple_flips_greedy,
    perturb_y_classification,
    multiple_flips_random,
)
from mlp import SimpleMlp

warnings.filterwarnings("ignore", category=ConvergenceWarning)
config = Config(path="config.yaml")

def efficiency():
    for ds_name, attr in config.datasets.items():
        x_train, x_test, y_train, y_test = attr["loader_function"]()
        random_seed = config.seed
        gamma = attr["gamma"]
        max_iter = attr["iter"]
        dataset = ds_name
        randomState = np.random.RandomState(random_seed)
        # x_train, x_test, y_train, y_test = ds.mnist17_prep()
        # x_train, x_test, y_train, y_test = ds.mnist15_prep()
        # x_train, x_test, y_train, y_test = ds.mnist178_prep()
        # x_train, x_test, y_train, y_test = ds.rcv_prep_f()
        # x_train, x_test, y_train, y_test = ds.cifarBinary_prep()

        p_labelled = 0.05
        n_labelled = int(y_train.shape[0] * p_labelled)

        (
            y_train_shuffled,
            x_train_shuffled,
            labels_unlabelled,
        ) = ds.unlabel_shuffle_training_set(y_train, x_train, n_labelled, randomState)

        flips_p = 0.2
        n_flips = int(flips_p * n_labelled)
        repetition = 50
        results_gm = np.empty((repetition,))
        results_im = np.empty((repetition,))
        print(f"{dataset} loaded")
        print(f"[{ds_name}][{p_labelled}][fb:{flips_p}]")
        for i in range(repetition):
            # print(f"[{i}]")
            start_time = time.time()
            perturb_y_classification(
                n_labelled, x_train_shuffled, y_train_shuffled, n_flips, gamma=gamma
            )
            t_gm = time.time() - start_time
            print(
                f"[{i}/{repetition}][{ds_name}][{p_labelled}][fb:{flips_p}]\t PM {t_gm:.2f}s"
            )
            # print("---Probabilistic %s seconds ---" % t_gm)
            results_gm[i] = t_gm

            start_time = time.time()
            weights = pairwise.rbf_kernel(
                x_train_shuffled, x_train_shuffled, gamma=gamma
            )
            adversarial.best_to_flip(n_flips, weights, n_labelled)
            t_im = time.time() - start_time
            print(
                f"[{i}/{repetition}][{ds_name}][{p_labelled}][fb:{flips_p}]\t IM {t_im:2f}s"
            )
            # print("---Influence %s seconds ---" % t_im)
            results_im[i] = t_im
            np.savetxt(f"time_pm_{dataset}_{p_labelled}", results_gm)
            np.savetxt(f"time_im_{dataset}_{p_labelled}", results_im)

def transductive_accuracy():

    for ds_name, attr in config.datasets.items():
        x_train, x_test, y_train, y_test = attr["loader_function"]()
        random_seed = config.seed
        gamma = attr["gamma"]
        max_iter = attr["iter"]
        dataset = ds_name
        randomState = np.random.RandomState(random_seed)

        p_labelled = config.p_labelled
        flip_budgets = config.flip_budgets
        for i, p_l in enumerate(p_labelled):
            n_labelled = int(y_train.shape[0] * p_l)
            (
                y_train_shuffled,
                x_train_shuffled,
                labels_unlabelled,
            ) = ds.unlabel_shuffle_training_set(
                y_train, x_train, n_labelled, randomState
            )

            lp = LabelPropagationCached(gamma=gamma, max_iter=max_iter)
            # ls = LabelSpreading(gamma=gamma, max_iter=10)
            gssl = "lp"
            weights = pairwise.rbf_kernel(
                x_train_shuffled, x_train_shuffled, gamma=gamma
            )
            flip_budget = int(0.2 * n_labelled)
            result_gm_t = np.empty((len(flip_budgets),))
            result_im_t = np.empty((len(flip_budgets),))
            result_gm_i = np.empty((len(flip_budgets),))
            result_im_i = np.empty((len(flip_budgets),))

            print(
                f"Transductive & inductive study on {dataset} [{p_l} labelled] for max_flip {flip_budget} with {gssl}"
            )
            for i, flips_p in enumerate(flip_budgets):
                n_flips = int(flips_p * n_labelled)
                print(f"{flips_p}:{n_flips}/{n_labelled}")
                acc_t, acc_i = multiple_flips_greedy(
                    x_train_shuffled,
                    labels_unlabelled,
                    lp,
                    x_train_shuffled[n_labelled:],
                    y_train_shuffled[n_labelled:],
                    n_flips,
                    n_labelled,
                    gamma=gamma,
                    x_test_t=x_test,
                    y_test_t=y_test,
                    clf=SimpleMlp(
                        random_state=2020, impl="tf", input_shape=(x_train.shape[1],)
                    ),
                )
                print(
                    f"Greedy method : [{i}]=>[transduction {acc_t}][induction {acc_i}]"
                )
                result_gm_t[i] = acc_t
                result_gm_i[i] = acc_i

                # acc_t, acc_i = adversarial.multiple_flips_accuracy_sklearn(
                #     x_train_shuffled,
                #     labels_unlabelled,
                #     lp,
                #     x_train_shuffled[n_labelled:],
                #     y_train_shuffled[n_labelled:],
                #     n_flips,
                #     weights=weights,
                #     x_test_t=x_test,
                #     y_test_t=y_test,
                #     clf=SimpleMlp(
                #         random_state=2020, impl="tf", input_shape=(x_train.shape[1],)
                #     ),
                # )
                # result_im_t[i] = acc_t
                # result_im_i[i] = acc_i
                # print(
                #     f"Influence method : [{i}]=>[transduction {acc_t}][induction {acc_i}]"
                # )
                np.savetxt(f"{gssl}_{dataset}_{p_l}_gm_t", result_gm_t)
                # np.savetxt(f"{gssl}_{dataset}_{p_l}_im_t", result_im_t)
                np.savetxt(f"{gssl}_{dataset}_{p_l}_gm_i", result_gm_i)
                # np.savetxt(f"{gssl}_{dataset}_{p_l}_im_i", result_im_i)

def inductive_accuracy():
    randomState = np.random.RandomState(random_seed)
    # x_train, x_test, y_train, y_test = ds.mnist17_prep()
    # x_train, x_test, y_train, y_test = ds.mnist15_prep()
    # x_train, x_test, y_train, y_test = ds.mnist178_prep()
    x_train, x_test, y_train, y_test = ds.rcv_prep_f()
    # x_train, x_test, y_train, y_test = ds.cifarBinary_prep()
    dataset = "rcv"

    p_labelled = 0.25
    flip_budgets = [0, 0.05, 0.10, 0.15, 0.2]
    n_labelled = int(y_train.shape[0] * p_labelled)

    (
        y_train_shuffled,
        x_train_shuffled,
        labels_unlabelled,
    ) = ds.unlabel_shuffle_training_set(y_train, x_train, n_labelled, randomState)

    lp = LabelPropagationCached(gamma=gamma, max_iter=max_iter)
    # ls = LabelSpreading(gamma=gamma, max_iter=10)
    # weights = pairwise.rbf_kernel(x_train_shuffled, x_train_shuffled, gamma=gamma)
    gssl = "lp"
    result_gm = np.empty((len(flip_budgets),))
    result_im = np.empty((len(flip_budgets),))
    result_rm = np.empty((len(flip_budgets),))
    flip_budget = int(0.2 * n_labelled)
    print(f"Inductive study on {dataset} for flip budget [{flip_budgets}] with {gssl}")
    for i, flips_p in enumerate(flip_budgets):
        n_flips = int(flips_p * n_labelled)
        print(f"{flips_p}:{n_flips}/{n_labelled}")
        acc = multiple_flips_random(
            x_train_shuffled,
            labels_unlabelled,
            lp,
            x_test,
            y_test,
            n_flips,
            n_labelled,
            seed=2020,
        )
        print(f"random [{i}] [{acc}]")
        result_rm[i] = acc
        _, acc = multiple_flips_greedy(
            x_train_shuffled,
            labels_unlabelled,
            lp,
            x_test,
            y_test,
            n_flips,
            n_labelled,
            gamma=gamma,
        )
        print(f"greedy [{i}] [{acc}]")
        result_gm[i] = acc

        # acc = adversarial.multiple_flips_accuracy_rfc(
        #     x_train_shuffled,
        #     labels_unlabelled,
        #     ls,
        #     x_test,
        #     y_test,
        #     i,
        #     weights=weights,
        # )
        # result_im[i] = acc
        # print(f"influence [{i}] [{acc}]")

        np.savetxt(f"rfc_{gssl}_{dataset}_{p_labelled}_rm", result_rm)
        np.savetxt(f"rfc_{gssl}_{dataset}_{p_labelled}_gm", result_gm)
        # np.savetxt("induc_im_15_ls", result_im)

def main():
    # efficiency()
    transductive_accuracy()
    # inductive_accuracy()

if __name__ == "__main__":
    main()


import json
import os
from typing import Any, Dict, List

import appdirs
from loguru import logger

class Settings:
    app_name = "cable-guy"
    settings_path = appdirs.user_config_dir(app_name)
    settings_file = os.path.join(settings_path, "settings.json")

    def __init__(self) -> None:
        self.root: Dict[str, Any] = {"version": 0, "content": {}}

    def settings_exist(self) -> bool:
        """Check if settings file exist

        Returns:
            bool: True if it exist
        """
        return os.path.isfile(self.settings_file)

    def load(self) -> bool:
        """Load settings from file

        Returns:
            bool: False if failed
        """
        if not self.settings_exist():
            logger.warning(f"User settings does not exist: {self.settings_file}")
            return False

        data = None
        try:
            with open(self.settings_file, encoding="utf-8") as file:
                data = json.load(file)
                if data["version"] != self.root["version"]:
                    logger.warning("User settings does not match with our supported version.")
                    return False

                self.root = data
        except Exception as exception:
            logger.warning(f"Failed to fetch data from file ({self.settings_file}): {exception}")
            logger.warning(data)

        return True

    def save(self, content: List[Any]) -> None:
        """Save content to file

        Args:
            content (list): Configuration list
        """
        # We don't want to write in disk if there is nothing different to write
        if self.root["content"] == content:
            return

        self.root["content"] = content

        if not os.path.exists(self.settings_path):
            os.makedirs(self.settings_path)

        with open(self.settings_file, "w+", encoding="utf-8") as file:
            logger.debug(f"Updating settings file: {self.settings_file}")
            json.dump(self.root, file, sort_keys=True, indent=4)


import sys
import os
import json
import torch
import numpy as np
import time

from opts import parse_args
from utils.logger import Logger
from data_funcs.data_loader import load_data
from utils.model_funcs import get_training_elements, evaluate_model, \
    get_lr_scheduler, get_optimiser, run_one_epoch, prox_step
from utils.checkpointing import save_checkpoint
from utils.utils import create_model_dir, create_metrics_dict, \
    metric_to_dict, init_metrics_meter, extend_metrics_dict

from models import RNN_MODELS

# os.environ["WANDB_API_KEY"] = 'INSERT KEY'
# os.environ["CUDA_LAUNCH_BLOCKING"] = '1'

def main(args):
    # system setup
    global CUDA_SUPPORT

    Logger.setup_logging(args.loglevel, logfile=args.logfile)
    logger = Logger()

    logger.debug(f"CLI args: {args}")

    if torch.cuda.device_count():
        CUDA_SUPPORT = True
    else:
        logger.warning('CUDA unsupported!!')
        CUDA_SUPPORT = False

    if not CUDA_SUPPORT:
        args.gpu = "cpu"

    if args.deterministic:
        # import torch.backends.cudnn as cudnn
        import os
        import random

        if CUDA_SUPPORT:
            # cudnn.deterministic = args.deterministic
            # cudnn.benchmark = not args.deterministic
            torch.cuda.manual_seed(args.manual_seed)
            torch.cuda.manual_seed_all(args.manual_seed)

        random.seed(args.manual_seed)
        np.random.seed(args.manual_seed)
        os.environ['PYTHONHASHSEED'] = str(args.manual_seed)
        torch.manual_seed(args.manual_seed)

    logger.info(f"Model: {args.model}, Dataset:{args.dataset}")

    # In case of DataParallel for .to() to work
    args.device = args.gpu[0] if type(args.gpu) == list else args.gpu

    # Load data sets
    trainset, testset, GC = load_data(args.data_path, args.dataset, args.seq_len)

    # It should work, it seems problem is only locally on my machine
    testloader = torch.utils.data.DataLoader(testset,
                                             batch_size=args.batch_size,
                                             num_workers=args.num_workers,
                                             shuffle=False,
                                             persistent_workers=True
                                             )
    args.input_size = next(iter(testloader))[0].shape[-1]

    if not args.evaluate:  # Training mode
        trainloader = torch.utils.data.DataLoader(
            trainset,
            batch_size=args.batch_size,
            num_workers=args.num_workers,
            shuffle=True,
            persistent_workers=True
        )

        init_and_train_model(args, trainloader, testloader, GC)

    else:  # Evaluation mode
        model, criterion, round = get_training_elements(
            args.model_name, args.input_size, args.seq_len, args.resume_from,
            args.load_best, args.gpu)

        metrics = evaluate_model(
            model, testloader, criterion, args.device, round,
            print_freq=10, metric_to_optim=args.metric,
            is_rnn=args.model in RNN_MODELS, GC=GC)

        metrics_dict = create_metrics_dict(metrics)
        logger.info(f'Validation metrics: {metrics_dict}')

def init_and_train_model(args, trainloader, testloader, GC):
    full_metrics = init_metrics_meter()
    model_dir = create_model_dir(args)
    # don't train if setup already exists
    if os.path.isdir(model_dir):
        Logger.get().info(f"{model_dir} already exists.")
        Logger.get().info("Skipping this setup.")
        return
    # create model directory
    os.makedirs(model_dir, exist_ok=True)
    # init wandb tracking
    # wandb.init(
    #     project="insert_project_name", entity="insert_entity", config=vars(args),
    #     name=str(create_model_dir(args, lr=False)), reinit=True)
    # save used args as json to experiment directory
    with open(os.path.join(create_model_dir(args), 'args.json'), 'w') as f:
        json.dump(vars(args), f, indent=4)

    is_rnn = args.model in RNN_MODELS
    model, criterion, current_round = get_training_elements(
        args.model, args.input_size, args.seq_len, args.resume_from, args.load_best, args.gpu)

    optimiser = get_optimiser(
        model.parameters(), args.optimiser, args.lr,
        args.momentum, args.weight_decay)

    scheduler = get_lr_scheduler(
        optimiser, args.rounds, args.lr_type)

    metric_to_optim = args.metric
    best_metric = np.inf
    train_time_meter = 0

    for i in range(args.rounds):
        start = time.time()
        metrics_meter = run_one_epoch(
            model, trainloader, criterion, optimiser, args.device,
            current_round, is_rnn, args.clip_grad, args.prox, args.gc_penalty)
        # Prox step to better detect Granger Causality
        if args.prox:
            prox_step(model, scheduler, args.gc_penalty)
        extend_metrics_dict(
            full_metrics, metric_to_dict(metrics_meter, i+1, 'train'))
        # wandb.log(metric_to_dict(metrics_meter, i+1, 'train', False))
        train_time = time.time() - start
        train_time_meter += train_time
        # Track timings across epochs
        Logger.get().debug(f'Epoch train time: {train_time}')

        if i % args.eval_every == 0 or i == (args.rounds - 1):
            metrics = evaluate_model(
                model, testloader, criterion, args.device, current_round,
                print_freq=10, metric_to_optim=metric_to_optim, is_rnn=is_rnn,
                GC=GC)
            extend_metrics_dict(
                full_metrics, metric_to_dict(metrics, i+1, 'test'))
            # wandb.log(metric_to_dict(metrics, i+1, 'test', False))
            avg_metric = metrics[metric_to_optim].get_avg()
            # Save model checkpoint
            model_filename = (f"{args.model}_{args.run_id}_checkpoint"
                              f"_{current_round:0>2d}.pth.tar")
            is_best = avg_metric < best_metric
            save_checkpoint(model, model_filename, is_best=is_best, args=args,
                            metrics=metrics, metric_to_optim=metric_to_optim)
            if is_best:
                best_metric = avg_metric

            if np.isnan(metrics['loss'].get_avg()):
                Logger.get().info(
                    'NaN loss detected, aborting training procedure.')
                return

        Logger.get().info(
            f'Current lr: {scheduler.get_last_lr()}')
        scheduler.step()
        current_round += 1
        torch.cuda.empty_cache()

    Logger.get().debug(
        f'Average epoch train time: {train_time_meter / args.rounds}')
    #  store the run
    with open(os.path.join(
            create_model_dir(args), 'full_metrics.json'), 'w') as f:
        del full_metrics['train_GC']
        del full_metrics['test_GC']
        del full_metrics['train_GC_full']
        del full_metrics['test_GC_full']
        json.dump(full_metrics, f, indent=4)

    Logger.get().debug(f'GC model:{model.GC()}')
    Logger.get().debug(f'GC true:{GC}')

if __name__ == "__main__":
    args = parse_args(sys.argv)
    main(args)
    torch.cuda.empty_cache()
    assert torch.cuda.memory_allocated() == 0


import urllib
import json
from urllib.request import urlopen

def eliminar_tildes(palabra):
    '''. Recibe una palabra.
    . Devuelve la palabra sin tildes.
    '''
    palabra = palabra.replace('á','a').replace('é','e').replace('í','i').replace('ó','o').replace('ú','u')
    return palabra

def comprobar_datos(pronostico, ubicacion):
    '''Comprueba que la ciudad y provincia ingresada por el usuario exista en las
       ciudades y provincias del pronostico
    . Recibe la informacion del pronostico y una lista llamada ubicacion con la
    ciudad y provincia ingresada por el usuario.
    . En caso de existir esa ubicacion, la retorna.
    '''
    continuar = True
    while continuar:
        for zona in pronostico:
            if eliminar_tildes(zona['name'].lower()) in ubicacion[0] and eliminar_tildes(zona['province'].lower()) in ubicacion[1]:
                continuar = False

        if continuar:
            print("Los datos ingresados no son validos, ingrese nuevamente")
            ciudad = input("Ingrese ciudad: ")
            provincia = input("Ingrese provincia: ")
            ubicacion = [eliminar_tildes(ciudad.lower()), eliminar_tildes(provincia.lower())]
    return ubicacion

def mostrar_pronostico(pronostico, ubicacion):
    '''Recibe informacion del pronostico y lo muestra para una ubicacion determinada.
    '''
    for zona in pronostico:
        if eliminar_tildes(zona['name'].lower()) in ubicacion[0] and eliminar_tildes(zona['province'].lower()) in ubicacion[1]:
            print(f"Pronostico de {zona['name']},{zona['province']} a {zona['weather']['day']} dias")
            print(".Temperatura en la mañana: ", zona['weather']['morning_temp'], "°C")
            print(".Pronostico de la mañana: ", zona['weather']['morning_desc'].replace(".",".\n"))
            print(".Temperatura en la tarde: ", zona['weather']['afternoon_temp'], "°C")
            print(".Pronostico de la tarde: ", zona['weather']['afternoon_desc'].replace(".",".\n"))

def mostrar_alertas(alertas, provincia, ciudad):
    lista_alertas = []
    for dicc in alertas:
        for ubicacion in dicc["zones"]:
            if provincia in eliminar_tildes(dicc["zones"][ubicacion].lower()):
                lista_alertas.append(dicc["title"])

    if len(lista_alertas)>=1:
        lista_alertas.append("Fuertes lloviznas")
        print(f"\nLas alertas en {ciudad.capitalize()}, {provincia.capitalize()} son: {', '.join(lista_alertas)}")
    else:
        print(f"\n{ciudad.capitalize()}, {provincia.capitalize()} no sufre ninguna alerta")

def pronostico_extendido(): #main
    try:
        with urlopen("https://ws.smn.gob.ar/map_items/forecast/1") as page:
            source = page.read()
        un_dia = json.loads(source)
        with urlopen("https://ws.smn.gob.ar/map_items/forecast/2") as page:
            source = page.read()
        dos_dias = json.loads(source)
        with urlopen("https://ws.smn.gob.ar/map_items/forecast/3") as page:
            source = page.read()
        tres_dias = json.loads(source)
        with urlopen("https://ws.smn.gob.ar/alerts/type/AL") as page:
            source = page.read()
        alertas = json.loads(source)

        ciudad = input("Ingrese ciudad: ")
        provincia = input("Ingrese provincia: ")
        ubicacion = [eliminar_tildes(ciudad.lower()), eliminar_tildes(provincia.lower())]
        ubicacion = comprobar_datos(un_dia, ubicacion)

        print("")
        mostrar_pronostico(un_dia, ubicacion)
        mostrar_pronostico(dos_dias, ubicacion)
        mostrar_pronostico(tres_dias, ubicacion)
        mostrar_alertas(alertas, eliminar_tildes(provincia.lower()), eliminar_tildes(ciudad.lower()))
    except urllib.error.URLError or socket.gaierror:
        print("Checke su conexion a internet por favor, el programa tormenta.py necesita de internet para funcionar,"
              "si su conexion a internet esta funcionando puede ser que el servidor del servicio meteorologico de la "
              "ciudad este teniendo problemas, en tal caso intente más tarde.")


#!/usr/bin/env python
"""ViewDeleteSubstitutes module."""
from colorama import init  # type: ignore
from termcolor import colored

from app.views.view import View
from app import settings as s

init(autoreset=True)

class ViewDeleteSubstitutes(View):
    """View delete substitute class."""

    def __init__(self):
        super().__init__()

    def display_delete_substitutes(self, substituted_name, substitute_name):
        """Display an alert message to confirm the deletion."""
        print(self.header())
        print(
            f'\n{self.justify("Souhaitez-vous vraiment supprimer", 3)}\
 {colored(substituted_name, "cyan")}\n{self.justify("et", 3)}\
 {colored(substitute_name, "cyan")} de vos favoris ?\n\n{self.del_alert}\n'
        )

    def input_delete_substitutes(self):
        """Input the choices."""
        self.menu_choices[0] = s.DELETE_ITEM
        choice = "\n".join(
            [
                f"{colored(index, 'yellow')}. {item}"
                for index, item in enumerate(self.menu_choices, 1)
            ]
        )
        print(choice)
        print(f"{s.CR}{self.footer}")
        if s.ERROR:
            print(self.msg_error)
        print(self.back, self.quit)


#! python
'''A simple way to run your python code from the CLI
The module uses introspection to try and expose your code to the command line. It won't work in all cases, it depends on the complexity of your code.

This is the executable part.
'''

import logging
import sys

LOGGER = logging.getLogger(__name__)

try:
	import simplifiedapp
except ModuleNotFoundError:
	import __init__ as simplifiedapp

target, sys_argv = simplifiedapp, []
if len(sys.argv) > 1:
	if ((len(sys.argv[1]) == 2) and sys.argv[1].startswith('-')) or sys.argv[1].startswith('--'):
		sys_argv = sys.argv[1:]
	elif len(sys.argv) > 2:
		target = sys.argv[1]
		sys_argv = sys.argv[2:]

simplifiedapp.main(target = target, sys_argv = sys_argv)


import requests

weather_endpoint = "https://api.openweathermap.org/data/2.5/onecall?"

weather_params = {
    "lat": 59.917180,
    "lon": 10.695490,
    "appid": "228b2c371fca593249846b5f5d78b1e2",
}

response = requests.get(weather_endpoint, params=weather_params)
onecall = response.json()
# print the response status
print('http response status: {}'.format(response))

# Print the response to the console
print('json response: {}\n'.format(onecall))
# print(response.raise_for_status())
# print('http response status: {}'.format(response.status_code))
# print(response.json())


import numpy as np
from matplotlib import pyplot as plt

# Значения основных параметров Вариант 12 (2)
KSI = 0.25
MU1 = 0
MU2 = 1
x0 = 0
xn = 1
n = 10000
h = float(1/n)
v = np.zeros(n+1)
v[-1] = MU2
X = np.linspace(x0, xn, n+1)

# Коэффициенты ai
def ai(k1, k2, kmid, x):
    if KSI >= x:
        return k1(x - h/2)
    elif KSI <= x - h:
        return k2(x-h/2)
    else:
        return kmid(x)

def di(q1, q2, qmid, x):
    if KSI >= x+h/2:
        return q1(x)
    elif KSI <= x - h/2:
        return q2(x)
    else:
        return qmid(x)

def fi(f1, f2, fmid, x):
    if KSI >= x + h/2:
        return f1(x)
    elif KSI <= x - h/2:
        return f2(x)
    else:
        return fmid(x)

def k1TEST(x):
    return 0.5

def k2TEST(x):
    return 1.25

def q1TEST(x):
    return 1

def q2TEST(x):
    return 0.0625

def f1TEST(x):
    return 1

def f2TEST(x):
    return 2.5

def analytSol():
    a = 2**0.5
    b = 0.05**0.5
    c1 = 0.5874420413692708
    c2 = -1.5874420413692708
    c3 = -16.23731986542228
    c4 = -23.37825943652864
    g = np.zeros(n+1)
    for i in range(n+1):
        x = i * h
        if x < KSI:
            g[i] = c1 * np.exp(a*x) + c2 * np.exp(-a*x) + 1
        else:
            g[i] = c3 * np.exp(b*x) + c4 * np.exp(-b*x) + 40
    return g

def numericSol():
    alpha = [0]
    beta = [0]
    for i in range(1, n):
        x = i * h
        A = ai(k1TEST, k2TEST, k2TEST, x) / h**2
        B = ai(k1TEST, k2TEST, k2TEST, x+h) / h**2
        C = A + B + di(q1TEST, q2TEST, q2TEST, x)
        alpha.append(B / (C - A * alpha[i-1]))
        beta.append((fi(f1TEST, f2TEST, f2TEST, x) + A * beta[i-1]) / (C - A * alpha[i-1]))
    for i in range(n, 1, -1):
        v[i-1] = alpha[i-1] * v[i] + beta[i-1]
    y = analytSol()
    return v, y

v, y = numericSol()
Eps1 = max(abs(v - y))
print(Eps1)
plt.plot(X, v, 'r')
plt.plot(X, y, 'g')
plt.show()


import matplotlib.pyplot as plt
import matplotlib as mpl
import numpy as np
import pandas as pd
import random
import glob
import re
import os
import seaborn as sns
import scipy.stats as stats

# Function, ........................................................................................
def find_columns_with_selected_dtype(*, df, dtype="numerical"):
    """
        =================   ===============================================================================
        Property            Description
        =================   ===============================================================================

        * Function          Finds column names of columns with requested dtype numeric or text,

        Parameters/Input
        _________________   _______________________________________________________________________________

        . Input .
        * df                DataFrame, with unique column names
        * dtype             data type that we with to find {'text', 'numeric'}

        Returns
        _________________   _______________________________________________________________________________

        * col_names         list wiht column names, that have eaither numeric or text dtape
    """
    col_names = []

    if dtype=="numerical":
        from pandas.api.types import is_numeric_dtype # if is_string_dtype(df[j]):
        for col_name in list(df.columns):
            if is_numeric_dtype(df[col_name]):
                col_names.append(col_name)

    if dtype=="text":
        from pandas.api.types import is_string_dtype # if is_string_dtype(df[j]):
        for col_name in list(df.columns):
            if is_string_dtype(df[col_name]):
                col_names.append(col_name)

    return col_names

# Function, ..............................................................................................

def clean_data_and_find_correlations(*, df, row_Filter=None, colName_main_variable, verbose=False):

    """
        =================   ===============================================================================
        Property            Description
        =================   ===============================================================================

        * Function          This function, takes df, with numeric data in each column,
                            computes all column to one of them, called colName_main_variable
                            the data are formatted and cleaned in each pari of column individually, (by removing na)

        Parameters/Input
        _________________   _______________________________________________________________________________

        . Input .
        * df                DataFrame, with unique column names
        * row_filter        list[bool], with len(row_filter )==df.shape[0]
        * verbose           bool, if True, shows info

        Returns
        _________________   _______________________________________________________________________________

        * comparisons_dct   dictionary, where key=column name in df,
                            inside each entry, there is another dict wiht condition names, raw and filtered data
                            and results of correlation made with 3 different methods (pearson, spearman and kendal)

    """

    #### filter the data:
    if row_Filter==None:
        row_Filter = [True]*df.shape[0]
    # ........
    filtered_df_main_col    = pd.Series(df.loc[row_Filter,colName_main_variable]) # pd.Series
    filtered_df             = df.loc[row_Filter, :]                               # pd Series or pd.DataFrame

    #### rename column with main group to avoid having duplicates,
    filtered_df.rename(columns={str(colName_main_variable):"".join([str(colName_main_variable),"_"])}, inplace=True) # to ensute, that we have no columns with the same name,

    #### Loop over each columns and compare it with main group,
    comparisons_dct = dict()
    # .......
    for i, colName in enumerate(list(filtered_df.columns)):

        if verbose==True:
            print(i, colName_main_variable, " - with - ", colName)

        # some columns had to many repeats r nan. i remove them later, but warningn  were annoying, thus I removed them;
        import warnings
        warnings.filterwarnings("ignore", category=RuntimeWarning)

        # --------------------------
        # Prepare dct for the data,
        # --------------------------

        # new dict,
        one_comparison_dct = dict()
        one_comparison_dct["X_main_group_sliced_with"]    = row_Filter
        one_comparison_dct["X_main_group"]                = colName_main_variable
        one_comparison_dct["Y_compared_with"]             = colName

        # --------------------------
        # PREPARE THE DATA
        # --------------------------

        # ................... raw data, .........................

        # prepare the data,
        filtered_df_one_col         = filtered_df.loc[:,colName]
        data_for_comparison_df      = pd.concat([filtered_df_main_col, filtered_df_one_col], axis=1)
        data_for_comparison_df_full = data_for_comparison_df.copy() # for hitograms

        # remove missing data,
        data_for_comparison_df      = data_for_comparison_df.dropna(how="any",axis=0) # to create X/Y_cleaned

        # names, and basic data to display,
        sample_number               = int(data_for_comparison_df.shape[0])  # int
        atribute_names              = list(data_for_comparison_df.columns) # list

        # ................... control, ............................

        # check if you can continue,
        if sample_number <= 2:
            # add to dict,
            one_comparison_dct["X_total"]                = [None]
            one_comparison_dct["X_cleaned"]              = [None]
            one_comparison_dct["X_cleaned_log"]          = [None]
            # .......
            one_comparison_dct["Y_total"]                = [None]
            one_comparison_dct["Y_cleaned"]              = [None]
            one_comparison_dct["Y_cleaned_log"]          = [None]
            # .......
            one_comparison_dct["pearson_results"]        = [None]
            one_comparison_dct["sperman_results"]        = [None]
            one_comparison_dct["kendalltau_results"]     = [None]
            # ..........
            one_comparison_dct["linregress_results"]     = [None]
            one_comparison_dct["linregress_results_log"] = [None]

            if verbose==True:
                print(f"Caution, (column cobination nr {i}) -ie.- {atribute_names[0]},vs{atribute_names[1]}, has less then 3 items to compare !")
            ############################################################################
            comparisons_dct[colName] = one_comparison_dct
            ############################################################################

        # else,
        if sample_number > 2:

            # ... X,Y data for plots and correlation, ................

            # all data, without removing NaN in each row, - for hist,
            X_total = data_for_comparison_df_full.iloc[:, 0]
            X_total = X_total.dropna(how="any").values.flatten()
            Y_total = data_for_comparison_df_full.iloc[:, 1]
            Y_total = Y_total.dropna(how="any").values.flatten()

            # data,
            X_cleaned = data_for_comparison_df.iloc[:, 0].values.flatten()
            Y_cleaned = data_for_comparison_df.iloc[:, 1].values.flatten()

            # transform values into log(x+2), +2 to avoid having log= inf, or zero
            X_cleaned_log = np.log(X_cleaned+16)
            Y_cleaned_log = np.log(Y_cleaned+16)

            # .....

            # add to dict,
            one_comparison_dct["X_total"]       = X_total
            one_comparison_dct["X_cleaned"]     = X_cleaned
            one_comparison_dct["X_cleaned_log"] = X_cleaned_log
            # .......
            one_comparison_dct["Y_total"]       = Y_total
            one_comparison_dct["Y_cleaned"]     = Y_cleaned
            one_comparison_dct["Y_cleaned_log"] = Y_cleaned_log

            # --------------------------
            # FIND CORR.
            # --------------------------

            # ... Correlation, ....................................

            # correlations,
            pearson_results    = stats.pearsonr(X_cleaned_log, Y_cleaned_log)   # linear
            sperman_results    = stats.spearmanr(X_cleaned, Y_cleaned)          # rank, with rho value,
            kendalltau_results = stats.kendalltau(X_cleaned, Y_cleaned)   # rank,based on orientation of pairs of ranks
            # ............
            one_comparison_dct["pearson_results"]      = pearson_results
            one_comparison_dct["sperman_results"]      = sperman_results
            one_comparison_dct["kendalltau_results"]   = kendalltau_results

            # Compute a least-squares regression for two sets of measurements.
            LR_slope, LR_intercept, LR_r_value, LR_p_value, LR_std_err = stats.linregress(X_cleaned, X_cleaned)
            # ..........
            linregress_results = {"slope": LR_slope,
                                      "intercept": LR_intercept,
                                      "r_value": LR_r_value,
                                      "p_value": LR_p_value,
                                      "std_err": LR_std_err}
            # ..........
            one_comparison_dct["linregress_results"] = linregress_results

            # Compute a least-squares regression for two sets of measurements.
            LR_slope, LR_intercept, LR_r_value, LR_p_value, LR_std_err = stats.linregress(X_cleaned_log, Y_cleaned_log)
            # ..........
            linregress_results_log = {"slope": LR_slope,
                                      "intercept": LR_intercept,
                                      "r_value": LR_r_value,
                                      "p_value": LR_p_value,
                                      "std_err": LR_std_err}
            # ..........
            one_comparison_dct["linregress_results_log"] = linregress_results_log

            ############################################################################
            comparisons_dct[colName] = one_comparison_dct
            ############################################################################

    return comparisons_dct

# Function, ..............................................................................................

def make_plots_and_table_with_corr(*, data_dct, N_plots_to_show=5, display_table=True, display_plots=True,
                                   N_plots_from=0, N_plots_to=2, figure_size= (12, 4), N_rows_in_table_to_show=10,
                                   return_rable=False, select_group_name=None, verbose=False):

    """
        =================   ===============================================================================
        Property            Description
        =================   ===============================================================================

        * Function          Larger function that makes four things:
                            - takes data_dct ie. dictionary generated with clean_data_and_find_correlations()
                              it collects and order all the data, in df
                            - if display_plots=True, display requested number of plots wiht scatter plot
                              and LR line representing relationship etwen compared values, and two histogram plots,
                              showsing distribusion of values in a given variable before and sfter selection (na removal)
                              required for calculating correlation between these values
                            - if return_rable=False, it will return a df, with all values such as rvalua and tau value
                              that were collected form the dictionary,
                            - if display_table=True, it will display, top number of requested correlations with the hisest absolute values
                              (either negative or positive)

        Parameters/Input
        _________________   _______________________________________________________________________________

        . Input .
        * data_dct          distionary, created with clean_data_and_find_correlations()
        * display_plots     bool, =True, see above
        * N_plots_from      int, =0, from whereto start displaying plots, 1st pair (0), is a correlation between main value itself
        * N_plots_to        int, =3, how many plots fdo display, not included,
        * figure_size       tuple(int, int)
        * return_rable      bool, =False, see above
        * select_group_name str, depreciated....
        * N_rows_in_table_to_show=10,  : if display_table=True, how many rows to display,
        * verbose           bool, =False

        Returns
        _________________   _______________________________________________________________________________

        * ≥1 plots          each with 3 subplots, see above
        * df image          df.head(), see above
        * df                unordered, df with data used for plots, in all conditiona, even if not displayed.
    """

    # collect and order the data from dict,
    dct_keys          = list(data_dct.keys())
    dct_rvalues       = list()
    dct_pvalues       = list()
    dct_rhovalues     = list()
    dct_tauvalues     = list()
    dct_sample_number = list()
    # .......
    for i, key in enumerate(dct_keys):
        if data_dct[key]["X_total"][0]!=None:
            dct_rvalues.append(data_dct[key]["pearson_results"][0])
            dct_pvalues.append(data_dct[key]["pearson_results"][1])
            dct_rhovalues.append(data_dct[key]["sperman_results"][0])
            dct_tauvalues.append(data_dct[key]["kendalltau_results"][0])
            dct_sample_number.append(data_dct[key]["X_cleaned"].shape[0])

        else:
            dct_rvalues.append(np.nan)
            dct_pvalues.append(np.nan)
            dct_rhovalues.append(np.nan)
            dct_sample_number.append(np.nan)
            dct_tauvalues.append(np.nan)

    # ....... join them into df,
    order_in_df = pd.concat([pd.Series(dct_keys),
                             pd.Series(dct_rvalues),
                             pd.Series(dct_pvalues),
                             pd.Series(dct_rhovalues),
                             pd.Series(dct_tauvalues),
                             pd.Series(dct_sample_number)
                            ],axis=1) # to use functions below

    # ...... re-name,
    order_in_df.columns=["keys", "rvalues", "pvalues", "rho", "tau", "sample nr"] # for my info

    # ...... save to return
    df_with_all_results_unordered = order_in_df.copy()

    # ...... re-order, to find the
    order_in_df["rvalues_abs"] = pd.Series(np.abs(order_in_df.iloc[:,1].values)) # use absolute values
    order_in_df.sort_values(by="rvalues_abs", ascending=False, inplace=True) # to get tghe order

    # --------------------------
    # DATA FOR PLOTS
    # --------------------------
    if display_plots==True:

        # show plots, of N top values
        for i,key in enumerate(list(order_in_df.loc[:,"keys"])):

            if i >= N_plots_from and i<=N_plots_to:

                # --------------------------
                # DATA FOR PLOTS
                # --------------------------

                X_cleaned_log = data_dct[key]["X_cleaned_log"]
                Y_cleaned_log = data_dct[key]["Y_cleaned_log"]
                # ......
                X_cleaned = data_dct[key]["X_cleaned"]
                Y_cleaned = data_dct[key]["Y_cleaned"]
                # ......
                X_total = data_dct[key]["X_total"]
                Y_total = data_dct[key]["Y_total"]

                # ......
                atribute_names = [data_dct[key]["X_main_group"], data_dct[key]["Y_compared_with"]]

                # .....
                LR_intercept = data_dct[key]["linregress_results_log"]["intercept"]
                LR_slope = data_dct[key]["linregress_results_log"]["slope"]
                LR_r_value = data_dct[key]["linregress_results_log"]["r_value"]
                LR_p_value = data_dct[key]["linregress_results_log"]["p_value"]

                # .....
                pearson_results = data_dct[key]["pearson_results"]

                # INFO:
                if verbose==True:
                    print(i, key, atribute_names)

                # --------------------------
                # PLOTS
                # --------------------------

                # ... fig, ax, .............................................................................

                fig, axs = plt.subplots(nrows=1, ncols=3, figsize=figure_size, facecolor="white")

                if select_group_name==None:
                    plt.suptitle(f"[{i}] {atribute_names[0]} - vs - {atribute_names[1]}", fontsize=17)
                else:
                    plt.suptitle(f"[{i}] {atribute_names[0]} - vs - {atribute_names[1]}, ({select_group_name})", fontsize=17)

                # colors
                selected_data_color = "deepskyblue"
                total_data_color    = "black"
                scatter_color       = "forestgreen"
                line_color          = "black"

                # ... SUBPLOT 1, scatterplot ..............................................................

                # data,

                X = X_cleaned_log
                Y = Y_cleaned_log

                # title,
                subplot_title = f'Correlation and LR:\nr = {LR_r_value:.2f}, p = {np.round(list(pearson_results)[1],3)}\ny = {LR_intercept:.2f}+{LR_slope:.2f}x'

                # plot and add line
                axs[0].plot(X, Y, linewidth=0, marker='.', color=scatter_color )
                axs[0].plot(X, LR_intercept + LR_slope * X, color=line_color, linestyle="--", linewidth=3)
                axs[0].set_title(subplot_title)

                # axes labels
                axs[0].set_xlabel(str(atribute_names[0])+"\nlog values")
                axs[0].set_ylabel(str(atribute_names[1])+"\nlog values")
                axs[0].set_xlim(X.min(), X.max())
                axs[0].set_ylim(Y.min(), Y.max())

                # Remove ticks, and axes that you dot'n want, format the other ones,
                axs[0].yaxis.set_ticks_position('left')# shows only that
                axs[0].xaxis.set_ticks_position('bottom')# shows only that
                axs[0].spines['top'].set_visible(False) # remove ...
                axs[0].spines['right'].set_visible(False) # remove ...
                axs[0].spines['bottom'].set_linewidth(1) # x axis width
                axs[0].spines['left'].set_linewidth(1) # y axis width

                # grid and labels,
                axs[0].xaxis.grid(True,color='grey', linestyle='--', linewidth=0.5)
                axs[0].yaxis.grid(True,color='grey', linestyle='--', linewidth=0.5)

                #................................................................................
                # ... kwars for histograms ......................................................
                hist_kwargs         = dict(histtype="stepfilled", density=True, bins=20)
                # ... kwars for histograms ......................................................
                #................................................................................

                # ... SUBPLOT 2, X data histogram ....................................

                # title,
                perc_value = np.round((X_cleaned.shape[0]/X_total.shape[0])*100,2)
                subplot_title = f"{X_total.shape[0]}, non-NaN items in df\n{X_cleaned.shape[0]} items used for Corr ({perc_value}%)"

                # plot and add line
                axs[1].hist(X_total, color=total_data_color,edgecolor=total_data_color, alpha=0.6, **hist_kwargs, label="all data")
                axs[1].hist(X_cleaned, color=selected_data_color,edgecolor=selected_data_color, alpha=0.5, **hist_kwargs, label="data used for LR")
                axs[1].set_title(subplot_title)

                # axes labels
                axs[1].set_xlabel(str(atribute_names[0]))
                axs[1].set_ylabel("Density")

                # recompute x/y limits
                axs[1].relim() # recompute the ax.dataLim
                axs[1].autoscale_view() # update ax.viewLim using the new dataLim

                # Remove ticks, and axes that you dot'n want, format the other ones,
                axs[1].yaxis.set_ticks_position('left')# shows only that
                axs[1].xaxis.set_ticks_position('bottom')# shows only that
                axs[1].spines['top'].set_visible(False) # remove ...
                axs[1].spines['right'].set_visible(False) # remove ...
                axs[1].spines['bottom'].set_linewidth(1) # x axis width
                axs[1].spines['left'].set_linewidth(1) # y axis width

                # ... SUBPLOT 3, Y data histogram ....................................

                # title,
                perc_value = np.round((Y_cleaned.shape[0]/Y_total.shape[0])*100,2)
                subplot_title = f"{Y_total.shape[0]}, non-NaN items in df\n{Y_cleaned.shape[0]} items used for Corr ({perc_value}%)"

                # plot and add line
                axs[2].hist(Y_total, color=total_data_color,edgecolor=total_data_color, alpha=0.6, **hist_kwargs, label= "all data")
                axs[2].hist(Y_cleaned, color=selected_data_color,edgecolor=selected_data_color, alpha=0.5, **hist_kwargs, label="data used for LR")
                axs[2].set_title(subplot_title)

                # axes labels
                axs[2].set_xlabel(str(atribute_names[1]))
                axs[2].set_ylabel("Density")

                # recompute x/y limits
                axs[2].relim() # recompute the ax.dataLim
                axs[2].autoscale_view() # update ax.viewLim using the new dataLim

                # Remove ticks, and axes that you dot'n want, format the other ones,
                axs[2].yaxis.set_ticks_position('left')# shows only that
                axs[2].xaxis.set_ticks_position('bottom')# shows only that
                axs[2].spines['top'].set_visible(False) # remove ...
                axs[2].spines['right'].set_visible(False) # remove ...
                axs[2].spines['bottom'].set_linewidth(1) # x axis width
                axs[2].spines['left'].set_linewidth(1) # y axis width

                # .. create patch list for legend, ..................

                # . import
                import matplotlib.patches as mpatches
                patch_list_for_legend =[]

                # .1.
                ingr_patch      = mpatches.Patch(color=total_data_color, label=f"all available data")
                patch_list_for_legend.append(ingr_patch)

                # .2.
                ingr_patch      = mpatches.Patch(color=selected_data_color, label=f"data used for Corr(X,Y), and LR")
                patch_list_for_legend.append(ingr_patch)

                # . Add legend to the figure
                l = fig.legend(handles=patch_list_for_legend, loc="center", frameon=False, scatterpoints=1, ncol=2, bbox_to_anchor=(0.5, 0.82), title="", fontsize=12)
                l.set_title("histograms with:")
                l.get_title().set_fontsize('12')

                # ... FIG aestetics ....................................
                sns.despine()
                fig.tight_layout()
                fig.subplots_adjust(top=0.6)# to adjust for main title, amd legend
                plt.show();

    # .... DISPLAY TABLE WITH MANY RESULTS BELOW PLOTS, .................
    if display_table == True:

        # remove unwanted
        df_with_corr_results_ordered = order_in_df.drop(["rvalues_abs"], axis=1)
        df_with_corr_results_ordered.reset_index(drop=True, inplace=True)

        from IPython.display import display
        print(f"""{"".join(["-"]*80)}\nTable with all corelations, \n{"".join(["-"]*80)}""")
        display(df_with_corr_results_ordered.head(N_rows_in_table_to_show))

    # ... RETURN .................................... I PLACED IT HERE TO NOT WRITE ANY FUNCTIONS AGAIN !
    if return_rable == True:

        return df_with_all_results_unordered

# Function, ..............................................................................................
def corr_heatmap(*, corr_res_dct, method="rvalues"):
    """
        =================   ===============================================================================
        Property            Description
        =================   ===============================================================================

        * Function          plots a histogram with correlation values generated for multiple comparisons
                            clean_data_and_find_correlations(), and then with make_plots_and_table_with_corr()

        Parameters/Input
        _________________   _______________________________________________________________________________

        . Input .
        * corr_res_dct      pd dataFrame, rows = condition name,
                            make_plots_and_table_with_corr() from dict, generated with clean_data_and_find_correlations()
        * method            str, {'rvalues', 'rho', 'tau'}

        Returns
        _________________   _______________________________________________________________________________

        * heatmap

    """

    #### collect and transpose values for heatmap, .................................................

    # . loop
    for i, key in enumerate(list(corr_res_dct.keys())):

        # extract,
        corr_table_one_col         = corr_res_dct[key].loc[:,method].copy()
        corr_table_one_col_for_na  = corr_res_dct[key].loc[:,method]

        # replace na with zero (to simplify histogram making),
        corr_table_one_col[corr_table_one_col_for_na.isna()] = 0

        # add to final table
        if i==0:
            corr_table      =  corr_table_one_col
        else:
            corr_table      = pd.concat([corr_table, corr_table_one_col], axis=1)

    # . add index to df,
    corr_table["variable"]  = pd.Series(list(corr_res_dct[key]["keys"]))
    corr_table.set_index("variable", inplace=True)

    # . add column names,
    corr_table.columns = list(corr_res_dct.keys())

    # . re-order the rows, for hist:
    row_order          = pd.Series(corr_table.sum(axis=1))
    row_order          = list(row_order.sort_values(ascending=False).index)
    row_order
    # . .......
    corr_table         = corr_table.loc[row_order,:]

    # . transpose, to have better layout of a heatmap
    corr_table        = corr_table.transpose()

    #### create annotated heatmap, ..............................................................

    # .. figure,
    sns.set_context(None)
    fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(20,10), facecolor="white")
    im = plt.imshow(corr_table.values, aspect = 1, origin="upper", interpolation='nearest', cmap="seismic")
    fig.suptitle(f"Heatmap with correlation results: {method}", fontsize=20)

    # . We want to show all ticks...
    ax.set_xticks(list(range(corr_table.shape[1])))
    ax.set_yticks(list(range(corr_table.shape[0])))

    # . and label them with the respective list entries
    ax.set_xticklabels(list(corr_table.columns))
    ax.set_yticklabels(list(corr_table.index))

    # . Rotate the tick labels and set their alignment.
    plt.setp(ax.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor", fontsize=15)
    plt.setp(ax.get_yticklabels(), fontsize=15)

    # . Remove ticks, and axes that you dot'n want, format the other ones,
    ax.spines['top'].set_visible(False) # remove ...
    ax.spines['right'].set_visible(False) # remove ...
    ax.yaxis.set_ticks_position('none')# shows only that
    ax.xaxis.set_ticks_position('none')# shows only that

    # .. spine description
    ax.set_ylabel("PNNS group", fontsize=20)
    ax.set_xlabel("Variable", fontsize=20)

    # colorbar, .......................................................................................

    # . create an axes on the right side of ax. The width of cax will be 5%
    #   of ax and the padding between cax and ax will be fixed at 0.05 inch.
    from mpl_toolkits.axes_grid1 import make_axes_locatable
    divider = make_axes_locatable(ax)
    cax = divider.append_axes("top", size="10%", pad=0.5)
    clb = plt.colorbar(im, cax=cax, orientation="horizontal", label="Cor(X,Y), r value", extend="both")

    # control the stuff,
    fig.tight_layout()
    fig.subplots_adjust(top=0.85)# to adjust for main title, amd legend
    plt.show()


"""Slack actions for Orders."""
from briefy.choreographer.actions.slack import ISlack
from briefy.choreographer.actions.slack import Slack
from briefy.choreographer.events.leica.order import order as events
from zope.component import adapter
from zope.interface import implementer

import typing as t

class OrderSlack(Slack):
    """Base class for Slack message sent on Lead events."""

    entity = 'Order'
    weight = 100
    _channel = '#leica-orders'
    title = 'Order!'
    text = 'An Order'

    def transform(self) -> t.List[dict]:
        """Transform data."""
        payload = super().transform()
        payload_item = payload[0]
        data = self.data
        payload_item['title'] = self.title
        payload_item['text'] = self.text
        payload_item['username'] = 'Briefy Bot'
        payload_item['text'] = f'Order can be seen <{self._action_url}|here>'
        payload_item['data'] = {
            'fields': [
                {'title': 'Customer',
                 'value': data.get('customer', {}).get('title'),
                 'short': True,
                 },
                {'title': 'Project',
                 'value': data.get('project', {}).get('title'),
                 'short': True,
                 },
                {'title': 'Title',
                 'value': data.get('title'),
                 'short': True,
                 },
                {'title': 'ID',
                 'value': data.get('slug'),
                 'short': True,
                 },
            ]
        }
        return payload

@adapter(events.IOrderCreated)
@implementer(ISlack)
class OrderSubmit(OrderSlack):
    """After creating a new Order, post on Slack."""

    title = 'New Order was created, please take a look into the details:'

@adapter(events.IOrderWfCancel)
@implementer(ISlack)
class OrderWfCancel(OrderSlack):
    """After cancelling an Order, post on Slack."""

    title = 'An Order was cancelled'

@adapter(events.IOrderWfRefuse)
@implementer(ISlack)
class OrderWfRefuse(OrderSlack):
    """After refusing an Order, post on Slack."""

    title = 'A set was refused by a customer.'

@adapter(events.IOrderWfDeliver)
@implementer(ISlack)
class OrderWfDeliver(OrderSlack):
    """After delivering an Order, post on Slack."""

    title = 'A set was delivered by a customer.'

@adapter(events.IOrderWfAssign)
@implementer(ISlack)
class OrderWfAssign(OrderSlack):
    """After assigning an Order, post on Slack."""

    title = 'An order was assigned.'

@adapter(events.IOrderWfSchedule)
@implementer(ISlack)
class OrderWfSchedule(OrderSlack):
    """After scheduling an Order, post on Slack."""

    title = 'An order was scheduled.'

@adapter(events.IOrderWfRemoveAvailability)
@implementer(ISlack)
class OrderWfRemoveAvailability(OrderSlack):
    """Slack after removing availability of an Order."""

    title = 'Order has no availabilities'

@adapter(events.IOrderWfUnassign)
@implementer(ISlack)
class OrderWfUnassign(OrderSlack):
    """Slack after unassigned of an Order."""

    title = 'Order was unassigned'

@adapter(events.IOrderWfSetAvailability)
@implementer(ISlack)
class OrderWfSetAvailability(OrderSlack):
    """Slack after setting availability of an Order."""

    title = 'Availability was set for the Order'

@adapter(events.IOrderWfEditLocation)
@implementer(ISlack)
class OrderWfEditLocation(OrderSlack):
    """Slack after editing location of an Order."""

    title = 'Location was updated for the Order'

@adapter(events.IOrderWfEditRequirements)
@implementer(ISlack)
class OrderWfEditRequirements(OrderSlack):
    """Slack after editing requiments of an Order."""

    title = 'Requirements was updated for the Order'


import netCDF4 as nc
import numpy as np
import matplotlib.pyplot as plt
import csv
import sys
import os, re
from datetime import datetime
import matplotlib

# for TIR bands there is only one possible value for Universal Conversion Coefficients per channel
tir_ucc_K1_K2 = np.matrix(([[0.006822, 3040.136402, 1735.337945], [0.006780, 2482.375199, 1666.398761], [0.006590, 1935.060183, 1585.420044], [0.005693, 866.468575, 1350.069147], [0.005225, 641.326517, 1271.221673]]))

def get_tir_coef(bandname):
	if bandname == 'BT10':
		bn = 0
	if bandname == 'BT11':
		bn = 1
	if bandname == 'BT12':
		bn = 2
	if bandname == 'BT13':
		bn = 3
	if bandname == 'BT14':
		bn = 4
	#Set ucc value for specific band
	tir_ucc1 = tir_ucc_K1_K2[bn, 0]
	K1 = tir_ucc_K1_K2[bn, 1]
	K2 = tir_ucc_K1_K2[bn, 2]
	return tir_ucc1, K1, K2

def bt2rad(tir_bt, K1, K2):
	rad = np.ma.masked_where(tir_bt == 0.0, tir_bt)
	rad = K2 / tir_bt
	rad = np.exp(rad)
	rad -= 1.0
	rad = K1 / rad
	return rad

def stack_aster(filename, path):
	element_path = os.path.join(path, filename)
	nc_file = nc.Dataset(element_path)
	BT10 = np.array(nc_file.variables['BT_band10'])
	BT10[BT10 < 273] = np.nan
	tir_ucc, tir_K1, tir_K2 = get_tir_coef('BT10')
	rad10 = bt2rad(BT10, tir_K1, tir_K2)
	rad10[rad10 < 6.2] = np.nan
	#BT11 = np.array(nc_file.variables['BT_band11'])
	#BT12 = np.array(nc_file.variables['BT_band12'])
	#BT13 = np.array(nc_file.variables['BT_band13'])
	#BT14 = np.array(nc_file.variables['BT_band14'])
	my_cmap = matplotlib.cm.hot_r #coolwarm #BuPu #seismic #magma #coolwarm
	my_cmap.set_bad(color='olive', alpha=1.0) #olive #black #khaki #silver
	plt.imshow(BT10)
	plt.show()
	plt.imshow(rad10, cmap=my_cmap)
	plt.colorbar()
	plt.clim()
	plt.show()
	return rad10 #stack_radiance

###################################
###################################

filename = sys.argv[1] #Landsat_ncfiles.heysham
path = sys.argv[2] #Landsat_ncfiles.heysham_path
stack_radiance = stack_aster(filename, path)

"""
for i, (name, layer) in enumerate(zip(tir_names[2:], tir_stack[2:])):
	#print name
	#print layer
	tir_ucc, tir_K1, tir_K2 = get_tir_coef(name)
	#print 'tir ucc is' , tir_ucc
	# Convert from DN to Radiance
	#vnir_rad[i] = dn2rad(original_data[i])
	tir_rad[i] = dn2rad(layer, tir_ucc)
	tir_rad[i][tir_rad[i] == dn2rad(0, tir_ucc)] = 0
	# Convert from Radiance to BT
	tir_BT[i+2] = rad2bt(tir_rad[i], tir_K1, tir_K2)
"""

for i, layer in enumerate(stack_radiance):
	#meanBT = np.nanmean(layer)
	#print meanBT
	#layer_amplitude = layer - meanBT
	#layer_amplitude[np.isnan(layer_amplitude)] = np.nan #-999 #this is a new added line
	#print layer_amplitude
	#layer_amplitude[landmask] = np.nan #999

	#my_cmap = matplotlib.cm.hot_r #coolwarm #BuPu #seismic #magma #coolwarm
	#my_cmap.set_bad(color='tan', alpha=1.0) #olive #black #khaki #silver
	#my_cmap.set_over(color='khaki') #black #olive
	#my_cmap.set_under(color='khaki') #khaki #grey
	plt.imshow(layer)
	plt.colorbar()
	plt.clim()
	plt.show()


"""
Unit tests and logging for churn libraray

Author: Yue Chang

Date: 14 Nov 2022
"""
import os
import logging

import pandas as pd

import churn_library as churn_lib

logging.basicConfig(
    filename='./logs/churn_library.log',
    level=logging.INFO,
    filemode='w',
    format='%(name)s - %(levelname)s - %(message)s')

def test_import():
    '''
    test data import - this example is completed for you to assist with the other test functions
    '''
    try:
        df = churn_lib.import_data("./data/bank_data.csv")
        logging.info("Testing import_data: SUCCESS")
    except FileNotFoundError as err:
        logging.error("Testing import_eda: The file wasn't found")
        raise err

    try:
        assert df.shape[0] > 0
        assert df.shape[1] > 0
    except AssertionError as err:
        logging.error(
            "Testing import_data: The file doesn't appear to have rows and columns")
        raise err

def test_eda():
    '''
    test perform eda function
    '''
    try:
        df = churn_lib.import_data('./data/bank_data.csv')
        churn_lib.perform_eda(df)
        assert os.path.exists('images/eda')
        assert len(os.listdir('images/eda')) > 0
        logging.info("Testing perform_eda: SUCCESS")
    except AssertionError as err:
        logging.error("Testing perform_eda: Images not created correctly.")
        raise err

def test_encoder_helper():
    '''
    test encoder helper
    '''
    try:
        df = churn_lib.import_data('./data/bank_data.csv')
        category_list = [
            'Gender',
            'Education_Level',
            'Marital_Status',
            'Income_Category',
            'Card_Category'
        ]
        df['Churn'] = df['Attrition_Flag'].apply(
            lambda val: 0 if val == "Existing Customer" else 1)
        new_df = churn_lib.encoder_helper(df, category_list, 'Churn')
        assert isinstance(new_df, pd.DataFrame)
        assert len(new_df.columns) > len(df.columns)
        logging.info("Testing encoder_helper: SUCCESS")
    except AssertionError as err:
        logging.error(
            "Testing encoder_helper: Failed to return dataframe with new encoded columns")
        raise err

def test_perform_feature_engineering():
    '''
    test perform_feature_engineering
    '''
    try:
        df = churn_lib.import_data('./data/bank_data.csv')
        df['Churn'] = df['Attrition_Flag'].apply(
            lambda val: 0 if val == "Existing Customer" else 1)
        res = churn_lib.perform_feature_engineering(df, 'Churn')
        assert res is not None
        logging.info("Testing perform_feature_engineering: SUCCESS")
    except AssertionError as err:
        logging.error(
            "Testing perform_feature_engineering: Feature Enginnering process failed. ")
        raise err

def test_train_models():
    '''
    test train_models
    '''
    try:
        df = churn_lib.import_data('./data/bank_data.csv')
        df['Churn'] = df['Attrition_Flag'].apply(
            lambda val: 0 if val == "Existing Customer" else 1)
        res = churn_lib.perform_feature_engineering(df, 'Churn')
        churn_lib.train_models(*res)
        assert os.path.exists('images/results')
        assert len(os.listdir('images/results')) > 0
        assert os.path.exists('models')
        assert len(os.listdir('models')) > 0
        logging.info("Testing train_models: SUCCESS")
    except AssertionError as err:
        logging.error(
            "Testing train_models: models or result images not created correctly.")
        raise err


#!/usr/bin/python
# -*- coding: utf-8 -*-
from multiprocessing import cpu_count

import cvxpy as cp
import time
import numpy as np
import random
import networkx as nx

def solve_MIP(edges, vertex_num):
    global color_num
    build_start = time.time()
    dic = {50: 8, 70: 20, 100: 16, 250: 99, 500: 18, 1000: 124}  # change
    if vertex_num in dic.keys():
        color_num = dic[vertex_num]
    else:
        color_num = vertex_num
    X = cp.Variable((vertex_num, color_num), boolean=True)
    W = cp.Variable(color_num, boolean=True)
    constraints = []
    for item in edges:
        for col in range(color_num):
            constraints += [X[item[0], col] + X[item[1], col] <= W[col]]

    for node in range(vertex_num):
        exp = 0
        for col in range(color_num):
            exp += X[node, col]
        constraints += [exp == 1]

    obj = 0
    for i in range(color_num):
        obj += W[i]
    obj = cp.Minimize(obj)
    problem = cp.Problem(obj, constraints)
    build_finish = time.time()
    print(cp.installed_solvers())
    print("build time is ", build_finish - build_start)
    problem.solve(solver=cp.GUROBI)
    print("solve time is ", time.time() - build_finish)

    return obj.value, W.value, X.value

def color_node(edge_matrix, colored_dic, node_index, color_index):
    for index, val in enumerate(edge_matrix[node_index]):
        if val == 1 and colored_dic.get(index, -1) == color_index:
            return False
    for row_index, row in enumerate(edge_matrix):
        val = row[node_index]
        if val == 1 and colored_dic.get(row_index, -1) == color_index:
            return False
    return True

def solve_greedy(edges, vertex_num):
    edge_matrix = [[0 for i in range(vertex_num)] for j in range(vertex_num)]
    dic = {}
    for item in edges:
        edge_matrix[item[0]][item[1]] = 1
        edge_matrix[item[1]][item[0]] = 1
        # dic[item[0]] = dic.get(item[0], 0) + 1
        # dic[item[1]] = dic.get(item[1], 0) + 1
    # dic_ = sorted(dic.items(), key=lambda x:x[1], reverse=True)
    #
    # sorted_dic = [k[0] for k in dic_]

    color_index = 0
    uncolored_set = [i for i in range(vertex_num)]

    colored_dic = {}
    while uncolored_set:
        for i in reversed(uncolored_set):
            if color_node(edge_matrix, colored_dic, i, color_index):
                colored_dic[i] = color_index
                uncolored_set.remove(i)
            else:
                for j in range(color_index):
                    if color_node(edge_matrix, colored_dic, i, j):
                        colored_dic[i] = j
                        uncolored_set.remove(i)
        color_index += 1
    a = list(colored_dic.items())
    ret = [k[1] for k in sorted(a)]
    # print(colored_dic)
    return ret, colored_dic, edge_matrix, color_index

def tabucol(new_solution, edges, vertex_num, k):
    violations, vio_edge = evaluate(new_solution, edges)
    iter = 0
    tabu_list = []
    while violations != 0 and iter <= 5:
        new_solution, violations, vio_edge = swap_color(new_solution, violations, vio_edge, tabu_list, k, edges)
        iter += 1
        # tabu_list.extend(violations[1])
    if violations == 0:
        # print(new_solution)
        return True
    return False

def swap_color(new_solution, violations, vio_edge, tabu_list, k, edges):
    cur = new_solution
    min_vio_edge = []
    ret = []
    min_vio = 999
    for i in vio_edge:
        for j in range(k):
            cur[i[0]] = j
            violations, vio_edge = evaluate(cur, edges)
            if violations < min_vio:
                min_vio = violations
                ret = cur
                min_vio_edge = vio_edge
    return ret, min_vio, min_vio_edge

def evaluate(solution, edges):
    violation = 0
    vio_edge = []
    for edge in edges:
        if solution[edge[0]] == solution[edge[1]]:
            violation += 1
            vio_edge.append(edge)
    return violation, vio_edge

def random_init(vertex_num, k):
    colored_dic = {}
    random.seed(1)
    for i in range(vertex_num):
        random_col = random.randint(0, k-1)
        colored_dic[i] = random_col
    return colored_dic

def swap(edges, colored_dic, color_index_1, color_index_2):
    ret_dic = {}
    class_1 = []
    class_2 = []
    edges = np.array(edges)

    for item in colored_dic.items():
        if item[1] == color_index_1:
            class_1.append(item[0])
        if item[1] == color_index_2:
            class_2.append(item[0])
    neighbor_matrix = edges[np.ix_(class_1 + class_2, class_1 + class_2)]
    random_row = np.random.randint(0, len(class_1))
    neighbor_matrix = np.delete(neighbor_matrix, random_row, 1)
    neighbor_matrix = np.delete(neighbor_matrix, random_row, 0)
    return ret_dic

def greedy(node_count, edges):
    graph = nx.Graph()
    graph.add_nodes_from(range(node_count))
    graph.add_edges_from(edges)

    strategies = [nx.coloring.strategy_largest_first,
                  ]

    best_color_count, best_coloring = node_count, {i: i for i in range(node_count)}
    for strategy in strategies:
        print(graph.degree)
        curr_coloring = nx.coloring.greedy_color(G=graph, strategy=strategy)
        curr_color_count = max(curr_coloring.values()) + 1
        if curr_color_count < best_color_count:
            best_color_count = curr_color_count
            best_coloring = curr_coloring
    return best_color_count, 0, [best_coloring[i] for i in range(node_count)]

def solve_it(input_data):
    # Modify this code to run your optimization algorithm

    # parse the input
    lines = input_data.split('\n')

    first_line = lines[0].split()
    node_count = int(first_line[0])
    edge_count = int(first_line[1])

    edges = []
    for i in range(1, edge_count + 1):
        line = lines[i]
        parts = line.split()
        edges.append([int(parts[0]), int(parts[1])])

    # build a trivial solution
    # every node has its own color
    res = greedy(node_count, edges)
    # res = solve_gurobi(edges, node_count)
    # ret = solve_greedy(edges, node_count)
    # ret = []
    # eng = matlab.engine.start_matlab()
    # eng.cd(r"matlab")
    # eng.solve(edges, node_count)

    solution = []
    col_index = []
    ret = []
    # for row, row_val in enumerate(res[2]):
    #     for col, col_val in enumerate(res[2][row]):
    #         if res[2][row][col] == 1:
    #             if col not in col_index:
    #                 col_index.append(col)
    #             ret.append(col_index.index(col))

    # prepare the solution in the specified output format
    output_data = str(node_count) + ' ' + str(0) + '\n'
    output_data += ' '.join(map(str, res))

    return output_data

import sys

if __name__ == '__main__':
    import sys

    if len(sys.argv) > 1:
        file_location = sys.argv[1].strip()
        with open(file_location, 'r') as input_data_file:
            input_data = input_data_file.read()
        print(solve_it(input_data))
    else:
        print(
            'This test requires an input file.  Please select one from the data directory. (i.e. python solver.py ./data/gc_4_1)')


from typing import List

import cv2
import numpy as np
import torch

from utils.data_descriptors import COORD

trans_xyz = lambda x, y, z: np.array([[1, 0, 0, x],
                                      [0, 1, 0, y],
                                      [0, 0, 1, z],
                                      [0, 0, 0, 1]], dtype=np.float32)

yaw_rotation = lambda th: np.array([[np.cos(th), 0, np.sin(th), 0],
                                    [0, 1, 0, 0],
                                    [-np.sin(th), 0, np.cos(th), 0],
                                    [0, 0, 0, 1]], dtype=np.float32)

pitch_rotation = lambda th: np.array([[1, 0, 0, 0],
                                      [0, np.cos(th), -np.sin(th), 0],
                                      [0, np.sin(th), np.cos(th), 0],
                                      [0, 0, 0, 1]], dtype=np.float32)

roll_rotation = lambda th: np.array([[np.cos(th), -np.sin(th), 0, 0],
                                     [np.sin(th), np.cos(th), 0, 0],
                                     [0, 0, 1, 0],
                                     [0, 0, 0, 1]], dtype=np.float32)

def _get_camera_to_world_matrix(coordinates: COORD) -> np.ndarray:
    """
    Converting camera poses into transformation matrix.
    """

    # Rotation matrices
    R_roll = roll_rotation(coordinates.roll / 180.0 * np.pi)
    R_pitch = pitch_rotation(coordinates.pitch / 180.0 * np.pi)
    R_yaw = yaw_rotation(coordinates.yaw / 180.0 * np.pi)

    # Combined rotation matrix
    R = R_roll @ R_pitch @ R_yaw

    # Translation matrix
    T = trans_xyz(coordinates.x, coordinates.y, coordinates.z)

    # Final camera-to-world transformation matrix
    c2w = R @ T

    return c2w

def get_camera_poses_from_list_of_coordinates(init_coordinates: COORD, coordinates: List[COORD]) -> torch.Tensor:
    """
    Getting camera-to-world poses from list of coordinates.
    """

    Ts_c2w = []
    for coord in coordinates:
        extrinsic_matrix = _get_camera_to_world_matrix(init_coordinates).reshape((4, 4))

        # Creating rotation matrices using OpenCV's Rodrigues function
        horizontal_rotation_matrix, _ = cv2.Rodrigues(np.array([0, 0, coord.yaw / 180.0 * np.pi]))
        vertical_rotation_matrix, _ = cv2.Rodrigues(np.array([coord.pitch / 180.0 * np.pi, 0, 0]))

        # Applying the rotations to the existing rotation matrix in the extrinsic matrix
        new_rotation_matrix = horizontal_rotation_matrix @ vertical_rotation_matrix @ extrinsic_matrix[:3, :3]

        # Updating the rotation component in the extrinsic matrix
        extrinsic_matrix[:3, :3] = new_rotation_matrix

        Ts_c2w.append(extrinsic_matrix)

    Ts_c2w = np.asarray(Ts_c2w, dtype=np.float32).reshape((-1, 4, 4))

    return torch.tensor(Ts_c2w)


import os
from typing import Union, List

import h5py
import numpy as np
import torch

from .light_field_dataset import LightFieldDataset

class LightField6x6Dataset(LightFieldDataset):
    """
    Dataset wrapping images and targets for the low res inputs
    """

    def __init__(self,
                 dataset_loc: os.path,
                 testing_set_idx_stop: int,
                 testing_set_idx_start: int,
                 is_training: bool = True,
                 is_timing: bool = False,
                 is_x_cached: bool = False,
                 is_y_cached: bool = False,
                 data_key: str = 'images'):
        super(LightField6x6Dataset, self).__init__()
        self.is_x_cached = is_x_cached
        self.is_y_cached = is_y_cached
        self.dataset_loc = dataset_loc
        self.is_training = is_training
        self.is_timing = is_timing

        self.h5_file = h5py.File(self.dataset_loc, 'r')

        self.x_images = self.h5_file[data_key + '_x']

        self.y_images = self.h5_file[data_key + '_y']

        print("Found x dataset with shape {}".format(self.x_images.shape))
        print("Found y dataset with shape {}".format(self.y_images.shape))

        self.num_samples = self.x_images.shape[-5]
        self.num_channels = self.x_images.shape[-3]

        self.num_views_x = self.x_images.shape[-4]
        self.height_x = self.x_images.shape[-2]
        self.width_x = self.x_images.shape[-1]

        self.num_views_y = self.y_images.shape[-4]
        self.height_y = self.y_images.shape[-2]
        self.width_y = self.y_images.shape[-1]

        self.all_indices = np.arange(self.num_samples, dtype=np.int32)

        self.testing_indices = np.arange(testing_set_idx_start,
                                         testing_set_idx_stop,
                                         dtype=np.int32)

        self.training_indices = \
            np.array(list(set(self.all_indices) - set(self.testing_indices)))

        if self.is_training:
            self.dataset_type = "Train"
            self.image_indices = self.training_indices
        else:
            self.dataset_type = "Test"
            self.image_indices = self.testing_indices

        print("{} idxs {} ... {}".format(self.dataset_type,
                                         self.image_indices[:10],
                                         self.image_indices[-10:]))

        if self.is_x_cached:
            print("Caching x dataset please wait...")
            self.x_images = self.x_images[self.image_indices.tolist()]

        if self.is_y_cached and not self.is_timing:
            print("Caching y dataset please wait...")
            self.y_images = self.y_images[self.image_indices.tolist()]

        self.item_fn = self.calc_item_fn()

    def calc_item_fn(self):
        if self.is_x_cached and self.is_y_cached:
            item_fn = self._get_sample_x_y_cached
        elif self.is_x_cached:
            item_fn = self._get_sample_x_cached
        elif self.is_y_cached:
            item_fn = self._get_sample_y_cached
        else:
            item_fn = self._get_sample_no_cached

        return item_fn

    def set_only_x_dataset(self):
        if not self.is_x_cached:
            print("Caching x dataset please wait...")
            self.x_images = self.x_images[self.image_indices.tolist()]
            self.is_x_cached = True

        self.item_fn = self._get_only_x

    def revert_only_x_dataset(self):
        self.item_fn = self.calc_item_fn()

    def _get_only_x(self, idx):
        return self.x_images[idx]

    def get_only_y(self, idx):
        if self.is_y_cached:
            image_idx = idx
        else:
            image_idx = self.image_indices[idx]

        return self.y_images[image_idx]

    def __len__(self):
        return self.image_indices.shape[0]

    def _get_sample_x_cached(self, idx):
        image_idx_y = self.image_indices[idx]

        return self.x_images[idx], self.y_images[image_idx_y]

    def _get_sample_y_cached(self, idx):
        image_idx_x = self.image_indices[idx]

        return self.x_images[image_idx_x], self.y_images[idx]

    def _get_sample_x_y_cached(self, idx):
        return self.x_images[idx], self.y_images[idx]

    def _get_sample_no_cached(self, idx):
        image_idx = self.image_indices[idx]

        return self.x_images[image_idx], self.y_images[image_idx]

    def _get_sample_timing_cached(self, idx):
        return self.x_images[idx]

    def __getitem__(self, idx):
        return self.item_fn(idx=idx)

    def get_x_images(self, to_list=True):
        if self.is_x_cached:
            if to_list:
                return self.x_images.tolist()
            else:
                return self.x_images
        else:
            print("Caching x dataset please wait...")
            if to_list:
                return self.x_images[self.image_indices.tolist()].tolist()
            else:
                return self.x_images[self.image_indices.tolist()]

    def get_y_images(self, to_list: bool = True) -> Union[np.ndarray, List]:
        if self.is_x_cached:
            if to_list:
                return self.x_images.tolist()
            else:
                return self.x_images
        else:
            print("Caching y dataset please wait...")
            if to_list:
                return self.x_images[self.image_indices.tolist()].tolist()
            else:
                return self.x_images[self.image_indices.tolist()]

    def fast_collate(self, batch):
        input_lfs = [lf[0] for lf in batch]
        output_lfs = [lf[1] for lf in batch]

        input_batch_tensor = torch.empty(
            (len(input_lfs),
             self.num_views_x,
             self.num_channels,
             self.height_x,
             self.width_x),
            dtype=torch.uint8,
            requires_grad=False
        )

        for idx, input_lf in enumerate(input_lfs):
            input_batch_tensor[idx] = torch.from_numpy(input_lf)

        output_batch_tensor = torch.empty(
            (len(output_lfs),
             self.num_views_y,
             self.num_channels,
             self.height_y,
             self.width_y),
            dtype=torch.uint8,
            requires_grad=False
        )

        for idx, output_lf in enumerate(output_lfs):
            output_batch_tensor[idx] = torch.from_numpy(output_lf)

        return input_batch_tensor, output_batch_tensor


from typing import List, Optional

import random
import csv
import boto3
import uuid
import pytz
from datetime import datetime, timezone, timedelta
from boto3.dynamodb.conditions import Key, Attr
import constants

dynamodb = boto3.resource('dynamodb')
survey_table = dynamodb.Table('timbre_survey')

def put_to_na(row):
    if row['clip_a'] not in na[row['attribute']]:
        na[row['attribute']][row['clip_a']] = 1
    else:
        na[row['attribute']][row['clip_a']] += 1

    if row['clip_b'] not in na[row['attribute']]:
        na[row['attribute']][row['clip_b']] = 1
    else:
        na[row['attribute']][row['clip_b']] += 1

def put_to_dontknow(row):
    if row['clip_a'] not in dontknow[row['attribute']]:
        dontknow[row['attribute']][row['clip_a']] = 1
    else:
        dontknow[row['attribute']][row['clip_a']] += 1

    if row['clip_b'] not in dontknow[row['attribute']]:
        dontknow[row['attribute']][row['clip_b']] = 1
    else:
        dontknow[row['attribute']][row['clip_b']] += 1

na = {}
for attr in constants.attributes:
    na[attr] = {}

dontknow = {}
for attr in constants.attributes:
    dontknow[attr] = {}

more = {}
for attr in constants.attributes:
    more[attr] = {}

less = {}
for attr in constants.attributes:
    less[attr] = {}

response = survey_table.scan(
    # FilterExpression=Attr("email").eq("sihyun06@gmail.com")
    )

print("Downloaded Survey data...\n")

for row in response['Items']:
    if row['others'] is None or row['others'] == []:
        continue

    qchanged = datetime(2021, 7, 19, 17, 19, 0, tzinfo=timezone.utc) # In JST

    ansclip = 'clip_a'
    if datetime.strptime(row['reg_ts'], '%a %b %d %H:%M:%S %Y').replace(tzinfo=pytz.timezone('Japan')) > qchanged and row['answer'] == 'clip_b': # Tue Jul 20 10:39:37 2021
        ansclip = row['answer']

    for attr in row['others']:
        if row[ansclip] not in more[attr]:
            more[attr][row[ansclip]] = 1
        else:
            more[attr][row[ansclip]] += 1

print("Loaded the other tags data...\n")

for row in response['Items']:
    if row['answer'] == 'na':
        # print(row)
        put_to_na(row)
        continue
    if row['answer'] == 'dontknow':
        # print(row)
        put_to_dontknow(row)
        continue
    if row['answer'] == 'clip_a':

        if row['clip_a'] not in more[row['attribute']]:
            more[row['attribute']][row['clip_a']] = 1
        else:
            more[row['attribute']][row['clip_a']] += 1

        if row['clip_b'] not in less[row['attribute']]:
            less[row['attribute']][row['clip_b']] = 1
        else:
            less[row['attribute']][row['clip_b']] += 1
        continue

    if row['answer'] == 'clip_b':

        if row['clip_b'] not in more[row['attribute']]:
            more[row['attribute']][row['clip_b']] = 1
        else:
            more[row['attribute']][row['clip_b']] += 1

        if row['clip_a'] not in less[row['attribute']]:
            less[row['attribute']][row['clip_a']] = 1
        else:
            less[row['attribute']][row['clip_a']] += 1
        continue

print("Created more, less and na, arrays...\n")

## Creating features Table

features = {}

for i in range(1, constants.CLIPS_NUM+1):
    for j in range(1, 13):
        clip_id = ('%05d' % i) + "-" + ('%02d' % j)
        features[clip_id] = {}

        for attr in constants.attributes:
            features[clip_id][attr] = 10

            # About More
            if clip_id in more[attr]:
                features[clip_id][attr] += more[attr][clip_id]

            # Less
            if clip_id in less[attr]:
                features[clip_id][attr] -= less[attr][clip_id]

            # NA
            if clip_id in na[attr]:
                features[clip_id][attr] -= (na[attr][clip_id] * 2)

print("Created features array...\n")

with open('mycsvfile.csv', 'w') as f:  # You will need 'wb' mode in Python 2.x
    w = csv.DictWriter(f, features.keys())
    w.writeheader()
    w.writerow(features)

csv = []
for k in features:
    csv.append(list(features[k].values()))

with open('features.csv', 'w') as f:  # You will need 'wb' mode in Python 2.x
    for row in csv:
        string_ints = [str(i) for i in row]
        f.write(",".join(string_ints) + "\n")

print("===========================\n\n\n")
print(na)
print("===========================\n\n\n")
print(dontknow)
print("===========================\n\n\n")
print(less)
print("===========================\n\n\n")
print(more)
print("===========================\n\n\n")

print("Now inserting...")

na_table = dynamodb.Table('notapplicable')
dontknow_table = dynamodb.Table('dontknow')
more_table = dynamodb.Table('more')
less_table = dynamodb.Table('less')

with na_table.batch_writer() as batch:
    for attr in constants.attributes:
        batch.put_item(
            Item={
                'attribute': attr,
                'weights': na[attr]
            }
        )

with dontknow_table.batch_writer() as batch:
    for attr in constants.attributes:
        batch.put_item(
            Item={
                'attribute': attr,
                'weights': dontknow[attr]
            }
        )
with less_table.batch_writer() as batch:
    for attr in constants.attributes:
        batch.put_item(
            Item={
                'attribute': attr,
                'weights': less[attr]
            }
        )
with more_table.batch_writer() as batch:
    for attr in constants.attributes:
        batch.put_item(
            Item={
                'attribute': attr,
                'weights': more[attr]
            }
        )


from typing import Any, Dict, Optional

import parse
import pydantic
from pydantic.fields import Field

from datahub.configuration.common import AllowDenyPattern
from datahub.configuration.source_common import EnvBasedSourceConfigBase
from datahub.ingestion.source.aws.aws_common import AwsSourceConfig
from datahub.ingestion.source.aws.s3_util import is_s3_uri
from datahub.ingestion.source.data_lake.profiling import DataLakeProfilerConfig

class DataLakeSourceConfig(EnvBasedSourceConfigBase):

    base_path: str = Field(
        description="Path of the base folder to crawl. Unless `schema_patterns` and `profile_patterns` are set, the connector will ingest all files in this folder."
    )
    platform: str = Field(
        default="",
        description="Autodetected.  Platform to use in namespace when constructing URNs. If left blank, local paths will correspond to `file` and S3 paths will correspond to `s3`.",
    )  # overwritten by validator below

    use_relative_path: bool = Field(
        default=False,
        description="Whether to use the relative path when constructing URNs. Has no effect when a `path_spec` is provided.",
    )
    ignore_dotfiles: bool = Field(
        default=True,
        description="Whether to ignore files that start with `.`. For instance, `.DS_Store`, `.bash_profile`, etc.",
    )

    aws_config: Optional[AwsSourceConfig] = Field(
        default=None, description="AWS details"
    )

    schema_patterns: AllowDenyPattern = Field(
        default=AllowDenyPattern.allow_all(),
        description="regex patterns for tables to filter for ingestion.",
    )
    profile_patterns: AllowDenyPattern = Field(
        default=AllowDenyPattern.allow_all(),
        description="regex patterns for tables to profile ",
    )

    path_spec: Optional[str] = Field(
        default=None,
        description="Format string for constructing table identifiers from the relative path. See the above setup section for details.",
    )

    profiling: DataLakeProfilerConfig = Field(
        default=DataLakeProfilerConfig(), description="Profiling configurations"
    )

    spark_driver_memory: str = Field(
        default="4g", description="Max amount of memory to grant Spark."
    )

    max_rows: int = Field(
        default=100,
        description="Maximum number of rows to use when inferring schemas for TSV and CSV files.",
    )

    @pydantic.root_validator()
    def ensure_profiling_pattern_is_passed_to_profiling(
        cls, values: Dict[str, Any]
    ) -> Dict[str, Any]:
        profiling = values.get("profiling")
        if profiling is not None and profiling.enabled:
            profiling.allow_deny_patterns = values["profile_patterns"]
        return values

    @pydantic.validator("platform", always=True)
    def validate_platform(cls, value: str, values: Dict[str, Any]) -> Optional[str]:
        if value != "":
            return value

        if is_s3_uri(values["base_path"]):
            return "s3"
        return "file"

    @pydantic.validator("path_spec", always=True)
    def validate_path_spec(
        cls, value: Optional[str], values: Dict[str, Any]
    ) -> Optional[str]:
        if value is None:
            return None

        if not value.startswith("./"):
            # enforce this for semantics
            raise ValueError("Path_spec must start with './'")

        name_indices = sorted([x[0] for x in parse.findall("{{name[{:d}]}}", value)])

        if len(name_indices) == 0:
            raise ValueError("Path spec must contain at least one name identifier")

        if name_indices != list(range(max(name_indices) + 1)):
            raise ValueError(
                "Path spec must contain consecutive name identifiers, starting at 0"
            )

        return value


#文字コード順でソートするプログラム

import sys,glob,MeCab,string
from os.path import basename

from collections import defaultdict
dic = defaultdict(int)
word_total = 0

for loop in ["{0:02d}".format(num) for num in range(0,100)]:
    n = int(sys.argv[1])
    sentence = ""
    for source in glob.glob("./" + str(loop) + "*.txt"):
        with open(basename(source), encoding = "utf-8") as a_file:
            sentence += "".join([arg1.strip() for arg1 in a_file])
    print(str(loop) + "*.txt")

    t = MeCab.Tagger("-F %f[6]\\n")
    lists = []
    for words in t.parse(sentence).split("\n"):
        if (words == "" or words == "EOS"):
            continue
        lists.append(words)

    i = 0
    lis_len = len(lists)
    for i in range(lis_len - n + 1):
        dic["-".join(lists[i:i+n])] += 1
        word_total += 1

gram_total = len(dic)
print("〜ファイル書き出し中〜")
with open("../output.txt", "w") as f:
    seikika = 0.0
    for i,j in sorted(dic.items()):
        f.write("{:20}\t{}\t{}\n".format(i,j,j/word_total))
        seikika = j /word_total + seikika
    f.write("グラム総要素数=" + str(gram_total) + " 正規化値合計=" + str(seikika) + "\n")
print("〜完了〜")


# -*- coding: utf-8 -*-
# Created on Sat Mar 10 2018 16:4:34
# Author: WuLC
# EMail: liangchaowu5@gmail.com

# hashmap to record position
from collections import defaultdict
class Solution(object):
    def numMatchingSubseq(self, S, words):
        """
        :type S: str
        :type words: List[str]
        :rtype: int
        """
        indices = defaultdict(list)
        for i in xrange(len(S)):
            indices[S[i]].append(i)
        count=0
        for word in words:
            record = {k:0 for k in indices.keys()}
            legal, idx = True, -1
            for c in word:
                if c not in indices or record[c] == len(indices[c]) or idx > indices[c][-1]:
                    legal = False
                    break
                for i in xrange(record[c], len(indices[c])):
                    if idx > indices[c][i]:
                        continue
                    else:
                        idx = indices[c][i]
                        record[c] = i+1
                        break
            if legal:
                count += 1
        return count

Algorithm/Python/792. Number of Matching Subsequences.py