# Corporation: HKCLR
# Project: AutoStore picking robots
# Author: Chang Liu
# Some codes are refered from https://github.com/facebookresearch/pycls

"""MMDet library's model training, testing, etc. functions.
"""

import cv2
import numpy as np
import mmcv
import os
import PIL.Image as pil_image
import torch
import time
from mmcv import Config
from mmcv.runner import load_checkpoint
from mmcv.parallel import MMDataParallel
from mmdet.apis import set_random_seed, init_detector, inference_detector
from mmdet.core import encode_mask_results
from mmdet.datasets import (
    build_dataset,
    build_dataloader,
    replace_ImageToTensor
)
from mmdet.models import build_detector
from mmdet.apis import train_detector, single_gpu_test
from typing import Optional, Tuple, List, NewType, Dict, Any, Callable
from pathlib import Path
from PIL.Image import Image
from autostore.core.logging import get_logger, setup_logging
from autostore.core.io import load_img_paths
from autostore.datasets.pre_process import (
    rsz_imgs_from_csv_info,
    rsz_paste_imgs_from_csv_info
)
from autostore.datasets.post_process import (
    restore_bboxes,
    restore_seg_mask,
    remove_irregular_bboxes,
    remove_irregular_seg_masks
)

EvaluateResultType = NewType(
    "EvaluateResultType",
    List[Tuple[List[np.ndarray], List[List[Dict[str, Any]]]]]
)

def train_model(
        mmdet_cfg: str,
        weight_path: str,
        output_dir: str
    ) -> None:
    """Train a mmdet model with provided config file and pretrained weight,
    and output the trained weight to output_dir.

    Args:
        mmdet_cfg (str):
            MMDet config file path.
        weight_path (str):
            Pretrained weight path.
        output_dir (str):
            The path where trained weight will be save at.

    Returns:
        None
    """
    os.makedirs(output_dir, exist_ok=True)
    setup_logging(os.path.join(output_dir, "train.log"))
    _logger = get_logger(__name__)
    cfg = Config.fromfile(mmdet_cfg)
    cfg.load_from = weight_path
    _logger.info(f"Loaded config file from {mmdet_cfg}.")
    try:
        # This del operation is to avoid exception that exclusively happened
        # on loading mask2former_r50 models
        del cfg.data.val.ins_ann_file
        del cfg.data.test.ins_ann_file
    except:
        pass
    cfg.device = "cuda"
    cfg.work_dir = output_dir
    cfg.seed = 0
    set_random_seed(0, deterministic=False)
    cfg.gpu_ids = [0]

    datasets = [build_dataset(cfg.data.train)]
    model = build_detector(cfg.model)
    model.CLASSES = datasets[0].CLASSES
    mmcv.mkdir_or_exist(os.path.abspath(cfg.work_dir))
    train_detector(model, datasets, cfg, distributed=False, validate=True)

    try:
        checkpoint_name = 'epoch_'+str(cfg.runner.max_epochs)
    except:
        checkpoint_name = 'iter_'+str(cfg.runner.max_iters)
    checkpoint_path = os.path.join(cfg.work_dir, f"{checkpoint_name}.pth")
    checkpoint = torch.load(checkpoint_path)
    os.remove(checkpoint_path)
    meta = checkpoint['meta']
    checkpoint['optimizer'] = None
    weights = checkpoint['state_dict']
    state_dict = {"state_dict": weights, "meta": meta}
    new_checkpoint_path = os.path.join(cfg.work_dir, f"{checkpoint_name}_small.pth")
    torch.save(state_dict, new_checkpoint_path)

def test_imgs(
        mmdet_cfg: str,
        weight_path: str,
        show_score_thr: float,
        output_dir: Optional[str] = None,
        font_size:int = 10,
    ) -> None:
    os.makedirs(output_dir, exist_ok=True)
    setup_logging(os.path.join(output_dir, "test.log"))
    _logger = get_logger(__name__)
    cfg = Config.fromfile(mmdet_cfg)
    try:
        # This del operation is to avoid exception that exclusively happened
        # on loading mask2former_r50 models
        del cfg.data.val.ins_ann_file
        del cfg.data.test.ins_ann_file
    except:
        pass

    if 'pretrained' in cfg.model:
        cfg.model.pretrained = None
    elif 'init_cfg' in cfg.model.backbone:
        cfg.model.backbone.init_cfg = None

    if cfg.model.get('neck'):
        if isinstance(cfg.model.neck, list):
            for neck_cfg in cfg.model.neck:
                if neck_cfg.get('rfp_backbone'):
                    if neck_cfg.rfp_backbone.get('pretrained'):
                        neck_cfg.rfp_backbone.pretrained = None
        elif cfg.model.neck.get('rfp_backbone'):
            if cfg.model.neck.rfp_backbone.get('pretrained'):
                cfg.model.neck.rfp_backbone.pretrained = None

    # Set seed thus the results are more reproducible
    cfg.seed = 0
    set_random_seed(0, deterministic=False)
    cfg.gpu_ids = [0]

    # in case the test dataset is concatenated
    if isinstance(cfg.data.test, dict):
        cfg.data.test.test_mode = True
        samples_per_gpu = cfg.data.test.pop('samples_per_gpu', 1)
        if samples_per_gpu > 1:
            # Replace 'ImageToTensor' to 'DefaultFormatBundle'
            cfg.data.test.pipeline = replace_ImageToTensor(
                cfg.data.test.pipeline)
    elif isinstance(cfg.data.test, list):
        for ds_cfg in cfg.data.test:
            ds_cfg.test_mode = True
        samples_per_gpu = max(
            [ds_cfg.pop('samples_per_gpu', 1) for ds_cfg in cfg.data.test])
        if samples_per_gpu > 1:
            for ds_cfg in cfg.data.test:
                ds_cfg.pipeline = replace_ImageToTensor(ds_cfg.pipeline)

    # Build dataset
    dataset = build_dataset(cfg.data.test)
    data_loader = build_dataloader(
            dataset,
            samples_per_gpu=samples_per_gpu,
            workers_per_gpu=cfg.data.workers_per_gpu,
            dist=False, #distributed
            shuffle=False)

    # Build the detector
    cfg.model.train_cfg = None
    model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg'))
    checkpoint = load_checkpoint(model, weight_path, map_location='cpu')

    if 'CLASSES' in checkpoint.get('meta', {}):
        model.CLASSES = checkpoint['meta']['CLASSES']
    else:
        model.CLASSES = dataset.CLASSES

    model = MMDataParallel(model, device_ids=cfg.gpu_ids)
    outputs = single_gpu_test(
        model,
        data_loader,
        False,
        None,
        show_score_thr,
        font_size
    )
    kwargs = {}
    eval_kwargs = cfg.get('evaluation', {}).copy()
    # hard-code way to remove EvalHook args
    for key in ['interval', 'tmpdir', 'start',
        'gpu_collect', 'save_best', 'rule', 'dynamic_intervals']:
        eval_kwargs.pop(key, None)
    eval_kwargs.update(dict(metric = ["bbox", "segm"], **kwargs))
    metric = dataset.evaluate(outputs, **eval_kwargs)
    _logger.info(metric)
    return model, outputs

def rsz_and_test_imgs(
        mmdet_cfg: str,
        weight_path: str,
        show_score_thr: float,
        mode: str,
        input_size: Optional[Tuple[int, int]] = None,
        output_dir: Optional[str] = None,
        background_img_path: str = None,
    ) -> None:
    cfg = Config.fromfile(mmdet_cfg)
    test_img_dir = cfg.data.test.img_prefix
    test_ann_file = cfg.data.test.ann_file
    if mode == "rsz_inference_imgs":
        model, results = rsz_inference_imgs(
            test_img_dir,
            mmdet_cfg,
            weight_path,
            input_size,
            show_score_thr,
            output_dir
        )
    elif mode == "rsz_paste_inference_imgs":
        model, results = rsz_paste_inference_imgs(
            test_img_dir,
            mmdet_cfg,
            weight_path,
            background_img_path,
            show_score_thr,
            output_dir
        )
    test_dataset = build_dataset(cfg.data.test)
    kwargs = {}
    eval_kwargs = cfg.get('evaluation', {}).copy()
    # hard-code way to remove EvalHook args
    for key in ['interval', 'tmpdir', 'start',
        'gpu_collect', 'save_best', 'rule',
        'dynamic_intervals']:
        eval_kwargs.pop(key, None)
    eval_kwargs.update(dict(metric = ["bbox", "segm"], **kwargs))
    metric = test_dataset.evaluate(results, **eval_kwargs)
    print(metric)
    return model, results

def inference_imgs(
        img_dir: str,
        mmdet_cfg: str,
        weight_path: str,
        show_score_thr: float,
        output_dir: Optional[str] = None,
        show_font_size: Optional[int] = None,
        thickness: Optional[int] = None
    ) -> None:
    """Perform inference on images with a mmdet model.

    Args:
        img_dir (str):
            Image folder paths.
        mmdet_cfg (cfg):
            MMDet config file path.
        weight_path:
            Pretrained weight path.
        show_score_thr (float):
            BBox with score lower than this value will not show
            on inference result visualization.
        output_dir (Optional[str]):
            The path to save visualization results. if None, visualization
            results will not be saved.

    Returns:
        None
    """
    model = init_detector(mmdet_cfg, weight_path, device='cuda:0')
    time_deltas = []
    results = []
    for img_path, img_name, img_ext in load_img_paths(img_dir):
        img = cv2.imread(img_path)
        t1 = time.time()
        result = inference_detector(model, img)
        t2 = time.time()
        time_deltas.append(t2 - t1)
        results.append(result)
        if output_dir:
            os.makedirs(Path(output_dir), exist_ok=True)
            visual_save_path = os.path.join(output_dir, f"{img_name}_inf.{img_ext}")
            show_font_size = show_font_size if show_font_size else 10
            thickness = thickness if thickness else 2
            if show_font_size:
                model.show_result(
                    img_path,
                    result,
                    score_thr=show_score_thr,
                    out_file=visual_save_path,
                    font_size=show_font_size,
                    thickness = thickness
                )
    time_deltas = np.array(time_deltas)
    print(f"avg time consumed for inferencing {len(time_deltas)} images is {time_deltas.mean()}")
    return model, results

def rsz_inference_imgs(
        img_dir: str,
        mmdet_cfg: str,
        weight_path: str,
        input_size: Tuple[int, int],
        show_score_thr: float,
        output_dir: Optional[str] = None,
    ) -> EvaluateResultType:
    model = init_detector(mmdet_cfg, weight_path, device='cuda:0')
    time_deltas = []
    results = []
    i = 0
    for img_path, img, rsz_size in rsz_imgs_from_csv_info(img_dir, input_size):
        img = np.asarray(img).astype(np.uint8)
        t1 = time.time()
        result = inference_detector(model, img)
        t2 = time.time()
        time_deltas.append(t2 - t1)
        bboxes, seg_masks = result[0][0], result[1][0]
        org_bboxes = restore_bboxes(
            bboxes,
            rsz_size,
            input_size
        )
        valid_bbox_idx = remove_irregular_bboxes(
            org_bboxes,
            input_size,
            0.8
        )
        org_seg_masks = restore_seg_mask(
            seg_masks,
            rsz_size,
            input_size
        )
        valid_seg_mask_idx = remove_irregular_seg_masks(
            org_seg_masks,
            input_size,
            0.8
        )
        valid_idx = set(valid_bbox_idx).intersection(set(valid_seg_mask_idx))
        valid_org_bboxes = [org_bboxes[i] for i in valid_idx]
        valid_org_bboxes = np.concatenate(valid_org_bboxes, axis=0)
        valid_org_seg_masks = [org_seg_masks[i] for i in valid_idx]
        result[0][0] = valid_org_bboxes
        result[1][0] = valid_org_seg_masks
        if output_dir:
            try:
                os.makedirs(Path(output_dir), exist_ok=True)
                img_name, img_ext = Path(img_path).name.split(".")
                visual_save_path = os.path.join(output_dir, f"{img_name}_pred.{img_ext}")
                model.show_result(
                    img_path,
                    result,
                    score_thr=show_score_thr,
                    out_file=visual_save_path,
                    font_size=9
                )
            except Exception as e:
                print(e)
                print(img_path)
                print(valid_bbox_idx)
                print(valid_seg_mask_idx)

        result_ = [([valid_org_bboxes], [valid_org_seg_masks])]
        encoded_result = [(bbox_result, encode_mask_results(mask_result))
            for bbox_result, mask_result in result_]
        results.extend(encoded_result)
        i += 1

    time_deltas = np.array(time_deltas)
    _logger.info(f"avg time consumed for inferencing \
                {len(time_deltas)} images is {time_deltas.mean()}")
    return model, results

def rsz_paste_inference_imgs(
        img_dir: str,
        mmdet_cfg: str,
        weight_path: str,
        background_img_path: str,
        show_score_thr: float,
        output_dir: Optional[str] = None,
    ) -> EvaluateResultType:
    back_img = pil_image.open(background_img_path).convert("RGB")
    input_size = (back_img.height, back_img.width)
    model = init_detector(mmdet_cfg, weight_path, device='cuda:0')
    time_deltas = []
    results = []
    i = 0
    for img_path, img, rsz_size in rsz_paste_imgs_from_csv_info(img_dir, back_img):
        img = np.asarray(img).astype(np.uint8)
        t1 = time.time()
        result = inference_detector(model, img)
        t2 = time.time()
        time_deltas.append(t2 - t1)
        bboxes, seg_masks = result[0][0], result[1][0]
        org_bboxes = restore_bboxes(
            bboxes,
            rsz_size,
            input_size
        )
        valid_bbox_idx = remove_irregular_bboxes(
            org_bboxes,
            input_size,
            0.8
        )
        org_seg_masks = restore_seg_mask(
            seg_masks,
            rsz_size,
            input_size
        )
        valid_seg_mask_idx = remove_irregular_seg_masks(
            org_seg_masks,
            input_size,
            0.8
        )
        valid_idx = set(valid_bbox_idx).intersection(set(valid_seg_mask_idx))
        valid_org_bboxes = [org_bboxes[i] for i in valid_idx]
        valid_org_bboxes = np.concatenate(valid_org_bboxes, axis=0)
        valid_org_seg_masks = [org_seg_masks[i] for i in valid_idx]
        result[0][0] = valid_org_bboxes
        result[1][0] = valid_org_seg_masks
        if output_dir:
            try:
                os.makedirs(Path(output_dir), exist_ok=True)
                img_name, img_ext = Path(img_path).name.split(".")
                visual_save_path = os.path.join(output_dir, f"{img_name}_pred.{img_ext}")
                model.show_result(
                    img_path,
                    result,
                    score_thr=show_score_thr,
                    out_file=visual_save_path,
                    font_size=9
                )
            except Exception as e:
                print(e)
                print(img_path)
                print(valid_bbox_idx)
                print(valid_seg_mask_idx)

        result_ = [([valid_org_bboxes], [valid_org_seg_masks])]
        encoded_result = [(bbox_result, encode_mask_results(mask_result))
            for bbox_result, mask_result in result_]
        results.extend(encoded_result)
        i += 1

    time_deltas = np.array(time_deltas)
    _logger.info(f"avg time consumed for inferencing \
                {len(time_deltas)} images is {time_deltas.mean()}")
    return model, results


import tkinter as tk
from tkinter import messagebox
from PIL import Image, ImageTk

class FMSVotingApp:
    def __init__(self, root):
        self.root = root
        self.root.title("FMS Voting App")

        # Cargar la imagen de fondo
        background_image = Image.open("/Users/axeloner/Pictures/Banner FMS España/header-fms-5.jpg")
        self.background_photo = ImageTk.PhotoImage(background_image)

        self.competitors = ["MC 1", "MC 2"]
        self.rounds = ["Easy Mode", "Hard Mode", "Temática 1", "Temática 2", "Random Mode", "Minuto 1", "Minuto 2", "Deluxe"]
        self.current_round_index = 0

        self.votes = {competitor: {round_name: [tk.StringVar() for _ in range(9)] for round_name in self.rounds} for competitor in self.competitors}
        self.round_totals = {round_name: {competitor: tk.IntVar() for competitor in self.competitors} for round_name in self.rounds}
        self.total_rounds = {competitor: tk.IntVar() for competitor in self.competitors}

         # Configurar el fondo de la ventana con la imagen cargada
        self.background_label = tk.Label(self.root, image=self.background_photo)
        self.background_label.place(relwidth=1, relheight=1)

        self.show_round()

        self.root.mainloop()

    def calculate_total(self, competitor, round_name):
        total = sum(int(value.get()) if value.get().isdigit() else 0 for value in self.votes[competitor][round_name])
        self.round_totals[round_name][competitor].set(total)

    def show_round(self):
        round_name = self.rounds[self.current_round_index]
        self.root.title(round_name)

        for widget in self.root.winfo_children():
            widget.destroy()

        for competitor in self.competitors:
            frame = tk.Frame(self.root)
            frame.pack(side="top", padx=20, pady=10)

            label_competitor = tk.Label(frame, text=competitor, font=("Helvetica", 14))
            label_competitor.pack(side="left", padx=10)

            pattern_entries = []
            for i in range(9):
                pattern_frame = tk.Frame(frame)
                pattern_frame.pack(side="left", padx=5)

                if i < 6:
                    tk.Label(pattern_frame, text=f"P{i+1}").pack(side="top")
                elif i == 6:
                    tk.Label(pattern_frame, text="Skills").pack(side="top")
                elif i == 7:
                    tk.Label(pattern_frame, text="Flow").pack(side="top")
                elif i == 8:
                    tk.Label(pattern_frame, text="P.Escena").pack(side="top")

                pattern_entry = tk.Entry(pattern_frame, width=5, textvariable=self.votes[competitor][round_name][i])
                pattern_entry.insert(0, "0")
                pattern_entry.pack(side="top")
                pattern_entries.append(pattern_entry)

        self.button_next = tk.Button(self.root, text="Next", command=self.next_round)
        self.button_next.pack(side="bottom", padx=20, pady=10)

        self.root.update_idletasks()
        self.root.geometry(f"{self.root.winfo_reqwidth()}x{self.root.winfo_reqheight()}")

    def next_round(self):
        round_name = self.rounds[self.current_round_index]
        for competitor in self.competitors:
            self.calculate_total(competitor, round_name)

        self.current_round_index += 1

        if self.current_round_index < len(self.rounds):
            self.show_round()
            if round_name == "Deluxe":
                self.button_next.config(text="Results", command=self.show_final_results)
        else:
            self.show_final_results()

    def show_final_results(self):
        for round_name in self.rounds:
            for competitor in self.competitors:
                self.calculate_total(competitor, round_name)

        for competitor in self.competitors:
            total_rounds = sum(self.round_totals[round_name][competitor].get() for round_name in self.rounds)
            self.total_rounds[competitor].set(total_rounds)

        for widget in self.root.winfo_children():
            widget.destroy()

        winner_diff = self.total_rounds["MC 1"].get() - self.total_rounds["MC 2"].get()

        if winner_diff >= 5:
            winner_msg = "Ganador: MC 1" if winner_diff > 0 else "Ganador: MC 2"
            self.button_close = tk.Button(self.root, text="Cerrar", command=self.root.destroy)
            self.button_close.pack(padx=20, pady=10)
        else:
            winner_msg = "Réplica"
            self.button_next_replica = tk.Button(self.root, text="Ronda Réplica", command=self.show_replica)
            self.button_next_replica.pack(padx=20, pady=10)

        for competitor in self.competitors:
            label = tk.Label(self.root, text=f"{competitor}: {self.total_rounds[competitor].get()}", font=("Helvetica", 12))
            label.pack()

        winner_label = tk.Label(self.root, text=winner_msg, font=("Helvetica", 14, "bold"))
        winner_label.pack(pady=10)

    def show_replica(self):
        self.current_round_index = len(self.rounds) - 1
        self.show_round()

if __name__ == "__main__":
    root = tk.Tk()
    app = FMSVotingApp(root)


import os
from os import chdir
chdir("/mnt/VegaSlowDataDisk/c3po_interface_mark/test_Yolo/6_classes_loss/")
import tensorflow as tf
import sys
import time
import cv2
import numpy as np
#import common_2 as common
import common_2 as common
import config
import model
import pandas as pd
from sklearn.model_selection import train_test_split

batch_size=16
nbr_entrainement=1

imagettes=pd.read_csv("/mnt/VegaSlowDataDisk/c3po/Images_aquises/imagettes.csv")
imagettes=common.to_reference_labels (imagettes,"classe")
index_train,index_test=common.split(imagettes)
imagettes_train=imagettes[imagettes["filename"].isin(index_train)]

images, labels, labels2=common.read_imagettes(imagettes_train,nbr=1)
images=np.array(images, dtype=np.float32)/255
labels=np.array(labels, dtype=np.float32)
print("Nbr images:", len(images))
train_ds=tf.data.Dataset.from_tensor_slices((images, labels)).batch(batch_size)

Model=model.model(config.nbr_classes, config.nbr_boxes, config.cellule_y, config.cellule_x)

string="/mnt/VegaSlowDataDisk/c3po_interface_mark/Materiels/Models/Yolo_models/training/"

optimizer=tf.keras.optimizers.Adam(learning_rate=1E-4)
checkpoint=tf.train.Checkpoint(model=Model)
train_loss=tf.keras.metrics.Mean()
checkpoint=tf.train.Checkpoint(model=model)
checkpoint.restore(tf.train.latest_checkpoint(string))

start=time.time()
end=time.time()
print((end-start)/60)
checkpoint.save(file_prefix=string)
#common.train(train_ds, nbr_entrainement,string,labels2)

common.train(train_ds, nbr_entrainement,string,labels2_train,optimizer,model,train_loss,checkpoint)

optimizer=tf.keras.optimizers.Adam(learning_rate=1E-4)
checkpoint=tf.train.Checkpoint(model=model)
train_loss=tf.keras.metrics.Mean()

checkpoint=tf.train.Checkpoint(model=model)
checkpoint.restore(tf.train.latest_checkpoint(string))

#train(train_ds, 400)
#train_test_split(train_ds)
start=time.time()
train(train_ds, 600,string,labels2_train)
end=time.time()
print((end-start)/60)
checkpoint.save(file_prefix=string)


"""
Actual helper script to execute code.
It takes care of proper error handling (e.g. if you press CTRL+C) and the difference between
running code on the robot vs. in simulation.

Usage:
  python3 run.py --sim lab1 [for simulation]
  python3 run.py lab1 [to run on a robot]
"""

import sys
import argparse
import importlib

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("run", help="run specified module")
    parser.add_argument("--sim", help="Run using VREP simulation", action="store_true")
    args = parser.parse_args()
    clientID = None
    if args.sim:
        from factory import FactorySimulation
        factory = FactorySimulation()
    else:
        from factory import FactoryCreate
        factory = FactoryCreate()
    try:
        if args.run.endswith(".py"):
            args.run = args.run[:-3]
        mod = importlib.import_module(args.run)
        Run = getattr(mod, "Run")
        r = Run(factory)
        r.run()
    except KeyboardInterrupt:
        pass
    except:
        factory.close()
        raise

    factory.close()

    # quit
    sys.exit()


from django.shortcuts import render
from django.http import HttpResponse
from django.views.generic import TemplateView
from usertemplates import models, forms

class TemplateEditorView(TemplateView):
    http_method_names = ['get', 'post', ]
    template_name = 'templateEditorView.html'

    def get(self, request, template_id=None, *args, **kwargs):
        context = self.get_context_data()
        context['page_title'] = 'Template Editor'

        if template_id is None:
            context['template_form'] = forms.TemplateForm()
        else:
            template = models.Template.objects.get(pk=template_id)
            context['template_form'] = forms.TemplateForm(instance=template)
            context['template_id'] = template_id
        return self.render_to_response(context=context)

    def post(self, request, template_id=None, *args, **kwargs):
        if template_id is None:
            form = forms.TemplateForm(request.POST)
        else:
            template = models.Template.objects.get(pk=template_id)
            form = forms.TemplateForm(request.POST,
                                      instance=template)
        if form.is_valid():
            form.clean()
            form.save()
            return HttpResponse(content="Tudo certo...")
        else:
            return HttpResponse(content="Verifique os campos...")


#preprocessing the data from online lending
import pandas as pd
import numpy as np
import gzip, csv
import os
import scipy.sparse as sp
import math
import random
from sklearn.linear_model import LogisticRegression, LinearRegression

VW_DS_DIR = '../../datasets/OnlineAutoLoan/'

feature_column = ['intercept', 'CarType', 'Primary_FICO', 'Term', 'Competition_rate', 'onemonth']
feature_column_interaction = [feature + '_interact' for feature in feature_column]
# encode the original data into vw format
def save_vw_dataset(X, y, did, ds_dir, choice = True):
    # change a bit as we change the data structure of y
    n_classes = max(y) + 1
    print("check step")
    fname = 'ds_loan{}_{}'.format(did, n_classes)
    #add it to the csv path for the use of bypassing the monster

#    with gzip.open(os.path.join(ds_dir, fname + '.vw.gz'), 'w') as f:
#        #if the matrix is sparse then we use the nonzero value to fill.
#        if sp.isspmatrix_csr(X):
#            for i in range(X.shape[0]):
#                str_output = '{} | {}\n'.format(y[i] + 1, ' '.join(
#                    '{}:{:.6f}'.format(j, val) for j, val in zip(list(X.loc[i].index), list(X.loc[i]))))
#                f.write(str_output.encode())
#        else:
#            for i in range(X.shape[0]):
#                print('yes')
#                str_output = '{} | {}\n'.format(y[i] + 1, ' '.join(
#                    '{}:{:.6f}'.format(j, val) for j, val in enumerate(X.loc[i])))
#                f.write(str_output.encode())
#    if choice == False:
#        return
    X['class'] = pd.Series(y)
    X.to_csv(VW_DS_DIR + fname + '.csv', index = None)
# map the xvalue and yvalue into discrete type
def map_class(X):

    ## map x
    for feature in ['CarType']:
        feature_type = list(set(list(X[feature])))
        feature_type = list(pd.Series(feature_type).dropna())
        feature_number = len(feature_type)
        feature_label = []
        feature_map = dict(zip(feature_type, list(range(1, feature_number + 1))))
        for item in list(X[feature]):
            #map the null value into zero
            if item == '' or item == None or (type(item) == float and np.isnan(item) == True):
                feature_label.append(0)
            else:
                feature_label.append(feature_map[item])
        X[feature] = pd.Series(feature_label)

    return X

#shuffle the value of x and y to a stochastic order
def shuffle(X, y):
    n = X.shape[0]
    perm = np.random.permutation(n)
    #print(n, perm)
    X_shuf = X.loc[perm]
    y_shuf = y[perm]
    return X_shuf, y_shuf

def preprocessing():
    df = pd.read_csv(VW_DS_DIR + 'Original/Data.csv')
    feature_column = ['Tier', 'State', 'termclass', 'Type', 'CarType', 'partnerbin', 'Primary_FICO', 'Term', 'rate', 'Competition_rate',
                      'onemonth', 'rate1', 'rel_compet_rate', 'mp', 'mp_rto_amtfinance', 'Amount_Approved']
    decision_column = ['apply']
    item_number = 50000
    for i in range(4):
        df_temp = df[feature_column + decision_column].loc[item_number*i: item_number*(i + 1)]
        df_temp.to_csv(VW_DS_DIR + 'loan' + str(i + 1) + '.csv', index = None)

def change_vw(ds_dir, fname, X, y):
    with gzip.open(os.path.join(ds_dir, fname + '.vw.gz'), 'w') as f:
    #if the matrix is sparse then we use the nonzero value to fill.
        if sp.isspmatrix_csr(X):
            for i in range(X.shape[0]):
                str_output = '{} | {}\n'.format(y[i] + 1, ' '.join(
                        '{}:{:.6f}'.format(j, val) for j, val in zip(list(X.loc[i].index), list(X.loc[i]))))
                f.write(str_output.encode())
        else:
            for i in range(X.shape[0]):
                print('yes')
                str_output = '{} | {}\n'.format(y[i] + 1, ' '.join(
                '{}:{:.6f}'.format(j, val) for j, val in enumerate(X.loc[i])))
                f.write(str_output.encode())

def price_compute(MP, Rate, Term, LoanAmount):
    #average monthly LIBOR
    #LIBOR = math.pow(1.0177, 1/12) - 1
    LIBOR = np.array(Rate)*0.0001
    price = -1*np.array(LoanAmount)
    for i in range(len(Rate)):
        for j in range(1, Term[i] + 1):
            price[i] += MP[i]/((1 + LIBOR[i])**j)
    #print(price)
    return price

def feature_normalize(feature_column, price):
    X = map_class(df[feature_column])

    for column in X.columns:
        X[column] = pd.Series(np.array(list(X[column]))/np.mean(X[column]))

    Y = np.zeros(X.shape)
    for index in range(len(X)):
        Y[index] = np.array(list(X.loc[index]))*price[index]

    Y = Y.T
    feature_norm = []
    for i in range(Y.shape[0]):
        feature_norm.append(np.mean(Y[i]))
        Y[i] = Y[i]/np.mean(Y[i])
    comb = pd.concat([X, pd.DataFrame(Y.T)], axis = 1)
    return X, comb, feature_norm

price_arm = 10
price_range = 1000

def simulation_data_generation(lr_model, X_original, price, feature_norm, id_norm):

    noise_std = 0.004

    csv_loan = open(VW_DS_DIR + 'test/ds_loan_' + str(price_arm) + '_' + str(price_range) + '.csv', 'w', newline = '')

    csv_writer = csv.writer(csv_loan)

    csv_writer.writerow(['curr_arm', 'reward'] + list(range(1, 2*len(X_original.columns) + 1)))

    arm_price = np.array(list(range(1, price_arm + 1)))*price_range
    Y = np.zeros(X_original.shape[1])
    reward = np.zeros((X_original.shape[0], price_arm))

    for t in range(X_original.shape[0]):
        for i in range(price_arm):
            Y = np.array(list(X_original.loc[t]))*arm_price[i]
            #rint(Y, feature_norm)
            Y = [Y[i]/feature_norm[i] for i in range(len(feature_norm))]
            X_temp = X_original.loc[t:t]
            X_temp.index = [0]
            comb = pd.concat([X_temp, pd.DataFrame(Y).T], axis = 1)
            comb.loc[0] = comb.loc[0]/id_norm['max_norm']
            #print(comb)
            prob = 1/(1 + math.exp(np.dot(-1*lr_model.coef_[0], np.array(list(comb.loc[0]))))) + np.random.normal(0, noise_std)
            if prob >= random.random():
                reward[t][i] = arm_price[i]
            csv_writer.writerow([i + 1, reward[t][i]] + list(comb.loc[0]))

    csv_loan.close()
    return reward

#convert the csv format to the gz for vw
def csv_to_gz():
    df = pd.read_csv(VW_DS_DIR + 'test/' + str(price_arm) +'_' + str(price_range) + '.csv')
    feature_column = list(df.columns)[2:]
    X_feature = df[feature_column]
    curr_arm = list(df['curr_arm'])
    reward = list(df['reward'])
    fname = 'ds_{}_{}'.format('loan2', price_arm)
    #add it to the csv path for the use of bypassing the monster
    with gzip.open(os.path.join(VW_DS_DIR, fname + '.vw.gz'), 'w') as f:
        #if the matrix is sparse then we use the nonzero value to fill.
        for i in range(X_feature.shape[0]):
            str_output = '{}:{} | {}\n'.format(curr_arm[i], reward[i], ' '.join(
                    '{}:{:.3f}'.format(j, val) for j, val in zip(list(feature_column), list(X_feature.loc[i]))))
            f.write(str_output.encode())
            if curr_arm[i] == price_arm:
                f.write("\n".encode())

#normalize the features into 0-1 intervals
def norm_normalization(X_comb):
    #the default norm across the complex feature dataframe
    id_norm = {"max_id":0, "max_norm":0}
    #find the maximum one in the feature
    for i in range(len(X_comb)):
        temp_vector = list(X_comb.loc[i])
        temp_norm = math.sqrt(np.sum([temp_vect**2 for temp_vect in temp_vector]))
        if id_norm["max_norm"] < temp_norm:
            id_norm['max_id'] = i
            id_norm['max_norm'] = temp_norm
    #normalize with each row
    for i in range(len(X_comb)):
        temp_vector = np.array(list(X_comb.loc[i]))
        temp_vector = pd.Series(list(temp_vector/id_norm['max_norm']))
        temp_vector.index = feature_column + feature_column_interaction
        X_comb.loc[i] = pd.Series(temp_vector)
    return X_comb, id_norm
if __name__ == '__main__':
#    preprocessing()
#    for i in range(4):
#        df_temp = pd.read_csv(VW_DS_DIR + 'loan' + str(i + 1) + '.csv')
#        X, y = shuffle(df_temp[list(df_temp.columns)[:-1]], df_temp['apply'])
#        X = map_class(X)
#        save_vw_dataset(X, list(y), i + 1, VW_DS_DIR)

#    feature_column = ['intercept', 'CarType', 'Primary_FICO', 'Term', 'rate', 'Competition_rate',
#                      'onemonth',  'Amount_Approved']
    feature_column = ['intercept', 'CarType', 'Primary_FICO', 'Term', 'Competition_rate', 'onemonth']
    feature_add = ['Type', 'partnerbin', 'rate']
    feature_column_interaction = [feature + '_interact' for feature in feature_column]
    df = pd.read_csv(VW_DS_DIR + 'Original/Data.csv')[0:2000]
    df['intercept'] = pd.Series(list(np.ones(len(df))))
    price = price_compute(list(df['mp']), list(df['rate']), list(df['Term']), list(df['Amount_Approved']))
    X_original, X_comb, feature_norm = feature_normalize(feature_column, price)
    # normalize further for further use of feature columns
    X_comb.columns = feature_column + feature_column_interaction
    X_comb_revise, id_norm = norm_normalization(X_comb)
    Y = np.array([[np.array(df['apply'])[i]] for i in range(len(df['apply']))])
    lr_model = LogisticRegression(fit_intercept = False)
    lr_model.fit(X_comb_revise, Y)
    simulation_data_generation(lr_model, X_original, price, feature_norm, id_norm)
#    print(lr_model.coef_)
#    simulation_data_generation(lr_model, X_original, price)
    #    csv_to_gz()

#    for i in range(4):
#        df = pd.read_csv(VW_DS_DIR + 'ds_loan' + str(i + 1) +'_2.csv')
#        X = df[feature_column]
#        y = list(df['class'])
#        change_vw(VW_DS_DIR, 'ds_loan' + str(i + 1) +'_2', X, y)


from typing import TextIO, Sequence
from unittest import mock

from pytest import fixture

from qtrex.models import QueryRef
from qtrex.store import Store
from qtrex.config import BaseConfig, YAMLConfig

@fixture
def mock_yaml_fixture() -> TextIO:
    """mock_yaml_fixture builds and returns a TextIO interface with sample YAML data."""
    with mock.patch(
        "{0}.open".format(__name__),
        create=True,
        new_callable=mock.mock_open(
            read_data="""
            params:
                -   key: my_key
                    value: [1, 2, 3]
            """
        ),
    ) as f:
        return f

@fixture
def invalid_top_level_config_fixture() -> TextIO:
    """Invalid top level config for YAML format."""
    with mock.patch(
        "{0}.open".format(__name__),
        create=True,
        new_callable=mock.mock_open(
            read_data="""
            template_me:
                -   key: my_key
                    value: [1, 2, 3]
            """
        ),
    ) as f:
        return f

@fixture
def invalid_key_valid_pair_yaml_fixture() -> TextIO:
    """Invalid key value params for YAML format."""
    with mock.patch(
        "{0}.open".format(__name__),
        create=True,
        new_callable=mock.mock_open(
            read_data="""
            params:
                -   job: teacher
            """
        ),
    ) as f:
        return f

@fixture
def mock_empty_yaml_fixture() -> TextIO:
    """Empty template params for YAML format."""
    with mock.patch(
        "{0}.open".format(__name__),
        create=True,
        new_callable=mock.mock_open(
            read_data="""
            params: []
            """
        ),
    ) as f:
        return f

@fixture
def mock_query_ref() -> QueryRef:
    """Fixture for QueryRef."""
    return QueryRef(
        "/mnt/test1.sql",
        "SELECT * FROM UNNEST({{ params.test_array_key }})",
        "test1.sql",
    )

@fixture
def mock_query_store(mock_query_ref) -> Store:
    store = Store()
    store[mock_query_ref.name] = mock_query_ref
    store["test2.sql"] = QueryRef(
        "/tmp/qtrex/test.sql", "SELECT {{ params.test_string_key }}", "test2.sql"
    )

    return store

@fixture
def mock_config_fixture() -> BaseConfig:
    with mock.patch(
        "{0}.open".format(__name__),
        create=True,
        new_callable=mock.mock_open(
            read_data="""
            params:
                - key: test_string_key
                  value: "string_value"
                - key: test_array_key
                  value: [1,2,3]
                - key: test_dict_key
                  value:
                    first: 1
                    two: 2
                    three: 3
                - key: my_project_id
                  value: test_gcp_project_id
                - key: my_dataset
                  value: test_dataset
                - key: my_table_name
                  value: test_table
            """
        ),
    ) as f:
        return YAMLConfig(f)

@fixture
def mock_query_file() -> TextIO:
    with mock.patch(
        "{0}.open".format(__name__),
        create=True,
        new_callable=mock.mock_open(
            read_data="""
            SELECT {{ params.test_string_key }}, {{ params.test_dict_key.first }}
            FROM UNNEST({{ params.test_array }})
            """
        ),
    ) as f:
        return f


"""Creates alerts from a csv"""

import csv
import irflow_client
import pprint
import logging
from time import gmtime, strftime

pp = pprint.PrettyPrinter(indent=4)

irflowAPI = irflow_client.IRFlowClient(config_file="./api.conf")
if irflowAPI.debug:
    logging.basicConfig(level=logging.DEBUG)
else:
    logging.basicConfig(level=logging.INFO)

logger = logging.getLogger('IR-Flow: CSV Example')

if irflowAPI.debug:
    irflowAPI.dump_settings()

alert_type = 'ds_test'
description = 'New CSV Alert on ' + strftime("%Y-%m-%d %H:%M:%S", gmtime())

with open('sample_csv_of_alerts.csv', 'rb') as csv_file:
    csv_reader = csv.DictReader(csv_file)
    for row in csv_reader:
        result = irflowAPI.create_alert(row, description=description, incoming_field_group_name=alert_type)
        if result['success']:
            logger.info('Created Alert_Num: ' + str(result['data']['alert']['alert_num']))
        else:
            logger.error('Failed to create alert')
            logger.error('Success == ' + str(result['success']))
            pp.pprint(result)


import RPi.GPIO as GPIO
from time import sleep
from datetime import datetime

# Pins
light = 22 # Pin for light relay control

mp_in1 = 24 # Direction control for main pump
mp_in2 = 23 # Direction control for main pump
mp_en = 25 # main pump

np_in1 = 17 # nutrients direction 1
np_in2 = 27 # nutrients direction 2
np_en = 18 # nutrients enable

GPIO.setmode(GPIO.BCM) # Chooses board

# Setup output pins for pumps and relays
GPIO.setup(mp_in1,GPIO.OUT)
GPIO.setup(mp_in2,GPIO.OUT)
GPIO.setup(mp_en,GPIO.OUT)
GPIO.setup(light,GPIO.OUT)
GPIO.setup(np_in1,GPIO.OUT)
GPIO.setup(np_in2,GPIO.OUT)
GPIO.setup(np_en,GPIO.OUT)

# intialize output pin states
GPIO.output(light,GPIO.LOW)
GPIO.output(mp_in1,GPIO.HIGH)
GPIO.output(mp_in2,GPIO.LOW)
GPIO.output(np_in1,GPIO.LOW)
GPIO.output(np_in2,GPIO.HIGH)

# Run main pump at normal rate
p=GPIO.PWM(mp_en,500)
p.start(35)

n = GPIO.PWM(np_en,500)
n.start(40)

light_flag = False # initialize light state to off

# Constants
midnight = datetime(1,1,1,23,0,0).time()
morning = datetime(1,1,1,5,0,0).time()

while (1):
    time = datetime.now().time() # Get current time

    # Set lights to proper staet
    if light_flag == True and (time > midnight or time < morning):
        GPIO.output(light,GPIO.LOW)
        light_flag = False
    elif light_flag == False and not (time > midnight or time < morning):
        GPIO.output(light,GPIO.HIGH)
        light_flag = True


from selenium import webdriver
from bs4 import BeautifulSoup
import os
import time
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from scrapy.selector import Selector
#import mysql入库 as mysqltool

def get_data(driver,name):
    data = Selector(text=driver.page_source)

    for res in data.xpath('//*[@id="poilist"]/div'):
        url=res.xpath('a/@href').extract_first()
        shopname=res.xpath('div[@class="content"]/*/div[1]/div/text()').extract_first()
        if 'None'==shopname:
            continue
        star=','.join(res.xpath('div[@class="content"]/*/div[2]/div[1]/i/@class').extract())
        shopsold=res.xpath('div[@class="content"]/*/div[2]/div[2]/text()').extract_first()
        shopright=res.xpath('div[@class="content"]/*/div[2]/i[1]/text()').extract_first()
        shoptime=res.xpath('div[@class="content"]/*/div[2]/i[3]/text()').extract_first()
        sendprice=''.join(res.xpath('div[@class="content"]/*/div[3]/span[1]/em/text()').extract())
        deliveryfee=''.join(res.xpath('div[@class="content"]/*/div[3]/span[2]/text()').extract())

        shopdata='%s[}%s[}%s[}%s[}%s[}%s[}%s[}%s[}%s' %(url,shopname,star,shopsold,shopright,shoptime,sendprice,deliveryfee,name)
        with open('C:/Users/Administrator/Desktop/美团外卖/shoplist_%s_1.txt' %name, 'a', encoding='utf-8') as f:
            f.write(shopdata)
            f.write('\n')

'''
    for i in range(0,9):
        driver.find_element_by_xpath('//*[@id="loading"]/div').click()
        time.sleep(5)

    data=driver.page_source
    soup=BeautifulSoup(data,"html.parser")
    list_data=soup.find('ul',attrs={'class':'list clearfix'}).find_all('li',attrs={'class':'fl rest-li'})
    for res in list_data:
        shop_id=res.find('a',attrs={'class':'rest-atag'}).get('href')
        with open('C:/Users/Administrator/Desktop/list.txt', 'a', encoding='utf-8') as f:
            f.write(shop_id)
            f.write('\n')

#url='http://i.waimai.meituan.com/home?lat=30.541111&lng=104.059155'
urllist=[#'http://i.waimai.meituan.com/home?lat=30.627222&lng=104.06055,玉林',
         #'http://i.waimai.meituan.com/home?lat=30.634774&lng=104.031921,紫荆',
         #'http://i.waimai.meituan.com/home?lat=30.619286&lng=104.075643,科华',
         #'http://i.waimai.meituan.com/home?lat=30.622109&lng=104.098407,锦华',
         #'http://i.waimai.meituan.com/home?lat=30.632749&lng=104.029313,红牌楼',
         #'http://i.waimai.meituan.com/home?lat=30.645977&lng=104.033134,双楠',
         #'http://i.waimai.meituan.com/home?lat=30.670267&lng=103.978478,光华',
         #'http://i.waimai.meituan.com/home?lat=30.681726&lng=104.012659,金沙',
         #'http://i.waimai.meituan.com/home?lat=30.705653&lng=104.013817,茶店子',
         #'http://i.waimai.meituan.com/home?lat=30.690691&lng=104.047379,老会展',
         #'http://i.waimai.meituan.com/home?lat=30.697218&lng=104.073694,北站',
         #'http://i.waimai.meituan.com/home?lat=30.677707&lng=104.096474,府青',
         #'http://i.waimai.meituan.com/home?lat=30.669313&lng=104.11071,建设路',
         #'http://i.waimai.meituan.com/home?lat=30.644231&lng=104.119188,万年场',
         'http://i.waimai.meituan.com/home?lat=30.646149&lng=104.109381,双桥',
        ]

url='http://waimai.meituan.com/home/wm6n2ev4q1x5'
chrome_driver = os.path.abspath(r"C:/Program Files (x86)/Google/Chrome/Application/chromedriver2.30.exe")
os.environ["webdriver.chrome.driver"] = chrome_driver
driver = webdriver.Chrome(chrome_driver)
driver.get(url)
get_data(driver)
driver.quit()

file_name='shoplist_%s' %urllist[0].split(',')[1]
for url in urllist:
        mobileEmulation = {'deviceName': 'iPhone 6 Plus'}
        options = webdriver.ChromeOptions()
        options.add_experimental_option('mobileEmulation', mobileEmulation)
        driver = webdriver.Chrome(executable_path='C:/Program Files (x86)/Google/Chrome/Application/chromedriver2.30.exe', chrome_options=options)
        driver.set_page_load_timeout(100)
        driver.set_script_timeout(100)
        driver.get(url.split(',')[0])
        time.sleep(5)
        #driver.maximize_window()
        driver.find_element_by_xpath('//*[@id="primary_item_bar_new"]/a[1]/img').click()
        time.sleep(5)
        for i in range(80):
            #driver.execute_script("window.scrollBy(0,200)", "")  # 向下滚动200px
            driver.execute_script("window.scrollBy(0,document.body.scrollHeight)","")#向下滚动到页面底部
            time.sleep(3)
        get_data(driver,url.split(',')[1])
        driver.quit()

#sql='insert into meituanshop(url,shopname,star,shopsold,shopright,shoptime,startprice,deliveryfee,CBD) values'
#mysqltool.get_sql(sql,file_name)
'''
def get_img(driver,line):
    id = line.split('[}')[1].replace('\n', '')
    url = 'http://www.dianping.com/shop/%s' % str(id)
    time.sleep(1)
    driver.get(url)
    data=Selector(text=driver.page_source)
    id = url.split('/')[-3]
    id1 = line.split('[}')[0].replace('\n', '')
    name = line.split('[}')[2].replace('\n', '')
    imglist = []
    try:
        for sel in data.xpath('//div[@class="shop-tab-recommend J-panel"]/ul/li'):
            imglist.append(sel)
        for sel in imglist:
            imgurl = sel.xpath('img/@src').extract_first()
            goods = sel.xpath('img/@alt').extract_first()
            if goods == None:
                continue
            img = '%s_photo_%d.jpg' % (name, imglist.index(sel))
            print(goods, imgurl)
            imgdata = '%s[}%s[}%s[}%s[}%s' % (id1, id, name, goods,img)
            print(imgdata)
            # tool.GetImg(imgdir3, imgurl, img)
            # with open('C:/Users/Administrator/Desktop/shangping_20171106.txt', 'a', encoding='utf-8') as f:
            #    f.write(imgdata)
            #    f.write('\n')
    except:
        print(url)

chrome_driver = os.path.abspath(r"C:/Program Files (x86)/Google/Chrome/Application/chromedriver2.30.exe")
os.environ["webdriver.chrome.driver"] = chrome_driver
driver = webdriver.Chrome(chrome_driver)
driver.get('http://www.dianping.com/citylist')

start_urls=[]
with open('C:/Users/Administrator/Desktop/20171103.txt', 'r', encoding='utf-8') as f:
    for line in f:
        start_urls.append(line.replace('\n', ''))

get_img(driver,'183[}10333429[}MorningHouse')


import collections

def tripleToList(dependencies, length, multipleParents=False, ignoreOutOfBounds=False):
    '''take in a list of triples (gov, dep, rel)
    and return a list of doubles (rel, gov)
    where the dep is in sentence order'''
    ret = [None] * length
    for gov,dep,rel in dependencies:
        if dep >= length and ignoreOutOfBounds:
            continue
        if multipleParents:
            if ret[dep] is None:
                ret[dep] = []
            ret[dep].append((rel.lower(), gov))
        else:
            ret[dep] = [rel.lower(), gov]
    return ret

    #TODO: handle cases with more than one parent

def combineDependencies(*dependenciesList):
    offset = 0
    combined = []
    for dependencies in dependenciesList:
        for dependency in dependencies:
            if dependency is None:
                combined.append(None)
            else:
                combined.append((dependency[0], dependency[1]+offset))
        offset += len(dependencies)
    return combined

def combineDependenciesMultiple(*dependenciesList):
    offset = 0
    combined = []
    for dependencies in dependenciesList:
        for dependency in dependencies:
            if dependency is None:
                combined.append(None)
            else:
                appendee = []
                for parent in dependency:
                    appendee.append((parent[0], parent[1]+offset))
                combined.append(appendee)
        offset += len(dependencies)
    return combined

def splitDependencies(dependencies, connectiveIndices):
    '''split the dependency tree into two trees,
    removing any links that cross the connective
    and adding new root nodes as necessary
    '''
    start, end = connectiveIndices
    altlexIndices = set(range(start, end))
    newDependencies = {'prev': [None]*start,
                       'altlex': [None]*(end-start),
                       'curr': [None]*(len(dependencies)-end)}
    for dep,rel_gov in enumerate(dependencies):
        if rel_gov is None:
            continue
        rel,gov = rel_gov
        #if the relationship crosses a barrier, remove it
        if gov == -1:
            if dep < start:
                newDependencies['prev'][dep] = rel,gov
            elif dep >= end:
                newDependencies['curr'][dep-end] = rel,gov
            elif dep in altlexIndices:
                newDependencies['altlex'][dep-start] = rel,gov
        elif dep < start and gov < start:
            newDependencies['prev'][dep] = rel,gov
        elif dep >= end and gov >= end:
            newDependencies['curr'][dep-end] = rel,gov-end
        elif dep in altlexIndices and gov in altlexIndices:
            newDependencies['altlex'][dep-start] = rel,gov-start

    #finally, make sure all the new dependencies have a root node

    #print(newDependencies)

    for section in newDependencies:
        if any(i == ('root', -1) for i in newDependencies[section]):
            continue
        root = 0
        for dependency in newDependencies[section]:
            if dependency is not None:
                #if dependency[1] >= len(newDependencies[section]):
                #    continue
                if dependency[-1] != -1 and newDependencies[section][dependency[1]] is None:
                    #if root < len(newDependencies[section]):
                    root = dependency[1]
                    newDependencies[section][root] = 'root',-1 #UNDO
        if len(newDependencies[section]) and newDependencies[section][root] is None:
            newDependencies[section][root] = 'root',-1
    return newDependencies

def getRoot(parse):
    for i in range(len(parse)):
        if parse[i] is not None and parse[i][0] == 'root':
            return i
    return None

def getRootMultiple(parse):
    for i in range(len(parse)):
        if parse[i] is not None and len(parse[i]) == 1 and parse[i][0][0] == 'root':
            return i
    return None

def iterDependencies(parse):
    '''iterate over the dependency structure from
    the leaves to the root
    (useful for dependency embeddings)'''
    pass

def getEventAndArguments(parse):
    '''take in a dependency parse and return the
    main event (usually a verb) and any corresponding
    (noun) arguments'''
    root = getRoot(parse)
    if root is None:
        return None,[]
    arguments = []
    for i in range(len(parse)):
        if parse[i] is not None and parse[i][1] == root:
            arguments.append(i)
    return root,arguments

def getEventAndArgumentsMultiple(parse):
    '''take in a dependency parse and return the
    main event (usually a verb) and any corresponding
    (noun) arguments'''
    root = getRoot(parse)
    if root is None:
        return None,[]
    arguments = []
    for i in range(len(parse)):
        if parse[i] is not None and any(j[1] == root for j in parse[i]):
            arguments.append(i)
    return root,arguments

#modify a dependency parse so that compounds/names/mwes are combined
def getCompounds(parse):
    ret = collections.defaultdict(list)
    for i in range(len(parse)):
        if parse[i] is None:
            continue
        if type(parse[i][0]) in (str, unicode):
            if parse[i][0] in ('compound', 'name', 'mwe'):
                ret[parse[i][1]].append(i)
        else:
            for j in parse[i]:
                if j[0] in ('compound', 'name', 'mwe'):
                    ret[j[1]].append(i)
    return ret

def getAllEventsAndArguments(parse):
    '''given a dependency parse return a list of all
    events (verbs) and their nsubj,nsubjpass,dobj,and iobj'''

    #use subjects and objects to find the predicates
    #0 is rel, 1 is gov
    ret = collections.defaultdict(dict)
    for i in range(len(parse)):
        if parse[i] is None:
            print('Problem with parse: {}'.format(parse))
            continue
        if type(parse[i][0]) in (str, unicode):
            if parse[i][0] in ('nsubj', 'nsubjpass', 'dobj', 'iobj'):
                if parse[i][0] not in ret[parse[i][1]]:
                    ret[parse[i][1]][parse[i][0]] = []

                ret[parse[i][1]][parse[i][0]].append(i)
        else:
            for j in parse[i]:
                if j[0] in ('nsubj', 'nsubjpass', 'dobj', 'iobj'):
                    if j[0] not in ret[j[1]]:
                        ret[j[1]][j[0]] = []

                    ret[j[1]][j[0]].append(i)

    return ret

def makeDependencies(words, deps):
    #make the root of the altlex the new root
    #for every edge on the path from the original root to the new root flip the direction
    altlexStart = len(words[0])
    altlexEnd = len(words[0]+words[1])
    altlexRoot = None

    #print()
    #print(datum['altlex'], altlexStart, altlexEnd, words[altlexStart:altlexEnd])
    #print(zip(words, deps))

    if len(words[1])==1:
        altlexRoot = altlexStart
    else:
        for i in range(altlexStart, altlexEnd):
            rel,gov = deps[i]
            #if the gov is the parent of the altlex root or there is none
            if gov in range(altlexStart, altlexEnd) and (altlexRoot is None or i == altlexRoot):
                altlexRoot = gov

    if altlexRoot is None:

        for i in range(altlexStart, altlexEnd):
            if deps[i][0] != 'det':
                altlexRoot = i
                break

    assert(altlexRoot is not None)

    parent = -1
    rel = 'root'
    child = altlexRoot
    while child != -1:
        dep, gov = deps[child]
        deps[child][0] = rel
        deps[child][1] = parent

        parent = child
        child = gov
        rel = dep + '_rev'

    #print(zip(words, deps))

    final_words = words[0] + words[1] + words[2]
    d = [('ROOT', None, None)]
    for i,j in enumerate(deps):
        if j is None:
            d.append((None,None,None))
        else:
            d.append((final_words[i].lower(), j[0], j[1]+1))

    return d


from math import floor
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
#import tensorflow as tf

import fuzzy_ART
import AE_class_2

fuzzy_ART_MODE = "TRAIN"
results_dir = "results/fuzzyART_w_AE-2/"

################################ Helper Functions #############################

def get_data():
    mnist = pd.read_csv("datasets/MNIST/mnist_test.csv").values
    data = {}
    for i in range(0,10):
        mask = mnist[:,0]==i
        temp = mnist[mask]
        temp = temp/255
        np.random.shuffle(temp)
        temp = temp[0:5, 1:]      # 5 images of each digit
        data[str(i)] = temp
    return data

def preprocess_for_AE(data_dict):
    data_array = []
    for i in range(0,10):
        data_array.append( data_dict[str(i)] )
    data_array = np.array(data_array).reshape(10*data_dict[str(i)].shape[0], 784)

    data_array = AE_class_2.data_preprocess(data_array)

    return data_array

def preprocess_for_ART(data_array):
    data_array = data_array.reshape(-1,32)
    return data_array

def train_fuzzy_ART(model, data_array, shuffle_data=False):
    if shuffle_data:
        np.random.shuffle(data_array)

    for i in range(data_array.shape[0]):
        Z, k = model.train(data_array[i])

        enc_learned_wts = Z[:32].reshape(4,8)
        plt.imshow(enc_learned_wts) #display learned expectations
        plt.title("Viz. of learned encoded digit: {}, sample:{}".format(str( floor( i/(data_array.shape[0]/10) ) ),
                                                                        str( i%(data_array.shape[0]/10) + 1)) )
        #plt.savefig( results_dir+"encoded_wts/"+"{}_{}.png".format(str(floor(i/5)),str(i%5 + 1)) )
        plt.show()

        print("CLASS: ", k)

    return model

def test(model):
    test_data = pd.read_csv("datasets/MNIST/mnist_test.csv").values
    np.random.shuffle(test_data)
    test_data = test_data[:1001, :]
    test_data_labels = np.array(test_data[:,0])
    test_data = test_data[:,1:]

    test_data = test_data.reshape(-1, 28, 28)
    test_data = np.expand_dims(test_data, 3)

    test_data = AE_class_2.data_preprocess(test_data)
    encoded_test_data = AE_model.autoencode(test_data, True)

    encoded_test_data = preprocess_for_ART(encoded_test_data)

    ART_output = []
    for i in range(0,encoded_test_data.shape[0]):
        op = ART_model.infer( encoded_test_data[i] )
        #if op[0] is None and op[1] is None:
            #op = [0, ART_model.N]

        ART_output.append(op[1])
    ART_output = np.array(ART_output)

    accuracy = np.mean( np.equal(test_data_labels, ART_output) ) * 100

    return accuracy, ART_output, test_data_labels

def Purity(model):
    _,  ART_output, test_data_labels = test(ART_model)

    #combined = np.vstack([ART_output, test_data_labels]).T
    categories = {}
    purity_list = []
    for i in range(np.max(ART_output)+1):

        mask = ART_output==i
        temp = test_data_labels[mask]  # labels of data belonging to category i

        categories[str(i)] = temp
        if temp.shape[0] > 0:
            most_freq_label = np.argmax( np.bincount(temp) )
            purity_for_curr_category = np.mean( temp==most_freq_label ) *100
        else:
            purity_for_curr_category = 0
        purity_list.append( purity_for_curr_category)

    purity = np.mean( np.array(purity_list) )
    return purity, categories

############### Load pre-trained Tensorflow AutoEncoder model #################
AE_model = AE_class_2.AutoEncoder()
AE_model.load_model("models/TF-AE/")

#####################################  TEST  ##################################

if fuzzy_ART_MODE is "TEST":
    # Load pre-trained fuzzy art model
    ART_rho = 0.95  # vigilance parameter
    ART_model = fuzzy_ART.fuzzy_ART(32,
                                    c_max=20,
                                    rho=ART_rho,
                                    alpha=0.00001,
                                    beta=1)
    ART_model.load_params("models/fuzzyART_w_AE-2_weights")

    accuracy, _, _ = test(ART_model)
    print("Test accuracy: ", accuracy)

################################# or TRAIN ####################################

elif fuzzy_ART_MODE is "TRAIN":
    '''
    In this case, the fuzzy ART model will be trained on the data encoded by the
    pretrained Autoencoder
    '''
    np.random.seed(0)
    #initialize a fuzzy ART model for training
    ART_rho = 0.999830 # vigilance parameter
    ART_model = fuzzy_ART.fuzzy_ART(32,
                                    c_max=20,
                                    rho=ART_rho,
                                    alpha=0.000001,
                                    beta=1)

    ART_train_data = get_data()

    preprocessed_data =  preprocess_for_AE(ART_train_data)
    encoded_train_data = AE_model.autoencode(preprocessed_data, True)

    encoded_train_data = preprocess_for_ART(encoded_train_data)

    n_epochs = 1
    #acc_list = []
    purity_list = []
    for e in range(n_epochs):
        ART_model = train_fuzzy_ART(ART_model, encoded_train_data, shuffle_data=False)
        #accuracy, _, _ = test(ART_model)
        purity, Categories = Purity(ART_model)

        #print("Test accuracy at epoch {}: {}".format(e+1, accuracy))
        print("Purity at epoch {}: {}".format(e+1, purity))
        #acc_list.append(accuracy)
        purity_list.append(purity)

    if n_epochs > 1:
        plt.plot(purity_list)
        plt.show()

    '''
    n = np.random.randint(encoded_train_data.shape[0])
    sample = encoded_train_data[n]
    inference = ART_model.infer(sample)
    '''
    #ART_model.save_params("models/fuzzyART_w_AE-2_weights")

    ######################### save preprocessed_data ##########################
    '''
    for i in range(preprocessed_data.shape[0]):
        img = preprocessed_data[i].squeeze()
        plt.imshow(img) #display learned expectations
        plt.title("ART train data -- digit: {}, sample:{}".format(str(floor(i/5)),str(i%5 + 1)))
        plt.savefig( results_dir+"ART_train_data/"+"{}_{}.png".format(str(floor(i/5)),str(i%5 + 1)) )
        plt.show()
    '''

    '''

    #ART_rho_list = [0.99978, 0.9997048, 0.9997052, 0.9997054, 0.9997055, 0.9997057, 0.9997058, 0.999706, 0.9997065, 0.9997068]
    ART_rho_list = ( np.random.rand(5)*2 - 1 )*0.00001 + 0.999817
    ART_rho_list = np.sort(ART_rho_list)
    #ART_rho_list = [0.99983, 0.999831, 0.999832]
    #alpha_list = ( np.random.rand(5)*2 - 1 )*0.07 +0.1
    purity_list=[]
    #n_epochs = 1

    np.random.seed(0)

    for rho in ART_rho_list:
        ART_model = fuzzy_ART.fuzzy_ART(32,
                                    c_max=20,
                                    rho=rho,
                                    alpha=0.0001,
                                    beta=1)

        ART_model = train_fuzzy_ART(ART_model, encoded_train_data, shuffle_data=False)

        purity, Categories = Purity(ART_model)

        print("Purity for rho={}:  {} ".format(rho, purity))

        purity_list.append(purity)

    plt.plot(ART_rho_list, purity_list)
    plt.show()

    '''


import logging
import os
import time
import sys
from telemetry_aggregator import TelemetryAggregator

import rticonnextdds_connector as rti

# Setup logging to stdout.
LOGGER = logging.getLogger(__name__)
logging.basicConfig(
    stream=sys.stdout,
    level=logging.DEBUG,
    format="%(asctime)s|%(levelname)s| %(message)s",
    datefmt="%Y-%m-%d %H:%M:%S")

class DDSTelemetryPublisher():
    """
    A server used to collect telemetry data.
    """
    DDS_DOMAIN_PARTICIPANT = "TelemetryParticipantLibrary::TelemetryDom0Participant"
    DDS_WRITER = "TelemetryPublisher::TelemetryWriter"
    # Skew in seconds over which we reinitialize the DDS Writer.
    TIME_SKEW = 10

    def __init__(self):
        """
        Class constructor
        """
        self.__init_writer()
        self.timestamp_prev = None

    def __init_writer(self):
        """
        Initialize the DDS Writer.
        """
        rti_connector = rti.Connector(self.DDS_DOMAIN_PARTICIPANT,
                                      os.path.dirname(os.path.realpath(__file__)) + os.path.sep + "dds_telemetry.xml")
        self.__dds_writer = rti_connector.get_output(self.DDS_WRITER)
        LOGGER.info("Initialized DDS Writer")

    def __check_timestamp(self):
        """
        Check if the timestamp has increased more than TIME_SKEW,
        and if so reinitialize the DDS Writer.
        """
        if self.timestamp_prev and abs(time.time() - self.timestamp_prev) > 10:
            self.__init_writer()

        self.timestamp_prev = time.time()

    def send(self, data):
        """
        Sends data to subscriber
        :param data: Payload to be sent to the subscriber.
        """
        self.__dds_writer.instance.set_string("stats", data)
        self.__dds_writer.write()

    def run(self):
        """
        Class runnable, will be called whenever the user wants to start sending data
        """
        while True:
            try:
                self.__check_timestamp()

                self.send(AGGREGATOR.get_stats())
                time.sleep(1)
            # pylint: disable=broad-exception-caught
            except Exception as exception:
                LOGGER.error("Failed to send data, exception: %s", exception)

if __name__ == "__main__":
    # Start the collector
    AGGREGATOR = TelemetryAggregator()
    AGGREGATOR.run()
    DDSTelemetryPublisher().run()


import ssl, socket, logging, threading, time

logging.basicConfig(filename="proxy.log", level=logging.DEBUG, format="%(asctime)s - %(message)s")
# ssl.create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None)
# In server mode, if you want to authenticate your clients using the SSL layer you’ll also have to specify CERT_REQUIRED
# and similarly check the client certificate. Passing SERVER_AUTH as purpose sets verify_mode to CERT_REQUIRED.
context = ssl.create_default_context(cafile="ca.crt")
context.load_cert_chain(certfile="server.crt", keyfile="server.key")
# hostname don`t match with hostname in cert.
context.check_hostname = False
# Load the key generation parameters for Diffie-Helman (DH) key exchange. This setting doesn’t apply to client sockets.
context.load_dh_params("dh.pem")

bindsocket = socket.socket()
bindsocket.bind(("", 7788))
bindsocket.settimeout(5)
bindsocket.listen(10)
lock = threading.Lock()
event = threading.Event()
clients = {}

class ProxySocket(threading.Thread):
    def __init__(self):
        super().__init__(daemon = True)
        self.running = True

    def stop(self):
        self.running = False

    def run(self):
        logging.critical("Start proxy")
        while self.running:
            try:
                newsocket, fromaddr = bindsocket.accept()
                print(newsocket)
            except socket.timeout:
                continue
            sslsocket = False
            try:
                sslsocket = context.wrap_socket(newsocket, server_side=True)
                #sslsocket = newsocket
                sslsocket.settimeout(5)
                logging.critical("New connection: " + str(fromaddr))
            except (ssl.SSLError, ConnectionResetError) as e:
                logging.critical(str(fromaddr) + " " + str(e))
            if sslsocket:
                name = fromaddr[0] + ":" + str(fromaddr[1])
                # for every host own worker
                clients[name] = Client(sslsocket)
                clients[name].start()
        logging.critical("Stop proxy")

class Client(threading.Thread):
    def __init__(self, conn):
        super().__init__(daemon = True)
        self.running = True
        self.s_socket = conn
        self.ip, self.port = conn.getpeername()
        self.name = self.ip + ":" + str(self.port)
        self.log_msg = "Client " + self.name
        self.crypto = False

    def stop(self):
        self.running = False

    def connect(self):
        self.server = Server(self)
        self.server.start()

    def run(self):
        logging.critical(self.log_msg + " Start")
        self.connect()
        while not event.is_set():
            time.sleep(1)
        while self.running:
            try:
                data = self.s_socket.recv(1024)
                if data:
                    logging.critical(self.log_msg + " > " + str(data))
                    self.server.d_socket.send(data)
                else:
                    logging.critical(self.log_msg + ' empty message')
                    self.running = False
            except socket.timeout:
                continue
            except Exception as e:
                logging.critical(str(e), exc_info=True)
                self.running = False
        # deleting worker from list of workers
        del clients[self.ip + ":" + str(self.port)]
        self.server.running = False
        self.s_socket.shutdown(socket.SHUT_RDWR)
        self.s_socket.close()
        logging.critical(self.log_msg + " close connection")

class Server(threading.Thread):
    def __init__(self, parent):
        super().__init__(daemon = True)
        self.parent = parent
        self.running = True
        self.log_msg = "Server " + parent.name
        self.dest_host = '' # server
        self.dest_port = 3333

    def stop(self):
        self.running = False

    def connect(self):
        logging.critical("%s connect to %s:%d" % (self.log_msg,self.dest_host,self.dest_port))
        self.d_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        self.d_socket.connect((self.dest_host, self.dest_port))
        self.d_socket.settimeout(5)
        event.set()

    def run(self):
        logging.critical(self.log_msg + " Start")
        try:
            self.connect()
        except:
            self.running = False
        while self.running:
            try:
                data = self.d_socket.recv(1024)
                if data:
                    logging.critical(self.log_msg + " < " + str(data))
                    self.parent.s_socket.send(data)
                else:
                    logging.critical(self.log_msg + ' Empty message')
                    self.running = False
            except socket.timeout:
                continue
            except Exception as e:
                logging.critical(str(e), exc_info=True)
                self.running = False
        self.parent.running = False
        event.clear()
        self.d_socket.shutdown(socket.SHUT_RDWR)
        self.d_socket.close()
        logging.critical(self.log_msg + " Close connection")

proxy = ProxySocket()
proxy.start()
while True:
    time.sleep(5)


from bs4 import BeautifulSoup
from celery import shared_task

import requests
from decimal import Decimal

from currency import model_choices as mch
from currency.data_source.service import previous_record_check_and_save, log

@shared_task()
def _finance_i_ua():
    log.info('finance.i.ua parser started')
    page = requests.get("https://finance.i.ua")
    soup = BeautifulSoup(page.content, 'html.parser')
    currency_bank = soup.find('div', class_='widget-currency_bank')  # div with bank currency rates
    table_rows = currency_bank.select("div tr")  # css selector -> in <div> select all <tr>
    for row in table_rows:
        if row.find('th'):
            currency_name = row.find('th').text
            if currency_name in {'USD', 'EUR'}:
                currency = {
                    'USD': mch.CURR_USD,
                    'EUR': mch.CURR_EUR,
                }[currency_name]

                rates = row.select('span span')  # spans inside spans

                rate_kwargs = {
                    'currency': currency,
                    'buy': round(Decimal(rates[0].text), 2),
                    'sale': round(Decimal(rates[2].text), 2),
                    'source': mch.SR_FINANCE_I_UA,
                }

                previous_record_check_and_save(rate_kwargs)


"""feltres URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.9/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Import the include() function: from django.conf.urls import url, include
    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
"""
from django.conf.urls import url
from django.contrib import admin

import resultater.views

urlpatterns = [
    url(r'^admin/', admin.site.urls),
    url(r'^xml/(?P<år>[0-9]{4})/(?P<klubb>[^/]+)/(?P<skytter>[^/]+)$', resultater.views.stevner_xml, name='stevner-xml'),
    url(r'^xml/(?P<år>[0-9]{4})/(?P<klubb>[^/]+)$',                    resultater.views.stevner_xml, name='stevner-xml'),
    url(r'^xml/(?P<år>[0-9]{4})$',                               resultater.views.stevner_xml, name='stevner-xml'),
    url(r'^xml/$',                                               resultater.views.stevner_xml, name='stevner-xml'),
    url(r'^(?P<stevnenr>[0-9]+).xlsx$',                          resultater.views.stevne_xlsx, name='stevne-xlsx'),
    url(r'^(?P<stevnenr>[0-9]+).xml$',                           resultater.views.stevner_xml, name='stevne-xml'),
    url(r'^(?P<stevnenr>[0-9]+)/$',                              resultater.views.stevne_html, name='stevne-html'),
    url(r'^$',                                                   resultater.views.index)
]


# -*- coding: utf-8 -*-
"""
Created on Thu Mar 18 10:09:02 2021

@author: Ernesto

K-Means Clustering Sandbox
"""

# import libraries
import numpy as np
from skimage import io
import matplotlib.pyplot as plt
import skimage
from matplotlib import colors

# load sample image
img_yeast = io.imread('./images/yeast_USC.tif')
img_book = io.imread('./images/book-cover.tif')
img_flowers = io.imread('./images/flowers-red.tif')
img_chalk = io.imread('./images/chalk-RGB.tif')
img_square = io.imread('./images/RGB-blue-yellow-squares.tif')

#img = img_yeast
#img = img_book
#img = img_flowers
#img = img_chalk
img = img_square

# specify number of means
k = 4

# Step 1) Initialize the algorithm. Specify an initial set of means
means = np.random.choice(np.unique(img.ravel()), size=k)
#means = np.array([45, 120, 181])

idx_x = np.random.uniform(low=0, high=img.shape[1], size=3).astype(int)
idx_y = np.random.uniform(low=0, high=img.shape[2], size=3).astype(int)

if len(img.shape) > 2:
    # generate idx to get pixels from
    smallest_dim = np.sort(img.shape)[1]
    #idx = np.random.uniform(low=0, high=smallest_dim, size=(k,2)).astype(int)

    idx_x = np.random.uniform(low=0, high=img.shape[0], size=k).astype(int)
    idx_y = np.random.uniform(low=0, high=img.shape[1], size=k).astype(int)

    #idx = np.expand_dims(idx, axis=-1)
    #means_red = np.random.choice(np.unique(img[:, :, 0].ravel()), size=k).reshape(-1, 1)
    #means_green = np.random.choice(np.unique(img[:, :, 1].ravel()), size=k).reshape(-1, 1)
    #means_blue = np.random.choice(np.unique(img[:, :, 2].ravel()), size=k).reshape(-1, 1)

    #means = np.dstack((means_red, means_green, means_blue))
    means = img[idx_x, idx_y, :]

    #means = np.expand_dims(means, axis=1)
    #red_idx = np.tile(np.array([True, False, False]), reps=k)
    #green_idx = np.tile(np.array([False, True, False]), reps=k)
    #blue_idx = np.tile(np.array([False, False, True]), reps=k)

else:
    means = np.random.choice(np.unique(img.ravel()), size=k)

num_iter = 15
means_delta = list()
means_list = list()
means_list.append(means.copy())

for i in range(num_iter):
#old_means = means

    # I want to vectorize the algorithm as much as possible, create mean array stacks
    means_stack = list()

    if len(img.shape) > 2:

        for mean in range(k):
            #means_stack.append(means[mean] * np.ones(img.shape))
            #means_stack.append(np.expand_dims(means[:, :, mean], axis=0) * np.ones(img.shape))
            means_stack.append(np.expand_dims(means[mean, :, :], axis=0) * np.ones(img.shape))

        means_stack = np.dstack(means_stack)

        norms_list = list()
        norms_stack = (np.tile(img, reps=k) - means_stack) ** 2

        for mean in range(k):
            # generate indices
            idx = np.arange(3) + mean
            norms_list.append(np.sum(norms_stack[:, :, idx], axis=-1))

        norms_stack = np.dstack(norms_list)

        clustered = np.argmin(norms_stack, axis = -1)

        for cluster in range(k):
            # create masks for each mean

            #mask = np.where(clustered == cluster, 1, 0)
            mask = np.where(clustered == cluster, 1, 0)

            mask = np.tile(np.expand_dims(mask,axis=-1), reps=3)

            # update the means vector
            #print(np.sum(img[mask.astype(bool)]), np.sum(mask))
            #means[cluster] = np.sum(img[mask]) / (np.sum(mask) + 1e-10)
            #means[cluster] = np.sum(img[mask.astype(bool)]) / (np.sum(mask) + 1e-10)
            cluster_ma = np.ma.masked_array(img, mask.astype(bool))

            # compute mean vector
            cluster_means = np.ma.mean(cluster_ma, axis = (0, 1))

            #means[:, :, cluster] = cluster_means.reshape(1, -1)
            means[cluster, :, :] = cluster_means.reshape(1, -1)

        #if means_delta[i] < 5:
        #    print("Soppted at Iter: ", i)
        #   break

    else:
        for mean in range(k):
            means_stack.append(means[mean] * np.ones(img.shape))
            #means_stack.append(np.expand_dims(means[:, :, mean], axis=0) * np.ones(img.shape))

        means_stack = np.dstack(means_stack)

        norms_stack = (np.expand_dims(img, axis=-1) - means_stack)**2

        clustered = np.argmin(norms_stack, axis = -1)

        for cluster in range(k):
            # create masks for each mean

            mask = np.where(clustered == cluster, 1, 0)
            #mask = np.where(clustered == cluster, 1, 0)

            #mask = np.tile(np.expand_dims(mask,axis=-1), reps=3)

            # update the means vector
            #print(np.sum(img[mask.astype(bool)]), np.sum(mask))
            #means[cluster] = np.sum(img[mask]) / (np.sum(mask) + 1e-10)
            means[cluster] = np.sum(img[mask.astype(bool)]) / (np.sum(mask) + 1e-10)

            #cluster_ma = np.ma.masked_array(img, mask.astype(bool))

            # compute mean vector
            #cluster_means = np.ma.mean(cluster_ma, axis = (0, 1))

            #means[:, :, cluster] = cluster_means.reshape(1, -1)
            #means[cluster, :, :] = cluster_means.reshape(1, -1)
    # Step 2) Assign samples to clusters: Assign each sample to the cluster set
    # whose mean is the closest (ties are resolved arbitrarily, but samples are
    # assigned to only one cluster)

    # first compute distance to clusters
    #norms_list = list()

    #for mean in range(k):
    #    norms_list = np.sqrt((img - means[mean])**2)

    #norms_stack = np.sqrt((np.expand_dims(img, axis=-1) - means_stack)**2)

    '''
    if len(img.shape) > 2:
        norms_list = list()
        norms_stack = (np.tile(img, reps=k) - means_stack) ** 2

        for mean in range(k):
            # generate indices
            idx = np.arange(3) + mean
            norms_list.append(np.sum(norms_stack[:, :, idx], axis=-1))

        norms_stack = np.dstack(norms_list)
    else:
        norms_stack = (np.expand_dims(img, axis=-1) - means_stack)**2
    '''
    #test_1 = norms_stack[:, :, 0]
    #test_2 = np.sqrt((img - means_stack[:, :, 0])**2)
    #np.isclose(test_1, test_2)
    #print(np.all(np.isclose(test_1, test_2)))

    # find the argmin along the means axis of the stack (axis = -1)
    # clustered = np.argmin(norms_stack, axis=-1).astype(np.uint8)

    '''
    if len(img.shape) > 2:

        red_clustered = np.argmin(norms_stack[:, :, red_idx], axis=-1)
        green_clustered = np.argmin(norms_stack[:, :, green_idx], axis=-1)
        blue_clustered = np.argmin(norms_stack[:, :, blue_idx], axis=-1)

        clustered = np.dstack((red_clustered, green_clustered, blue_clustered))

        for cluster in range(k):
            # create masks for each mean
            mask = np.where(clustered == np.tile(cluster, reps=k), 1, 0)

            # update the means vector
            print(np.sum(img[mask.astype(bool)]), np.sum(mask))
            #means[cluster] = np.sum(img[mask]) / (np.sum(mask) + 1e-10)
            #means[cluster] = np.sum(img[mask.astype(bool)]) / (np.sum(mask) + 1e-10)
            #a = np.sum(img[mask.astype(bool)]) / (np.sum(mask) + 1e-10)
            cluster_ma = np.ma.masked_array(img, mask.astype(bool))

            # compute mean vector
            cluster_means = np.ma.mean(cluster_ma, axis = (0, 1))

            means[:, :, cluster] = cluster_means.reshape(1, -1)
    '''
    #clustered = np.argmin(norms_stack, axis = -1)
    #else:
    #clustered = np.argmin(norms_stack, axis=-1)

    # Step 3) Update the cluster centers (means)
    # Now I need to cycle through the means

    #mask = np.where(clustered == 0,)
    #mask = np.where(clustered == 1, 1, 0)
    #print(np.sum(mask))
    #print(np.unique(mask, return_counts=True))

    #test = np.sum(img[mask.astype(bool)])
    '''
    for cluster in range(k):
        # create masks for each mean

        #mask = np.where(clustered == cluster, 1, 0)
        mask = np.where(clustered == cluster, 1, 0)

        mask = np.tile(np.expand_dims(mask,axis=-1), reps=3)

        # update the means vector
        #print(np.sum(img[mask.astype(bool)]), np.sum(mask))
        #means[cluster] = np.sum(img[mask]) / (np.sum(mask) + 1e-10)
        #means[cluster] = np.sum(img[mask.astype(bool)]) / (np.sum(mask) + 1e-10)
        cluster_ma = np.ma.masked_array(img, mask.astype(bool))

        # compute mean vector
        cluster_means = np.ma.mean(cluster_ma, axis = (0, 1))

        #means[:, :, cluster] = cluster_means.reshape(1, -1)
        means[cluster, :, :] = cluster_means.reshape(1, -1)
    '''
    means_list.append(means.copy())
    #means_delta.append(np.sqrt((means - old_means)**2))
    means_delta.append(np.sqrt(np.sum((means_list[i + 1] - means_list[i])**2)))

    if means_delta[i] < 5:
        print("Soppted at Iter: ", i)
        break

# update means
#means_in_clustered, counts_per_mean = np.unique(clustered, return_counts=True)

plt.figure()
plt.plot(means_delta)
plt.figure()
plt.imshow(clustered)
plt.show()

#from matplotlib import cm
#color_arr = plt.cm.get_cmap('hsv', k)

#plt.figure()
#plt.imshow(img)
#plt.imshow(clustered, cmap=color_arr)


import sqlite3
import os
import json
from util import str_list_to_int

MEDIA_PATH = os.path.join("data", "media")
MEDIA_DB_PATH = os.path.join("data", "media.db")

def main():
    filelist = os.listdir(MEDIA_PATH)

    if filelist is None or (len(filelist)) <= 0:
        print("no files found")
        return

    filelist_int = str_list_to_int(filelist)
    filelist = [str(x) for x in sorted(filelist_int)]

    with sqlite3.connect(MEDIA_DB_PATH) as conn:
        cur = conn.cursor()
        for filename in filelist:
            with open(os.path.join(MEDIA_PATH, filename), "r") as mfile:
                for line in mfile.readlines():
                    data = json.loads(line)
                    if type(data) != dict:
                        continue
                    if "title" not in data:
                        data["title"] = ""
                    cur.execute("insert into media (type, title, url) values (?, ?, ?)",
                            (data["type"], data["title"], data["url"]))
        conn.commit()

if __name__ == "__main__":
    main()


from __future__ import annotations

from typing import TYPE_CHECKING

from cleo.io.null_io import NullIO
from packaging.utils import canonicalize_name
from poetry.core.packages.package import Package

from poetry.factory import Factory
from tests.helpers import get_package
from tests.mixology.helpers import add_to_repo
from tests.mixology.helpers import check_solver_result
from tests.mixology.version_solver.conftest import Provider

if TYPE_CHECKING:
    from poetry.core.packages.project_package import ProjectPackage

    from poetry.repositories import Repository
    from poetry.repositories import RepositoryPool

def test_with_compatible_locked_dependencies(
    root: ProjectPackage, repo: Repository, pool: RepositoryPool
) -> None:
    root.add_dependency(Factory.create_dependency("foo", "*"))

    add_to_repo(repo, "foo", "1.0.0", deps={"bar": "1.0.0"})
    add_to_repo(repo, "foo", "1.0.1", deps={"bar": "1.0.1"})
    add_to_repo(repo, "foo", "1.0.2", deps={"bar": "1.0.2"})
    add_to_repo(repo, "bar", "1.0.0")
    add_to_repo(repo, "bar", "1.0.1")
    add_to_repo(repo, "bar", "1.0.2")

    locked = [get_package("foo", "1.0.1"), get_package("bar", "1.0.1")]
    provider = Provider(root, pool, NullIO(), locked=locked)

    check_solver_result(
        root,
        provider,
        result={"foo": "1.0.1", "bar": "1.0.1"},
    )

def test_with_incompatible_locked_dependencies(
    root: ProjectPackage, repo: Repository, pool: RepositoryPool
) -> None:
    root.add_dependency(Factory.create_dependency("foo", ">1.0.1"))

    add_to_repo(repo, "foo", "1.0.0", deps={"bar": "1.0.0"})
    add_to_repo(repo, "foo", "1.0.1", deps={"bar": "1.0.1"})
    add_to_repo(repo, "foo", "1.0.2", deps={"bar": "1.0.2"})
    add_to_repo(repo, "bar", "1.0.0")
    add_to_repo(repo, "bar", "1.0.1")
    add_to_repo(repo, "bar", "1.0.2")

    locked = [get_package("foo", "1.0.1"), get_package("bar", "1.0.1")]
    provider = Provider(root, pool, NullIO(), locked=locked)

    check_solver_result(
        root,
        provider,
        result={"foo": "1.0.2", "bar": "1.0.2"},
    )

def test_with_unrelated_locked_dependencies(
    root: ProjectPackage, repo: Repository, pool: RepositoryPool
) -> None:
    root.add_dependency(Factory.create_dependency("foo", "*"))

    add_to_repo(repo, "foo", "1.0.0", deps={"bar": "1.0.0"})
    add_to_repo(repo, "foo", "1.0.1", deps={"bar": "1.0.1"})
    add_to_repo(repo, "foo", "1.0.2", deps={"bar": "1.0.2"})
    add_to_repo(repo, "bar", "1.0.0")
    add_to_repo(repo, "bar", "1.0.1")
    add_to_repo(repo, "bar", "1.0.2")
    add_to_repo(repo, "baz", "1.0.0")

    locked = [get_package("baz", "1.0.1")]
    provider = Provider(root, pool, NullIO(), locked=locked)

    check_solver_result(
        root,
        provider,
        result={"foo": "1.0.2", "bar": "1.0.2"},
    )

def test_unlocks_dependencies_if_necessary_to_ensure_that_a_new_dependency_is_satisfied(
    root: ProjectPackage, repo: Repository, pool: RepositoryPool
) -> None:
    root.add_dependency(Factory.create_dependency("foo", "*"))
    root.add_dependency(Factory.create_dependency("newdep", "2.0.0"))

    add_to_repo(repo, "foo", "1.0.0", deps={"bar": "<2.0.0"})
    add_to_repo(repo, "bar", "1.0.0", deps={"baz": "<2.0.0"})
    add_to_repo(repo, "baz", "1.0.0", deps={"qux": "<2.0.0"})
    add_to_repo(repo, "qux", "1.0.0")
    add_to_repo(repo, "foo", "2.0.0", deps={"bar": "<3.0.0"})
    add_to_repo(repo, "bar", "2.0.0", deps={"baz": "<3.0.0"})
    add_to_repo(repo, "baz", "2.0.0", deps={"qux": "<3.0.0"})
    add_to_repo(repo, "qux", "2.0.0")
    add_to_repo(repo, "newdep", "2.0.0", deps={"baz": ">=1.5.0"})

    locked = [
        get_package("foo", "2.0.0"),
        get_package("bar", "1.0.0"),
        get_package("baz", "1.0.0"),
        get_package("qux", "1.0.0"),
    ]
    provider = Provider(root, pool, NullIO(), locked=locked)

    check_solver_result(
        root,
        provider,
        result={
            "foo": "2.0.0",
            "bar": "2.0.0",
            "baz": "2.0.0",
            "qux": "1.0.0",
            "newdep": "2.0.0",
        },
    )

def test_with_compatible_locked_dependencies_use_latest(
    root: ProjectPackage, repo: Repository, pool: RepositoryPool
) -> None:
    root.add_dependency(Factory.create_dependency("foo", "*"))
    root.add_dependency(Factory.create_dependency("baz", "*"))

    add_to_repo(repo, "foo", "1.0.0", deps={"bar": "1.0.0"})
    add_to_repo(repo, "foo", "1.0.1", deps={"bar": "1.0.1"})
    add_to_repo(repo, "foo", "1.0.2", deps={"bar": "1.0.2"})
    add_to_repo(repo, "bar", "1.0.0")
    add_to_repo(repo, "bar", "1.0.1")
    add_to_repo(repo, "bar", "1.0.2")
    add_to_repo(repo, "baz", "1.0.0")
    add_to_repo(repo, "baz", "1.0.1")

    locked = [
        get_package("foo", "1.0.1"),
        get_package("bar", "1.0.1"),
        get_package("baz", "1.0.0"),
    ]
    provider = Provider(root, pool, NullIO(), locked=locked)

    check_solver_result(
        root,
        provider,
        result={"foo": "1.0.2", "bar": "1.0.2", "baz": "1.0.0"},
        use_latest=[canonicalize_name("foo")],
    )

def test_with_compatible_locked_dependencies_with_extras(
    root: ProjectPackage, repo: Repository, pool: RepositoryPool
) -> None:
    root.add_dependency(Factory.create_dependency("foo", "^1.0"))

    package_foo_0 = get_package("foo", "1.0.0")
    package_foo_1 = get_package("foo", "1.0.1")
    bar_extra_dep = Factory.create_dependency(
        "bar", {"version": "^1.0", "extras": "extra"}
    )
    for package_foo in (package_foo_0, package_foo_1):
        package_foo.add_dependency(bar_extra_dep)
        repo.add_package(package_foo)

    bar_deps = {"baz": {"version": "^1.0", "extras": ["extra"]}}
    add_to_repo(repo, "bar", "1.0.0", bar_deps)
    add_to_repo(repo, "bar", "1.0.1", bar_deps)
    add_to_repo(repo, "baz", "1.0.0")
    add_to_repo(repo, "baz", "1.0.1")

    locked = [
        get_package("foo", "1.0.0"),
        get_package("bar", "1.0.0"),
        get_package("baz", "1.0.0"),
    ]
    provider = Provider(root, pool, NullIO(), locked=locked)

    check_solver_result(
        root,
        provider,
        result={"foo": "1.0.0", "bar": "1.0.0", "baz": "1.0.0"},
    )

def test_with_yanked_package_in_lock(
    root: ProjectPackage, repo: Repository, pool: RepositoryPool
) -> None:
    root.add_dependency(Factory.create_dependency("foo", "*"))

    add_to_repo(repo, "foo", "1")
    add_to_repo(repo, "foo", "2", yanked=True)

    # yanked version is kept in lock file
    locked_foo = get_package("foo", "2")
    assert not locked_foo.yanked
    provider = Provider(root, pool, NullIO(), locked=[locked_foo])
    result = check_solver_result(
        root,
        provider,
        result={"foo": "2"},
    )
    assert result is not None
    foo = result.packages[0]
    assert foo.yanked

    # without considering the lock file, the other version is chosen
    provider = Provider(root, pool, NullIO())
    check_solver_result(
        root,
        provider,
        result={"foo": "1"},
    )

def test_no_update_is_respected_for_legacy_repository(
    root: ProjectPackage, repo: Repository, pool: RepositoryPool
) -> None:
    root.add_dependency(Factory.create_dependency("foo", "^1.0"))

    foo_100 = Package(
        "foo", "1.0.0", source_type="legacy", source_url="http://example.com"
    )
    foo_101 = Package(
        "foo", "1.0.1", source_type="legacy", source_url="http://example.com"
    )
    repo.add_package(foo_100)
    repo.add_package(foo_101)

    provider = Provider(root, pool, NullIO(), locked=[foo_100])
    check_solver_result(
        root,
        provider,
        result={"foo": "1.0.0"},
    )


from core import app
from core.models.core.has_routes import HasRoutes

from core.models.auth.chimp import Chimp

from core.helpers.validation_rules import validation_rules
from core.helpers.json import to_json

from flask import request
from mailchimp3 import MailChimp

with app.app_context():
	class Newsletter(HasRoutes):

		endpoint = '/_newsletter'
		routes = [
			{
				'route': '/signup',
				'view_function': 'signup_view',
				'methods': ['POST']
			}
		]

		schema = {
			'email': validation_rules['email'],
			'shop': validation_rules['text']
		}

		@classmethod
		def signup_view(cls):
			data = cls.validate(cls._get_json_from_request())

			chimp = Chimp.get_where({'shop': data['shop']})
			client = MailChimp(chimp['username'], chimp['access_token'])

			try:
				response = client.lists.members.create(chimp['list_id'], {
					'email_address': data['email'],
					'status': 'subscribed'
				})

				return cls._format_response({
					'email': data['email'],
					'list_id': chimp['list_id'],
				})

			except Exception as e:
				return to_json({
					'email': data['email'],
					'list_id': chimp['list_id'],
					'error': e.response.json()['title']
				}, 400)


import esphome.codegen as cg
import esphome.config_validation as cv
from esphome import pins
from esphome.components import display
from esphome.const import (
    CONF_ID,
    CONF_INVERT,
    CONF_LAMBDA,
    CONF_PAGES,
)

DEPENDENCIES = ["esp32"]

CONF_GREYSCALE = "greyscale"

t547_ns = cg.esphome_ns.namespace("t547")
T547 = t547_ns.class_(
    "T547", cg.PollingComponent, display.DisplayBuffer
)

CONFIG_SCHEMA = cv.All(
    display.FULL_DISPLAY_SCHEMA.extend(
        {
            cv.GenerateID(): cv.declare_id(T547),
            cv.Optional(CONF_GREYSCALE, default=False): cv.boolean,
            cv.Optional(CONF_INVERT, default=True): cv.boolean,
        }
    )
    .extend(cv.polling_component_schema("5s")),
    cv.has_at_most_one_key(CONF_PAGES, CONF_LAMBDA),
    cv.only_with_arduino,
)

async def to_code(config):
    var = cg.new_Pvariable(config[CONF_ID])

    await cg.register_component(var, config)
    await display.register_display(var, config)

    if CONF_LAMBDA in config:
        lambda_ = await cg.process_lambda(
            config[CONF_LAMBDA], [(display.DisplayBufferRef, "it")], return_type=cg.void
        )
        cg.add(var.set_writer(lambda_))

    cg.add(var.set_greyscale(config[CONF_GREYSCALE]))
    cg.add(var.set_invert(config[CONF_INVERT]))

    cg.add_build_flag("-DBOARD_HAS_PSRAM")