"""
For reading and writing crap.
"""

import argparse
import csv
from typing import Tuple, List, Dict
import os
import time

import torch
from tqdm import tqdm

import constants  # for speed test script; not needed for data IO
import viewer

# lib functions

def which_exist(filenames: List[str]) -> List[str]:
    """
    Returns the subset of filenames that exist on disk.
    """
    return list(filter(lambda f: os.path.exists(f), filenames))

def split_tensor(
    data: torch.Tensor, classes: int = 1
) -> Tuple[torch.IntTensor, torch.FloatTensor]:
    """
    Splits 'all data' tensor into labels and features tensors.

    Returns 2-tuple of:
        (1) either a 1d (N) vector (if classes == 1),
                or a 2d (N x L) matrix (if classes > 1)
        (2) a 2d (N x D) matrix of datums x features
    """
    # have to be careful when selecting one column:
    # - selecting data[:, 0]  gives a 1d (N) vector
    # - selecting data[:, :1] gives a 2d (N x 1) matrix
    if classes == 1:
        labels = data[:, 0].type(torch.IntTensor)
    else:
        labels = data[:, :classes].type(torch.IntTensor)
    features = data[:, classes:]
    return labels, features

def csv_to_tensor(filename: str) -> torch.Tensor:
    """
    Loads all data from filename in csv format; returns in a single tensor.
    """
    with open(filename, "r") as f:
        rows = [r for r in csv.reader(f, quoting=csv.QUOTE_NONNUMERIC)]
    return torch.Tensor(rows)

def csv_to_tensors(filename: str) -> Tuple[torch.IntTensor, torch.FloatTensor]:
    """
    Loads data from filename in csv format; return label (col 0) and features
    (rest) tensors.
    """
    return split_tensor(csv_to_tensor(filename))

def tensor_to_csv(t: torch.Tensor, filename: str) -> None:
    """
    Writes tensor t to filename on disk in csv format, creating parent directories as needed.
    """
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with open(filename, "w") as f:
        w = csv.writer(f)
        w.writerows(t)

def tensor_to_bin(t: torch.Tensor, filename: str) -> None:
    """
    Writes tensor t to filename on disk in torch.save(...) format, creating
    directories as needed.
    """
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    torch.save(t, filename)

def bin_to_tensor(filename: str) -> torch.Tensor:
    """
    Loads tensor from disk at filename in torch.save(...) format.
    """
    return torch.load(filename)

def bin_to_tensors(
    filename: str, classes: int = 1
) -> Tuple[torch.IntTensor, torch.FloatTensor]:
    """
    Loads data from filename in torch.save(...) format; return label
    (classes) and features (rest) tensors.

    Returns 2-tuple of:
        (1) either a 1d (N) vector (if classes == 1),
                or a 2d (N x L) matrix (if classes > 1)
        (2) a 2d (N x D) matrix of datums x features
    """
    return split_tensor(bin_to_tensor(filename), classes)

def bias_tensor(t: torch.Tensor) -> torch.Tensor:
    """
    Adds bias column (all 1s) to `t`.

    Arguments:
        t 2D (N x D) tensor

    Retuns:
          2D (N x D+1) tensor, with final bias column (all 1s)
    """
    n = len(t)
    bias_col = torch.ones(n)
    return torch.cat([t, bias_col], dim=1)

def labels_to_onehot(labels: torch.IntTensor, classes: int) -> torch.IntTensor:
    """
    Turns 1d (N) class-label tensor `labels` into 2d (N x classes) onehot
    tensor and returns it.

    Arguments:
        labels: 1d (N) vector of class labels
        classes: number of class label options (will be output cols)

    Returns:
        2d (N x classes) onehot label matrix
    """
    n, = labels.size()
    label_idx = labels.type(torch.LongTensor).view(-1, 1)
    return torch.IntTensor(n, classes).zero_().scatter_(1, label_idx, 1)

# script

def speedtest(n: int = 3) -> None:
    """
    Tests speed of loading csv vs torch.load(...) on a tensor. Does `n`
    iterations of tests on several different file sizes. Plots results to
    visdom.
    """
    tests = [
        ("big file, csv", csv_to_tensor, constants.TRAIN_NORM),
        ("big file, tensor", bin_to_tensor, constants.TRAIN_TENSOR),
        ("small file, csv", csv_to_tensor, constants.VAL_NORM),
        ("small file, tensor", bin_to_tensor, constants.VAL_TENSOR),
    ]
    results = {}  # type: Dict[str, List[float]]
    for desc, fn, arg in tqdm(tests):
        results[desc] = []
        for i in range(n):
            start = time.perf_counter()
            res = fn(arg)
            results[desc].append(time.perf_counter() - start)
            del res
    viewer.plot_jitter(results, "Data Loading Speeds")

def convert() -> None:
    """
    Converts files from this project from csv to torch.save(...) format.
    """
    worklist = [
        (constants.TRAIN_NORM, constants.TRAIN_TENSOR),
        (constants.VAL_NORM, constants.VAL_TENSOR),
        (constants.TEST_NORM, constants.TEST_TENSOR),
    ]

    # pre-check: don't convert if any dest. files exist
    existing = which_exist([out for inp, out in worklist])
    if len(existing) > 0:
        print(
            "ERROR: Not converting because the following files already "
            "exist: {}".format(existing)
        )
        return

    print("dataio.convert :: start")
    for csv_fn, bin_fn in worklist:
        print("\t Converting {} to {}...".format(csv_fn, bin_fn))
        # This is the actual line of code that does the conversion.
        tensor_to_bin(csv_to_tensor(csv_fn), bin_fn)
    print("dataio.convert :: finish")

def bias() -> None:
    """
    Adds bias term to files from this project (output from convert()).
    """
    worklist = [
        (constants.TRAIN_TENSOR, constants.TRAIN_BIAS),
        (constants.VAL_TENSOR, constants.VAL_BIAS),
        (constants.TEST_TENSOR, constants.TEST_BIAS),
    ]

    existing = which_exist([out for inp, out in worklist])
    if len(existing) > 0:
        print(
            "ERROR: Not adding bias because the following files already "
            "exist: {}".format(existing)
        )
        return

    print("dataio.bias :: start")
    for tensor_fn, bias_fn in worklist:
        print("\t Converting {} to {}...".format(tensor_fn, bias_fn))
        # This is the actual line of code that does the biasing.
        tensor_to_bin(bias_tensor(bin_to_tensor(tensor_fn)), bias_fn)
    print("dataio.bias :: finish")

def onehot() -> None:
    """
    Transform labels to onehot for files from this project (output from
    bias()).
    """
    worklist = [
        (constants.TRAIN_BIAS, constants.TRAIN_ONEHOT),
        (constants.VAL_BIAS, constants.VAL_ONEHOT),
        (constants.TEST_BIAS, constants.TEST_ONEHOT),
    ]

    existing = which_exist([out for inp, out in worklist])
    if len(existing) > 0:
        print(
            "ERROR: Not creating onehots because the following files "
            "already exist: {}".format(existing)
        )
        return

    print("dataio.onehot :: start")
    for bias_fn, onehot_fn in worklist:
        print("\t Converting {} to {}...".format(bias_fn, onehot_fn))
        # This is the actual code that does the onehot'ing.
        class_labels, data = bin_to_tensors(bias_fn)
        onehot_labels = labels_to_onehot(class_labels, 10).type(torch.FloatTensor)
        tensor_to_bin(torch.cat([onehot_labels, data], dim=1), onehot_fn)
    print("dataio.onehot :: finish")

def main() -> None:
    """
    Data utils: speed test, conversion, biasing, making one-hot (see body).
    """
    parser = argparse.ArgumentParser(description="Data stuff.")
    choice = parser.add_mutually_exclusive_group(required=True)
    choice.add_argument("--speedtest", action="store_true")
    choice.add_argument("--convert", action="store_true")
    choice.add_argument("--bias", action="store_true")
    choice.add_argument("--onehot", action="store_true")
    args = parser.parse_args()
    if args.speedtest:
        speedtest()
    if args.convert:
        convert()
    if args.bias:
        bias()
    if args.onehot:
        onehot()

if __name__ == "__main__":
    main()


# -*- coding: utf-8 -*-
from flask.ext.assets import Bundle, Environment

css = Bundle(
    "libs/bootstrap/dist/css/bootstrap.css",
    "css/style.css",
    "css/bootstrap-multiselect.css",
    "css/bootstrap-tagsinput.css",
    "css/selectize.css",
    filters="cssmin",
    output="public/css/common.css"
)

js = Bundle(
    "libs/jQuery/dist/jquery.js",
    "libs/bootstrap/dist/js/bootstrap.js",
    "libs/ajax/ckeditor/ckeditor.js",
    "js/plugins.js",
    "js/bootstrap-multiselect.js",
    "js/bootstrap-tagsinput.js",
    "js/selectize.js",
    "js/script.js",
    filters='jsmin',
    output="public/js/common.js"
)

assets = Environment()

assets.register("js_all", js)
assets.register("css_all", css)


import argparse
import pickle

import matplotlib.pyplot as plt
import seaborn as sns
from sklearn import decomposition
from sklearn import pipeline
from sklearn import preprocessing
from sklearn import cluster
from sklearn import metrics

sns.set()

from typing import Dict, Any

def dummy_data():
    import numpy as np
    a = np.random.normal(0, 0.1, (100, 10))
    b = np.random.normal(0, 0.1, (100, 10))
    return np.vstack((a, b))

def main(logits_file_path: str) -> None:
    print(logits_file_path)  # load file here
    data = dummy_data()

    kmeans = cluster.KMeans(n_clusters=2)
    fun = pipeline.make_pipeline(preprocessing.StandardScaler(), decomposition.PCA(n_components=3))
    data_transformed = fun.fit_transform(data)

    ypred = kmeans.fit_predict(data_transformed)
    print("> silhouette score: {}".format(metrics.silhouette_score(data_transformed, ypred)))

    fig, (ax1, ax2) = plt.subplots(nrows=1, ncols=2)

    first = ax1.scatter(data_transformed[:100, 0], data_transformed[:100, 1], c=data_transformed[:100, 2], marker="o", label="Legitimate")
    second = ax1.scatter(data_transformed[100:200, 0], data_transformed[100:200, 1], c=data_transformed[100:200, 2], marker="x", label="Watermarked")
    ax1.figure.colorbar(second, ax=ax1, label="Third component")

    ax1.set_xlabel("First component")
    ax1.set_ylabel("Second component")
    ax1.set_title("Ground")
    for class_, m in zip([0, 1], ["o", "x"]):
        ax2.scatter(data_transformed[ypred == class_, 0], data_transformed[ypred == class_, 1], c=data_transformed[ypred == class_, 2], marker=m)

    ax2.figure.colorbar(second, ax=ax2, label="Third component")

    ax2.set_xlabel("First component")
    ax2.set_ylabel("Second component")
    ax2.set_title("Clustering labels")
    plt.legend()
    plt.show()

def load_file(file_path: str) -> Dict[str, Any]:
    with open(file_path, "rb") as f:
        return pickle.load(f)

def handle_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--logits_file",
        type=str,
        default=None,
        help="Results data file for plotting.")

    args = parser.parse_args()
    if args.logits_file is None:
        raise ValueError("Logits data file must be provided.")

    return args

if __name__ == "__main__":
    args = handle_args()
    main(args.logits_file)


import tweepy

class TwitterClient:
    def __init__(self,
                 consumer_key=None,
                 consumer_secret=None,
                 access_key=None,
                 access_secret=None):
        auth = tweepy.OAuthHandler(consumer_key, consumer_secret)
        auth.set_access_token(access_key, access_secret)
        self.client = tweepy.API(auth)

    def send(self, message):
        self.client.update_status(message)


import json
from typing import Optional

from fastapi import APIRouter, Depends
from sqlalchemy import select
from starlette.background import BackgroundTasks
from starlette.requests import Request
from starlette.templating import Jinja2Templates
from telegram.ext import Application
from telegram.helpers import escape_markdown

from shaas_common.model import Menu, MenuItem, Event, Order, EventState
from shaas_common.poll import close_poll_if_necessary
from shaas_common.settings import get_settings
from shaas_common.storage import Storage, get_db
from shaas_web.bot import get_bot

web_app_router = APIRouter()

templates = Jinja2Templates(directory="templates")

@web_app_router.get("/{menu_id}")
async def get_menu(
        menu_id: int,
        request: Request,
        s:Storage=Depends(get_db),
        user_id: Optional[int] = None,
        user_hash: Optional[str] = None,
        chat_id: Optional[int] = None
):
    menu = await s.menu_item.get_items(menu_id)

    current = await s.event.get_current(chat_id)
    active = current and current.state == EventState.collecting_orders

    data = {
        "request": request,
        "menu": menu,
        "user_id": user_id,
        "user_hash": user_hash,
        "active": "true" if active else "false",
        "event_id": current.id if current else 0
    }
    return templates.TemplateResponse("index.html", data)

@web_app_router.post("/order")
async def make_order(request: Request, background_tasks: BackgroundTasks, app = Depends(get_bot)):
    s = Storage()

    form = await request.form()

    user_id = int(form["user_id"])
    order_raw = json.loads(form["order_data"])
    event_id = int(form["event_id"])
    comment = form.get("comment")
    # comment = escape_markdown(comment)

    event = await s.event.get(event_id)

    if event.state != EventState.collecting_orders:
        await app.bot.send_message(
            chat_id=user_id,
            text="Это меню для события, которое уже закрыто.\n\nЗаказ не оформлен."
        )
        return {
            "ok": True
        }

    order_data = dict()
    for row in order_raw:
        key_id = int(row["id"])
        key = await s.menu_item.get(key_id)
        value = int(row["count"])
        order_data[key] = value

    await s.order.create_order(user_id, event.id, order_data, comment)
    await s.commit()

    items_str = []
    total_price = 0
    for item, count in order_data.items():
        price = count*item.price
        total_price += price
        items_str.append(
            f"{count}x {item.name} - {price} р."
        )

    msg = "Заказ принят!\n\n" + "\n".join(items_str) + f"\n\nИтого: {total_price} р."
    if comment:
        msg += f"\n\nКомментарий:\n{comment}"

    await app.bot.send_message(
        chat_id=user_id,
        text=msg
    )

    await close_poll_if_necessary(s, app.bot, event.id)
    # background_tasks.add_task(close_poll_if_necessary, s, app.bot, event_id)

    return {
        "ok": True
    }


# -*- coding: utf-8 -*-
import os
from setuptools import setup, find_packages

root = os.path.abspath(os.path.dirname(__file__))

version = __import__('events_watcher').__version__

with open(os.path.join(root, 'README.rst')) as f:
    README = f.read()

setup(
    name='django-events-watcher',
    version=version,
    description='Events watcher is an event packaging library for Django to track changes made in your models.',
    long_description=README,
    author='Florent Messa',
    author_email='florent.messa@gmail.com',
    url='http://github.com/thoas/django-events-watcher',
    packages=find_packages(),
    zip_safe=False,
    include_package_data=True,
    classifiers=[
        'Environment :: Web Environment',
        'Intended Audience :: Developers',
        'License :: OSI Approved :: MIT License',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3',
        'Topic :: Utilities',
    ]
)


#!/usr/bin/env python2

"""
AR.Drone's position controller.

PD controller. Needs slight tuning.

Tausif S., 2017

"""

from __future__ import division, print_function, absolute_import

# Import ROS libraries
import roslib
import rospy
import numpy as np

# Import class that computes the desired positions
from tf.transformations import euler_from_quaternion
from geometry_msgs.msg import TransformStamped, Twist

from std_msgs.msg import String
import math

class PositionController(object):
    """ROS interface for controlling the Parrot ARDrone in the Vicon Lab."""
    # write code here for position controller
    def __init__(self):
        """Initialize the PositionController class."""

        # Current position.
        self.current_trans_x = 0.
        self.current_trans_y = 0.
        self.current_trans_z = 0.

        # Current quaternion.
        self.current_rot_x = 0.
        self.current_rot_y = 0.
        self.current_rot_z = 0.
        self.current_rot_w = 0.

        self.current_yaw = 0

        # Desired postion for trajectory goal.
        self.desired_x = 0.
        self.desired_y = 0.
        self.desired_z = 0.
        self.desired_yaw = 0.

        # Placeholder for old variable for differentiation.
        self.x_old = 0.
        self.y_old = 0.
        self.z_old = 0.
        self.climb_rate_old = 0.

        self.x_desired_old = 0.
        self.y_desired_old = 0.
        self.z_desired_old = 0.

        self.desired_x_err = 1.
        self.desired_y_err = 1.
        self.desired_z_err = 1.
        self.desired_yaw_err = 1.

    def update_position_controller(self, dt):
        """Postion Controller.

        Parameters
        ----------
        dt: float
            Change in time (seconds) used for differentiation and intergration
            purposes.

        Returns
        -------
        controlled_data: 1x4 array
            This array stores the 4 computed by the position controller.
            [roll, pitch, yaw_rate, climb_rate]
        """

        g = 9.8                            # Acceleration due to gravity
        # damping_rt_x = 0.7                    # Damping ratio
        # damping_rt_y = 0.7
        rise_time_z = 1
        rise_time_yaw = 1.

        x_now = self.current_trans_x
        y_now = self.current_trans_y
        z_now = self.current_trans_z

        #---------------------------------------------------------------------
        # Step 1: Calculating current the mass-normalized thrust.
        #---------------------------------------------------------------------

        climb_rate_now = (z_now - self.z_old) / dt
        climb_accel = (climb_rate_now - self.climb_rate_old) / dt

        quaternion = np.array([self.current_rot_x,
                              self.current_rot_y,
                              self.current_rot_z,
                              self.current_rot_w])

        euler = euler_from_quaternion(quaternion)
        roll_angle = euler[0]
        pitch_angle = euler[1]
        yaw_angle = euler[2]

        self.current_yaw = yaw_angle

        # f = (climb_accel + g) / (np.cos(roll_angle) * np.cos(pitch_angle))
        f = (0 + g) / (math.cos(roll_angle) * math.cos(pitch_angle))

        self.z_old = z_now
        self.climb_rate_old = climb_rate_now

        #---------------------------------------------------------------------
        # Step 2: Determining the desired x/y accelerations.
        #---------------------------------------------------------------------
        x_rate_now = (x_now - self.x_old) / dt
        y_rate_now = (y_now - self.y_old) / dt

        # print("x/y rate: {0} {1}".format(x_rate_now, y_rate_now))

        desired_x_rate = (self.desired_x - self.x_desired_old)/dt
        desired_y_rate = (self.desired_y - self.y_desired_old)/dt
        # desired_z_rate = 0

        damping_rt_x = 1
        damping_rt_y = 1

        w_n_x = 1
        w_n_y = 1

        x_accel = 2.0*damping_rt_x*w_n_x*(desired_x_rate - x_rate_now) + math.pow(w_n_x, 2)*(self.desired_x - x_now)
        y_accel = 2.0*damping_rt_y*w_n_y*(desired_y_rate - y_rate_now) + math.pow(w_n_y, 2)*(self.desired_y - y_now)

        # print("x/y accel: {0} {1}".format(x_accel, y_accel))
        print("x/y/z des: {0} {1} {2}".format(self.desired_x, self.desired_y, self.desired_z))
        print("x/y/z now: {0} {1} {2}".format(x_now, y_now, z_now))

        self.x_old = x_now
        self.y_old = y_now
        self.x_desired_old = self.desired_x
        self.y_desired_old = self.desired_y

        #---------------------------------------------------------------------
        # Step 3: Calculating commanded roll and pitch.
        #---------------------------------------------------------------------
        roll_command_rt = -y_accel / f

        if roll_command_rt >= 1.:
            roll_command_rt = 1.
        if roll_command_rt <= -1.:
            roll_command_rt = -1.

        roll_command = math.asin(roll_command_rt)

        pitch_command_rt = x_accel / (f*math.cos(roll_command))

        if pitch_command_rt >= 1.:
            pitch_command_rt = 1.
        if pitch_command_rt <= -1.:
            pitch_command_rt = -1.

        pitch_command = math.asin(pitch_command_rt)

        if yaw_angle >= 0 or yaw_angle <= 0:
            roll_command_B = (roll_command*math.cos(yaw_angle)) + (pitch_command*math.sin(yaw_angle))
            pitch_command_B = (-roll_command*math.sin(yaw_angle)) + (pitch_command*math.cos(yaw_angle))
            roll_command = roll_command_B
            pitch_command = pitch_command_B

        #---------------------------------------------------------------------
        # Step 4: Calculating commanded climb and yaw rates.
        #---------------------------------------------------------------------
        tau_z = rise_time_z/2.2
        tau_yaw = rise_time_yaw/2.2

        climb_rate_command = (1/tau_z)*(self.desired_z - self.current_trans_z)

        if climb_rate_command >= 2:
            climb_rate_command = 2

        # Ensuring the yaw angles are in the respective quadrant.
        for i in range(0, 1):
            if ((355*(np.pi/180)) <= self.desired_yaw <= (5*(np.pi/180))):
                break
            elif (yaw_angle < 0) and ((6*(np.pi/180)) <= self.desired_yaw <= (354*(np.pi/180))):
                yaw_angle = yaw_angle + (2*np.pi)
                break
            else:
                pass

        yaw_rate_command = (0.85)*(self.desired_yaw - yaw_angle)

        # Updating the desired pose error.
        self.desired_x_err = np.absolute((np.absolute(x_now) - (self.desired_x + 0.5)) / (self.desired_x + 0.5))
        self.desired_y_err = np.absolute((np.absolute(y_now) - (self.desired_y + 0.5)) / (self.desired_y + 0.5))
        self.desired_z_err = np.absolute((z_now - (self.desired_z + 0.001)) / (self.desired_z + 0.001))
        self.desired_yaw_err = np.absolute((yaw_angle - (self.desired_yaw + 0.01)) / (self.desired_yaw + 0.01))

        # Returning the commanded roll/pitch/yaw/climb
        controlled_data = np.array([roll_command, pitch_command, yaw_rate_command, climb_rate_command])

        return controlled_data


"""Twitter URL Configuration

The `urlpatterns` list routes URLs to views. For more information please see:
    https://docs.djangoproject.com/en/1.8/topics/http/urls/
Examples:
Function views
    1. Add an import:  from my_app import views
    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
Class-based views
    1. Add an import:  from other_app.views import Home
    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
Including another URLconf
    1. Add an import:  from blog import urls as blog_urls
    2. Add a URL to urlpatterns:  url(r'^blog/', include(blog_urls))
"""
from django.conf.urls import include, url
from django.contrib import admin
from django.conf import settings
from main import views
from django.conf.urls.static import static

urlpatterns = [
    url(r'^admin/', include(admin.site.urls)),
    url('^registerForm$', views.register, name='register'),
    url('^$', views.main, name='main'),
    url('^addTweet/$', views.new_tweet, name='tweet'),
    url('^login$', views.login_page, name="login_page"),
    url('^profile/(?P<username>\w+)$', views.profile, name='profile'),
    url('^settings/$', views.settings, name='settings'),
    url('^profile_pic/$', views.profile_pic, name='profile_pic'),
    url(r'^logout/', views.log_out, name='log_out'),
    url(r'^follow/(?P<username>\w+)$', views.follow, name='follow'),
    url('^likes/(?P<username>\w+)/(?P<id>\d+)$', views.likes, name='likes'),
    url(r'^all_messages/(?P<username>\w+)$', views.all_messages, name='all_messages'),
    url(r'^show_message$', views.show_message, name='show_message'),
    url(r'^send_message$', views.send_message, name='send_message'),
    url(r'^delete_conversation', views.delete_conversation, name='delete_conversation'),
    url(r'^check_message', views.check_message, name='check_message'),
    ]

if settings.DEBUG:
    urlpatterns += static(settings.STATIC_URL, document_root=settings.STATIC_ROOT)
    urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


from datetime import datetime
from tensorboardX import SummaryWriter

import os
import logging

from utils.utils import create_logger, output_process, fix_random

class BaseTrainer(object):
    def __init__(self, experimental_name='debug', seed=None):
        # BASE
        self.current_time = datetime.now().strftime('%b.%d_%H.%M.%S')
        self.writer = None
        self.logger = None
        self.experimental_name = experimental_name
        self.seed = seed

        # SAVE PATH
        self.repo_path = os.getcwd()
        self.save_folder = f'{self.repo_path}/results/{experimental_name}'

        output_process(self.save_folder)  # create folder or not
        self._init_log()  # get log and writer
        if seed is not None:
            fix_random(seed)

    def _init_log(self):
        self.writer = SummaryWriter(log_dir=self.save_folder)
        self.logger = create_logger()
        fh = logging.FileHandler(filename=f'{self.save_folder}/log.txt')
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)


import pandas as pd
import numpy as np
import seaborn as sns
import lightgbm as lgb
import xgboost as xgb
from functools import partial
from bayes_opt import BayesianOptimization
import gc
from sklearn.metrics import accuracy_score , mean_squared_error , f1_score , average_precision_score , roc_auc_score , confusion_matrix , precision_recall_curve , roc_auc_score
from sklearn.model_selection import StratifiedKFold , train_test_split , KFold
from time import time
import warnings
warnings.filterwarnings("ignore", category=DeprecationWarning)
warnings.filterwarnings("ignore", category=UserWarning)
warnings.filterwarnings("ignore", category=RuntimeWarning)
warnings.filterwarnings("ignore", category=FutureWarning)

import lightgbm as lgb
global_regressor  = lgb.LGBMRegressor
global_classifier = lgb.LGBMClassifier

# mode = regression , classification
def get_max_f1_cutoff(model , xs_val , ys_val ):
    pred_train_val = model.predict_proba(xs_val)[:, 1]
    if np.isnan(pred_train_val).sum() > 0:
        return 0

    pr  ,re ,th= precision_recall_curve(ys_val , pred_train_val)
    th = np.concatenate([th, np.array([1])])
    df_prre = pd.DataFrame( np.array([pr , re , th]).T , columns = ['pr' , 're' , 'th'])
    df_prre['f1'] = 2*(df_prre['pr']*df_prre['re'])/(df_prre['pr']+df_prre['re'])
    df_prre = df_prre.sort_values(by = 'f1' , ascending = False)
    return df_prre.iloc[0,:]

def objective_fix(params, xs , ys , xtest , ytest , mode = 'regression',usef1 = False, cat_feature = []):
    if mode == 'regression':
        params['objective'] = 'regression'
        params['metric'] = 'l2'

        model = global_regressor(**params)
        model.fit(xs,ys,eval_set=[(xtest,ytest)] , categorical_feature = cat_feature)
        pred = model.predict(xtest)

        #evaluation
        result = -1 * mean_squared_error(ytest, pred)

    elif mode == 'classification':
        params['objective'] = 'multiclass'
        params['metric'] = 'multiclass'
        params['num_class'] = len(np.unique(ys))

        model = global_classifier(**params)
        model.fit(xs,ys,eval_set=[(xtest,ytest)] , categorical_feature = cat_feature)
        pred = model.predict(xtest)

        #evaluation
        result = None
        if usef1:
            result = f1_score(ytest, pred , average = 'micro')
        else:
            result = accuracy_score(ytest, pred)

    elif mode == 'binary':
        params['objective'] = 'binary'
        params['metric'] = 'auc'

        model = global_classifier(**params)
        model.fit(xs,ys,eval_set=[(xtest,ytest)] , categorical_feature = cat_feature)
        pred = model.predict(xtest)

        #evaluation
        result = None
        if usef1:
            result = get_max_f1_cutoff(model, xtest, ytest)[-1]  # f1 value
        else:
            result = roc_auc_score(ytest, model.predict_proba(xtest)[:, 1])  # auc value

    return result

def objective_fold(params,xs,ys,n_split = 5, mode = 'regression',usef1 = False, cat_feature = []):

    if mode == 'regression':
        kfold = KFold(n_splits=n_split, random_state=7, shuffle=True)
    elif (mode == 'classification') or (mode == 'binary'):
        kfold = StratifiedKFold(n_splits=n_split, random_state=7, shuffle=False)

    allscore = []
    for train_index, val_index  in kfold.split(xs,ys):
        xtrain, xtest = xs[train_index], xs[val_index]
        ytrain, ytest = ys[train_index], ys[val_index]

        if mode == 'regression':
            params['objective'] = 'regression' # 트레이닝에 사용
            params['metric'] = 'l2' # eval_set에 사용

            model = global_regressor(**params)
            model.fit(xtrain,ytrain,eval_set=[(xtest,ytest)], categorical_feature = cat_feature)
            pred = model.predict(xtest)

            #evaluation
            result = -1 * mean_squared_error(ytest, pred)

        elif mode == 'classification':
            params['objective'] = 'multiclass'
            params['metric'] = 'multiclass'
            params['num_class'] = len(np.unique(ys))

            model = global_classifier(**params)
            model.fit(xtrain,ytrain,eval_set=[(xtest,ytest)], categorical_feature=cat_feature )
            pred = model.predict(xtest)
            #pred_round = pred.round()
            result = None
            if usef1:
                result = f1_score(ytest, pred , average = 'micro')
            else:
                result = accuracy_score(ytest, pred)

        elif mode == 'binary':
            params['objective'] = 'binary'
            params['metric'] = 'auc'

            model = global_classifier(**params)
            model.fit(xtrain,ytrain,eval_set=[(xtest,ytest)] , categorical_feature = cat_feature)
            pred = model.predict(xtest)

            #evaluation
            result = None
            if usef1:
                result = get_max_f1_cutoff(model, xtest, ytest)[-1]  # f1 value
            else:
                result = roc_auc_score(ytest, model.predict_proba(xtest)[:, 1])  # auc value

        allscore.append(result)

    return np.asarray(allscore).mean()

def create_bysop_eval(params , mode ,  xdata , ydata , xtest = None , ytest = None, n_split = 5 , use_f1 = False, cat_feature= []):
    def lgbm_eval(num_iterations,
                  num_leaves,
                  colsample_bytree,
                  subsample,
                  max_depth,
                  reg_alpha,
                  reg_lambda,
                  min_split_gain,
                  min_child_weight,
                  min_data_in_leaf,
                  lambda_l1,
                  lambda_l2,
                  learning_rate,
                  bagging_fraction,
                  feature_fraction
                  ):
        params["num_iterations"] = int(num_iterations)
        params["num_leaves"] = int(num_leaves)
        params['colsample_bytree'] = max(min(colsample_bytree, 1), 0)
        params['subsample'] = max(min(subsample, 1), 0)
        params['max_depth'] = int(max_depth)
        params['reg_alpha'] = max(reg_alpha, 0)
        params['reg_lambda'] = max(reg_lambda, 0)
        params['min_split_gain'] = min_split_gain
        params['min_child_weight'] = min_child_weight
        params['min_data_in_leaf'] = int(min_data_in_leaf)
        params['lambda_l1'] = max(lambda_l1, 0)
        params['lambda_l2'] = max(lambda_l2, 0)
        params['learning_rate'] = max(learning_rate, 0.0001)
        params['bagging_fraction'] = min(bagging_fraction, 1.0)
        params['feature_fraction'] = min(feature_fraction, 1.0)
        #params['verbose']= -1

         # start scoring
        if (type(xtest) == type(None) ) and ( type(ytest) == type(None) ):
            cv_score = objective_fold(params, xdata, ydata, n_split=n_split, mode=mode, usef1=use_f1, cat_feature = cat_feature)  # 여기에 전역 데이터를 넣어야 한다.
        else:
            cv_score = objective_fix(params, xdata, ydata, xtest, ytest, mode=mode, usef1=use_f1, cat_feature = cat_feature)  # 여기에 전역 데이터를 넣어야 한다.
        gc.collect()
        return cv_score
    return lgbm_eval

# multi calss classification

params = dict()
params['task'] = 'train'
params['device'] = 'gpu'
params['boosting'] = 'gbdt'

params['is_unbalance'] = True
#params["learning_rate"] = 0.02
params['seed']=326
params['bagging_seed']=326
#params['early_stopping_rounds'] = 100

# Data and run
# Data Load
# exsample
from sklearn.datasets import fetch_california_housing, load_breast_cancer, load_iris

df_regression = fetch_california_housing()
df_binary = load_breast_cancer()
df_multi = load_iris()

df_list = {}
df_list['regression'] = df_regression
df_list['binary'] = df_binary
df_list['classification'] = df_multi

starttime = time()
for k, v in df_list.items():
    if k == 'regression':
        X_train, X_test, y_train, y_test = train_test_split(v.data, v.target, test_size=0.20,random_state= 517)
    else:
        X_train, X_test, y_train, y_test = train_test_split(v.data, v.target, test_size=0.20, stratify=v.target, random_state= 517)

    y_train = y_train.flatten()
    y_test = y_test.flatten()

    lgb_eval = create_bysop_eval(params ,k,X_train,y_train,X_test,y_test,5, use_f1 = True, cat_feature = [])
    clf_bo = BayesianOptimization(lgb_eval, {'num_iterations' : (2, 1000),
                                            'num_leaves': (42, 5000),
                                            'colsample_bytree': (0.001, 1),
                                            'subsample': (0.001, 1),
                                            'max_depth': (4, 15),
                                            'reg_alpha': (0, 20),
                                            'reg_lambda': (0, 10),
                                            'min_split_gain': (0, 1),
                                            'min_child_weight': (0, 45),
                                            'min_data_in_leaf': (1, 10),
                                           'lambda_l1': (0, 10),
                                           'lambda_l2': (0, 10),
                                           'learning_rate' : (0.6,0.001),
                                            'bagging_fraction' : (0.1,1.0),
                                            'feature_fraction' : (0.1,1.0)})
    clf_bo.maximize(init_points=1, n_iter=2)
    print(clf_bo.max)
    print('-' * 100)

    lgb_eval = create_bysop_eval(params ,k ,X_train,y_train,5, use_f1 = True, cat_feature = [])
    clf_bo = BayesianOptimization(lgb_eval, {'num_iterations' : ( 2 , 1000 ),
                                             'num_leaves': (42, 5000),
                                            'colsample_bytree': (0.001, 1),
                                            'subsample': (0.001, 1),
                                            'max_depth': (4, 15),
                                            'reg_alpha': (0, 20),
                                            'reg_lambda': (0, 10),
                                            'min_split_gain': (0, 1),
                                            'min_child_weight': (0, 45),
                                            'min_data_in_leaf': (0, 100),
                                            'lambda_l1': (0, 10),
                                            'lambda_l2': (0, 10),
                                            'learning_rate' : (0.6,0.001),
                                            'bagging_fraction' : (0.1,1.0),
                                            'feature_fraction' : (0.1,1.0)})
    clf_bo.maximize(init_points=1, n_iter=2)
    print(clf_bo.max)
    print('-' * 100)
print('all pass')
print('time : {}'.format(time() - starttime))

param_best = {'bagging_fraction': 0.21176541753913136,
 'colsample_bytree': 0.7140942216454612,
 'feature_fraction': 0.11946795064183889,
 'lambda_l1': 2.310135421593227,
 'lambda_l2': 7.468019368264845,
 'learning_rate': 0.22728585822684572,
 'max_depth': 9.240811767549072,
 'min_child_weight': 12.221204534872191,
 'min_data_in_leaf': 7.0330947275715765,
 'min_split_gain': 0.4964603965643156,
 'num_leaves': 4229.636789209139,
 'reg_alpha': 0.786784124586184,
 'reg_lambda': 7.287151182953648,
 'subsample': 0.024418195680870523}
param_best = clf_bo.max['params']
param_best['num_leaves'] = int(param_best['num_leaves'])
param_best['max_depth'] = int(param_best['max_depth'])
param_best['min_data_in_leaf'] = int(param_best['min_data_in_leaf'])
param_best['task'] = 'train'
param_best['device'] = 'cpu'
param_best['boosting'] = 'gbdt'
param_best['seed']=326
param_best['bagging_seed']=326
param_best['objective'] = 'regression' # 트레이닝에 사용
#param_best['metric'] = 'multi_logloss' # eval_set에 사용


import numpy as np
import os
import random

nx = 8096
ny = 7888

fs = os.stat('scroll.dat').st_size
assert fs % (nx*ny*2) == 0
nz = fs//(nx*ny*2)

# "uint16" is little-endian, at least on my machine ...
fp = np.memmap('scroll.dat', dtype='uint16', mode='r', shape=(nx,ny,nz))

# just quickly check that endianness
assert fp[0][0][0] == 0x60a9

pp = []
cc = []

SIZE = 100

# pick an arbitrary point for the center
cx = random.randint(0, nx-SIZE-1)
cy = random.randint(0, ny-SIZE-1)
cz = random.randint(0, nz-SIZE-1) if nz > SIZE else 0

POINTS = 1000000

pp = np.zeros((POINTS,3), dtype='float')
cc = np.zeros((POINTS,3), dtype='float')
for n in range(0,POINTS):
    v = 0
    while v < 32768:
        xx = cx + random.random() * (SIZE-1)
        yy = cy + random.random() * (SIZE-1)
        zz = cz + random.random() * (min(SIZE,nz)-1)
        v = fp[int(xx)][int(yy)][int(zz)]
        #v = interp([xx,yy,zz])[0]

    pp[n] = [xx,yy,zz]
    vv = (v-32768)/32768
    cc[n] = [vv,1-vv,0]

import open3d as o3d

pcd = o3d.geometry.PointCloud()
pcd.points = o3d.utility.Vector3dVector(pp)
pcd.colors = o3d.utility.Vector3dVector(cc)
o3d.visualization.draw_geometries([pcd])


from django.conf import settings
from django.shortcuts import render_to_response
from django.template import RequestContext, Context
from django.db.models import Q
from mash.experiments.models import Experiment, Configuration
import math

####################################################################################################
#
# CONSTANTS
#
####################################################################################################

NB_EXPERIMENTS_PER_PAGE             = 50
PAGES_NAVIGATION_BORDER_COUNT       = 5
PAGES_NAVIGATION_PREV_NEXT_COUNT    = 3

INITIALS = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z']

SORT_KEYS = { 'u': 'user__username', 'n': 'name', 'c': 'creation_date', 's': 'start', 'd': 'duration',
              't': 'configuration__task__name', 'h': 'heuristics_count', 'p': 'predictor' }

####################################################################################################
#
# EXPERIMENTS LIST MANAGEMENT
#
####################################################################################################

#---------------------------------------------------------------------------------------------------
# Represents an entry in the header of the table containing the list of experiments
#---------------------------------------------------------------------------------------------------
class HeaderEntry:

    def __init__(self, label, sort_key=None):
        self.label = label
        self.sort_key = sort_key

#---------------------------------------------------------------------------------------------------
# Holds the configuration used to customize the look and the behavior of a list of experiments
#---------------------------------------------------------------------------------------------------
class ListConfiguration:

    #-----------------------------------------------------------------------------------------------
    # Constructor
    #-----------------------------------------------------------------------------------------------
    def __init__(self, request, experiments_type, user=None):
        self.header_entries     = []
        self.url                = ''
        self.request            = request

        # Settings modifying the number of experiments in the list
        self.experiments_type   = experiments_type
        self.user               = user
        self.initial            = None
        self.query              = None

        # Settings modifying the order of the experiments in the list
        self.sort_key           = 'c'
        self.sort_ordering      = 'd'
        self.start              = 0

        self._initQuerySettings(request)
        self._initSortingSettings(request)

    #-----------------------------------------------------------------------------------------------
    # Add an entry in the header of the table containing the list of experiments
    #-----------------------------------------------------------------------------------------------
    def addHeaderEntry(self, entry):
        self.header_entries.append(entry)

    #-----------------------------------------------------------------------------------------------
    # Initialize the settings determining the number of experiments in the list from the parameters
    # of the request
    #-----------------------------------------------------------------------------------------------
    def _initQuerySettings(self, request):
        # Initial of the name of the experiment
        if request.GET.has_key('i'):
            self.initial = request.GET['i']
            if not(self.initial.upper() in INITIALS):
                self.initial = None

        # Query entered in the textfield
        if request.POST.has_key('query'):
            self.query = request.POST['query']
            if self.query in [ u'Search\u2026', u'Search...', 'Search...' ]:
                self.query = None

        # Query in the parameters
        if (self.query is None) and request.GET.has_key('q'):
            self.query = request.GET['q']

    #-----------------------------------------------------------------------------------------------
    # Initialize the settings determining the order of the experiments in the list from the parameters
    # of the request
    #-----------------------------------------------------------------------------------------------
    def _initSortingSettings(self, request):
        # Sort key
        if request.GET.has_key('sk'):
            self.sort_key = request.GET['sk']
            if (len(self.sort_key) != 1) or not(SORT_KEYS.has_key(self.sort_key)):
                self.sort_key = 'c'

        # Ordering (ascendant or descendant)
        if request.GET.has_key('so'):
            self.sort_ordering = request.GET['so']
            if (self.sort_ordering != 'a') and (self.sort_ordering != 'd'):
                if (self.sort_key == 'c') or (self.sort_key == 's'):
                    self.sort_ordering = 'd'
                else:
                    self.sort_ordering = 'a'

        # Start index (when the list spans on multiple pages)
        if request.GET.has_key('s'):
            try:
                self.start = int(request.GET['s'])
                self.start = (self.start / NB_EXPERIMENTS_PER_PAGE) * NB_EXPERIMENTS_PER_PAGE
            except:
                pass

    #-----------------------------------------------------------------------------------------------
    # Set the prefix of all the URLs that will be returned by the configuration from now
    #-----------------------------------------------------------------------------------------------
    def setupUrls(self, url, *args):
        self.url = self._buildUrl(url, *args)

    #-----------------------------------------------------------------------------------------------
    # Returns the base URL for the navigation links
    #-----------------------------------------------------------------------------------------------
    def navigationUrl(self):

        args = {'i': self.initial,
                'q': self.query,
                'sk': self.sort_key,
                'so': self.sort_ordering,
               }

        return self._buildUrl(self.url, args)

    #-----------------------------------------------------------------------------------------------
    # Returns the base URL for the links used to sort the list (in the header of the table)
    #-----------------------------------------------------------------------------------------------
    def headerUrl(self):

        args = {'i': self.initial,
                'q': self.query,
                's': self.start,
               }

        return self._buildUrl(self.url, args)

    #-----------------------------------------------------------------------------------------------
    # Returns the base URL for the links used to restrict the list to the experiments beginning with
    # a certain initial
    #-----------------------------------------------------------------------------------------------
    def initialsUrl(self):

        args = {'q': self.query,
                'sk': self.sort_key,
                'so': self.sort_ordering,
               }

        return self._buildUrl(self.url, args)

    #-----------------------------------------------------------------------------------------------
    # Returns the base URL for the query box
    #-----------------------------------------------------------------------------------------------
    def queryUrl(self):

        args = {'i': self.initial,
                'sk': self.sort_key,
                'so': self.sort_ordering,
               }

        return self._buildUrl(self.url, args)

    #---------------------------------------------------------------------------------------------------
    # Used to build an URL with a list of arguments (url?arg1=val1&arg2=val2...)
    #---------------------------------------------------------------------------------------------------
    def _buildUrl(self, url, *args):

        first = (url.find('?') < 0)

        for arg in args:

            if isinstance(arg, str) or isinstance(arg, unicode):
                if len(arg) > 0:
                    if first:
                        url += '?' + arg
                        first = False
                    else:
                        url += '&' + arg
            else:
                if len(arg) > 0:
                    for (name, value) in arg.items():
                        if value is not None:
                            if first:
                                url += '?' + name + '=' + str(value)
                                first = False
                            else:
                                url += '&' + name + '=' + str(value)

        return url

#---------------------------------------------------------------------------------------------------
# Can render a list of experiments with a specific configuration
#---------------------------------------------------------------------------------------------------
class ListRenderer:

    #---------------------------------------------------------------------------------------------------
    # Constructor
    #---------------------------------------------------------------------------------------------------
    def __init__(self, configuration):
        self.configuration = configuration

    #---------------------------------------------------------------------------------------------------
    # Render a list of experiments
    #---------------------------------------------------------------------------------------------------
    def renderExperimentsList(self, additional_args=''):

        # Retrieve the experiments
        (experiments, nb_experiments, current_page, nb_pages) = self._getExperimentsList()

        self.configuration.setupUrls(self.configuration.request.path, additional_args)

        # Set the list title
        title = 'Unknown experiments'
        if self.configuration.experiments_type == Configuration.PUBLIC:
            title = 'Public experiments'
        elif self.configuration.experiments_type == Configuration.CONSORTIUM:
            title = 'Consortium experiments'
        elif self.configuration.experiments_type == Configuration.PRIVATE:
            title = "Private experiments of '%s'" % self.configuration.user.username
        elif self.configuration.experiments_type == Configuration.CONTEST_BASE:
            title = 'Contest experiments'

        # Render the page
        return render_to_response('experiments/list.html',
                                  {'configuration': self.configuration,
                                   'title': title,
                                   'letters': INITIALS,
                                   'current_page': current_page,
                                   'nb_pages': nb_pages,
                                   'multiple_pages': (nb_pages > 1),
                                   'pages_navigation': self._getHTMLPagesNavigationControl(current_page, nb_pages),
                                   'nb_experiments': nb_experiments,
                                   'nb_experiments_per_page': NB_EXPERIMENTS_PER_PAGE,
                                   'table_header': self._getHTMLTableHeader(),
                                   'experiments': experiments,
                                   'display_username': (self.configuration.experiments_type == Configuration.CONSORTIUM),
                                  },
                                  context_instance=RequestContext(self.configuration.request))

    #---------------------------------------------------------------------------------------------------
    # Returns the list of experiments to display, according to the configuration
    #---------------------------------------------------------------------------------------------------
    def _getExperimentsList(self):

        # Compute the sort key
        actual_sort_key = SORT_KEYS[self.configuration.sort_key]
        if self.configuration.sort_ordering == 'd':
            actual_sort_key = '-' + actual_sort_key

        # Retrieve the experiments from the database
        queryset = Experiment.objects.exclude(status=Experiment.STATUS_DELETED)

        queryset = queryset.filter(configuration__experiment_type=self.configuration.experiments_type)
        if self.configuration.experiments_type == Configuration.PRIVATE:
            queryset = queryset.filter(user=self.configuration.user)

        if self.configuration.query is not None:
            queryset = queryset.filter(Q(user__username__search=self.configuration.query) |
                                       Q(name__search=self.configuration.query) |
                                       Q(user__username__icontains=self.configuration.query) |
                                       Q(name__icontains=self.configuration.query))

        if self.configuration.initial is not None:
            queryset = queryset.filter(name__istartswith=self.configuration.initial)

        if (actual_sort_key == 'duration') or (actual_sort_key == '-duration'):
            queryset = queryset.extra(select={'computed_duration': "end - start"})
            actual_sort_key = actual_sort_key.replace('duration', 'computed_duration')

        if (actual_sort_key == 'heuristics_count') or (actual_sort_key == '-heuristics_count'):
            queryset = queryset.extra(select={'heuristics_count': "SELECT COUNT(*) FROM experiments_configuration_heuristics_set AS hs WHERE hs.configuration_id=experiments_experiment.configuration_id"})

        queryset = queryset.extra(select={'predictor': "SELECT s.value FROM experiments_setting AS s WHERE s.configuration_id=experiments_experiment.configuration_id AND s.name='USE_PREDICTOR'"})

        experiments = queryset.order_by(actual_sort_key)

        return self._setupPages(experiments)

    def _setupPages(self, queryset):
        nb_items = queryset.count()

        items = queryset[self.configuration.start:self.configuration.start+NB_EXPERIMENTS_PER_PAGE]

        # Compute the current page and their count
        if self.configuration.start >= nb_items:
            self.configuration.start = (nb_items / NB_EXPERIMENTS_PER_PAGE) * NB_EXPERIMENTS_PER_PAGE

        current_page = (self.configuration.start / NB_EXPERIMENTS_PER_PAGE) + 1
        nb_pages = int(math.ceil(float(nb_items) / NB_EXPERIMENTS_PER_PAGE))

        # Return the results
        return (items, nb_items, current_page, nb_pages)

    #---------------------------------------------------------------------------------------------------
    # Returns the HTML code of the header of the table containing the list
    #---------------------------------------------------------------------------------------------------
    def _getHTMLTableHeader(self):
        html = '
\n'

        for entry in self.configuration.header_entries:
            if entry.sort_key is None:
                html += '%s	\n' % entry.label
            elif (entry.sort_key == self.configuration.sort_key) and (self.configuration.sort_ordering == 'd'):
                html += '%s	\n' % (self.configuration.headerUrl(), entry.sort_key, entry.label)
            else:
                html += '%s	\n' % (self.configuration.headerUrl(), entry.sort_key, entry.label)

        html += '
\n'

        return html

    #---------------------------------------------------------------------------------------------------
    # Returns the HTML code of the pages navigation control on the heuristics list page
    #---------------------------------------------------------------------------------------------------
    def _getHTMLPagesNavigationControl(self, current_page, nb_pages):

        url = self.configuration.navigationUrl()

        html = ''

        if nb_pages <= PAGES_NAVIGATION_BORDER_COUNT:
            for i in range(0, nb_pages):
                if i == current_page - 1:
                    html += '' + str(i+1) + ''
                else:
                    html += '' + str(i+1) + ''

        elif current_page < PAGES_NAVIGATION_PREV_NEXT_COUNT:
            for i in range(0, PAGES_NAVIGATION_BORDER_COUNT):
                if i == current_page - 1:
                    html += '' + str(i+1) + ''
                else:
                    html += '' + str(i+1) + ''

            if nb_pages > PAGES_NAVIGATION_BORDER_COUNT + 1:
                html += '...'

            html += '' + str(nb_pages) + ''

        elif current_page <= PAGES_NAVIGATION_BORDER_COUNT:
            max = current_page + PAGES_NAVIGATION_PREV_NEXT_COUNT
            if max > nb_pages:
                max = nb_pages

            for i in range(0, max):
                if i == current_page - 1:
                    html += '' + str(i+1) + ''
                else:
                    html += '' + str(i+1) + ''

            if nb_pages > current_page + PAGES_NAVIGATION_PREV_NEXT_COUNT:
                if nb_pages > current_page + PAGES_NAVIGATION_PREV_NEXT_COUNT + 1:
                    html += '...'

                html += '' + str(nb_pages) + ''

        elif current_page > nb_pages - PAGES_NAVIGATION_PREV_NEXT_COUNT + 1:
            if nb_pages - PAGES_NAVIGATION_BORDER_COUNT > 0:
                html += '1'

                if nb_pages - PAGES_NAVIGATION_BORDER_COUNT > 1:
                    html += '...'

            for i in range(nb_pages - PAGES_NAVIGATION_BORDER_COUNT, nb_pages):
                if i == current_page - 1:
                    html += '' + str(i+1) + ''
                else:
                    html += '' + str(i+1) + ''

        elif current_page > nb_pages - PAGES_NAVIGATION_BORDER_COUNT:
            if current_page - PAGES_NAVIGATION_BORDER_COUNT - 1 > 0:
                html += '1'

                if current_page - PAGES_NAVIGATION_BORDER_COUNT - 1 > 1:
                    html += '...'

            for i in range(current_page - PAGES_NAVIGATION_PREV_NEXT_COUNT - 1, nb_pages):
                if i == current_page - 1:
                    html += '' + str(i+1) + ''
                else:
                    html += '' + str(i+1) + ''

        html += ''

        return html


"""Using the human evaluation dataset to to sample code to test the effectiveness of this approach

Dataset from: https://github.com/openai/human-eval/tree/master
Paper: https://arxiv.org/pdf/2107.03374.pdf
"""

import glob
import json
from pathlib import Path

import tqdm
from typer import Typer

from prompt_to_code.agents.agents import run_agent
from prompt_to_code.config import PromptToCodeConfig

# requires human-eval is installed
# Also need to un-comment out the 'eval' function
try:
    from human_eval.data import HUMAN_EVAL, read_problems
    from human_eval.evaluation import evaluate_functional_correctness
except ImportError:
    print("=" * 80)
    print("Install human-eval from: https://github.com/openai/human-eval")
    print("$ git clone https://github.com/openai/human-eval")
    print("$ pip install -e human-eval")
    print("Then uncomment the import statement from /human_eval/execution.py")
    print("```#                         exec(check_program, exec_globals)```")
    print("=" * 80)
    raise ImportError("human-eval required")

def run_human_eval(
    agent="tdd",
    outdir: Path | str = "./examples/human_eval",
    num_samples_per_task=1,
    start_question: int = 0,
    start_ittr: int = 0,
):
    problems = read_problems(HUMAN_EVAL)
    outdir = Path(outdir)
    ittr = tqdm.tqdm(total=len(problems) * num_samples_per_task)
    problem_names = sorted(problems.keys())
    for i, example_id in enumerate(problem_names):
        if i < start_question:
            ittr.update(num_samples_per_task - start_ittr)
            continue
        example = problems[example_id]
        for loop_cnt in range(start_ittr, num_samples_per_task):
            filename = outdir / f"human_eval_{i:04}_{loop_cnt:04}.py"
            name = example["task_id"]
            prompt = example["prompt"]
            run_agent(agent, name, filename, prompt)
            ittr.update(1)

def aggregate_outputs(outdir: Path | str = "./examples/human_eval"):
    pattern = str(Path(outdir) / "human_eval_*.py")
    results = []
    task_ids = set()
    for fname in glob.glob(pattern):
        code = Path(fname).read_text()

        s = fname.split(str(outdir / "human_eval_"))[1].split(".py")[0]
        if "_" in s:
            task_number, *_ = s.split("_")
        else:
            task_number = s
        task_id = f"HumanEval/{int(task_number)}"
        task_ids.add(task_id)
        results.append({"task_id": task_id, "completion": code})
    with open(outdir / "human_eval_samples.jsonl", "w") as f:
        for problem in results:
            f.write(json.dumps(problem) + "\n")

    problems = read_problems(HUMAN_EVAL)
    with open(outdir / "human_eval_problems.jsonl", "w") as f:
        for problem in problems.values():
            if problem["task_id"] in task_ids:
                f.write(json.dumps(problem) + "\n")

def eval_tests(
    outdir="./examples/human_eval",
    k: list[int] | None = None,
    n_workers: int = 4,
    timeout: float = 3.0,
):
    if k is None:
        k = [1]
    outdir = Path(outdir)
    sample_file = outdir / "human_eval_samples.jsonl"
    problem_file = outdir / "human_eval_problems.jsonl"
    return evaluate_functional_correctness(
        str(sample_file), k, n_workers, timeout, str(problem_file)
    )

def score(num_samples_per_task, outdir):
    aggregate_outputs(outdir=outdir)
    # Eval
    k = [1, 2, 3, 4, 5, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100]
    results = eval_tests(
        outdir=outdir,
        k=[i for i in k if i <= num_samples_per_task],
        n_workers=4,
        timeout=3,
    )
    print(results)
    with open(outdir / "human_eval_results.json", "w") as f:
        json.dump(results, f)

app = Typer(name="Prompt-to-code: Human Eval")

@app.command()
def human_eval(
    agent: str = "tdd",
    outdir_root: str = "./examples",
    num_samples_per_task: int = 1,
    just_score: bool = False,
    start_question: int = 0,
    start_ittr: int = 0,
):
    PromptToCodeConfig()
    outdir = Path(outdir_root) / f"./human_eval_{agent}"

    print("Running human eval for agent", agent)
    print(f"saving output to {outdir}")
    print("WARNING THIS WILL COST $$$, please monitor OPENAI bill")
    if not just_score:
        run_human_eval(
            agent=agent,
            outdir=outdir,
            num_samples_per_task=num_samples_per_task,
            start_question=start_question,
            start_ittr=start_ittr,
        )
    score(num_samples_per_task=num_samples_per_task, outdir=outdir)

if __name__ == "__main__":
    app()


import sys; input = sys.stdin.readline
from collections import deque

def bfs(si, sj):
    q = deque([(si, sj)])
    visited[si][sj] = 1

    while q:
        i, j = q.popleft()
        for di, dj in [[1, 2], [2, 1], [2, -1], [1, -2], [-1, -2], [-2, -1], [-2, 1], [-1, 2]]:
            ni, nj = i+di, j+dj
            if 0 <= ni < l and 0 <= nj < l and not visited[ni][nj]:
                q.append((ni, nj))
                visited[ni][nj] = visited[i][j] + 1
                if (ni, nj) == (ei, ej):
                    return

T = int(input())
for t in range(1, T+1):
    l = int(input())
    si, sj = map(int, input().split())
    ei, ej = map(int, input().split())
    visited = [[0] * l for _ in range(l)]
    bfs(si, sj)
    print(visited[ei][ej] - 1)


# Created on 22 October 2019
# Handles all entities, including loose items

from random import randint
from pygame.display import get_surface
from NPCs.SpawnConditions import SpawnConditions
from Tools.constants import AIR_ID, BLOCK_W
import World as w
from Tools.lists import items

class EntityHandler:
    def __init__(self):
        self.entities = []
        self.items = []

    def move(self, dt, player_pos, pick_up):
        for entity in self.entities:
            entity.move(player_pos, dt)
        for item in self.items:
            item.move(dt)
            if item.pick_up_immunity <= 0 and pick_up(item):
                self.items.remove(item)

    def check_hit_entities(self, player_x, item):
        for entity in self.entities:
            # Check if we are swinging a weapon and the entity can be hit
            # and our sword collides with the entity
            if entity.immunity <= 0 and item.polygon.collides_polygon(entity.rect):
                # Check if we killed the entity
                if entity.hit(item.damage, player_x):
                    self.entities.remove(entity)

    def check_hit_player(self, player_rect):
        for entity in self.entities:
            if player_rect.colliderect(entity.rect):
                return entity.stats.dmg, entity.rect.centerx
        return 0, 0

    def get_display(self, rect):
        display = get_surface()
        for entity in self.entities:
            if rect.colliderect(entity.rect):
                display.blit(entity.surface, (entity.pos[0] - rect.x, entity.pos[1] - rect.y))
        for item in self.items:
            if rect.colliderect(item.rect):
                display.blit(items[item.idx].get_dropped_display(), (item.pos[0] - rect.x, item.pos[1] - rect.y))

    def collides_with_entity(self, rect):
        for entity in self.entities:
            if entity.rect.colliderect(rect):
                return True
        return False

    def spawn(self, player_pos):
        conditions = SpawnConditions()
        conditions.check_world()
        player_pos = [i // BLOCK_W for i in player_pos]
        for x in range(max(0, player_pos[0] - 50), min(w.blocks.shape[1], player_pos[0] + 50)):
            if x in w.spawners.keys():
                for y in range(max(0, player_pos[1] - 50, min(w.blocks.shape[0], player_pos[1] + 50))):
                    if y in w.spawners[x].keys() and randint(1, 10000) == 1:
                        entity = items[w.spawners[x][y]].spawn((x, y), conditions)
                        if entity is not None:
                            self.entities.append(entity)

# Maybe used in future
def get_spawn_spaces_with_hole(center, r_inner, r_outer, walking):
    # X range
    v1_min, v1_max = max(0, center[0] - r_outer), min(w.blocks.shape[1], center[0] + r_outer)
    # Y bounds
    v2_min1, v2_max1 = max(0, center[1] - r_outer), max(0, center[1] - r_inner)
    v2_min2, v2_max2 = min(w.blocks.shape[0], center[1] + r_inner), min(w.blocks.shape[0],
                                                                        center[1] + r_outer)
    v2_range_full = (range(v2_min1, v2_max2))
    v2_range_parts = (range(v2_min1, v2_max1), range(v2_min2, v2_max2))
    air = {}

    def add_val(v1_, v2_, val):
        if v1_ not in air.keys():
            air[v1_] = {}
        air[v1_][v2_] = val

    for v1 in range(v1_min, v1_max):
        for v2_range in v2_range_full if abs(v1 - center[1]) <= r_inner else \
                v2_range_parts:
            air_count = 0
            hit_block = False
            v2 = 0
            for v2 in v2_range:
                block = w.blocks[v2][v1] if walking else w.blocks[v1][v2]
                if block != AIR_ID:
                    hit_block = True
                    if air_count > 0:
                        add_val(v1, v2 - air_count, air_count)
                    air_count = 0
                elif hit_block:
                    air_count += 1
            if air_count > 0:
                add_val(v1, v2 - air_count, air_count)
    return air


from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys
from selenium.webdriver import ActionChains
import time
chrome_options = Options()
chrome_options.add_argument("--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Safari/537.36")
chrome_options.add_argument('--headless')
chrome_options.add_argument('--blink-settings=imagesEnabled=false')
chrome_options.add_argument("--no-sandbox")
chrome_options.add_argument("--disable-dev-shm-usage")
chrome_options.add_argument("--disable-gpu")
chrome_options.add_argument("--disable-extensions")
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
driver = webdriver.Chrome(options=chrome_options)
def get_data(url):
    try:
        driver.maximize_window()
        driver.get(url)
        time.sleep(1.5)
        product_elements = driver.find_elements(By.CSS_SELECTOR, 'li.product-listing__item')

        for click_el in product_elements:
            try:
                name = click_el.find_element(By.CSS_SELECTOR, 'span.product-listing__product-name').text
                print(name)

                phone_button = click_el.find_element(By.CSS_SELECTOR, 'div.js-company-contacts-settings')
                phone_button.click()
            except Exception:
                print("Phone button not found for this product")

        product_elements = driver.find_elements(By.CSS_SELECTOR, 'li.product-listing__item')
        for el in product_elements:
            try:
                name = el.find_element(By.CSS_SELECTOR, 'span.product-listing__product-name').text
            except Exception:
                name = ''
            try:
                company_name_element = el.find_element(By.CSS_SELECTOR, 'div.product-listing__company-name')
                company_name = company_name_element.find_element(By.CSS_SELECTOR, 'span').text.strip()
            except Exception:
                company_name = ''
            try:
                price = el.find_element(By.CSS_SELECTOR, 'div.product-listing__price-wrapper').text
            except Exception:
                price = ''
            try:
                phone_number = el.find_element(By.CLASS_NAME, 'js-company-contacts-settings').text
            except Exception:
                phone_number = ''

            print(phone_number)
    except Exception as ex:
        print(ex)
    finally:
        driver.quit()
        driver.close()

def main():
    get_data('https://perm.pulscen.ru/search/price/1003-kirpich-kamen-bloki?q=%D0%B3%D0%B0%D0%B7%D0%BE%D0%B1%D0%BB%D0%BE%D0%BA')

if __name__ == '__main__':
    main()


import os
import re
import sys
import glob
import sklearn as skl
import math
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn.cluster
from sklearn.decomposition import PCA
from sklearn.decomposition import NMF

from data_grid import DataGrid
from data_grid_TiNiSn import DataGrid, DataGrid_TiNiSn_500C, DataGrid_TiNiSn_600C
from NMF import nmf_func
from scipy.signal import find_peaks
from scipy.stats import pearsonr

def nmf_cluster(a=10,b=11,deal="None",use_peak=1,visualize=1,pear_threshold=0.8,peak_threshold=3,contri_threshold=0.1,dataset=1):

    '''
    Performs NMF clustering on the passed dataset and checks for peak shifting

    Parameters
    ------------------------

    a  : int
       Number of clusters (start)

    b  : int
       Number of clusters (end)

    deal : "None", "log", "square", "fft", "optimal"
        Dataset transformation before passing to NMF

        "None"    : no transformation
        "log"     : take natural log of dataset
        "square"  : square the dataset
        "fft"     : take fourier transform of the dataset
        "optimal" : discard the dataset and return the optimal W&H found from NMFk (see NMFk Julia.ipynb)

    use_peak : 0,1
        Check for the same number of peaks for highly correlated basis vectors?

        0 : no
        1 : yes

    visualize : 0,1
        Plot the basis vectors obtained from NMF?

        0 : no
        1 : yes

    pear_threshold : float64, range(0,1)
        Threshold for Pearson correlation
        Recommended: 0.7, 0.8, 0.9

    peak_threshold : float64
        Threshold for finding the number of peaks

    contri_threshold : float64, range(0,1)
        Threshold for the minimum conribution of phases at a composition point on the wafer map
        Recommended: 0.08, 0.1, 0.15, 0.25, 0.3

    dataset : 1,2
        Dataset desired

        1  : TiNiSn_500C
        2  : TiNiSn_600C

    Returns
    ------------------------
    Cluster grid(s) with the provided number of clusters

    labels : array (m_samples,)
        Labels of each sample in the dataset

    '''

    # Visualize all the patterns for the different cluster no.
    def clust_visuals(a,b,deal,dataset=1):
        off_set = 0
        fig,ax = plt.subplots(figsize=(8,10))

        if deal=="None":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+50))
                off_set+=100
            ax.set_title('Diffraction Clusters from W matrix')

        elif deal=="log":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+np.log(50)))
                off_set+=np.log(100)
            ax.set_title('Diffraction Clusters from W matrix')

        elif deal=="square":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+np.square(50)))
                off_set+=np.square(100)
            ax.set_title('Diffraction Clusters from W matrix')

        elif deal=="fft":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+np.square(50)))
                off_set+=100
            ax.set_title('Diffraction Clusters from W matrix')

        elif deal=="optimal":
            for j in range(a,b):
                W,H = nmf_func(j,deal,dataset)
                for i in range(np.shape(W)[1]):

                    ax.plot(dataGrid.data[1][:,0], W[:,i]+off_set)

                ax.annotate(j,(3.5,off_set+50))
                off_set+=100
            ax.set_title('Diffraction Clusters from W matrix')

    # Check for clusters with same number of peaks
    # Took time to finally write this code

    def peaks_func (W,H,peak_threshold):

        # Visualize the clusters and their peaks
        peaks_length=[]
        for i in range(0,len(H)):
            peaks, _ = find_peaks(W[:,i], height=peak_threshold)
            peaks_length.append(len(peaks))
            #fig,ax = plt.subplots()
            #ax.plot(dataGrid.data[1][:,0],W[:,i])
            #ax.plot((peaks*0.005)+1.5, W[peaks,i], "x")
            #ax.set_title(str(i+1))

        #print(peaks_length)

        same_peaks=[]
        for i in range(0,len(peaks_length)-1):
            smpeaks=[]

            if peaks_length[i] in list(peaks_length[:i]):
                #print(i)
                continue

            #print('Yes')
            for j in range(i+1,len(peaks_length)):
                    if peaks_length[i]==peaks_length[j]:
                        if len(smpeaks)==0:
                            smpeaks.append(i)
                        smpeaks.append(j)

            if len(smpeaks)!=0:
                same_peaks.append(smpeaks)

        #print(same_peaks)
#         for i in range(0,len(same_peaks)):
#             fig,ax = plt.subplots()
#             for j in same_peaks[i]:
#                 ax.plot(dataGrid.data[1][:,0],W[:,j])
#             ax.set_title('Same No. of Peaks '+str(same_peaks[i]))
        return same_peaks

    def Pearson_func(W,H,pear_threshold):

        # Pearson Correlation for each cluster against another
        p_corr = np.zeros((len(H),len(H)))

        for i in range(0,len(H)):
            for j in range(0,len(H)):
                corr, _ = pearsonr(W[:,i],W[:,j])
                p_corr[i,j]=corr

        # Find correlations higher than a certain threshold
        corr_idx = [[j,i] for j in range(len(p_corr)) for i in range(j+1,len(p_corr)) if np.abs(p_corr[j][i])>=pear_threshold]

        #print(corr_idx)
        #for i in range(0,len(corr_idx)):
         #   fig,ax = plt.subplots()
          #  for j in corr_idx[i]:
           #     ax.plot(W[:,j])
            #ax.set_title('Pearson Corr. '+ str(corr_idx[i]))

        return corr_idx

    def peak_pearson_match(corr_idx,same_peaks):
        #Check for clusters with high correlation and same peaks
        peak_shift = []
        for i in range(len(corr_idx)):
            for j in range(len(same_peaks)):
                if corr_idx[i][0] in same_peaks[j] and corr_idx[i][1] in same_peaks[j]:
                    print('.........Found Peak and Pearson match!!!..........')
                    peak_shift.append(corr_idx[i])

        return peak_shift

    def reshape_func(H,use_idx):

        #Combine the weights of all the peakshifted ones in H and also delete the peakshifted phases leaving the first phase in W
        #corr_idx=[[1,6],[2,3,4],[5,7]]

        H_reshaped1=np.zeros(shape=(len(H),177))
        for i in range(len(H)):
            H_reshaped1[i]=H[i]

        #H_reshaped1= H_reshaped  #this code will modify H_reshaped as H_reshaped1 is modified: very weird!!!!!!

        #This is done outside this function
        #use_idx =[]
        #for i in range(len(peak_shift)):
            #use_idx.append(peak_shift[i])

        for i in range(len(use_idx)):
            #print(i)
            b0 = H[use_idx[i][0]]

            for j in range(len(use_idx[i])):
                if j==0:
                    #print(j)
                    #print(b0)
                    continue

                #print(j)
                #if np.all(b0==H_reshaped[1]):
                 #   print('Yes')

                #print(b0)

                b0=b0 + H[use_idx[i][j]]

            H_reshaped1[use_idx[i][0]]=b0

        idx=[]
        for i in range(len(use_idx)):
            idx = idx + list(use_idx[i][1:])
        #idx = [i+1 for i in idx]
        print('Here are the clusters indices taken out', idx)
        new_H = np.delete(H_reshaped1,idx,0)
        print('Former H is of shape', np.shape(H_reshaped1))
        print('new_H is of shape', np.shape(new_H))

        return new_H

    def contributions(new_H,contri_threshold):

        #Reshape new_H cutting off those clusters contributing below a threshold

        H_reshape = np.zeros(shape=(np.shape(new_H)))

        for j in range(0,177):
            p = new_H[:,j]/np.sum(new_H[:,j])
            for i in range(len(p)):
                if p[i]<contri_threshold:
                    p[i]=0
            H_reshape[:,j]=p

        #Reshape the H where the 3 most abundant clusters remain for each datapoint/dot

        Big_locations=[]
        H_reshaped = np.zeros(shape=(np.shape(new_H)))

        for j in range(0,177):
            b = list(H_reshape[:,j])
            locations = []
            minimum = min(b)
            for i in range(3):
                maxIndex = b.index(max(b))
                if max(b)==0:
                    maxIndex=0
                    continue
                locations.append(maxIndex)
                b[maxIndex] = minimum
                H_reshaped[maxIndex,j]=H_reshape[maxIndex,j]

            Big_locations.append(locations)
            new_H_reshaped = H_reshaped
            #print(locations,j)

        #print(np.shape(new_H_reshaped))

        return new_H_reshaped,Big_locations

    def form_labels(new_H_reshaped,Big_locations):

        #Form labels for single-phases point
        labels1=[]
        labels_clust1=[]
        oo=-1

        for i in range(len(Big_locations)):
            if len(Big_locations[i])==1:
                #print(Big_locations[i],i,oo)

                if Big_locations[i] in list(labels_clust1[:]):
                    #print(Big_locations[i],i,oo)
                    continue
                oo+=1
                labels1.append(oo)
                labels_clust1.append(Big_locations[i])
                #print(Big_locations[i],i,oo,'Yes')

        #Form labels for double-phased points
        #This took a while :-( but I finally made it more efficient
        labels2=[]
        labels_clust2=[]
        ooo=oo

        for i in range(len(Big_locations)):
            p=0

            if len(Big_locations[i])==2:
                #print(i)

                for ii,v in enumerate(labels_clust2):
                    if set(Big_locations[i]) == set(v):   #Check if they are equal regardless of arrangement
                        #print(Big_locations[i],i,ooo)
                        p=1
                        continue

                if p==1:
                    continue

                ooo+=1
                labels2.append(ooo)
                labels_clust2.append(sorted(Big_locations[i]))
                #print(Big_locations[i],i,ooo,"Yes")

        #Form labels for triple-phased points
        #This took a while too :-(

        labels3=[]
        labels_clust3=[]
        oooo=ooo

        for i in range(len(Big_locations)):
            p=0

            if len(Big_locations[i])==3:
                #print(i)

                for ii,v in enumerate(labels_clust3):
                    if set(Big_locations[i]) == set(v):  #Check if they are equal regardless of arrangement
                        #print(Big_locations[i],i,oooo)
                        p=1
                        continue

                if p==1:
                    continue

                oooo+=1
                labels3.append(oooo)
                labels_clust3.append(sorted(Big_locations[i]))
                #print(Big_locations[i],i,oooo,"Yes")

        all_labels=labels1+labels2+labels3
        labels_clust = labels_clust1+labels_clust2+labels_clust3

        #Get the cluster for each point
        labels=[]
        for val in range(0,177):
            labels.append(labels_clust.index(sorted(Big_locations[val])))

        #print(labels)
        #print(list(labels_clust))

#         #This part of the code is a savior
#         new_labels=[None]*177
#         for ii in range(0,177):
#             x,y=dataGrid.coord(ii+1)
#             new_grid=dataGrid.grid_num(16-x,y)
#             new_labels[new_grid-1]=labels[ii]

        return labels,all_labels,labels_clust
        #return new_labels,all_labels,labels_clust

    def get_cluster_grid(all_labels,labels):

        hues = [float(float(x)/float(len(all_labels))) for x in range(1,len(all_labels)+1)]
        cluster_grid = np.zeros(shape = (15,15,3))
        #cluster_grid = np.ones(shape = (15,15,3))
        for val in range(1,178):
            x,y = dataGrid.coord(val)
            cluster = labels[val-1]
            #cluster_grid[15-y][x-1] = matplotlib.colors.hsv_to_rgb([hues[cluster],1,1])
            cluster_grid[y-1][x-1] = matplotlib.colors.hsv_to_rgb([hues[cluster],1,1])

        return cluster_grid

    def main_func(i,deal,pear_threshold,peak_threshold,contri_threshold,dataset=1):

        W,H=nmf_func(i,deal,dataset)

        corr_idx = Pearson_func(W,H,pear_threshold)
        same_peaks = peaks_func(W,H,peak_threshold)

        if same_peaks ==[]:
            print("No clusters with the same number of peaks for the given threshold and no. of clusters")
        if corr_idx ==[]:
            print("No clusters with correlation above or equal to the given correlation threshold. No peak shifting will be considered")

        print(np.shape(corr_idx),np.shape(same_peaks))

        if corr_idx==[]:

            new_H = H

        elif same_peaks==[] or use_peak==0:
            use_idx =[]
            for i in range(len(corr_idx)):
                use_idx.append(corr_idx[i])

            new_H=reshape_func(H,use_idx)

        elif use_peak==1:

            peak_shift = peak_pearson_match(corr_idx,same_peaks)

            if peak_shift==[]:

                print('No match between Pearson and same peaks found. Just Pearson will be considered')
                use_idx =[]
                for i in range(len(corr_idx)):
                    use_idx.append(corr_idx[i])

                new_H=reshape_func(H,use_idx)

            else:

                use_idx =[]
                for i in range(len(peak_shift)):
                    use_idx.append(peak_shift[i])

                new_H=reshape_func(H,use_idx)

        new_H_reshaped,Big_locations = contributions(new_H,contri_threshold)

        #print(np.shape(Big_locations))

        labels,all_labels,labels_clust = form_labels(new_H_reshaped,Big_locations)

        cluster_grid = get_cluster_grid(all_labels,labels)

        return cluster_grid,labels

   # This is where the interesting stuffs happen

    if dataset==1:
        dataGrid = DataGrid_TiNiSn_500C()
        data = dataGrid.get_data_array()
    elif dataset==2:
        dataGrid = DataGrid_TiNiSn_600C()
        data = dataGrid.get_data_array()

    if visualize==1:
            clust_visuals(a,b,deal,dataset)

    if b-a > 8: x_width=15
    else: x_width=10

    Big_labels=[]
    fig = plt.figure(figsize=(x_width,8))
    fig.tight_layout()

    for i in range(a,b):

        print('.....................................'+str(i)+' Clusters'+'...........................................')

        cg,labels = main_func(i,deal,pear_threshold,peak_threshold,contri_threshold,dataset)
        Big_labels.append(labels)
        ax = fig.add_subplot(2,np.ceil((b-a)/2),i-a+1)
        ax.imshow(cg)
        ax.invert_yaxis()
        #ax.invert_xaxis()
        ax.title.set_text(i)
        ax.axis("off")

    #print(".................There are %d clusters in total..........." %len(all_labels))
    k=.05
    plt.subplots_adjust(left=k,right=(1-k),bottom=k,top=(1-k),wspace=k,hspace=k)
    plt.show()

    return Big_labels


from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession

from app.core.db import get_async_session
from app.core.user import current_superuser, current_user
from app.crud.donation import donation_crud
from app.models import User
from app.schemas.donation import DonationCreate, DonationDB
from ...utils import DONATION_EXCLUDE_FIELS, investing

donation_router = APIRouter()

@donation_router.get(
    '/',
    response_model=list[DonationDB],
    dependencies=[Depends(current_superuser)],
    response_model_exclude_none=True
)
async def get_all_donations(
    session: AsyncSession = Depends(get_async_session)
):
    """Только для суперюзеров."""
    return await donation_crud.get_multi(session)

@donation_router.post(
    '/',
    response_model=DonationDB,
    response_model_exclude={*DONATION_EXCLUDE_FIELS},
    response_model_exclude_none=True
)
async def create_donation(
    donation: DonationCreate,
    session: AsyncSession = Depends(get_async_session),
    user: User = Depends(current_user),
):
    new_donation = await donation_crud.create_if_not_exists(
        donation, session, user
    )
    await investing(new_donation, session)
    return new_donation

@donation_router.get(
    '/my',
    response_model=list[DonationDB],
    response_model_exclude={*DONATION_EXCLUDE_FIELS}
)
async def get_user_donations(
    session: AsyncSession = Depends(get_async_session),
    user: User = Depends(current_user)
):
    return await donation_crud.get_by_user(
        session=session, user=user
    )


from SPARQLWrapper import SPARQLWrapper2
from SPARQLWrapper.SmartWrapper import Bindings
import pandas as pd
import urllib.error
from ooetl.extractors import RemoteExtractor

class SparqlExtractor(RemoteExtractor):
    def __init__(self, endpoint_url: str, query: str, max_retries: int = 10, legacy: bool = False, method: str = "GET"):
        """Create a new instance, setting the parameters

        Args:
            :endpoint_url: remote sparql endpoint
            :query: the query to post to the sparql endpoint
            :max_retries: the max retries to attempt
            :legacy: if the results are returned in a legacy format
            :method: "POST" or "GET"

        Returns:
            instance of a :class:`RemoteExtractor` subclass
        """
        super(SparqlExtractor, self).__init__(endpoint_url)
        self.remote_url = endpoint_url
        self.query = query
        self.max_retries = max_retries
        self.legacy = legacy
        self.method = method

    def extract(self, **kwargs):
        sparql_wrapper = SPARQLWrapper2(self.remote_url)
        sparql_wrapper.setQuery(self.query)
        sparql_wrapper.setMethod(self.method)

        # while loop to handle repeating in case of temporary errors.
        results = None
        max_retries = self.max_retries
        while max_retries > 0:

            try:
                result: Bindings = sparql_wrapper.query()
                if self.legacy:
                    results = result.fullResult["results"]["bindings"]
                else:
                    results = result.bindings
                break
            except urllib.error.HTTPError:
                max_retries -= 1
        if results is None:
            raise Exception(f"Endpoint {self.remote_url} failure.")

        # transform results into a simple dict
        # so that it can be directly used to create a DataFrame
        res = []
        for r in results:
            res.append(dict((k, r[k].value) for k in r.keys()))
        return pd.DataFrame(res)


"""Config flow for Nasa."""
from __future__ import annotations

from typing import Any

from aionasa import APOD
from aionasa.errors import APIException
import voluptuous as vol

from homeassistant import config_entries, exceptions
from homeassistant.const import CONF_API_KEY, CONF_SCAN_INTERVAL
from homeassistant.core import callback
from homeassistant.data_entry_flow import FlowResult
from homeassistant.helpers.aiohttp_client import async_get_clientsession
import homeassistant.helpers.config_validation as cv

from .const import (
    API_CATEGORY,
    CONF_API_CATEGORY,
    DEFAULT_SCAN_INTERVAL,
    DOMAIN,
    LOGGER,
)

STEP_USER_DATA_SCHEMA = vol.Schema({vol.Optional(CONF_API_KEY): str})

class ConfigFlow(config_entries.ConfigFlow, domain=DOMAIN):
    """Handle a config flow for Nasa."""

    VERSION = 1
    CONNECTION_CLASS = config_entries.CONN_CLASS_CLOUD_POLL

    async def _validate_input(self, data) -> None:
        """Validate the user input allows us to connect."""

        api_key = data.get(CONF_API_KEY)
        session = async_get_clientsession(self.hass)

        try:
            if api_key is not None:
                api = APOD(api_key=api_key, session=session)
                return await api.get(as_json=True)

        except APIException as error:
            raise InputValidationError("invalid_api_key") from error
        except OSError as error:
            raise InputValidationError("invalid_api_key") from error

    async def async_step_user(
        self,
        user_input: dict[str, Any] | None = None,
    ) -> FlowResult:
        """Handle the initial step."""
        errors = {}

        if self._async_current_entries():
            return self.async_abort(reason="single_instance_allowed")

        if user_input is not None:
            try:
                await self._validate_input(user_input)
            except InputValidationError as error:
                errors["base"] = error.base
            except Exception:  # pylint: disable=broad-except
                LOGGER.exception("Unexpected exception")
                errors["base"] = "unknown"
            if not errors:
                return self.async_create_entry(title=DOMAIN.title(), data=user_input)

        return self.async_show_form(
            step_id="user", data_schema=STEP_USER_DATA_SCHEMA, errors=errors
        )

    @staticmethod
    @callback
    def async_get_options_flow(
        config_entry: config_entries.ConfigEntry,
    ) -> OptionsFlowHandler:
        """Get the options flow for this handler."""
        return OptionsFlowHandler(config_entry)

class OptionsFlowHandler(config_entries.OptionsFlow):
    """Handle a option flow for Nasa."""

    def __init__(self, config_entry: config_entries.ConfigEntry) -> None:
        """Initialize options flow."""
        self.options = config_entry.options
        self.data = config_entry.data

    async def async_step_init(
        self,
        user_input: dict[str, Any] | None = None,
    ) -> FlowResult:
        """Handle options flow."""
        if not user_input:
            configured_api_category: list[str] = self.options.get(CONF_API_CATEGORY, [])

            return self.async_show_form(
                step_id="init",
                data_schema=vol.Schema(
                    {
                        vol.Required(
                            CONF_API_CATEGORY,
                            default=configured_api_category,
                        ): cv.multi_select({k: k for k in API_CATEGORY}),
                        vol.Optional(
                            CONF_SCAN_INTERVAL,
                            default=self.options.get(CONF_SCAN_INTERVAL, DEFAULT_SCAN_INTERVAL),
                        ): cv.positive_int,
                    }
                ),
            )
        LOGGER.debug("User input CONFIG OPTION: %s", user_input)
        return self.async_create_entry(title="", data=user_input)

class InputValidationError(exceptions.HomeAssistantError):
    """Error to indicate we cannot proceed due to invalid input."""

    def __init__(self, base: str) -> None:
        """Initialize with error base."""
        super().__init__()
        self.base = base


# import threading
# import time
#
# class MyThread(threading.Thread):
#
#     def run(self):
#         count = 0
#         while True:
#             count = count + 1
#             print " No of loop {} and Thread name is {} \n".format(count , self.name)
#             time.sleep(5)
#
#
# if __name__ == '__main__':
#
#     for i in range(0,100):
#         m = MyThread()
#         m.start()
#         print "print from main thread  {}".format(i)

import pymongo
from dateutil.parser import parse

from_date = parse('2016-01-01')
to_date = parse('2017-12-17')

db = pymongo.MongoClient('mongodb://10.1.4.64:27017')

print(db.db_apsma.tweet.find({'searchKey':'narendramodi','created_at':{'$lte': to_date,'$gte': from_date}}).count())
print(db.db_apsma.facebook.find({'searchKey':'narendramodi','created_at':{'$lte': to_date,'$gte': from_date}}).count())
print(db.db_apsma.video.find({'searchKey':'narendramodi'}).count())
print(db.db_apsma.news.find({'searchKey':'narendramodi'}).count())

db.close()