"""
Find nearest neighbor and got crop patches based on registration of nearest neighbor

Sample usage:

import torch
from utils import *
from torch.utils.data import Dataset, DataLoader
from utils.registration import *

def multi_masks2onehot(mask):
    D, H, W = mask.shape
    onehot_mask = np.zeros((2, D, H, W))
    for i in range(2):
        onehot_mask[i][mask == i] = 1

    return onehot_mask

class ValBrainImageDataset(Dataset):

    def __init__(self, ls, moving_ls, transform=None):

        self.ls = ls
        self.moving_ls = moving_ls
        self.transform = transform

    def __len__(self):
        return len(self.ls)

    def __getitem__(self, idx):
        print self.ls[idx]
#         start_time = time.time()
        roi, img = get_patient_img_and_contour(self.ls[idx])
        nearest_p = find_nearest_patient(self.ls[idx], self.moving_ls)
        fix_contour, fix_img = get_patient_img_and_contour(nearest_p)
        outTx = affine(img, fix_img)
        out = resample(img, fix_img, outTx)
        out_contour = resample(roi, fix_contour, outTx, interpolator = sitk.sitkNearestNeighbor)
        indice = find_border_dynamic_threshold(sitk.GetArrayFromImage(out_contour))
        indices = pad3D_indices(indice, sitk.GetArrayFromImage(out_contour))

        cropped_img =  img[indices[0][0]:indices[0][-1],indices[1][0]:indices[1][-1],indices[2][0]:indices[2][-1]].astype(np.float32)
        cropped_roi = multi_masks2onehot(roi[indices[0][0]:indices[0][-1],indices[1][0]:indices[1][-1],indices[2][0]:indices[2][-1]]).astype(np.float32)
        # print("--- %s seconds ---" % (time.time() - start_time))

        return torch.from_numpy(cropped_img[np.newaxis, ...]).type(torch.FloatTensor), torch.from_numpy(cropped_roi).type(torch.FloatTensor)
"""

import SimpleITK as sitk
import os
import numpy as np

def find_nearest_patient(p_name, p_ls):
    """
    First to loop through the training name list to get the nearest neighbor based on pixel intensity histogram and z-slices num
    Args:
        p_name: patient name(str)
        p_ls: list of patient names(str) to get nearest neighbor
    Returns:
        patient name of nearest neighbor(str)
    """
    contour, img = get_patient_img_and_contour(p_name)
    distance_dict = {}
    for i in range(len(p_ls)):
        if p_ls[i] != p_name:
#             print p_ls[i]
            contour2, img2 = get_patient_img_and_contour(p_ls[i])
            if np.abs(img.shape[0] - img2.shape[0]) <= 11:
                hist1 = np.histogram(img,bins=1000)[0]
                hist2 = np.histogram(img2,bins=1000)[0]
                distance_dict[p_ls[i]] = (calculateDistance(hist1, hist2))
    return min(distance_dict, key = lambda x: distance_dict.get(x))

def get_patient_img_and_contour(patient_name, contour = 'parotid r', img_path = '/mnt/hdd10T/htang6/data/brain_ai/preprocessed/'):
    """
    Read patient image and contour in np.array
    Args:
        patient_name: patient name(str)
        contour: name of contour(str)
        img_path: path to processed image data and contour data
    Returns:
        contour and ct image in np.array
    """
    for f in os.listdir(img_path):
        if f.startswith(patient_name) and contour in f.lower():
            interestarea = np.load(img_path + f)
        elif f.startswith(patient_name) and 'clean' in f.lower():
            image = np.load(img_path + f)

    return interestarea, image

def calculateDistance(i1, i2):
    return np.sum((i1-i2)**2)

def  find_border_dynamic_threshold(mask):
    """
    Find the border of the passed in mask
    Args:
        mask: mask in np.array
    Returns:
        indices of elements value are bigger than zero in np.array
    """
    threshold = 0
    indices = np.where(mask > threshold)
    return indices

def pad3D_indices(indices, original_img, shape= (30, 120, 120)):
    """
    Pass in the indices of border, pad indices into desired shape
    Args:
        indices: indices from find_border_dynamic_threshold function.
        original_img: original_img in np.array
        shape: desired shape of crop patches
    Returns:
        padded indices fit the desired shape(list of tuples)
    """
    m_max = indices[0].max()
    m_min = indices[0].min()
    n_max = indices[1].max()
    n_min = indices[1].min()
    r_max = indices[2].max()
    r_min = indices[2].min()
#     print a[0]
    m_diff = shape[0] - (m_max - m_min)
    n_diff = shape[1] - (n_max - n_min)
    r_diff = shape[2] - (r_max - r_min)
    z_min = m_min - m_diff/2
    z_max = m_max + (m_diff+1)/2
    y_min = n_min - n_diff/2 #- 10
    y_max = n_max + (n_diff+1)/2#- 10
    x_min = r_min - r_diff/2
    x_max = r_max + (r_diff+1)/2
    if z_max > original_img.shape[0]:
        z_max = original_img.shape[0]
        z_min = z_max - shape[0]
    if y_max > original_img.shape[1]:
        y_max = original_img.shape[1]
        y_min = y_max - shape[1]
    if x_max > original_img.shape[2]:
        x_max = original_img.shape[2]
        x_min = x_max - shape[2]
    if z_min < 0:
        z_min = 0
        z_max = shape[0]
    if y_min < 0:
        y_min = 0
        y_max = shape[1]
    if x_min < 0:
        x_min = 0
        x_max = shape[2]

    return [(z_min, z_max) ,(y_min, y_max), (x_min, x_max)]

def rigid(fixed, moving):
    """
    rigid registration on the crop patch
    Args:
        fixed: fixed image in np.array.
        moving: moving image in np.array.
    Returns:
        displacement field
    """
    R = sitk.ImageRegistrationMethod()
    R.SetMetricAsMeanSquares()
    R.SetOptimizerAsRegularStepGradientDescent(8.0, .01, 200 )
    R.SetInitialTransform(sitk.TranslationTransform(sitk.GetImageFromArray(fixed).GetDimension()))
    R.SetInterpolator(sitk.sitkNearestNeighbor)

    R.AddCommand( sitk.sitkIterationEvent, lambda: R )

    outTx = R.Execute(sitk.GetImageFromArray(fixed), sitk.GetImageFromArray(moving))

    return outTx

def affine(fixed, moving):
    """
    affine registration on the crop patch
    Args:
        fixed: fixed image in np.array.
        moving: moving image in np.array.
    Returns:
        displacement field
    """
    R = sitk.ImageRegistrationMethod()
    R.SetMetricAsMeanSquares()
#     initial_transform = sitk.CenteredTransformInitializer(sitk.GetImageFromArray(fixed),
#                                                       sitk.GetImageFromArray(moving),
#                                                       sitk.AffineTransform(sitk.GetImageFromArray(fixed).GetDimension()))
#     R.SetShrinkFactorsPerLevel([3,2,1])
#     R.SetSmoothingSigmasPerLevel([2,1,1])

#     R.SetMetricAsJointHistogramMutualInformation(20)
    R.MetricUseFixedImageGradientFilterOff()

    R.SetOptimizerAsGradientDescent(learningRate=0.5,
                                    numberOfIterations=200,
                                    estimateLearningRate = R.EachIteration)
    R.SetOptimizerScalesFromPhysicalShift()

    R.SetInitialTransform(sitk.AffineTransform(sitk.GetImageFromArray(fixed).GetDimension()))

    R.SetInterpolator(sitk.sitkLinear)
#     R.SetInitialTransform(sitk.AffineTransform(sitk.GetImageFromArray(fixed).GetDimension()))
#     R.SetInterpolator(sitk.sitkNearestNeighbor)

    R.AddCommand( sitk.sitkIterationEvent, lambda:R )

    outTx = R.Execute(sitk.GetImageFromArray(fixed), sitk.GetImageFromArray(moving))

    return outTx

def resample(fixed, moving, outTx, interpolator = sitk.sitkLinear):
    """
    Apply the displacement field onto moving image
    Args:
        fixed: fixed image in np.array.
        moving: moving image in np.array.
        outTx: displacement field.
        interpolator: sitk.sitkLinear by defalut
    Returns:
        moving image after registration in sitk.Image
    """
#   outTx = sitk.ReadTransform('out.txt')
    resampler = sitk.ResampleImageFilter()
    resampler.SetReferenceImage(sitk.GetImageFromArray(fixed));
    resampler.SetInterpolator(interpolator)
    resampler.SetDefaultPixelValue(0)
    resampler.SetTransform(outTx)
    out = resampler.Execute(sitk.GetImageFromArray(moving))

    return out


#!/usr/bin/env python3
# coding: utf-8
# File: quotesSpider.py
# Author: lxw
# Date: 4/16/17 15:36 PM

import scrapy

class QuotesSpider(scrapy.Spider):
    name = "quotes"

    """
    def start_requests(self):
        urls = [
            "http://quotes.toscrape.com/page/1/",
            "http://quotes.toscrape.com/page/2/",
        ]
        for url in urls:
            yield scrapy.Request(url=url, callback=self.parse)
    """
    start_urls = [
        "http://quotes.toscrape.com",
    ]

    def parse(self, response):
        print(type(response))   # <class 'scrapy.http.response.html.HtmlResponse'>
        for quote in response.css("div.quote"):
            content = quote.css("span.text::text").extract_first()
            author = quote.css("small.author::text").extract_first()
            tags = quote.css("div.tags a.tag::text").extract()
            yield dict(content=content, author=author, tags=tags)

        # next_link = response.css("li.next a::attr(href)").extract_first()
        # if next_link:
        #     next_link = response.urljoin(next_link)
        #     yield scrapy.Request(url=next_link, callback=self.parse)


import speech_recognition as sr
import subprocess
import sounddevice as sd
import soundfile as sf
from gtts import gTTS
# from gpt import gpt_answer
# from dht import runs
# from notifications import run
# from volume import increase_vol, decrease_vol
# from where_is import where_is
# from music import music_function

# GPIO.setmode(GPIO.BOARD)
# GPIO.setup(11, GPIO.OUT)
# GPIO.output(11, 1)

# Функция Текст-в-речь
def speak(text):
    tts = gTTS(text=text, lang="en")
    tts.save("output.mp3")
    subprocess.run(["mpg321", "output.mp3"])

def play_peep_sound():
    peep_sound_path = "peep.wav"  # Путь к файлу с звуком

    # Загрузите аудиофайл с использованием soundfile
    peep_data, sample_rate = sf.read(peep_sound_path)

    # Воспроизведите аудиофайл с использованием sounddevice
    sd.play(peep_data, sample_rate)
    sd.wait()

def listen():
    r = sr.Recognizer()

    with sr.Microphone() as source:
        print("Скажите что-нибудь:")
        audio = r.listen(source)

    try:
        text = r.recognize_google(audio, language="en-en")
        print("Вы сказали: " + text)
        return text
    except sr.UnknownValueError:
        print("Речь не распознана")
    except sr.RequestError as e:
        print("Ошибка при обращении к сервису распознавания речи: {0}".format(e))
    return None

if __name__ == "__main__":
    remember_mode = False

    while True:
        try:
            result = listen()

            if result:
                print("You said:")
                print(result)
                print(type(result))
                if "switch on" in result.lower():
                    play_peep_sound()
                    # GPIO.output(11, 0)
                    print("lights are switched on")
                elif "switch off" in result.lower():
                    play_peep_sound()
                    # GPIO.output(11, 1)
                    print("lights are switched off")
                # elif "increase volume" in result.lower():
                #     increase_vol()
                # elif "decrease volume" in result.lower():
                #     decrease_vol()

                if "remember" in result.lower():
                    print("Remember mode activated.")
                    remember_mode = True
                    continue

                # print("remember_mode", remember_mode)

                # if remember_mode:
                #     user_question = result  # Store the user's question for later use
                #     print("You talked:")
                #     print(result)
                #     print(type(result))
                #     remember_mode = (
                #         False  # Disable help mode after capturing the question
                #     )
                #     run(message=result)

                # if "where is" in result.lower():
                #     print("Notification mode activated.")
                #     notification = where_is()
                #     print("notification: ", notification)
                #     speak(notification)
                #     music_function()

                if "help me" in result.lower():
                    print(
                        "Help mode activated. Please ask a question within 5 seconds."
                    )
                    help_mode = True
                    continue

                # if "temperature" in result.lower():
                #     temperature = runs()

                # if help_mode:
                #     user_question = result  # Store the user's question for later use
                #     print("You asked:")
                #     print(result)
                #     # print(type(result))
                #     help_mode = False  # Disable help mode after capturing the question
                #     answer = gpt_answer(query=user_question)
                #     speak(answer)
                #     music_function()
        except KeyboardInterrupt:
            print("Exiting...")
            break
        except Exception as e:
            # Redirect ALSA error messages to /dev/null (suppress them)
            subprocess.run(["python3", "your_script.py"], stderr=subprocess.DEVNULL)


import os
import pyodbc
import time
import shutil
from datetime import datetime
from stat import S_IWRITE

from add_new_files import validate_file
from timer_dec import timer
from const import PRODUCTION, START_CATALOG, CURSOR, TRANSFER_FILE

def list_new_files():
    """ Adding to database drawing uploaded directly to PRODUCTION catalog."""
    source_cat = PRODUCTION
    query = 'Select Plik From Technologia;'
    try:
        result = CURSOR.execute(query)
    except pyodbc.Error:
        print('Connection to database failed.')
        return None

    new_files, current_db_files = [], []

    print('Data Base refreshed.')

    current_db_files = [file[0] for file in result]
    files_counter = 0
    pdf_list = ''

    for catalog in os.listdir(source_cat):
        if catalog in ['desktop.ini', 'Thumbs.db']:
            continue

        deep_path = os.path.join(source_cat, catalog)

        if not os.path.isdir(deep_path):
            continue

        current_files = os.listdir(deep_path)

        for file in current_files:
            if file.lower().endswith('pdf'):
                files_counter += 1
                pdf_list = f'{pdf_list}\n{file}'
                print(f'{files_counter} files has been analyzed.', end="\r")

    archive(pdf_list)

    print('\n', end='\r')

def archive(file_name):
    with open('W:/!!__PRODUKCJA__!!/2__Baza_Danych/pdf_list.txt', 'a', encoding='utf-8') as history_file:
        history_file.write(file_name)

def main():
    list_new_files()

if __name__ == '__main__':
    main()


from deeplake.core.dataset import Dataset

from deeplake.constants import MB
from deeplake.core.storage.gcs import GCSProvider
from deeplake.enterprise.util import raise_indra_installation_error  # type: ignore
from deeplake.core.storage import S3Provider
from deeplake.core.storage.azure import AzureProvider
from deeplake.util.remove_cache import get_base_storage
from deeplake.util.exceptions import EmptyTokenException

from deeplake.util.dataset import try_flushing  # type: ignore
import importlib
import jwt

# Load lazy to avoid cycylic import.
INDRA_API = None

def import_indra_api_silent():
    global INDRA_API
    if INDRA_API:
        return INDRA_API
    if not importlib.util.find_spec("indra"):
        return None
    try:
        from indra import api  # type: ignore

        INDRA_API = api
        return api
    except Exception as e:
        return e

def import_indra_api():
    api = import_indra_api_silent()

    if api is None:
        raise_indra_installation_error()  # type: ignore
    elif isinstance(api, Exception):
        raise_indra_installation_error(api)
    else:
        return api

INDRA_INSTALLED = bool(importlib.util.find_spec("indra"))

def _get_indra_ds_from_azure_provider(
    path: str,
    token: str,
    provider: AzureProvider,
):
    if provider is None:
        return None

    api = import_indra_api()
    account_name = provider.account_name
    account_key = provider.account_key
    sas_token = provider.get_sas_token()
    expiration = str(provider.expiration) if provider.expiration else None

    return api.dataset(
        path,
        origin_path=provider.root,
        token=token,
        account_name=account_name,
        account_key=account_key,
        sas_token=sas_token,
        expiration=expiration,
    )

def _get_indra_ds_from_gcp_provider(
    path: str,
    token: str,
    provider: GCSProvider,
):
    if provider is None:
        return None

    api = import_indra_api()
    creds = provider.get_creds()
    anon = creds.get("anon", "")
    expiration = creds.get("expiration", "")
    access_token = creds.get("access_token", "")
    json_credentials = creds.get("json_credentials", "")
    endpoint_override = creds.get("endpoint_override", "")
    scheme = creds.get("scheme", "")
    retry_limit_seconds = creds.get("retry_limit_seconds", "")

    return api.dataset(
        path,
        origin_path=provider.root,
        token=token,
        anon=anon,
        expiration=expiration,
        access_token=access_token,
        json_credentials=json_credentials,
        endpoint_override=endpoint_override,
        scheme=scheme,
        retry_limit_seconds=retry_limit_seconds,
    )

def _get_indra_ds_from_s3_provider(
    path: str,
    token: str,
    provider: S3Provider,
):
    if provider is None:
        return None

    api = import_indra_api()

    creds_used = provider.creds_used
    if creds_used == "PLATFORM":
        provider._check_update_creds()
        return api.dataset(
            path,
            origin_path=provider.root,
            token=token,
            aws_access_key_id=provider.aws_access_key_id,
            aws_secret_access_key=provider.aws_secret_access_key,
            aws_session_token=provider.aws_session_token,
            region_name=provider.aws_region,
            endpoint_url=provider.endpoint_url,
            expiration=str(provider.expiration),
        )
    elif creds_used == "ENV":
        return api.dataset(
            path,
            origin_path=provider.root,
            token=token,
            profile_name=provider.profile_name,
        )
    elif creds_used == "DICT":
        return api.dataset(
            path,
            origin_path=provider.root,
            token=token,
            aws_access_key_id=provider.aws_access_key_id,
            aws_secret_access_key=provider.aws_secret_access_key,
            aws_session_token=provider.aws_session_token,
            region_name=provider.aws_region,
            endpoint_url=provider.endpoint_url,
        )

def dataset_to_libdeeplake(hub2_dataset: Dataset):
    """Convert a hub 2.x dataset object to a libdeeplake dataset object."""
    try_flushing(hub2_dataset)
    api = import_indra_api()
    path: str = hub2_dataset.path

    token = (
        hub2_dataset.client.get_token()
        if (hub2_dataset._token is None or hub2_dataset._token == "")
        and hub2_dataset.client
        else hub2_dataset._token
    )
    if token is None or token == "":
        raise EmptyTokenException
    if hub2_dataset.libdeeplake_dataset is None:
        libdeeplake_dataset = None
        if path.startswith("gdrive://"):
            raise ValueError("Gdrive datasets are not supported for libdeeplake")
        elif path.startswith("mem://"):
            raise ValueError("In memory datasets are not supported for libdeeplake")
        elif path.startswith("hub://"):
            provider = hub2_dataset.storage.next_storage
            if isinstance(provider, S3Provider):
                libdeeplake_dataset = _get_indra_ds_from_s3_provider(
                    path=path, token=token, provider=provider
                )

            elif isinstance(provider, GCSProvider):
                libdeeplake_dataset = _get_indra_ds_from_gcp_provider(
                    path=path, token=token, provider=provider
                )

            elif isinstance(provider, AzureProvider):
                libdeeplake_dataset = _get_indra_ds_from_azure_provider(
                    path=path, token=token, provider=provider
                )
            else:
                raise ValueError("Unknown storage provider for hub:// dataset")

        elif path.startswith("s3://"):
            libdeeplake_dataset = _get_indra_ds_from_s3_provider(
                path=path, token=token, provider=hub2_dataset.storage.next_storage
            )

        elif path.startswith(("gcs://", "gs://", "gcp://")):
            provider = get_base_storage(hub2_dataset.storage)

            libdeeplake_dataset = _get_indra_ds_from_gcp_provider(
                path=path, token=token, provider=provider
            )

        elif path.startswith(("az://", "azure://")):
            az_provider = get_base_storage(hub2_dataset.storage)
            libdeeplake_dataset = _get_indra_ds_from_azure_provider(
                path=path, token=token, provider=az_provider
            )

        else:
            org_id = hub2_dataset.org_id
            org_id = (
                org_id or jwt.decode(token, options={"verify_signature": False})["id"]
            )
            libdeeplake_dataset = api.dataset(path, token=token, org_id=org_id)

        hub2_dataset.libdeeplake_dataset = libdeeplake_dataset
    else:
        libdeeplake_dataset = hub2_dataset.libdeeplake_dataset

    assert libdeeplake_dataset is not None
    libdeeplake_dataset._max_cache_size = max(
        hub2_dataset.storage.cache_size, libdeeplake_dataset._max_cache_size
    )
    commit_id = hub2_dataset.pending_commit_id
    libdeeplake_dataset.checkout(commit_id)
    slice_ = hub2_dataset.index.values[0].value
    if slice_ != slice(None):
        if isinstance(slice_, tuple):
            slice_ = list(slice_)
        libdeeplake_dataset = libdeeplake_dataset[slice_]
    return libdeeplake_dataset


import json

import os, cv2

from PIL import Image

import numpy as np

import json

import os, cv2

import math

import numpy as np

import matplotlib.pyplot as plt

from scipy.stats import pearsonr

def image_hist(image_path: str):
    # def image_hist(img):

    """
An image histogram is a statistical table that reflects the distribution of pixels in an image.
Its horizontal axis represents the type of pixels in the image, which can be grayscale or color.
The vertical axis represents the total number of pixels or percentage of all pixels in the image for each color value.
An image is composed of pixels, so a histogram that reflects the distribution of pixels can
often be an important feature of the image.,The display method of the histogram
is dark and bright on the left, with the left side used to describe the darkness
of the image and the right side used to describe the brightness of the image.

param image_ Path: Pass in the image file to search for pixels
return:  No return value

    """

    # One dimensional histogram (single channel histogram)

    img = cv2.imread(image_path, 0)

    color = ('blue', 'green', 'red')

    # Use the plt Intrinsic function to directly draw

    global hist

    hist = cv2.calcHist([img], [0], None, [256], [0, 256])

    plt.plot(hist, color="r")

def cal_res(img_path, image_name):
    tmp = []

    for i in range(256):
        tmp.append(0)

    val = 0

    k = 0

    res = 0

    I = Image.open(img_path)

    greyIm = I.convert('L')

    img = np.array(greyIm)

    x, y = img.shape

    dst = np.zeros([x, y])

    for i in range(x):

        for j in range(y):

            if img[i, j] > 150:

                dst[i, j] = 255

            else:

                dst[i, j] = 0

    m = 0

    for i in range(dst.shape[0]):

        for j in range(dst.shape[1]):

            # print(dst[i,j])

            if dst[i, j] == 255:
                m += 1

    for i in range(len(img)):

        for j in range(len(img[i])):
            val = img[i][j]

            tmp[val] = float(tmp[val] + 1)

            k = float(k + 1)

    for i in range(len(tmp)):
        tmp[i] = float(tmp[i] / k)

    tmp = np.array(tmp)

    for i in range(len(tmp)):

        if (tmp[i] == 0):

            res = res

        else:

            res = float(res - tmp[i] * (math.log(tmp[i]) / math.log(2.0)))

    return res, dst

def getFileList(dir, Filelist, ext=None):
    """
    Obtain a list of files in the folder and its subfolders.
    Input dir: Folder root directory.
    Input ext: Extension.
    Return: File path list
    """

    newDir = dir

    if os.path.isfile(dir):

        if ext is None:

            Filelist.append(dir)

        else:

            if ext in dir[-3:]:
                Filelist.append(dir)

    elif os.path.isdir(dir):

        for s in os.listdir(dir):
            newDir = os.path.join(dir, s)

            getFileList(newDir, Filelist, ext)

    return Filelist

def visualization_bbox1(num_image, json_path, img_path):
    with open(json_path) as annos:
        annotation_json = json.load(annos)

        image_name = annotation_json['images'][num_image - 1]['file_name']

        id = annotation_json['images'][num_image - 1]['id']  # Read Image ID

        image_path = os.path.join(img_path, str(image_name).zfill(5))

        res, dst = cal_res((str(image_path)), str(image_name))

        image = cv2.imread(image_path, 1)

        img = cv2.imread(image_path, 0)

        hist = cv2.calcHist([img], [0], None, [256], [0, 256])

        return hist, res

train_json = '/train1/train.json'
test_json = '/test1/test.json'
val_json = '/val1/val.json'

train_path = '/train1/train_image/'
test_path = '/test1/test_image/'
val_path = '/val1/val_image/'

imglist = getFileList(train_path, [], 'jpg')

# cal_res()

res1 = []

for imgpath in imglist:

    d = d + 1

    hist, res = visualization_bbox1(d, train_json, train_path)

    res1.append(res)

    print(imgpath)

    print("res:", res)
    if res >= 2 and res < 3:

        plt.plot(hist, color="y", alpha=1, linestyle='--', linewidth=0.5, marker='o', markersize=1)

    elif res >= 3 and res < 4:

        plt.plot(hist, color="g", alpha=1, linestyle='-.', linewidth=0.5, marker='>', markersize=1)

    elif res >= 4 and res < 5:

            plt.plot(hist, color="r", alpha=1, linestyle=':', linewidth=0.5, marker='*', markersize=1)
    elif res >= 5 and res < 6:

        plt.plot(hist, color="b", alpha=1, linestyle=':', linewidth=0.5, marker='x', markersize=1)

    elif res >= 6 and res < 7 :

        plt.plot(hist, color="c", alpha=1, linestyle=':', linewidth=0.5, marker='D', markersize=1)

    elif res >= 7:

        plt.plot(hist, color="m", alpha=1, linestyle=':', linewidth=0.5, marker='1', markersize=1)

    plt.legend()

print(max(res1), min(res1))

plt.legend(loc=1, fontsize='10')
plt.xlabel('Pixel')
plt.ylabel('Number')
plt.savefig('/media/ExtDisk/yxt/IMG/pic.jpg', dpi=500)


import os

os.environ["TF_CPP_MIN_LOG_LEVEL"] = '3'  # 只显示 Error
import cv2
import logging

import numpy as np

from data_loaders.image_preprocessor import ImagePreprocessor, ImagePreprocessorConfig
from data_loaders.image_set_loader import ImageSetDataLoader, ImageSetDataLoaderConfig
from models.cnn_model import CNNModel, CNNModelConfig
from trainers.universal_trainer import UniversalTrainer, UniversalTrainerConfig

logger = logging.getLogger(__name__)

IS_TRAINING = False
DATA_AUGMENTATION = True and IS_TRAINING

INPUT_SHAPE = (65, 125, 1)
INPUT_SHAPE_REVERSE = (125, 65, 1)

VALIDATION_IMAGE_CHAR_ROOT = "dataset/validation_image_char/"
VALIDATION_IMAGE_PROVINCE_ROOT = "dataset/validation_image_province/"

# split word classify
CHARACTER_LIST = [str(i) for i in range(10)] + list("QWERTYUPASDFGHJKLZXCVBNM")
PROVINCE_LIST = ['冀', '新', '鄂', '宁', '桂', '黑', '湘', '皖', '云', '豫', '蒙', '赣', '吉', '辽', '苏', '甘', '晋', '浙', '闽',
                 '渝', '贵', '陕', '粤', '川', '鲁', '琼', '青', '藏', '京', '津', '沪']

preprocessor_char_config = ImagePreprocessorConfig(
    dataset_file_root="dataset/character_data_46x90/",
    character_list=CHARACTER_LIST,
    augmentation_amount=500,
    out_image_size=INPUT_SHAPE,
    out_file_root="dataset/character_data_augmentation/",
    save_image_format_result=False
)

preprocessor_province_config = ImagePreprocessorConfig(
    dataset_file_root="dataset/character_data_46x90/",
    character_list=PROVINCE_LIST,
    augmentation_amount=500,
    out_image_size=INPUT_SHAPE,
    out_file_root="dataset/character_data_augmentation/",
    save_image_format_result=False
)

model_char_config = CNNModelConfig(input_shape=INPUT_SHAPE_REVERSE,
                                   output_len=len(CHARACTER_LIST),
                                   learning_rate=0.0001)
model_province_config = CNNModelConfig(input_shape=INPUT_SHAPE_REVERSE,
                                       output_len=len(PROVINCE_LIST),
                                       learning_rate=0.0001)

trainer_config = UniversalTrainerConfig(epoch=5)

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG)

    if IS_TRAINING:
        logger.info("Preprocess...")
        preprocessor_char = ImagePreprocessor(preprocessor_char_config)
        preprocessor_province = ImagePreprocessor(preprocessor_province_config)
        logger.info("Making dataset...")
        data_loader_char_config = ImageSetDataLoaderConfig(
            image_dict=preprocessor_char.get_image_dict(),
            image_shape=INPUT_SHAPE_REVERSE,
            shuffle_buffer_size=100000,
            batch_size=100
        )
        data_loader_province_config = ImageSetDataLoaderConfig(
            image_dict=preprocessor_province.get_image_dict(),
            image_shape=INPUT_SHAPE_REVERSE,
            shuffle_buffer_size=100000,
            batch_size=100
        )

        data_loader_char = ImageSetDataLoader(data_loader_char_config)
        data_loader_province = ImageSetDataLoader(data_loader_province_config)

        model_char = CNNModel(model_char_config)
        model_province = CNNModel(model_province_config)

        trainer_char = UniversalTrainer(model_char.get_model(),
                                        data_loader_char.get_dataset(),
                                        trainer_config)
        trainer_province = UniversalTrainer(model_province.get_model(),
                                            data_loader_province.get_dataset(),
                                            trainer_config)

        logger.info("Training...")
        trainer_char.train()
        trainer_province.train()
        trainer_char.save("log/test_split_char.h5")
        trainer_province.save("log/test_split_province.h5")
    else:
        model_char = CNNModel(model_char_config)
        trainer_char = UniversalTrainer(model_char.get_model(), None, trainer_config)
        trainer_char.load("log/test_split_char.h5")

        model_province = CNNModel(model_province_config)
        trainer_province = UniversalTrainer(model_province.get_model(), None, trainer_config)
        trainer_province.load("log/test_split_province.h5")

        # province
        valid_image_province_list = os.listdir(VALIDATION_IMAGE_PROVINCE_ROOT)
        logger.debug("Validation image list: %s", valid_image_province_list)
        for image_file_name in valid_image_province_list:
            image_file = os.path.join(VALIDATION_IMAGE_PROVINCE_ROOT, image_file_name)
            image = cv2.imread(image_file, flags=cv2.IMREAD_COLOR)
            image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
            image = cv2.adaptiveThreshold(image, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 51, 0)
            image = cv2.resize(image, INPUT_SHAPE[:2])
            image = np.array([image, ]) / 255
            image = np.reshape(image, np.shape(image) + (1,))
            predict_result = np.argmax(trainer_char.model.predict(image, batch_size=1), axis=-1)[0]

            logger.info("Predict result: %s %s",
                        predict_result,
                        PROVINCE_LIST[predict_result], )

            cv2.imshow("Validation image", image[0])
            cv2.waitKey()

        # char
        valid_image_char_list = os.listdir(VALIDATION_IMAGE_CHAR_ROOT)
        logger.debug("Validation image list: %s", valid_image_char_list)
        for image_file_name in valid_image_char_list:
            image_file = os.path.join(VALIDATION_IMAGE_CHAR_ROOT, image_file_name)
            image = cv2.imread(image_file, flags=cv2.IMREAD_COLOR)
            image = cv2.cvtColor(image, cv2.COLOR_RGB2GRAY)
            image = cv2.adaptiveThreshold(image, 255, cv2.ADAPTIVE_THRESH_MEAN_C, cv2.THRESH_BINARY, 51, 0)
            image = cv2.resize(image, INPUT_SHAPE[:2])
            image = np.array([image, ]) / 255
            image = np.reshape(image, np.shape(image) + (1,))
            predict_result = np.argmax(trainer_char.model.predict(image, batch_size=1), axis=-1)[0]

            logger.info("Predict result: %s %s",
                        predict_result,
                        CHARACTER_LIST[predict_result], )

            cv2.imshow("Validation image", image[0])
            cv2.waitKey()


from avatar3d import *
from torch_dataset import *
from torch.utils.data import DataLoader
from tqdm import tqdm
from time import sleep
import os
import os.path as osp

def train(
    data,
    data_path,
    batch_size=32,
    epoch = 1000,
    learning_rate=1e-4,
    momentum=0.1,
    print_loss=50,
    save_model=True,
    save_freq=1000,
    save_path='./checkpoints',
    device='cuda:0'):

    save_count = 0
    if not osp.exists(save_path):
        os.mkdir(save_path)

    train_set = ImageDataset(data, data_path, mode='train')
    train_dataloader = DataLoader(train_set, batch_size=batch_size)
    model = Avatar3D()
    model = model.to(device)
    loss_fn = nn.MSELoss()

    optimizer = torch.optim.SGD(model.parameters(), lr = learning_rate, momentum = momentum)
    pbar = tqdm(total = epoch)
    epoch_loss = 0.
    for e in range(epoch):
        running_loss = 0.
        for i, batch in enumerate(train_dataloader):
            img, keypoints, pose = batch['image'].to(device), batch['keypoints'].to(device), batch['pose'].to(device)
            optimizer.zero_grad()
            output = model(img, keypoints)
            loss = loss_fn(output, pose)
            loss.backward()
            optimizer.step()
            running_loss += loss.item()
            #sleep(0.01)
        avg_batch_loss = running_loss/(i+1)
        epoch_loss += avg_batch_loss
        if e % print_loss == 0:
            print("epoch:%d, running loss: %f" % (e,epoch_loss/(e+1)))

        if save_model and e % save_freq == 0:
            save_count += 1
            torch.save({
                'model_state_dict':model.state_dict(),
                'optimizer_state_dict': optimizer.state_dict(),
                'loss': epoch_loss/(e+1),
                'epoch': e,
                'batch_size': batch_size
            }, osp.join(save_path, f'checkpoint%d.pth'%save_count))
        pbar.update()

    torch.save({
                'model_state_dict':model.state_dict(),
                'optimizer_state_dict': optimizer.state_dict(),
                'loss': epoch_loss/(e+1),
                'epoch': e,
                'batch_size': batch_size
            }, osp.join(save_path, f'checkpoint_final.pth'))
    return model


import os, django
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "KingBestMall.settings")
django.setup()
from category.models import *
from django.db import connections
import requests

# data=requests.get('https://backend.shopnroar.com/products/GetAllCategories').json()
# target=data['Categories']
# for i in target:
#         main=Main_Categories.objects.create(Cat_Name=i['Category'],Pic=i['path'],Cat_Des='ShopnRoarCategory')
#         print('cate     ',main.id)
#         sub=i['SubCategories']
#         for j in sub:
#             print(j)
#             subdata=Sub_Categories.objects.create(Sub_Cat_Name=j['SubCat'],Subcat_Des='ShopnRoarSubCategory',Cat_ID=main)
#             print('sub      ',subdata.Cat_ID)#     # print(i)

# import pandas
# df = pandas.read_csv('FieldsName.csv', error_bad_lines=False)
# # sub=Sub_Categories.objects.all()
# # for i in sub:
# #     print(i.Sub_Cat_Name)
# # df.head()
# # print(df)
# for j in df:
#     # print(j)
#     for k in j:
#         print(k)

import csv

# print("here")
# MixList = []
oldfile = open('/home/rana/Desktop/kingbestmallbackend/KingBestMall/category/FieldsName.csv')
read = csv.reader(oldfile)
# print(read)
count=0
for data in read:
    # print(data)
    sub=Sub_Categories.objects.get(Sub_Cat_Name=data[0])
    print('name         ',sub.Sub_Cat_Name)
    for j in data:
        if j!='' and j!=sub.Sub_Cat_Name:
            # count=count+1
            # feature=Features.objects.create(Feature_Name=j)
            print(j)

    print('//////////////')


import datetime
import logging
import time

from google.appengine.api.app_identity import get_application_id
from google.appengine.api import users
from google.appengine.ext import ndb
from google.appengine.ext.webapp.util import login_required
import webapp2

from gcloud.datastore import SCOPE
from gcloud.datastore import connection
from oauth2client import client

APP_NAME = get_application_id()
IMPLICIT_CREDENTIALS = client.GoogleCredentials.get_application_default()
SCOPED_CREDENTIALS = IMPLICIT_CREDENTIALS.create_scoped(SCOPE)
CONNECTION = connection.Connection(credentials=SCOPED_CREDENTIALS)
if APP_NAME is None:
    DATASET = None
else:
    DATASET = CONNECTION.dataset(APP_NAME)

class Foo(ndb.Model):
    _use_memcache = False
    _use_cache = False

    now = ndb.DateTimeProperty(auto_now_add=True)
    nickname = ndb.BlobProperty(indexed=True)
    email = ndb.BlobProperty(indexed=True)
    user_id = ndb.BlobProperty(indexed=True)

def make_entity(user):
    entity = DATASET.entity('Foo')
    entity['now'] = datetime.datetime.utcnow()
    entity['nickname'] = user.nickname()
    entity['email'] = user.email()
    entity['user_id'] = user.user_id()
    start = time.time()
    entity.save()
    duration = time.time() - start
    logging.debug('Saving with gcloud-python %r took %f seconds.',
                  entity.key(), duration)
    return entity

def make_ndb_entity(user):
    foo_entity = Foo(nickname=user.nickname(),
                     email=user.email(),
                     user_id=user.user_id())
    start = time.time()
    foo_entity.put()
    duration = time.time() - start
    logging.debug('Saving with NDB %r took %f seconds.',
                  foo_entity.key, duration)
    return foo_entity

class MainHandler(webapp2.RequestHandler):

    @login_required
    def get(self):
        user = users.get_current_user()
        entity = make_entity(user)
        key_id = entity.key().id()
        message = 'Key saved: %d' % (key_id,)
        self.response.write(message)

class NDBHandler(webapp2.RequestHandler):

    @login_required
    def get(self):
        user = users.get_current_user()
        foo_entity = make_ndb_entity(user)
        key_id = foo_entity.key.id()
        message = 'Key saved: %d' % (key_id,)
        self.response.write(message)

app = webapp2.WSGIApplication([
    ('/', MainHandler),
    ('/ndb', NDBHandler),
], debug=True)


from shimpy.core import Extension, Themelet, Event
from shimpy.core.models import Image, Tag
from shimpy.core.utils import SparseList, make_link
from shimpy.core.context import context

from sqlalchemy import not_, func

import structlog
import re
from webhelpers.html import literal
from .theme import IndexTheme

log = structlog.get_logger()

class SearchTermParseEvent(Event):
    def __init__(self, results, term, negative):
        self.results = results
        self.term = term
        self.negative = negative
        self.filter_added = False

    def add_filter(self, f):
        self.filter_added = True
        if self.negative:
            f = not_(f)
        self.results = self.results.filter(f)

class SearchTermParseException(Exception):
    pass

class PostListBuildingEvent(Event):
    def __init__(self, search_terms):
        self.search_terms = search_terms
        self.parts = SparseList()

    def add_control(self, html, position=50):
        self.parts.insert(position, html)

class Index(Extension):
    priority = 80  # STPE fallback needs to be last of the STPE handlers

    def __init__(self):
        self.theme = IndexTheme()
        #self.theme = Template(filename='shimpy/ext/index/theme.mako')

    def onInitExt(self, event):
        # context.config.set_default_int("index_images", 24)
        # context.config.set_default_bool("index_tips", True)
        pass

    def onPageRequest(self, event):
        page = context.page
        user = context.user
        request = context.request
        database = context.database
        send_event = context.server.send_event

        if event.page_matches("post/list"):
            if "search" in request.args:
                search = request.args["search"]
                page.mode = "redirect"
                if search:
                    page.redirect = make_link("post/list/" + search + "/1")
                else:
                    page.redirect = make_link("post/list/1")
                return

            search_terms = event.get_search_terms()
            page_number = event.get_page_number()
            page_size = event.get_page_size()
            count_search_terms = len(search_terms)

            try:
                total_pages = Image.count_pages(search_terms)
                images = Image.find_images((page_number-1)*page_size, page_size, search_terms)
            except SearchTermParseException:
                total_pages = 0
                images = []

            count_images = len(images)

            if count_search_terms == 0 and count_images == 0 and page_number == 1:
                self.theme.display_intro(page)

            elif count_search_terms > 0 and count_images == 1 and page_number == 1:
                page.mode = "redirect"
                page.redirect = make_link("post/view/" + images[0].id)

            else:
                plbe = PostListBuildingEvent(search_terms)
                send_event(plbe)

                self.theme.set_page(page, page_number, total_pages, search_terms)
                self.theme.display_page(page, images)

                if plbe.parts:
                    self.theme.display_admin_block(page, plbe.parts)

    def onSetupBuilding(self, event):
        sb = SetupBlock("Index Options", position=20)

        sb.add_label("Show ")
        sb.add_int_option("index_images")
        sb.add_label(" images on the post list")

        event.panel.add_block(sb)

    def onSearchTermParse(self, event):
        # TODO: search by tag count
        # TODO: search by ratio
        # TODO: search by filesize
        # TODO: search by id
        # TODO: search by hash / md5
        # TODO: search by filetype / ext
        # TODO: search by file name
        # TODO: search by source
        # TODO: search by posted
        # TODO: search by size

        # TODO: move to user ext
        m = re.match("^user_id=(\d+)$", event.term)
        if m:
            event.add_filter(Image.user_id == m.group(1))

        # TODO: move to numeric score ext
        m = re.match("^score([<>=]+)(\d+)$", event.term)
        if m:
            event.add_filter(Image.score.op(m.group(1))(m.group(2)))

        if not event.filter_added:
            # re.match("^[a-zA-Z0-9\'\_\-\.\(\)]+$", event.term):
            tag = Tag.get(event.term)
            if tag:
                #log.debug("Adding filter for plain tag", term=event.term, tag=tag.name)
                event.add_filter(Image.tags.contains(tag))
            else:
                if not event.negative:
                    # if the tag is negative, it doesn't
                    # matter that we couldn't find it
                    log.debug("Tag not found", term=event.term)
                    event.add_filter(False)


from app import app, db
from flask import render_template, redirect, request, flash, jsonify
import urllib2
from .models import devices, timer, log
from time import strftime
from datetime import datetime, timedelta
import temperature
#from datetime import datetime

def write_log(deviceid, state):
    newlog = log()
    newlog.devices_id = deviceid
    newlog.state = state
    newlog.time = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
    db.session.add(newlog)
    db.session.commit()

def check_status(deviceip):
    status_url = "http://" + deviceip + "/cgi-bin/relay.cgi?state"
    try:
        status_check = urllib2.urlopen(status_url, timeout=0.2).read()
        status = status_check.strip()
    except:
        status = "OFF"
    return status

def turn_on(deviceip, deviceid):
    on_url = "http://" + deviceip + "/cgi-bin/relay.cgi?on"
    try:
        on_status = check_status(deviceip)
        if check_status(deviceip) == "OFF":
            on_check = urllib2.urlopen(on_url, timeout=0.1).read()
            on_status = on_check.strip()
            write_log(deviceid, check_status(deviceip))
        else:
            return on_status
    except:
        on_status = "OFF"
    return on_status

def turn_off(deviceip, deviceid):
    off_url = "http://" + deviceip + "/cgi-bin/relay.cgi?off"
    try:
        off_status = check_status(deviceip)
        if off_status == "ON":
            off_check = urllib2.urlopen(off_url, timeout=0.3).read()
            off_status = off_check.strip()
            write_log(deviceid, check_status(deviceip))
        else:
            return off_status
    except:
        off_status = "OFF"
    return off_status

def toggle_device(deviceip, deviceid):
    toggle_url = "http://" + deviceip + "/cgi-bin/relay.cgi?toggle"
    try:
        toggle_check = urllib2.urlopen(toggle_url, timeout=0.3).read()
        toggle_status = toggle_check.strip()
        write_log(deviceid, check_status(deviceip))
    except:
        toggle_status = "NA"
    return toggle_status

@app.route('/toggle')
def toggle():
    deviceip = request.args.get('deviceip')
    deviceid = request.args.get('deviceid')
    status = toggle_device(deviceip, deviceid)
    return status

@app.route('/index-bs')
def indexbs():
    #state_bed1 = urllib2.urlopen('http://192.168.1.101/cgi-bin/relay.cgi?state').read()
    datimers = timer.query.all()
    dadevices = devices.query.all()
    device_status_list = []
    cur_temp = str(round(temperature.read_temp(),1))
    for check_devices in dadevices:
        state_bed1 = check_status(check_devices.ip)
        state_bed1 = state_bed1.strip()
        if state_bed1 == "ON":
            button_status="btn-success"
        else:
            button_status="btn-danger"
        device_status_list.append([check_devices.id, check_devices.name, button_status, check_devices.ip, check_devices.temp])

    return render_template('index-bs.html', title='Home',
                                         device_list = device_status_list,
                                         timer = datimers,
                                         cur_temp = cur_temp,
                                         state_bed1=state_bed1)
@app.route('/')
@app.route('/index')
def index():
    #state_bed1 = urllib2.urlopen('http://192.168.1.101/cgi-bin/relay.cgi?state').read()
    datimers = timer.query.all()
    dadevices = devices.query.all()
    device_status_list = []
    cur_temp = str(round(temperature.read_temp(),1))
    for check_devices in dadevices:
        state_bed1 = check_status(check_devices.ip)
        state_bed1 = state_bed1.strip()
        if state_bed1 == "ON":
            button_status="mdl-color--green-300"
        else:
            button_status="mdl-color--red-300"
        device_status_list.append([check_devices.id, check_devices.name, button_status, check_devices.ip, check_devices.temp])

    return render_template('index.html', title='Home',
                                         device_list = device_status_list,
                                         timer = datimers,
                                         cur_temp = cur_temp,
                                         state_bed1=state_bed1)

@app.route('/add_timer', methods=['POST'])
def add_timer():
    """
    Retrieves input form and writes to the db
    """
    start = request.form['starttime']
    end = request.form['endtime']
    nameid = request.form['deviceid']
    timertype = request.form['timertype']
    if end == '24:00':
        end = '00:00'
    newtimer = timer()
    newtimer.name_id = nameid
    newtimer.start_time = start
    newtimer.end_time = end
    newtimer.timer_type = timertype
    db.session.add(newtimer)
    db.session.commit()
    #flash('Entry was added')
    return redirect('/index')
    #return render_template('delete.html', title='Home', starter=start, ender=end)

@app.route('/delete_timer/<postID>', methods=['POST'])
def delete_timer(postID):
    timer.query.filter(timer.id == postID).delete()
    db.session.commit()
    return redirect('/index')

@app.route('/add_temp', methods=['POST'])
def add_temp():
    """
    Retrieves input form and writes to the db
    """
    temperature = request.form['temperature']
    deviceid = request.form['deviceid']

    device = devices.query.get(deviceid)
    device.temp = temperature
    db.session.commit()
    #flash('Entry was added')
    return redirect('/index')

@app.route('/check_device_status', methods=['POST'])
def check_device_status():
    deviceid = request.form['deviceid']
    device = devices.query.get(deviceid)
    state = check_status(device.ip)
    return jsonify(staus=state)

@app.route('/check_temp', methods=['POST'])
def check_temp():
    cur_temp = str(round(temperature.read_temp(),1))
    return jsonify(temp=cur_temp)

@app.route('/check_timers')
def check_timers():
    flag = "NO"
    deviceip="notset"
    cur_time = strftime("%H:%M")
    datimers = timer.query.all()
    dastatus = check_status("192.168.1.101")
    testlist = []
    #maxtemp = 23
    for timer_entry in datimers:
        deviceip=devices.query.filter_by(id=timer_entry.name_id).first().ip
        maxtemp=float(devices.query.filter_by(id=timer_entry.name_id).first().temp)
        if maxtemp == 0.0:
            maxtemp=99
        if timer_entry.start_time < cur_time and timer_entry.end_time > cur_time and temperature.read_temp() < maxtemp:
            if check_status(deviceip) =="OFF":
                turn_on(deviceip, timer_entry.name_id)
        #teststatus = "OFF"
        testv = datetime.strptime(cur_time, '%H:%M') - datetime.strptime(timer_entry.end_time, '%H:%M')
        if testv > timedelta(minutes=1) and testv < timedelta(minutes=5) or temperature.read_temp() > maxtemp:
            flag = "YES " + str(testv) + " " + (deviceip)
            if check_status(deviceip) == "ON":
                turn_off(deviceip, timer_entry.name_id)
            if testv > timedelta(minutes=1) and timer_entry.timer_type == 0:
                delete_timer(timer_entry.id)
        #time.strptime(stamp, '%I:%M')
        teststatus = check_status(deviceip)
        testlist.append([timer_entry.name_id,timer_entry.start_time,timer_entry.end_time,teststatus,testv,maxtemp])

    return render_template('test.html', time=cur_time,
                                        status=dastatus,
                                        flag=flag,
                                        testv=testv,
                                        testc=timedelta(minutes=1),
                                        timers=testlist)

@app.route('/log')
def test():
    dalog = log.query.all()
    dadevices = devices.query.all()
    log_list = []
    test_log_list = []
    for entry in dalog:
        test_log_list.append([entry.id,entry.devices_id,entry.state,entry.time])
    for device in dadevices:
        off_count = 0
        on_count = 0
        total_time = 0
        for entry in dalog:
            if entry.devices_id == device.id:
                if entry.state == "ON":
                    on_count += 1
                    on_time = datetime.strptime(entry.time, '%Y-%m-%d %H:%M:%S')
                    #log_list.append([entry.devices_id, entry.state, entry.time])
                if entry.state == "OFF":
                    off_count += 1
                    accrued_time = datetime.strptime(entry.time, '%Y-%m-%d %H:%M:%S') - on_time
                    if accrued_time.total_seconds() < 0:
                        accrued_time = 0
                    else:
                        accrued_time = accrued_time.total_seconds()
                    total_time = (total_time + accrued_time)
        total_time = total_time / 3600
        elec_cost = 0.215
        power = 1.3
        total_cost = power * elec_cost * total_time
        log_list.append([device.id, on_count, off_count, round(total_time, 2), round(total_cost, 2)])
    return render_template('testbed2.html', title='testbed2',
                                            log = log_list,
                                            test = test_log_list)

def convert(seconds):
    hours = int(seconds/3600)
    minutes = int((seconds - (hours * 3600))/60)
    time = str(hours) + '.' + str(minutes) + ' hrs'
    return time

def get_date_list():
    dalog = log.query.all()
    log_list = []
    logdate = ""
    for entry in dalog:
        full_date = datetime.strptime(entry.time, '%Y-%m-%d %H:%M:%S')
        date = full_date.date()
        if date != logdate:
            logdate = date
            log_list.append([logdate])
    return log_list

@app.route('/daylog')
def logger():
    dalog = log.query.all()
    dadevices = devices.query.all()
    date_list = get_date_list()
    log_list = []
    last_state = "OFF"
    for device in dadevices:
        for thedate in date_list:
            off_count = 0
            on_count = 0
            total_time = 0
            for entry in dalog:
                full_date = datetime.strptime(entry.time, '%Y-%m-%d %H:%M:%S')
                entrydate = full_date.date()
                if entry.devices_id == device.id and entrydate == thedate[0]:
                    if entry.state == "ON" and last_state == "OFF":
                        on_count += 1
                        on_time = datetime.strptime(entry.time, '%Y-%m-%d %H:%M:%S')
                        #log_list.append([entry.devices_id, entry.state, entry.time])
                        last_state = "ON"
                    if entry.state == "OFF" and last_state == "ON":
                        off_count += 1
                        accrued_time = datetime.strptime(entry.time, '%Y-%m-%d %H:%M:%S') - on_time
                        if accrued_time.total_seconds() < 0:
                            accrued_time = 0
                        else:
                            accrued_time = accrued_time.total_seconds()
                        total_time = (total_time + accrued_time)
                        last_state = "OFF"
            #total_time = total_time / 3600
            elec_cost = 0.215
            power = 1.3
            total_cost = power * elec_cost * total_time
            log_list.append([thedate[0], device.id, total_time, round(total_cost, 2)])
            log_list.sort()
    date = ""
    logger_list = []
    for line in log_list:
        value1 = 0
        value2 = 0
        value3 = 0
        if line[0] != date:
            date = line[0]
            for moreline in log_list:
                if moreline[0] == date:
                    if moreline[1] == 1:
                        value1 = convert(moreline[2])
                    if moreline[1] == 2:
                        value2 = convert(moreline[2])
                    if moreline[1] == 3:
                        value3 = convert(moreline[2])
            logger_list.append([date, value1, value2, value3])

    return render_template('log.html', title='log',
                                            log = logger_list)


from selenium import webdriver
from selenium.common import exceptions
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from webdriver_manager.chrome import ChromeDriverManager
from webdriver_manager.utils import ChromeType

from msrecorder.app.config.config_service import ConfigService

class BrowserService:
    instance = None

    @staticmethod
    def get_instance():
        if BrowserService.instance is None:
            BrowserService.instance = BrowserService()

        return BrowserService.instance

    def __init__(self):
        self.config = ConfigService.get_instance().config
        self.browser = self.__get_browser()

    def __get_browser(self):
        browser: webdriver.Chrome = None

        chrome_options = self.__get_chrome_options()
        chrome_type = self.__get_chrome_type()

        browser = webdriver.Chrome(ChromeDriverManager(
            chrome_type=chrome_type).install(), options=chrome_options)

        return browser

    def __get_chrome_options(self):
        chrome_options = webdriver.ChromeOptions()
        chrome_options.add_argument('--ignore-certificate-edrrors')
        chrome_options.add_argument('--ignore-ssl-errors')
        chrome_options.add_argument("--use-fake-ui-for-media-stream")
        chrome_options.add_argument("start-maximized")
        chrome_options.add_experimental_option(
            'excludeSwitches', ['enable-logging'])

        if self.config.mute_audio:
            chrome_options.add_argument("--mute-audio")

        return chrome_options

    def __get_chrome_type(self):
        chrome_type = ChromeType.GOOGLE
        # chrome_type = self.config.get

        if self.config.chrome_type:
            if self.config.chrome_type == "chromium":
                chrome_type = ChromeType.CHROMIUM
            elif self.config.chrome_type == "msedge":
                chrome_type = ChromeType.MSEDGE

        return chrome_type


import requests
import json
import time

page = 1
job_data = []

response = requests.get('http://212.159.75.17:8000/jobs?page={0}'.format(page), auth=('admin', 'media101')).json()
numberOfJobs = response['count']
numberOfPages = round(numberOfJobs / 10, 0)

while page <= numberOfPages:

	print("Getting jobs from page: {0}".format(page))
	response = requests.get('http://212.159.75.17:8000/jobs?page={0}'.format(page), auth=('admin', 'media101')).json()

	job_data.extend(response['jobs'])

	page += 1

print("Jobs Number: {0}".format(len(job_data)))

with open('jobs_data.json', 'w') as outfile:
    json.dump(job_data, outfile, indent=4)


import pandas as pd
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('-l', '--lipid')
parser.add_argument('-s', '--system')
parser.add_argument('-c', '--chains', nargs='+')
parser.add_argument('-cp', '--chainspos', nargs='+')
args = parser.parse_args()

dataset = pd.read_csv('Interaction_{}/Dataset_{}/Dataset.csv'.format(args.lipid, args.lipid))      # pylipid generated dataset file
dataset['Residue Name'] = dataset['Residue'].apply(lambda name: name[-3:])
new_dataset_file = '{}_{}_chains_dataset.csv'.format(args.lipid, args.system)           # name for new dataset file
pdb_cg = 'step5_charmm2gmx.pdb'                                         # raw structure file generated with charmm-gui
prot_chains = ['PROA', 'PROB', 'PROC', 'PROD', 'PROE']                  # segnames for protein chains in structure file
chains_type = dict(zip(prot_chains, args.chains))
chains_typepos = dict(zip(prot_chains, args.chainspos))

#chains_type = {'PROA': 'b2', 'PROB': 'a1', 'PROC': 'b2', 'PROD': 'a1', 'PROE': 'g2'}
#chains_typepos = {'PROA': 'b2_1', 'PROB': 'a1_1', 'PROC': 'b2_2', 'PROD': 'a1_2', 'PROE': 'g2_1'}

residues = []
residueIds = []
residueChains = []

# this parser is hardcoded, should work with charmm-gui generated martini files
with open(pdb_cg) as f:
    prev_resnum = 0
    residueId = 0
    for line in f:
        if any(chain in line for chain in prot_chains):
            atom = line.strip().split()
            if atom[4] != prev_resnum:
                residues.append(atom[4]+atom[3])
                residueChains.append(atom[10])
                residueIds.append(residueId)
                prev_resnum = atom[4]
                residueId += 1

resid_data = pd.DataFrame(list(zip(residues, residueIds, residueChains)), columns=['Residue', 'Residue ID', 'Residue Chain'])
resid_data['Lipid'] = args.lipid
resid_data['Residue Num'] = resid_data['Residue'].apply(lambda num: int(num[0:-3]))
resid_data['Residue Name'] = resid_data['Residue'].apply(lambda name: name[-3:])
resid_data['System'] = args.system
resid_data['Residue Chain Type'] = resid_data['Residue Chain'].apply(lambda chain: chains_type[chain])
resid_data['Residue Chain TypePos'] = resid_data['Residue Chain'].apply(lambda chain: chains_typepos[chain])

# dataset.drop(['Residue'], axis=1, inplace=True)             # in case of pdb input for pylipid messed up numbering
print(resid_data)
print(dataset)
new_dataset = resid_data.merge(dataset, on=['Residue','Residue ID', 'Residue Name'])

# reads raw pickles with all duration times for each replica

resid_times = pd.read_pickle('Interaction_{}/Dataset_{}/Duration.pickle'.format(args.lipid, args.lipid))
resid_times = pd.DataFrame.from_dict(resid_times, orient='index')
resid_times['Residue ID'] = resid_times.index
resid_times['Durations All'] = resid_times.apply(lambda row: row[0] + row[1] + row[2] + row[3], axis=1)
resid_times.rename(columns={0: 'Durations Sys1', 1: 'Durations Sys2', 2: 'Durations Sys3', 3: 'Durations Sys4'}, inplace=True)

new_dataset = new_dataset.merge(resid_times, on=['Residue ID'])

new_dataset.to_csv(new_dataset_file, index=False)

# print(new_dataset[new_dataset['Residue ID'] == 951])
# print(resid_times[951])

if len(dataset) != len(new_dataset):
    print('Problem with residue data merging, some residues omitted!')

# selected = new_dataset.iloc[:, 0:15]
# selected.drop(axis=1, inplace=True, labels=['Occupancy std', 'Lipid Count', 'Lipid Count std'])
# print(selected.sort_values(by=['Occupancy'], ascending=False).iloc[0:50, :])

# identify at which chain(s) binding site 0 is located
# print(new_dataset[new_dataset['Binding Site ID'] == 0])


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jun 21 10:11:05 2019

@author: matteo, mmilde

This file contains default parameter for OCTA hierarchical BB.
Parameters are stored as dictionaries.
For more information look into
M. B. Milde (2019) Spike-Based Computational Primitives for
Vision-Based Scene Understanding, PhD diss., Institute of
Neuroinformatics, University of Zurch and ETH Zurich,
Zurich Switzerland.

"""

from brian2 import pF, nS, mV, ms, pA, nA

wta_params = {'we_inp_exc': 100,
              'we_exc_inh': 300,  # 55/50
              'wi_inh_exc': -200,  # -300
              'we_exc_exc': 10.0,  # 45
              'sigm': 2,
              'rp_exc': 1 * ms,
              'rp_inh': 1 * ms,
              'wi_inh_inh': -100,
              'ei_connection_probability': 0.5,
              'ie_connection_probability': 0.66,
              'ii_connection_probability': 0.1
              }

octa_params = {'distribution': 'gamma',
               'dist_param_init': 0.5,  # shape for gamma < 0.5
               'scale_init': 1.0,  # sigma for gamma 1.0
               'dist_param_re_init': 0.4,
               'scale_re_init': 0.9,
               're_init_index': None,
               're_init_threshold': 0.2,
               'buffer_size_plast': 200,
               'noise_weight': 30.0,
               'variance_th_c': 0.5,
               'variance_th_p': 0.4,
               'learning_rate': 0.007,
               'inh_learning_rate': 0.01,
               'decay': 150,
               'tau_stdp': 10 * ms,
               'tau_pred': 1.5 * ms,
               'seed' : 42,
               }

# Dictionaries for mismatch

mismatch_neuron_param = {
    'Inoise': 0,
    'Iconst': 0,
    'kn': 0,
    'kp': 0,
    'Ut': 0,
    'Io': 0,
    'Cmem': 0.2,
    'Iath': 0.2,
    'Iagain': 0.2,
    'Ianorm': 0.2,
    'Ica': 0.2,
    'Itauahp': 0.2,
    'Ithahp': 0.2,
    'Cahp': 0.2,
    'Ishunt': 0.2,
    'Ispkthr': 0.2,
    'Ireset': 0.2,
    'Ith': 0.2,
    'Itau': 0.2,
    'refP': 0.2,
}

mismatch_synap_param = {
    'I_syn': 0,
    'kn_syn': 0,
    'kp_syn': 0,
    'Ut_syn': 0,
    'Csyn': 0.2,
    'I_tau': 0.2,
    'I_th': 0.2,
    'Io_syn': 0.2,
    'w_plast': 0,
    'baseweight': 0.2,
}


import pandas as pd
import numpy as np
from math import radians, cos, sin, asin, sqrt, floor
import time
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.neighbors import KNeighborsClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import GradientBoostingClassifier
import matplotlib.pyplot as plt

def get_distance(lng1, lat1, lng2, lat2):
    lng1, lat1, lng2, lat2 = map(radians, [lng1, lat1, lng2, lat2])
    dlon = lng2 - lng1
    dlat = lat2 - lat1
    a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
    dis = 2 * asin(sqrt(a)) * 6371 * 1000
    return dis

def get_gongcan_map(gongcanDf):
    tmpMap = {}
    for i in range(len(gongcanDf)):
        key = gongcanDf.iloc[i]['RNCID'] + '_' + gongcanDf.iloc[i]['CellID']
        value = [gongcanDf.iloc[i]['Longitude'], gongcanDf.iloc[i]['Latitude']]
        tmpMap[key] = value
    return tmpMap

def process_data(dataDf):
    rnc_ids = dataDf.groupby(['RNCID_1', 'CellID_1'])
    rnc_id_nos = list(rnc_ids.indices.keys())
    datas = []
    data_labels = []
    # Statistic the each point
    for index in range(len(rnc_ids)):
        key = rnc_id_nos[index][0] + '_' + rnc_id_nos[index][1]
        base_longitude = gongcanMap[key][0]
        base_latitude = gongcanMap[key][1]

        rncDf = rnc_ids.get_group(rnc_id_nos[index])
        labels = []
        data = np.zeros((len(rncDf), 25))
        for j in range(len(rncDf)):
            lon = rncDf.iloc[j]['Longitude']
            lat = rncDf.iloc[j]['Latitude']
            label = [int((lon - base_longitude) * pow(10, 20)), int((lat - base_latitude) * pow(10, 20)), lon, lat]
            labels.append(label)
            num_connected = rncDf.iloc[j]['Num_connected']
            tmp = [rncDf.iloc[j]['IMSI'], rncDf.iloc[j]['MRTime']] + [base_longitude, base_latitude]
            for k in range(num_connected):
                location = gongcanMap.get(
                    rncDf.iloc[j]['RNCID_' + str(k + 1)] + '_' + rncDf.iloc[j]['CellID_' + str(k + 1)])
                if location is not None:
                    tmp += [location[0], location[1], rncDf.iloc[j]['RSSI_' + str(k + 1)]]
            for item in range(len(tmp)):
                data[j][item] = tmp[item]
        datas.append(data)
        data_labels.append(np.array(labels))
    return datas, data_labels

def make_picture(errors, labels, color):
    plt.figure(figsize=(15, 8))
    for index in range(len(errors)):
        x = range(len(errors[index]))
        plt.plot(x, errors[index], label='group_' + str(labels[index]), linewidth=0.5, color=color, marker='o',
                 markerfacecolor='blue', markersize=1)
    plt.xlabel('number')
    plt.ylabel('average error')
    plt.title('Average error probability distribution diagram')
    plt.legend()
    plt.show()

def make_time_picture(times):
    plt.figure(figsize=(15, 8))
    plt.bar(x=range(5, 15), height=times, width=0.7, color="lightblue")
    plt.xlabel('number')
    plt.ylabel('cost(s)')
    plt.title('Time performance bar diagram')
    plt.legend()
    plt.show()

def make_total_picture(errors):
    plt.figure(figsize=(15, 8))
    x = range(len(errors[0]))
    for index in range(len(errors)):
        plt.plot(x, errors[index], label='K=' + str(index + 5), linewidth=0.5, color='r', marker='o',
                 markerfacecolor='blue', markersize=1)
    plt.xlabel('number')
    plt.ylabel('average error')
    plt.title('Average error probability distribution diagram')
    plt.legend()
    plt.show()

def train(classifier, data, label, top_data=None, top_data_label=None):
    distances = []
    for number in range(10):
        # split data
        X_train, X_test, y_train, y_test = train_test_split(data, label, test_size=0.2, random_state=number)
        if top_data is not None and top_data_label is not None:
            X_train = np.vstack((X_train, top_data))
            y_train = np.vstack((y_train, top_data_label))
        classifier.fit(X_train[:, 4:], y_train[:, 0:2])
        y_pred = classifier.predict(X_test[:, 4:])
        locations = []
        for index in range(len(y_pred)):
            locations.append([X_test[index][2] + y_pred[index][0] / pow(10, 20),
                              X_test[index][3] + y_pred[index][1] / pow(10, 20)])
        tmp = []
        for index in range(len(y_pred)):
            tmp.append(get_distance(y_test[index][2], y_test[index][3],
                                    locations[index][0],
                                    locations[index][1]))
        distances.append(sorted(tmp))
    errors = []
    matrix = np.array(distances)
    for index in range(matrix.shape[1]):
        errors.append(np.mean(matrix[:, index]))
    return errors

def merge_data(group_medians, datas, data_labels, k):
    sorted_group_median = sorted(group_medians.items(), key=lambda d: d[1])
    merged_datas = []
    merged_data_labels = []
    top_k_data = datas[sorted_group_median[0][0]]
    top_k_label = data_labels[sorted_group_median[0][0]]
    for index in range(k):
        merged_datas.append(datas[sorted_group_median[index][0]])
        merged_data_labels.append(data_labels[sorted_group_median[index][0]])
        # merge
        if index > 0:
            top_k_data = np.vstack((datas[sorted_group_median[index][0]], top_k_data))
            top_k_label = np.vstack((data_labels[sorted_group_median[index][0]], top_k_label))
    for index in range(k, len(datas)):
        # merge_data = np.vstack((datas[sorted_group_median[index][0]], top_k_data))
        # merge_data_label = np.vstack((data_labels[sorted_group_median[index][0]], top_k_label))
        merged_datas.append(datas[sorted_group_median[index][0]])
        merged_data_labels.append(data_labels[sorted_group_median[index][0]])
    return merged_datas, merged_data_labels, top_k_data, top_k_label

if __name__ == "__main__":
    # read data
    dataDf = pd.read_csv('../data_2g.csv', header=0, dtype={'RNCID_1': np.object, 'CellID_1': np.object,
                                                         'RNCID_2': np.object, 'CellID_2': np.object,
                                                         'RNCID_3': np.object, 'CellID_3': np.object,
                                                         'RNCID_4': np.object, 'CellID_4': np.object,
                                                         'RNCID_5': np.object, 'CellID_5': np.object,
                                                         'RNCID_6': np.object, 'CellID_6': np.object,
                                                         'RNCID_7': np.object, 'CellID_7': np.object})
    gongcanDf = pd.read_csv('../2g_gongcan.csv', header=0, dtype={'RNCID': np.object, 'CellID': np.object})
    # get gongcan data
    gongcanMap = get_gongcan_map(gongcanDf)

    size = len(dataDf)
    # Get the matrix
    minLatitude = dataDf['Latitude'].min()
    maxLatitude = dataDf['Latitude'].max()
    minLongitude = dataDf['Longitude'].min()
    maxLongitude = dataDf['Longitude'].max()

    datas, data_labels = process_data(dataDf)
    average_errors = []
    group_medians = {}
    for index in range(len(datas)):
        rfc = RandomForestClassifier()
        average_error = train(rfc, datas[index], data_labels[index])
        median = np.median(average_error)
        group_medians[index] = median
        average_errors.append(average_error)

    # top k data
    total_errors = []
    exec_times = []
    for k in range(5, 15):
        merged_datas, merged_data_labels, top_k_data, top_k_label = merge_data(group_medians, datas, data_labels, k)
        total_error = []
        average_errors = []
        start = time.time()
        for index in range(len(merged_datas)):
            rfc = RandomForestClassifier()
            if index >= k:
                average_error = train(rfc, merged_datas[index], merged_data_labels[index],
                                      top_data=top_k_data, top_data_label=top_k_label)
            else:
                average_error = train(rfc, merged_datas[index], merged_data_labels[index])
            average_errors.append(average_error)
            total_error += average_error
        color = 'r'
        end = time.time()
        exec_time = round(end - start, 2)
        exec_times.append(exec_time)
        # make_picture(average_errors, range(len(merged_datas)), color)
        print('K:', k, 'Total error:', sum(total_error))
        total_errors.append(sorted(total_error))
    # make_total_picture(total_errors)
    make_time_picture(exec_times)


#!/usr/bin/env python
# coding: utf-8

# In[1]:

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.cluster import KMeans
from sklearn.manifold import TSNE
import plotly.graph_objs as go
import plotly .offline as offline
import plotly.figure_factory as ff
from sklearn.preprocessing import LabelEncoder

# In[2]:

# Importing dataset and examining it
dataset = pd.read_csv("Patches.csv")
print(dataset.head())
print(dataset.shape)
print(dataset.info())
print(dataset.describe())

# In[3]:

#Converting cetegorical to numerical values
dataset['Tree'] = dataset['Tree'].map({'Other':0, 'Spruce':1})

print(dataset.info())

# In[4]:

# Plotting Correlation Heatmap
corrs = dataset.corr()
figure = ff.create_annotated_heatmap(
    z=corrs.values,
    x=list(corrs.columns),
    y=list(corrs.index),
    annotation_text=corrs.round(2).values,
    showscale=True)
offline.plot(figure,filename='corrheatmap.html')

# In[5]:

#Examining the dataset
print(dataset.info())

# In[15]:

# Dividing data into subsets
#Hydrology Data
subset1 = dataset[['Elevation','Slope','Horizontal_Distance_To_Hydrology','Vertical_Distance_To_Hydrology','Tree']]

#Roadways Data
subset2 = dataset[['Elevation','Slope','Horizontal_Distance_To_Roadways','Tree']]

#Fire_Points Data
subset3 = dataset[['Elevation','Slope','Horizontal_Distance_To_Fire_Points','Tree']]

# In[16]:

# Normalizing numerical features so that each feature has mean 0 and variance 1
feature_scaler = StandardScaler()
X1 = feature_scaler.fit_transform(subset1)
X2 = feature_scaler.fit_transform(subset2)
X3 = feature_scaler.fit_transform(subset3)

# In[8]:

#Verifying if there are any NA values in the dataset
dataset.isna().sum()
dataset.isna()

# In[9]:

# Analysis on subset1 - Hydrology Data
# Finding the number of clusters (K) - Elbow Plot Method
inertia = []
for i in range(1,11):
    kmeans = KMeans(n_clusters = i, random_state = 100)
    kmeans.fit(X1)
    inertia.append(kmeans.inertia_)

plt.plot(range(1, 11), inertia)
plt.title('The Elbow Plot')
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')
plt.show()

# In[10]:

# Running KMeans to generate labels
kmeans = KMeans(n_clusters = 4)
kmeans.fit(X1)

# In[24]:

# Implementing t-SNE to visualize first subset
tsne = TSNE(n_components = 2, perplexity =30,n_iter=5000)
x_tsne = tsne.fit_transform(X1)

Elevation = list(dataset['Elevation'])
Slope = list(dataset['Slope'])
Horizontal_Distance_To_Hydrology = list(dataset['Horizontal_Distance_To_Hydrology'])
Vertical_Distance_To_Hydrology = list(dataset['Vertical_Distance_To_Hydrology'])
Tree = list(dataset['Tree'])

data = [go.Scatter(x=x_tsne[:,0], y=x_tsne[:,1], mode='markers',
                    marker = dict(color=kmeans.labels_, colorscale='Rainbow', opacity=0.5),
                                text=[f'Elevation: {a}; Slope: {b}; Horizontal_Distance_To_Hydrology:{c}, Vertical_Distance_To_Hydrology:{d}, Tree:{e}' for a,b,c,d,e in list(zip(Elevation,Slope,Horizontal_Distance_To_Hydrology,Vertical_Distance_To_Hydrology,Tree))],
                                hoverinfo='text')]

layout = go.Layout(title = 't-SNE Dimensionality Reduction', width = 700, height = 700,
                    xaxis = dict(title='First Dimension'),
                    yaxis = dict(title='Second Dimension'))
fig = go.Figure(data=data, layout=layout)
offline.plot(fig,filename='t-SNE1.html')

# In[17]:

# Analysis on subset1 - Roadways Data
# Finding the number of clusters (K) - Elbow Plot Method
inertia = []
for i in range(1,11):
    kmeans = KMeans(n_clusters = i, random_state = 100)
    kmeans.fit(X2)
    inertia.append(kmeans.inertia_)

plt.plot(range(1, 11), inertia)
plt.title('The Elbow Plot')
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')
plt.show()

# In[18]:

# Running KMeans to generate labels
kmeans = KMeans(n_clusters = 2)
kmeans.fit(X2)

# In[20]:

# Implementing t-SNE to visualize the second subset
tsne = TSNE(n_components = 2, perplexity =30,n_iter=5000)
x_tsne = tsne.fit_transform(X2)

Elevation = list(dataset['Elevation'])
Slope = list(dataset['Slope'])
Horizontal_Distance_To_Roadways = list(dataset['Horizontal_Distance_To_Roadways'])
#Vertical_Distance_To_Hydrology = list(dataset['Vertical_Distance_To_Hydrology'])
Tree = list(dataset['Tree'])

data = [go.Scatter(x=x_tsne[:,0], y=x_tsne[:,1], mode='markers',
                    marker = dict(color=subset2['Tree'], colorscale='Rainbow', opacity=0.5),
                                text=[f'Elevation: {a}; Slope: {b}; Horizontal_Distance_To_Roadways:{c}, Tree:{d}' for a,b,c,d in list(zip(Elevation,Slope,Horizontal_Distance_To_Roadways,Tree))],
                                hoverinfo='text')]

layout = go.Layout(title = 't-SNE Dimensionality Reduction', width = 700, height = 700,
                    xaxis = dict(title='First Dimension'),
                    yaxis = dict(title='Second Dimension'))
fig = go.Figure(data=data, layout=layout)
offline.plot(fig,filename='t-SNE1.html')

# In[21]:

# Analysis on subset1 - Fire_Points Data
# Finding the number of clusters (K) - Elbow Plot Method
inertia = []
for i in range(1,11):
    kmeans = KMeans(n_clusters = i, random_state = 100)
    kmeans.fit(X3)
    inertia.append(kmeans.inertia_)

plt.plot(range(1, 11), inertia)
plt.title('The Elbow Plot')
plt.xlabel('Number of clusters')
plt.ylabel('Inertia')
plt.show()

# In[22]:

# Running KMeans to generate labels
kmeans = KMeans(n_clusters = 2)
kmeans.fit(X3)

# In[23]:

# Implementing t-SNE to visualize the third Subset
tsne = TSNE(n_components = 2, perplexity =30,n_iter=5000)
x_tsne = tsne.fit_transform(X3)

Elevation = list(dataset['Elevation'])
Slope = list(dataset['Slope'])
Horizontal_Distance_To_Fire_Points = list(dataset['Horizontal_Distance_To_Fire_Points'])
#Vertical_Distance_To_Hydrology = list(dataset['Vertical_Distance_To_Hydrology'])
Tree = list(dataset['Tree'])

data = [go.Scatter(x=x_tsne[:,0], y=x_tsne[:,1], mode='markers',
                    marker = dict(color=subset3['Tree'], colorscale='Rainbow', opacity=0.5),
                                text=[f'Elevation: {a}; Slope: {b}; Horizontal_Distance_To_Fire_Points:{c}, Tree:{d}' for a,b,c,d in list(zip(Elevation,Slope,Horizontal_Distance_To_Fire_Points,Tree))],
                                hoverinfo='text')]

layout = go.Layout(title = 't-SNE Dimensionality Reduction', width = 700, height = 700,
                    xaxis = dict(title='First Dimension'),
                    yaxis = dict(title='Second Dimension'))
fig = go.Figure(data=data, layout=layout)
offline.plot(fig,filename='t-SNE1.html')

# In[ ]:


from telebot import TeleBot
import  requests
import time

bot = TeleBot("810863369:AAHo_ve5b4_nD9-hnL2uLe1699drXmFZMrw")
@bot.message_handler(commands=["start"])
def send_welcome(message):
		bot.reply_to(message, text="welcome bot created By @GGVVGG\n(Just send user)")

@bot.message_handler(func=lambda m:True)
def Send_info(message2):
	user = message2.text
	try:
		infos = requests.get("https://www.instagram.com/{}/?__a=1".format(user)).json()
		userid = str(infos['logging_page_id'])
		hid = userid.split('_')[1]
		bot.send_message(message2.chat.id,text="id : "+hid)
		time.sleep(0.5)
		bio = str(infos['graphql']['user'].get('biography'))
		bot.send_message(message2.chat.id,text=bio)
		time.sleep(0.5)
		link = str(infos['graphql']['user'].get('external_url'))
		bot.send_message(message2.chat.id, text=link)
		time.sleep(0.5)
		followes = str(infos['graphql']['user'].get('edge_followed_by').get('count'))
		bot.send_message(message2.chat.id, text="followers : "+followes)
		fullname = str(infos['graphql']['user'].get('full_name'))
		bot.send_message(message2.chat.id,text=fullname)
		local = str(infos['graphql']['user'].get('is_business_account'))
		bot.send_message(message2.chat.id,text="is bussiness : "+local)
		joined = str(infos['graphql']['user'].get('is_joined_recently'))
		bot.send_message(message2.chat.id,text="is new : "+joined)
		pic = str(infos['graphql']['user'].get('profile_pic_url'))
		bot.send_photo(message2.chat.id,photo=pic)
		private = str(infos['graphql']['user'].get('is_private'))
		bot.send_message(message2.chat.id,text="is private : "+private)
	except Exception as e:
		print(e)
		pass
bot.polling(True)


# import
from filterpy.kalman import ExtendedKalmanFilter as EKF
from numpy import dot, array, sqrt, matrix, cos, sin, tan
import sympy
import math
import numpy as np
from sympy import symbols, Matrix
from matplotlib import pyplot as plt

import plotHelp

class RobotEKF2(EKF):
    def __init__(self, dt, wheelbase, std_acc, std_steer):
        EKF.__init__(self, 5, 1, 2)
        self.dt = dt
        self.wheelbase = wheelbase
        self.std_acc = std_acc
        self.std_steer = std_steer

        self.M = array([[self.std_acc**2, 0], [0, self.std_steer**2]])

    def move(self, x, u, dt):
        hdg = x[3, 0]
        a = u[0,    0]
        vel = x[2, 0]
        dtheta = x[4, 0]
        steering_angle = u[1, 0]
        dist = vel
        x[4, 0] = 0.0
        if abs(steering_angle) > 0.01:
            beta = (dist/self.wheelbase) * tan(steering_angle)
            r = self.wheelbase / tan(steering_angle)

            dx = np.matrix([
                [-r*sin(hdg) + r*sin(hdg+dtheta*dt)],
                [r*cos(hdg) - r*cos(hdg+dtheta*dt)],
                [a*dt],
                [dtheta*dt],
                [beta]])
        else:
            dx = np.matrix([
                [vel*self.dt*cos(hdg)],
                [vel*self.dt*sin(hdg)],
                [a*dt],
                [dtheta],
                [0]])
        return x + dx

    def F_x(self, x, u):
        px = x[0, 0]
        py = x[1, 0]
        v = x[2, 0]
        theta = x[3, 0]
        dtheta = x[4, 0]
        alpha = u[1, 0]
        acc = u[0, 0]
        r = self.wheelbase/np.tan(alpha)

        if alpha < 0.0001:
            alpha = 0.0001

        return np.matrix([[1, 0, 0, -r*np.cos(theta)+r*np.cos(dtheta*self.dt+theta),
                           self.dt*r*np.cos(dtheta*self.dt+theta)],
                          [0, 1, 0, -r*np.sin(theta)+r*np.sin(dtheta*self.dt+theta),
                           self.dt*r*np.sin(dtheta*self.dt+theta)],
                          [0, 0, 1, 0, 0],
                          [0, 0, 0, 1, self.dt],
                          [0, 0, np.tan(alpha)/self.wheelbase, 0, 0]])

    def F_u(self, x, u):
        px = x[0, 0]
        py = x[1, 0]
        v = x[2, 0]
        theta = x[3, 0]
        dtheta = x[4, 0]
        alpha = u[1, 0]
        acc = u[0, 0]
        if alpha < 0.0001:
            alpha = 0.0001

        return np.matrix([
            [0, -self.wheelbase*(-np.tan(alpha)**2-1)*np.sin(theta)/np.tan(
                alpha)**2 + self.wheelbase*(-np.tan(alpha)**2-1)*np.sin(theta+dtheta*self.dt)/np.tan(alpha)**2],
            [0, self.wheelbase*(-np.tan(alpha)**2-1)*np.cos(theta)/np.tan(
                alpha)**2 - self.wheelbase*(-np.tan(alpha)**2-1)*np.cos(theta+dtheta*self.dt)/np.tan(alpha)**2],
            [self.dt, 0],
            [0, 0],
            [0, v*(np.tan(alpha)**2+1)/self.wheelbase]])

    def predict(self, u=array([[0.0], [0.0]])):
        newX = self.move(self.x, u, self.dt)

        #
        F_x = self.F_x(self.x, u)
        F_u = self.F_u(self.x, u)

        self.P = dot(F_x, self.P).dot(F_x.T) + dot(F_u, self.M).dot(F_u.T)
        self.x = newX

    def Hx(X):
        return array([[X[2, 0]], [X[4, 0]]])

    def H_of(X):
        return array([[0.0, 0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 1.0]])

    def update(self, z, R):
        super(RobotEKF2, self).update(z=z, HJacobian=RobotEKF2.H_of, Hx=RobotEKF2.Hx, R=R)

class RobotEKF(EKF):
    def __init__(self, dt, wheelbase, std_acc, std_steer):
        EKF.__init__(self, 5, 2, 2)
        self.dt = dt
        self.wheelbase = wheelbase
        self.std_acc = std_acc
        self.std_steer = std_steer

        alpha, a, x, y, v, w, theta, dtheta, time = symbols(
            'alpha, a, x, y, v, w, theta, dtheta, t')

        d = v
        # + a/2*time**2
        beta = (d/w)*sympy.tan(alpha)
        r = w/sympy.tan(alpha)

        self.fxu = Matrix(
            [[x - r * sympy.sin(theta) + r * sympy.sin(theta + dtheta*time)],
                [y + r * sympy.cos(theta) - r * sympy.cos(theta + dtheta*time)],
                [v + a * time],
                [theta + dtheta*time],
                [beta]])

        self.F_x = self.fxu.jacobian(Matrix([x, y, v, theta, dtheta]))
        self.F_u = self.fxu.jacobian(Matrix([a, alpha]))

        print('F_x', sympy.simplify(self.F_x))
        print('F_u', sympy.simplify(self.F_u))

        self.subs = {x: 0, y: 0, v: 0, alpha: 0, a: 0, time: dt, w: wheelbase, theta: 0}
        self.x_x, self.x_y = x, y

        self.v, self.alpha, self.theta, self.dtheta, self.a = v, alpha, theta, dtheta, a

    def move(self, x, u, dt):
        hdg = x[3, 0]
        a = u[0,    0]
        vel = x[2, 0]
        dtheta = x[4, 0]
        steering_angle = u[1, 0]
        dist = vel
        x[4, 0] = 0.0
        if abs(steering_angle) > 0.001:
            beta = (dist/self.wheelbase) * tan(steering_angle)
            r = self.wheelbase / tan(steering_angle)

            dx = np.matrix([
                [-r*sin(hdg) + r*sin(hdg+dtheta*dt)],
                [r*cos(hdg) - r*cos(hdg+dtheta*dt)],
                [a*dt],
                [dtheta*dt],
                [beta]])
        else:
            dx = np.matrix([
                [vel*self.dt*cos(hdg)],
                [vel*self.dt*sin(hdg)],
                [a*dt],
                [dtheta],
                [0]])
        return x + dx

    def predict(self, u=array([[0.0], [0.0]])):
        self.x = self.move(self.x, u, self.dt)
        # Update the state value
        self.subs[self.alpha] = u[1]
        self.subs[self.a] = u[0]

        #
        F_x = array(self.F_x.evalf(subs=self.subs)).astype(float)
        F_u = array(self.F_u.evalf(subs=self.subs)).astype(float)

        self.subs[self.x_x] = self.x[0, 0]
        self.subs[self.x_y] = self.x[1, 0]
        self.subs[self.v] = self.x[2, 0]
        self.subs[self.theta] = self.x[3, 0]
        self.subs[self.dtheta] = self.x[4, 0]

        M = array([[self.std_acc**2, 0], [0, self.std_steer**2]])
        self.P = dot(F_x, self.P).dot(F_x.T) + dot(F_u, M).dot(F_u.T)

    def Hx(X):
        return array([[X[2, 0]], [X[4, 0]]])

    def H_of(X):
        return array([[0.0, 0.0, 1.0, 0.0, 0.0], [0.0, 0.0, 0.0, 0.0, 1.0]])

    def update(self, z, R):
        super(RobotEKF, self).update(z=z, HJacobian=RobotEKF.H_of, Hx=RobotEKF.Hx, R=R)

def generateTest(std_vel, std_accel, std_steer, std_theta, std_dtheta):
    rob1 = RobotEKF(dt=0.1, wheelbase=26, std_acc=std_accel, std_steer=std_steer)
    rob2 = RobotEKF2(dt=0.1, wheelbase=26, std_acc=std_accel, std_steer=std_steer)
    X = rob1.x
    X = rob2.x
    # State = X
    u = matrix([[90], [math.pi/180*20]])
    U_a = u

    X = rob1.move(x=X, u=u, dt=rob1.dt)
    # print('State', X)
    State = X
    # State = np.concatenate((State, X), axis=1)
    u = matrix([[0], [math.pi/180*2]])
    for i in range(100):

        X = rob1.move(x=X, u=u, dt=rob1.dt)
        U_a = np.concatenate((U_a, u), axis=1)
        State = np.concatenate((State, X), axis=1)

        # State.append(X)
    vel_err = State[2, :]+np.random.normal(0.0, std_vel, State.shape[1])
    theta_err = State[3, :]+np.random.normal(0.0, std_theta, State.shape[1])
    dtheta_err = State[4, :]+np.random.normal(0.0, std_dtheta, State.shape[1])

    plt.figure()
    plt.plot(theta_err[0, :].tolist()[0])
    plt.plot(State[3, :].tolist()[0])

    plt.figure()
    plt.plot(dtheta_err[0, :].tolist()[0])
    plt.plot(State[4, :].tolist()[0])
    plt.show()

    U_a[0, :] += np.random.normal(0.0, std_accel, U_a.shape[1])
    U_a[1, :] += np.random.normal(0.0, std_steer, U_a.shape[1])
    return State, U_a, rob1, rob2, vel_err, theta_err, dtheta_err

def main():
    std_vel = 0.0
    std_theta = np.radians(1)
    std_dtheta = np.radians(0.001)
    print("S", std_dtheta)
    State, U_a, rob1, rob2, vel_err, theta_err, dtheta_err = generateTest(
        std_vel=std_vel, std_accel=0.05, std_steer=np.radians(1.e-4), std_theta=std_theta, std_dtheta=std_dtheta)
    rob1.P = np.diag([.0, .1, .1, np.radians(0.001), 0.0])
    rob2.P = np.diag([.0, .1, .1, np.radians(0.001), 0.0])

    X = rob1.x
    X1 = rob2.x

    for u, vel_mes, theta_e, dtheta_e in zip(U_a.T, vel_err.tolist()[0], theta_err.tolist()[0], dtheta_err.tolist()[0]):

        u = u.T
        # rob1.predict(u)
        # rob1.update(array([[vel_mes], [dtheta_e]]), R=matrix([[std_vel**2, 0], [0, std_dtheta**2]]))
        X = np.concatenate((X, rob1.x), axis=1)
        print(u)
        rob2.predict(u)
        rob2.update(array([[vel_mes], [dtheta_e]]), R=matrix([[std_vel**2, 0], [0, std_dtheta**2]]))
        X1 = np.concatenate((X1, rob2.x), axis=1)

    plt.figure()
    plt.title('Velocity')
    plt.plot(X[2, :].tolist()[0], '--r')
    plt.plot(State[2, :].tolist()[0], 'b')
    plt.plot(X1[2, :].tolist()[0], '--g')

    plt.figure()
    plt.title('Oriantation')
    plt.plot(X[3, :].tolist()[0], '--r')
    plt.plot(State[3, :].tolist()[0], 'b')
    plt.plot(X1[3, :].tolist()[0], '--g')

    plt.figure()
    plt.title('Angular velocity')
    plt.plot(X[4, :].tolist()[0], '--r')
    plt.plot(State[4, :].tolist()[0], 'b')
    plt.plot(dtheta_err.tolist()[0], '--g')
    plt.plot(X1[4, :].tolist()[0], '-^g')
    plt.legend(['EKF', 'Real', 'Error'])

    plt.figure()
    plt.plot(X[0, :].tolist()[0], X[1, :].tolist()[0], '--r')
    plt.plot(State[0, :].tolist()[0], State[1, :].tolist()[0], 'b')
    plt.plot(X1[0, :].tolist()[0], X1[1, :].tolist()[0], '--g')
    plt.show()

if __name__ == '__main__':
    main()


import glfw
from OpenGL.GL import *
from math import *

W = 4
Y_MAX = 1
MARGIN_X = 0.2
MARGIN_Y = 0.1
N = 20
H = W/N

def display():
    glClear(GL_COLOR_BUFFER_BIT)

    glLineWidth(1)
    glBegin(GL_LINE_LOOP)
    glVertex2d(-W, -Y_MAX)
    glVertex2d( W, -Y_MAX)
    glVertex2d( W,  Y_MAX)
    glVertex2d(-W,  Y_MAX)
    glEnd()

    glLineWidth(1)
    glBegin(GL_LINES)
    glVertex2d(-W, 0.0)
    glVertex2d( W, 0.0)
    glVertex2d(0.0,  Y_MAX)
    glVertex2d(0.0,  -Y_MAX)
    glEnd()

    glLineWidth(2)
    glBegin(GL_LINE_STRIP)
    for i in range(-N, N+1):
        x = i*H
        glVertex2d(x, sin(x))
    glEnd()

    glfw.swap_buffers(window)

def init():
    gray = 0.95
    glClearColor(gray, gray, gray, 1.0)
    glColor3d(0.0, 0.0, 0.0)
    glOrtho(-W - MARGIN_X, W + MARGIN_X, -Y_MAX - MARGIN_Y, Y_MAX + MARGIN_Y, -1.0, 1.0)
    display()   # necessary only on Windows

def window_refresh(window):
    display()

if not glfw.init():
    raise RuntimeError('Could not initialize GLFW3')

window = glfw.create_window(300, 300, 'Parabola', None, None)
if not window:
    glfw.terminate()
    raise RuntimeError('Could not create an window')

glfw.set_window_refresh_callback(window, window_refresh)
glfw.make_context_current(window)

init()

while not glfw.window_should_close(window):
    glfw.wait_events()

glfw.terminate()


#=======================================
# REDES NEURONALES ARTIFICIALES - MLFFNN
#=======================================
from model import Model
from data import Data
from util import Util

#-----------
# PARAMETROS
#-----------
DATASET_FILE = '../dataset/clientes_data.csv'

EPOCHS = 500
BATCH_SIZE = 25
TEST_SIZE = 0.2 # 20% Test Set

NAME_MODEL_TO_SAVE = 'ann_model-250-50.h5'
NAME_LABELENCODER_X1_TO_SAVE = 'labelEncoder_X_1.save'
NAME_LABELENCODER_X2_TO_SAVE = 'labelEncoder_X_2.save'
NAME_SCALER_TO_SAVE = 'stdScaler.save'

#------------------------------------
# PARTE I - PREPROCESAMIENTO DE DATOS
#------------------------------------
[X_train, X_test, y_train, y_test] = Data().preProcessData(DATASET_FILE,
                                        TEST_SIZE,
                                        NAME_LABELENCODER_X1_TO_SAVE,
                                        NAME_LABELENCODER_X2_TO_SAVE,
                                        NAME_SCALER_TO_SAVE)
print("X_train: ", X_train)
print("y_train: ", y_train)
print("X_test: ", X_test)
print("y_test: ", y_test)

#---------------------------------------------------
# PARTE II - CONSTRUYENDO LA RED NEURONAL ARTIFICIAL
#---------------------------------------------------
model = Model()
model.trainNetwork(X_train, y_train, EPOCHS, BATCH_SIZE)
# Summarize model.
model.neural_network.summary()
model.graphModel("./model/model_graph.png")
# Guardar la RNA en disco
model_file_name = model.saveModel(NAME_MODEL_TO_SAVE)
print("Modelo guardado en disco >> ", model_file_name)

#-----------------------------------------------------
# PARTE III - PREDICCIONES Y VISUALIZACION DE RESULTADOS
#-----------------------------------------------------

FILENAME_MODEL_TO_LOAD = NAME_MODEL_TO_SAVE
FILENAME_SCALER_TO_LOAD = NAME_SCALER_TO_SAVE
FILENAME_LABELENCODER_X1_TO_LOAD = NAME_LABELENCODER_X1_TO_SAVE
FILENAME_LABELENCODER_X2_TO_LOAD = NAME_LABELENCODER_X2_TO_SAVE

# Cargar la RNA desde disco
from model import Model
new_model = Model()
loaded_model = new_model.loadModel(FILENAME_MODEL_TO_LOAD)
print("Modelo cargado de disco << ", loaded_model)
# Summarize model.
loaded_model.summary()

new_model.setModel(loaded_model)

new_util = Util()
# Cargar los parametros usados
loaded_scaler = new_util.loadFile(FILENAME_SCALER_TO_LOAD)
loaded_labelEncoderX1 = new_util.loadFile(FILENAME_LABELENCODER_X1_TO_LOAD)
loaded_labelEncoderX2 = new_util.loadFile(FILENAME_LABELENCODER_X2_TO_LOAD)

# Validar y predecir
[cm, y_pred] = new_model.validateModel(X_test, y_test)
print ("\nMatriz de Confusión: \n", cm)

# Graficando la Matriz de Confusión
title = "Matriz de Confusión: Abandono de clientes"
new_util.plot_confusion_matrix(y_test, y_pred, normalize=False,title=title)