import os
from os.path import join
from PIL import Image
import numpy as np
from typing import Tuple
from scipy.ndimage import rotate
import random
import psutil
import matplotlib.pyplot as plt
from collections import defaultdict
import cv2
from tensorflow.keras import backend as K
import tensorflow as tf
from skimage.filters.edges import sobel

class ImageGenerator(object):

    def __init__(self, batch_size, path_to_x, path_to_y, match_string_x, match_string_y, aug_fn, splits,
                 random_generation=True,random_crop=None,resize_toggle=True) -> object:
        """
        Generator class for generating inout,label pairs
        @param batch_size: Batch size
        @param path_to_x: Path to input images
        @param path_to_y: Path to segmented images
        @param match_string_x: String to match input image
        @param match_string_y: String to match segmented image
        @param aug_fn: Function to augment the images
        @param random_generation: Switch to generate random batches

        @return object
        """
        self.batch_size = batch_size
        self.match_string_x = match_string_x
        self.match_string_y = match_string_y
        self.path_x = path_to_x
        self.path_y = path_to_y
        self.aug_fn = aug_fn
        self.random_gen = random_generation
        self.curr_index = 0
        self.list_paths_x = self.get_paths(path_to_x, match_string_x)
        # self.splits = splits
        self.resize_toggle = resize_toggle
        self.random_crop = random_crop
        # self.list_paths_y = self.get_paths(path_to_y,match_string_y)

    def get_paths(self, path, match_string) -> list:
        """
        Returns list of paths to all the images in the dataset that can be loaded into memory by the generator.
        @rtype: list
        """
        list_paths = []
        for i in os.listdir(path):
            for j in os.listdir(str(join(path, i))):
                if (j.split('_')[-1] == match_string + '.png'):
                    list_paths.append(str(join(path, i, j)))

        return list_paths
    def __iter__(self):
        return self

    def __len__(self) -> int:
        """
        Returns the length of the generator
        @return: int
        """
        return (len(self.list_paths_x)) // self.batch_size

    def __next__(self) -> Tuple[np.ndarray, np.ndarray]:
        """
        Returns batch of images and their corresponding segmentation images
        @rtype: Tuple[np.ndarray,np.ndarray]
        """
        batch_x = []
        batch_y = []
        batch_bmap = []
        if (self.random_gen):
            idx = np.random.uniform(low=0, high=len(self.list_paths_x), size=(self.batch_size)).astype(int)
        else:
            if self.curr_index==self.__len__():
                self.curr_index=0
            idx = np.arange(start=self.curr_index, stop=self.curr_index + self.batch_size)
            self.curr_index += self.batch_size
        for i in idx:
            x_sample = np.asarray(Image.open(self.list_paths_x[i]).convert('RGB')) / 255
            y_path = self.get_path_y(self.list_paths_x[i])
            y_sample = np.asarray(Image.open(y_path).convert('RGB'))
            if self.resize_toggle==True:
                x_sample,y_sample = self.resize(x_sample,y_sample)
            if self.aug_fn!=None:
                x_sample, y_sample = self.aug_fn(x_sample, y_sample,crop_size=self.random_crop)
            y_sample = mask_to_arr(y_sample)
            # if self.bmap:
            #     bmap_sample = self.seg2bmap(y_sample)
            #     bmap_sample[bmap_sample>0] = 1
            # if self.splits>1:
            #     x_sample = split(x_sample,self.splits)
            #     y_sample = split(y_sample,self.splits)
            #     for i in range(self.splits):
            #         batch_x.append(x_sample[i])
            #         batch_y.append(y_sample[i])
            # else:
            # if self.bmap:
            #     batch_y.append(y_sample)
            #     batch_x.append(x_sample)
            #     batch_bmap.append(bmap_sample)

            batch_y.append(y_sample)
            batch_x.append(x_sample)
        return np.asarray(batch_x), np.asarray(batch_y)

    # def seg2bmap(self,seg, width=None, height=None):
    #  sobel_arr = []
    #  for i in range(seg.shape[-1]):
    #     sobel_arr.append(sobel(seg[...,i]))
    #  return np.asarray(np.sum(sobel_arr,axis=0))

    def resize(self,x,y,shape=(1024,512))-> Tuple[np.ndarray,np.ndarray]:
        """
        Resize function to reduce the dimension of the image,label pair
        Enables training on smaller GPUs
        @param x: Image
        @param y: Label
        @param shape: Desired output shape tupple
        @return: resized image,label pair
        """
        x = cv2.resize(x,shape,interpolation=cv2.INTER_NEAREST)
        y = cv2.resize(y,shape,interpolation=cv2.INTER_NEAREST)
        return x,y

    def get_path_y(self,path) -> str:
        """
        Returns path to label image for a given input image
        @param path: path to the input image
        @rtype: str
        """
        if psutil.WINDOWS:
            path_split = '\\'
        else:
            path_split = '/'

        y_path = path.split(path_split)
        y_path[0] = self.path_y.split(path_split)[0]
        filename = y_path[-1].split('_')
        filename[-1] = self.match_string_y + '.png'
        y_path[-1] = '_'.join(filename)
        y_path = path_split.join(y_path)
        return y_path

# Labeling from original Cityscapes Scripts
# def get_label():
#     Label = namedtuple('Label', [
#         'name',
#         'id',
#         'trainId',
#         'category',
#         'categoryId',
#         'hasInstances',
#         'ignoreInEval',
#         'color',
#     ])
#
#
#     labels = [
#         #Label('unlabeled', 0, 255, 'void', 0, False, True, (0, 0, 0)),
#         #Label('ego vehicle', 1, 255, 'void', 0, False, True, (0, 0, 0)),
#         #Label('rectification border', 2, 255, 'void', 0, False, True, (0, 0, 0)),
#         #Label('out of roi', 3, 255, 'void', 0, False, True, (0, 0, 0)),
#         #Label('static', 4, 255, 'void', 0, False, True, (0, 0, 0)),
#         #Label('dynamic', 5, 255, 'void', 0, False, True, (111, 74, 0)),
#         #Label('ground', 6, 255, 'void', 0, False, True, (81, 0, 81)),
#         Label('road', 1, 0, 'flat', 1, False, False, (128, 64, 128)),
#         Label('sidewalk', 2, 1, 'flat', 1, False, False, (244, 35, 232)),
#         #Label('parking', 9, 255, 'flat', 1, False, True, (250, 170, 160)),
#         #Label('rail track', 10, 255, 'flat', 1, False, True, (230, 150, 140)),
#         Label('building', 3, 2, 'construction', 2, False, False, (70, 70, 70)),
#         Label('wall', 4, 3, 'construction', 2, False, False, (102, 102, 156)),
#         Label('fence', 5, 4, 'construction', 2, False, False, (190, 153, 153)),
#         # Label('guard rail', 14, 255, 'construction', 2, False, True, (180, 165, 180)),
#         # Label('bridge', 15, 255, 'construction', 2, False, True, (150, 100, 100)),
#         # Label('tunnel', 16, 255, 'construction', 2, False, True, (150, 120, 90)),
#         Label('pole', 6, 5, 'object', 3, False, False, (153, 153, 153)),
#         # Label('polegroup', 18, 255, 'object', 3, False, True, (153, 153, 153)),
#         Label('traffic light', 7, 6, 'object', 3, False, False, (250, 170, 30)),
#         Label('traffic sign', 8, 7, 'object', 3, False, False, (220, 220, 0)),
#         Label('vegetation', 9, 8, 'nature', 4, False, False, (107, 142, 35)),
#         Label('terrain', 10, 9, 'nature', 4, False, False, (152, 251, 152)),
#         Label('sky', 11, 10, 'sky', 5, False, False, (70, 130, 180)),
#         Label('person', 12, 11, 'human', 6, True, False, (220, 20, 60)),
#         Label('rider', 13, 12, 'human', 6, True, False, (255, 0, 0)),
#         Label('car', 14, 13, 'vehicle', 7, True, False, (0, 0, 142)),
#         Label('truck', 15, 14, 'vehicle', 7, True, False, (0, 0, 70)),
#         Label('bus', 16, 15, 'vehicle', 7, True, False, (0, 60, 100)),
#         # Label('caravan', 29, 255, 'vehicle', 7, True, True, (0, 0, 90)),
#         # Label('trailer', 30, 255, 'vehicle', 7, True, True, (0, 0, 110)),
#         Label('train', 17, 16, 'vehicle', 7, True, False, (0, 80, 100)),
#         Label('motorcycle', 18, 17, 'vehicle', 7, True, False, (0, 0, 230)),
#         Label('bicycle', 19, 18, 'vehicle', 7, True, False, (119, 11, 32)),
#         # Label('license plate', -1, -1, 'vehicle', 7, False, True, (0, 0, 142)),
#     ]
#     return labels

def get_label() -> dict:
    """
    Returns label dictonary for specified RGB values
    @return: label dictionary
    """
    label_dict = {
        (0,0,0):0,
        (128, 64, 128):1,
        (244, 35, 232):2,
        (70, 70, 70):3,
        (102, 102, 156):4,
        (190, 153, 153):5,
        (153, 153, 153):6,
        (250, 170, 30):7,
        (220, 220, 0):8,
        (107, 142, 35):9,
        (152, 251, 152):10,
        (70, 130, 180):11,
        (220, 20, 60):12,
        (255, 0, 0):13,
        (0, 0, 142):14,
        (0, 0, 70):15,
        (0, 60, 100):16,
        (0, 80, 100):17,
        (0, 0, 230):18,
        (119, 11, 32):19

    }
    label_dict_exception = defaultdict(lambda: 0,label_dict)
    return label_dict_exception

def get_color() -> dict:
    """
    Returns inverted label dictionary
    @return: inverted label dictionary
    """
    label_dict = get_label()
    color_dict = {value: key for (key, value) in label_dict.items()}
    return  color_dict

def augmentation_fn(x, y, crop_size) -> Tuple[np.ndarray, np.ndarray]:
    """
    Augments the image,label pair
    This augmentation helps in learning more robust features
    @param x: Input Image
    @param y: Segmentation Image
    @rtype: Tuple[np.ndarray,np.ndarray]
    """
    h,w = x.shape[0],x.shape[1]
    # Random cropping
    new_h,new_w = int(h*crop_size),int(w*crop_size)
    start_h,start_w = int(np.random.uniform(0,h-new_h)), int(np.random.uniform(0,w-new_w))
    x_new,y_new = x[start_h:(start_h+new_h),start_w:(start_w+new_w),:], y[start_h:(start_h+new_h),start_w:(start_w+new_w),:]
    i = np.random.choice([0,90,180,270])
    x_new = rotate(x_new,i)
    y_new = rotate(y_new,i)
    i = np.random.choice([True,False])
    if i:
        x_new = np.fliplr(x_new)
        y_new = np.fliplr(y_new)
    return x_new, y_new

def arr_to_categorical(image):
    # cp = np.zeros()
    # print(image.dtype)

    cp = np.eye(len(get_label()))[image.astype(int)]
    # for i in range(image.shape[0]):
    #     for j in range(image.shape[1]):
    #         cp[i,j,int(image[i,j])] = 1
    return cp.reshape((image.shape[0],image.shape[1],len(get_label().items())))

def mask_to_arr(image)-> np.ndarray:
    """
    Converts a RGB image to array containing class for each pixel in the RGB image.
    @param image: RGB image
    @return: categorical array
    """
    label_dict = get_label()
    arr = np.zeros((image.shape[0],image.shape[1],1))
    for i,j in label_dict.items():
        x,y= np.where(np.sum(image,axis=-1)==np.sum(i))[:2]
        arr[x,y] = j
    # x,y = np.where(np.argmax(arr,axis=-1)==0)
    # z = np.zeros_like(x)
    # arr[x,y,z] = 1
    # for i in range(image.shape[0]):
    #     for j in range(image.shape[1]):
    #         try:
    #             arr[i,j,label_dict[tuple(image[i,j,:])]-1] = 1
    #         except:
    #             continue
    return arr_to_categorical(arr)

def categorical_to_img(arr)-> np.ndarray:
    """
    Converts categorical array to image
    input: softmax array
    output: image
    @param arr: input softmax image from the model
    @return: RGB image
    """
    s,h,w = arr.shape[:-1]
    argmax_arr = np.argmax(arr,axis=-1)
    img = np.zeros(shape=(s,h,w,3))
    color_dict = get_color()
    for i in range(s):
        for j in color_dict.keys():
            img[i][argmax_arr[i]==j] = color_dict[j]
    return img

# if __name__ == '__main__':
#     test_gen = ImageGenerator(1, join('leftImg8bit', 'train'), join('gtFine', 'train'), 'leftImg8bit', 'gtFine_color',
#                               augmentation_fn, 1, True,random_crop=0.75,bmap=True)
#
#     images, labels= next(test_gen)
#     # arr = mask_to_arr(labels)
#     img = categorical_to_img(labels['output'])
#     # print(img[0])
#     for i in range(img.shape[0]):
#         plt.imshow(img[i]/255)
#         plt.show()
#         plt.imshow(labels['edge'][i,:,:],cmap='gray')
#         plt.show()
#         plt.clf()
#         # plt.imshow(images[i])
#         # plt.show()
#         # plt.clf()
#         # plt.imshow(labels[0,:,:,1])
#         # plt.show()
#     # print(images.shape)
#     # for epoch in range(2):
#     #     for i in tqdm(range(len(test_gen))):
#     #         images, labels = next(test_gen)
#
#     # plt.imshow(labels[1,:,:,14])
#     # plt.show()


import os

import requests

from settings import images_directory, logger, nasa_token
from utils import get_and_save_image_to_disk

def get_nasa_apod_pictures_urls(url, params):
    response = requests.get(url, params=params)
    response.raise_for_status()
    answer = response.json()
    images_urls = []
    for picture_item in answer:
        url = picture_item.get('url')
        if url:
            images_urls.append(url)
    return images_urls

def fetch_nasa_apod_pictures(token, images_directory, count=20):
    api_url_base = 'https://api.nasa.gov{route}'
    image_filename_template = 'nasa_apod_{}'
    params = {'api_key': token,
              'count': count,
              'thumbs': True}
    apod_route = '/planetary/apod'
    api_url = api_url_base.format(route=apod_route)
    try:
        images_urls = get_nasa_apod_pictures_urls(api_url, params=params)
    except requests.ConnectionError as e:
        logger.error(f'url: {api_url} Connection error NASA APOD: {e}')
        return
    except requests.HTTPError as e:
        logger.error(f'url: {api_url} HTTP error NASA APOD: {e}')
        return

    for num, url in enumerate(images_urls, start=1):
        filepath_template = os.path.join(images_directory, image_filename_template.format(num))
        try:
            get_and_save_image_to_disk(url, filepath_template)
        except requests.ConnectionError as e:
            logger.error(f'url: {url} Connection error NASA APOD: {e}')
        except requests.HTTPError as e:
            logger.error(f'url: {url} HTTP error NASA APOD: {e}')
        except requests.exceptions.MissingSchema as e:
            logger.error(f'url: {url} MissingSchema error NASA APOD, bad url: {e}')
    if not images_urls:
        logger.error(f'url: {api_url} No NASA APOD found')

def main():
    fetch_nasa_apod_pictures(nasa_token, images_directory)

if __name__ == '__main__':
    main()


from typing import List

def compute_answer(lines: List[str]) -> int:
    result = 0
    grid = [[False] * 1000 for _ in range(1000)]
    for line in lines:
        if line.startswith("turn on "):
            line = line[8:]
            fn = lambda x: True
        elif line.startswith("turn off "):
            line = line[9:]
            fn = lambda x: False
        elif line.startswith("toggle "):
            line = line[7:]
            fn = lambda x: not x
        a, b = line.split(" through ")
        ar, ac = [int(_) for _ in a.split(",")]
        br, bc = [int(_) for _ in b.split(",")]
        for r in range(ar, br + 1):
            for c in range(ac, bc + 1):
                grid[r][c] = fn(grid[r][c])
    for r in range(len(grid)):
        for c in range(len(grid[0])):
            if grid[r][c]:
                result += 1
    return result


# Pytorch dataset class

import os
import torch
from torch.utils.data import Dataset, DataLoader
import numpy as np
import h5py
from sklearn.preprocessing import OneHotEncoder

class BCMDataset(Dataset):
    """BCM dataset"""

    def __init__(self, file_path, window_size = 3, stride = 1, MFCC_stride = 0.005, transform=None):
        """
        Args:
        ----------
            file_path : str
                Path to the h5 file.
            window_size : int, optional
                The size of the window in seconds. The default is 3.
            stride : int, optional
                The stride of the window in seconds. The default is 1.
            MFCC_stride : float, optional
                The stride of the MFCC in seconds. The default is 0.01.
        """
        self.transform = transform
        self.file_path = file_path
        self.window_size = window_size
        self.stride = stride
        self.MFCC_stride = MFCC_stride

        #Empty list to store the data
        self.data = np.load(file_path)

        y = np.zeros(len(self.data))
        i = 0
        while i < len(self.data):
            # Generate random number
            n = int(500 + np.random.rand()*500)
            y[i:i+n] = int(np.random.rand()*5)
            i += n

        # Use sklearn one hot encoding
        onehot_encoder = OneHotEncoder(sparse=False)
        y = y.reshape(len(y), 1)
        y = onehot_encoder.fit_transform(y)
        self.y = y

    def __len__(self):
        return int(len(self.data))

    def __getitem__(self, idx):
        return torch.from_numpy(self.data[idx]).float().cpu(), torch.from_numpy(self.y[idx]).float().cpu()

    def concat_train_test_datasets(path): # Uses all files in folder to concatenate test and train datasets
        # os walk to get all files in folder
        training_set_list = []
        val_set_list = []
        file_count = 0
        for file in sorted(os.walk(path)):
            # Add every fourth file to validation set
            if file_count % 4 == 0:
                val_set_list.append(BCMDataset(file))
            else:
                training_set_list.append(BCMDataset(file))
            file_count += 1

# Print dataset version
print("Dataset version:", 0.12)

#Testing
if False:
    dataset = BCMDataset('data/mfcc_array2.npy')
    print(len(dataset))
    print(dataset[0])

    pass


"""Methods for determining the Koeppen climate classification of a location."""

# pylint: disable=E0401
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path

# pylint: enable=E0401

import pandas as pd
from ladybug.epw import EPW, Location
from scipy import spatial
from ..bhom.analytics import bhom_analytics

@dataclass
class KoeppenClassification:
    """Summary of KoeppenClassification dataclass."""

    classification: str = field(init=True, default=None)
    description: str = field(init=True, default=None)
    group: str = field(init=True, default=None)
    precipitation_type: str = field(init=True, default=None)
    level_of_heat: str = field(init=True, default=None)

    def __post_init__(self):
        classifications = [
            "Af",
            "Am",
            "As",
            "Aw",
            "BSh",
            "BSk",
            "BWh",
            "BWk",
            "Cfa",
            "Cfb",
            "Cfc",
            "Csa",
            "Csb",
            "Csc",
            "Cwa",
            "Cwb",
            "Cwc",
            "Dfa",
            "Dfb",
            "Dfc",
            "Dfd",
            "Dsa",
            "Dsb",
            "Dsc",
            "Dwa",
            "Dwb",
            "Dwc",
            "Dwd",
            "EF",
            "ET",
        ]
        if self.classification not in classifications:
            raise ValueError(f"Invalid Koeppen classification: {self.classification}")

    @bhom_analytics()
    def to_dict(self):
        """Return a dictionary representation of the KoeppenClassification."""
        return {
            "classification": self.classification,
            "description": self.description,
            "group": self.group,
            "precipitation_type": self.precipitation_type,
            "level_of_heat": self.level_of_heat,
        }

class KoeppenClimateClassifications(Enum):
    """Summary of KoeppenClimateClassifications enum."""

    AF = KoeppenClassification(
        classification="Af",
        description="Tropical rainforest climate",
        group="Tropical",
        precipitation_type="Rainforest",
        level_of_heat=None,
    )
    AM = KoeppenClassification(
        classification="Am",
        description="Tropical monsoon climate",
        group="Tropical",
        precipitation_type="Monsoon",
        level_of_heat=None,
    )
    AS = KoeppenClassification(
        classification="As",
        description="Tropical dry savanna climate",
        group="Tropical",
        precipitation_type="Savanna, Dry",
        level_of_heat=None,
    )
    AW = KoeppenClassification(
        classification="Aw",
        description="Tropical savanna, wet",
        group="Tropical",
        precipitation_type="Savanna, Wet",
        level_of_heat=None,
    )
    BSH = KoeppenClassification(
        classification="BSh",
        description="Hot semi-arid (steppe) climate",
        group="Arid",
        precipitation_type="Steppe",
        level_of_heat="Hot",
    )
    BSK = KoeppenClassification(
        classification="BSk",
        description="Cold semi-arid (steppe) climate",
        group="Arid",
        precipitation_type="Steppe",
        level_of_heat="Cold",
    )
    BWH = KoeppenClassification(
        classification="BWh",
        description="Hot deserts climate",
        group="Arid",
        precipitation_type="Desert",
        level_of_heat="Hot",
    )
    BWK = KoeppenClassification(
        classification="BWk",
        description="Cold desert climate",
        group="Arid",
        precipitation_type="Desert",
        level_of_heat="Cold",
    )
    CFA = KoeppenClassification(
        classification="Cfa",
        description="Humid subtropical climate",
        group="Temperate",
        precipitation_type="Without dry season",
        level_of_heat="Hot summer",
    )
    CFB = KoeppenClassification(
        classification="Cfb",
        description="Temperate oceanic climate",
        group="Temperate",
        precipitation_type="Without dry season",
        level_of_heat="Warm summer",
    )
    CFC = KoeppenClassification(
        classification="Cfc",
        description="Subpolar oceanic climate",
        group="Temperate",
        precipitation_type="Without dry season",
        level_of_heat="Cold summer",
    )
    CSA = KoeppenClassification(
        classification="Csa",
        description="Hot-summer Mediterranean climate",
        group="Temperate",
        precipitation_type="Dry summer",
        level_of_heat="Hot summer",
    )
    CSB = KoeppenClassification(
        classification="Csb",
        description="Warm-summer Mediterranean climate",
        group="Temperate",
        precipitation_type="Dry summer",
        level_of_heat="Warm summer",
    )
    CSC = KoeppenClassification(
        classification="Csc",
        description="Cool-summer Mediterranean climate",
        group="Temperate",
        precipitation_type="Dry summer",
        level_of_heat="Cold summer",
    )
    CWA = KoeppenClassification(
        classification="Cwa",
        description="Monsoon-influenced humid subtropical climate",
        group="Temperate",
        precipitation_type="Dry winter",
        level_of_heat="Hot summer",
    )
    CWB = KoeppenClassification(
        classification="Cwb",
        description="Subtropical highland climate or temperate oceanic climate with dry winters",
        group="Temperate",
        precipitation_type="Dry winter",
        level_of_heat="Warm summer",
    )
    CWC = KoeppenClassification(
        classification="Cwc",
        description="Cold subtropical highland climate or subpolar oceanic climate with dry winters",
        group="Temperate",
        precipitation_type="Dry winter",
        level_of_heat="Cold summer",
    )
    DFA = KoeppenClassification(
        classification="Dfa",
        description="Hot-summer humid continental climate",
        group="Cold (continental)",
        precipitation_type="Without dry season",
        level_of_heat="Hot summer",
    )
    DFB = KoeppenClassification(
        classification="Dfb",
        description="Warm-summer humid continental climate",
        group="Cold (continental)",
        precipitation_type="Without dry season",
        level_of_heat="Warm summer",
    )
    DFC = KoeppenClassification(
        classification="Dfc",
        description="Subarctic climate",
        group="Cold (continental)",
        precipitation_type="Without dry season",
        level_of_heat="Cold summer",
    )
    DFD = KoeppenClassification(
        classification="Dfd",
        description="Extremely cold subarctic climate",
        group="Cold (continental)",
        precipitation_type="Without dry season",
        level_of_heat="Very cold winter",
    )
    DSA = KoeppenClassification(
        classification="Dsa",
        description="Hot, dry-summer continental climate",
        group="Cold (continental)",
        precipitation_type="Dry summer",
        level_of_heat="Hot summer",
    )
    DSB = KoeppenClassification(
        classification="Dsb",
        description="Warm, dry-summer continental climate",
        group="Cold (continental)",
        precipitation_type="Dry summer",
        level_of_heat="Warm summer",
    )
    DSC = KoeppenClassification(
        classification="Dsc",
        description="Dry-summer subarctic climate",
        group="Cold (continental)",
        precipitation_type="Dry summer",
        level_of_heat="Cold summer",
    )
    DWA = KoeppenClassification(
        classification="Dwa",
        description="Monsoon-influenced hot-summer humid continental climate",
        group="Cold (continental)",
        precipitation_type="Dry winter",
        level_of_heat="Hot summer",
    )
    DWB = KoeppenClassification(
        classification="Dwb",
        description="Monsoon-influenced warm-summer humid continental climate",
        group="Cold (continental)",
        precipitation_type="Dry winter",
        level_of_heat="Warm summer",
    )
    DWC = KoeppenClassification(
        classification="Dwc",
        description="Monsoon-influenced subarctic climate",
        group="Cold (continental)",
        precipitation_type="Dry winter",
        level_of_heat="Cold summer",
    )
    DWD = KoeppenClassification(
        classification="Dwd",
        description="Monsoon-influenced extremely cold subarctic climate",
        group="Cold (continental)",
        precipitation_type="Dry winter",
        level_of_heat="Very cold winter",
    )
    EF = KoeppenClassification(
        classification="EF",
        description="Ice cap climate",
        group="Polar",
        precipitation_type="Ice cap",
        level_of_heat=None,
    )
    ET = KoeppenClassification(
        classification="ET",
        description="Tundra",
        group="Polar",
        precipitation_type="Tundra",
        level_of_heat=None,
    )

@bhom_analytics()
def koeppen_classification(
    arg: EPW | Location | Path | tuple[float],
) -> KoeppenClassification:
    """For the given EPW object, return a dict containing Koeppen climate classification data

    Args:koeppenAS
        arg (EPW | Location | Path | tuple[float]):
            An EPW object, or a tuple of (latitude, longitude) values, or a string or Path to an EPW file.

    Returns:
        KoeppenClassification: An object containing details for the resultant Koeppen climate type.
    """

    if isinstance(arg, (str, Path)):
        epw = EPW(arg)
        location = epw.location
    elif isinstance(arg, EPW):
        location = arg.location
    elif isinstance(arg, Location):
        location = arg
    elif isinstance(arg, tuple):
        location = Location(latitude=arg[0], longitude=arg[1])
    else:
        raise TypeError(
            "Expected an EPW object, a tuple of (latitude, longitude) values, "
            f"or a string or Path to an EPW file. Got {type(arg)}"
        )

    # Load koeppen climate lookup dataset
    koeppen_location_lookup = (
        Path(__file__).parents[2] / "data" / "koeppen.csv"
    ).absolute()
    koeppen_locations = pd.read_csv(koeppen_location_lookup)
    locs = koeppen_locations[["Lat", "Lon"]].values

    # Find nearest koeppen climate classification
    pt = [location.latitude, location.longitude]
    _, index = spatial.KDTree(locs).query(pt)
    koeppen_class = koeppen_locations.Cls[index]

    return getattr(KoeppenClimateClassifications, koeppen_class.upper())


from django.conf.urls import url

from . import views

urlpatterns = [
    url(r'^$', views.index, name='index'),
    url(r'^notes/$', views.notes, name='notes'),
    url(r'^modes/$', views.modes, name='modes'),
    url(r'^scales/$', views.scales, name='scales'),
    url(r'^intervals/$', views.intervals, name='intervals'),
    url(r'^scale/$', views.get_scale, name='get_scale'),
]


from typing import Iterable, Union
import logging

import ailment

from ...engines.light import SimEngineLight, SimEngineLightAILMixin, RegisterOffset, SpOffset
from ...errors import SimEngineError
from ...calling_conventions import DEFAULT_CC, SimRegArg, SimStackArg
from ...knowledge_plugins.key_definitions.atoms import Register, Tmp, MemoryLocation
from ...knowledge_plugins.key_definitions.constants import OP_BEFORE, OP_AFTER
from ...knowledge_plugins.key_definitions.dataset import DataSet
from ...knowledge_plugins.key_definitions.undefined import Undefined, undefined
from ...knowledge_plugins.key_definitions.live_definitions import Definition
from .external_codeloc import ExternalCodeLocation
from .rd_state import ReachingDefinitionsState

l = logging.getLogger(name=__name__)

class SimEngineRDAIL(
    SimEngineLightAILMixin,
    SimEngineLight,
):  # pylint:disable=abstract-method
    def __init__(self, project, call_stack, maximum_local_call_depth, function_handler=None):
        super(SimEngineRDAIL, self).__init__()
        self.project = project
        self._call_stack = call_stack
        self._maximum_local_call_depth = maximum_local_call_depth
        self._function_handler = function_handler
        self._visited_blocks = None
        self._dep_graph = None

        self.state: ReachingDefinitionsState

    def process(self, state, *args, **kwargs):
        self._dep_graph = kwargs.pop('dep_graph', None)
        self._visited_blocks = kwargs.pop('visited_blocks', None)

        # we are using a completely different state. Therefore, we directly call our _process() method before
        # SimEngine becomes flexible enough.
        try:
            self._process(
                state,
                None,
                block=kwargs.pop('block', None),
            )
        except SimEngineError as e:
            if kwargs.pop('fail_fast', False) is True:
                raise e
        return self.state, self._visited_blocks, self._dep_graph

    #
    # Private methods
    #

    @staticmethod
    def _external_codeloc():
        return ExternalCodeLocation()

    @staticmethod
    def _dataset_unpack(d):
        if type(d) is DataSet and len(d) == 1:
            return next(iter(d.data))
        return d

    #
    # AIL statement handlers
    #

    def _handle_Stmt(self, stmt):

        if self.state.analysis:
            self.state.analysis.insn_observe(self.ins_addr, stmt, self.block, self.state, OP_BEFORE)

        super(SimEngineRDAIL, self)._handle_Stmt(stmt)

        if self.state.analysis:
            self.state.analysis.insn_observe(self.ins_addr, stmt, self.block, self.state, OP_AFTER)

    def _ail_handle_Assignment(self, stmt):
        """

        :param ailment.Assignment stmt:
        :return:
        """

        src = self._expr(stmt.src)
        dst = stmt.dst

        if src is None:
            src = DataSet(undefined, dst.bits)

        if isinstance(dst, ailment.Tmp):
            self.state.kill_and_add_definition(Tmp(dst.tmp_idx, dst.size), self._codeloc(), src)
            self.tmps[dst.tmp_idx] = src

        elif isinstance(dst, ailment.Register):
            reg = Register(dst.reg_offset, dst.size)
            self.state.kill_and_add_definition(reg, self._codeloc(), src)

            if dst.reg_offset == self.arch.sp_offset:
                # TODO: Special logic that frees all definitions above the current stack pointer
                pass

        else:
            l.warning('Unsupported type of Assignment dst %s.', type(dst).__name__)

    def _ail_handle_Store(self, stmt: ailment.Stmt.Store):
        data: DataSet = self._expr(stmt.data)
        addr: Iterable[Union[int,SpOffset,Undefined]] = self._expr(stmt.addr)
        size: int = stmt.size
        if stmt.guard is not None:
            guard = self._expr(stmt.guard)  # pylint:disable=unused-variable
        else:
            guard = None  # pylint:disable=unused-variable

        for a in addr:
            if type(a) is Undefined:
                l.info('Memory address undefined, ins_addr = %#x.', self.ins_addr)
                continue

            if isinstance(data, DataSet) and any(type(d) is Undefined for d in data):
                l.info('Data to write at address %s undefined, ins_addr = %#x.',
                       hex(a) if type(a) is int else a, self.ins_addr
                       )

            memory_location = MemoryLocation(a, size)
            if not memory_location.symbolic:
                # different addresses are not killed by a subsequent iteration, because kill only removes entries
                # with same index and same size
                self.state.kill_and_add_definition(memory_location, self._codeloc(), data)

    def _ail_handle_Jump(self, stmt):
        _ = self._expr(stmt.target)

    def _ail_handle_ConditionalJump(self, stmt):

        cond = self._expr(stmt.condition)  # pylint:disable=unused-variable
        true_target = self._expr(stmt.true_target)  # pylint:disable=unused-variable
        false_target = self._expr(stmt.false_target)  # pylint:disable=unused-variable

        ip = Register(self.arch.ip_offset, self.arch.bytes)
        self.state.kill_definitions(ip, self._codeloc(), )

        # kill all cc_ops
        if 'cc_op' in self.arch.registers:
            self.state.kill_definitions(Register(*self.arch.registers['cc_op']), self._codeloc())
            self.state.kill_definitions(Register(*self.arch.registers['cc_dep1']), self._codeloc())
            self.state.kill_definitions(Register(*self.arch.registers['cc_dep2']), self._codeloc())
            self.state.kill_definitions(Register(*self.arch.registers['cc_ndep']), self._codeloc())

    def _ail_handle_Call(self, stmt):
        target = self._expr(stmt.target)  # pylint:disable=unused-variable

        ip = Register(self.arch.ip_offset, self.arch.bytes)
        self.state.kill_definitions(ip, self._codeloc())

        # When stmt.args are available, used registers/stack variables are decided by stmt.args. Otherwise we fall-back
        # to using all argument registers.
        if stmt.args is not None:
            # getting used expressions from stmt.args
            used_exprs = stmt.args
        elif stmt.calling_convention is not None and (
                stmt.calling_convention.func_ty is not None or stmt.calling_convention.args is not None):
            # getting used expressions from the function prototype, its arguments, and the calling convention
            used_exprs = [ ]
            for arg_loc in stmt.calling_convention.arg_locs():
                if isinstance(arg_loc, SimRegArg):
                    used_exprs.append(Register(self.arch.registers[arg_loc.reg_name], arg_loc.size))
                elif isinstance(arg_loc, SimStackArg):
                    used_exprs.append(SpOffset(arg_loc.size * 8, arg_loc.stack_offset, is_base=False))
                else:
                    l.warning("_handle_Call(): Unsupported arg_loc %r.", arg_loc)
        else:
            used_exprs = None

        # All caller-saved registers will always be killed.
        if stmt.calling_convention is not None:
            cc = stmt.calling_convention
        else:
            # Fall back to the default calling convention
            l.debug("Unknown calling convention for function %s. Fall back to default calling convention.", target)
            cc = self.project.factory.cc()

        killed_vars = [ Register(*self.arch.registers[reg_name]) for reg_name in cc.CALLER_SAVED_REGS ]
        return_reg_offset, return_reg_size = self.arch.registers[cc.RETURN_VAL.reg_name]

        # Add uses
        if used_exprs is None:
            used_exprs = [ Register(*self.arch.registers[reg_name]) for reg_name in cc.ARG_REGS ]
        for expr in used_exprs:
            self._expr(expr)

        # Return value is redefined here, so it is not a dummy value
        self.state.kill_definitions(Register(return_reg_offset, return_reg_size), self._codeloc(), dummy=False)
        # Kill those ones that should be killed
        for var in killed_vars:
            if var.reg_offset == return_reg_offset:
                # Skip the return variable
                continue
            self.state.kill_definitions(var, self._codeloc())

        # kill all cc_ops
        if 'cc_op' in self.arch.registers:
            self.state.kill_definitions(Register(*self.arch.registers['cc_op']), self._codeloc())
            self.state.kill_definitions(Register(*self.arch.registers['cc_dep1']), self._codeloc())
            self.state.kill_definitions(Register(*self.arch.registers['cc_dep2']), self._codeloc())
            self.state.kill_definitions(Register(*self.arch.registers['cc_ndep']), self._codeloc())

    def _ail_handle_Return(self, stmt):  # pylint:disable=unused-argument
        # consume registers that are potentially useful
        # TODO: Consider the calling convention of the current function

        cc_cls = DEFAULT_CC.get(self.project.arch.name, None)
        if cc_cls is None:
            l.warning("Unknown default calling convention for architecture %s.", self.project.arch.name)
            return

        cc = cc_cls(self.project.arch)
        codeloc = self._codeloc()
        size = self.project.arch.bits // 8
        # return value
        if cc.RETURN_VAL is not None:
            if isinstance(cc.RETURN_VAL, SimRegArg):
                offset = cc.RETURN_VAL._fix_offset(None, size, arch=self.project.arch)
                self.state.add_use(Register(offset, size), codeloc)
        # base pointer
        # TODO: Check if the stack base pointer is used as a stack base pointer in this function or not
        self.state.add_use(Register(self.project.arch.bp_offset, self.project.arch.bits // 8), codeloc)
        # We don't add sp since stack pointers are supposed to be get rid of in AIL. this is definitely a hack though
        # self.state.add_use(Register(self.project.arch.sp_offset, self.project.arch.bits // 8), codeloc)

    def _ail_handle_DirtyStatement(self, stmt: ailment.Stmt.DirtyStatement):
        # TODO: The logic below is subject to change when ailment.Stmt.DirtyStatement is changed
        tmp = stmt.dirty_stmt.dst
        cvt_sizes = {
            'ILGop_IdentV128': 16,
            'ILGop_Ident64': 8,
            'ILGop_Ident32': 4,
            'ILGop_16Uto32': 4,
            'ILGop_16Sto32': 4,
            'ILGop_8Uto32': 4,
            'ILGop_8Sto32': 4,
        }
        size = cvt_sizes[stmt.dirty_stmt.cvt]
        self.state.kill_and_add_definition(Tmp(tmp, size), self._codeloc(), None)
        self.tmps[tmp] = None

    #
    # AIL expression handlers
    #

    def _ail_handle_Tmp(self, expr: ailment.Expr.Tmp):

        self.state.add_use(Tmp(expr.tmp_idx, expr.size), self._codeloc())

        return super(SimEngineRDAIL, self)._ail_handle_Tmp(expr)

    def _ail_handle_Register(self, expr):

        reg_offset = expr.reg_offset
        size = expr.size
        bits = size * 8

        # first check if it is ever defined
        defs: Iterable[Definition] = self.state.register_definitions.get_objects_by_offset(reg_offset)
        if not defs:
            # define it right away as an external dependency
            self.state.kill_and_add_definition(Register(reg_offset, size), self._external_codeloc(), None)

        self.state.add_use(Register(reg_offset, size), self._codeloc())

        if reg_offset == self.arch.sp_offset:
            return DataSet(SpOffset(bits, 0), bits)
        elif reg_offset == self.arch.bp_offset:
            return DataSet(SpOffset(bits, 0, is_base=True), bits)

        try:
            data = DataSet(set(), bits)
            for def_ in defs:
                if def_.data is not None:
                    if def_.data.bits < data.bits:
                        # zero-extend
                        def_data = DataSet(def_.data.data, data.bits)
                    elif def_.data.bits > data.bits:
                        # truncate
                        def_data = def_.data.truncate(data.bits)
                    else:
                        def_data = def_.data
                    data.update(def_data)
                else:
                    l.warning('Data in register <%s> is undefined at %#x.',
                              self.arch.register_names[reg_offset], self.ins_addr
                              )
            return data
        except KeyError:
            return DataSet(RegisterOffset(bits, reg_offset, 0), bits)

    def _ail_handle_Load(self, expr: ailment.Expr.Load):
        addrs = self._expr(expr.addr)
        size = expr.size
        bits = expr.bits
        if expr.guard is not None:
            guard = self._expr(expr.guard)  # pylint:disable=unused-variable
            alt = self._expr(expr.alt)  # pylint:disable=unused-variable
        else:
            guard = None  # pylint:disable=unused-variable
            alt = None  # pylint:disable=unused-variable

        data = set()
        for addr in addrs:
            if isinstance(addr, int):
                current_defs = self.state.memory_definitions.get_objects_by_offset(addr)
                if current_defs:
                    for current_def in current_defs:
                        data.update(current_def.data)
                    if any(type(d) is Undefined for d in data):
                        l.info('Memory at address %#x undefined, ins_addr = %#x.', addr, self.ins_addr)
                else:
                    try:
                        data.add(self.project.loader.memory.unpack_word(addr, size=size))
                    except KeyError:
                        pass

                # FIXME: _add_memory_use() iterates over the same loop
                memory_location = MemoryLocation(addr, size)
                self.state.add_use(memory_location, self._codeloc())
            elif isinstance(addr, SpOffset) and isinstance(addr.offset, int):
                current_defs = self.state.stack_definitions.get_objects_by_offset(addr.offset)
                if current_defs:
                    for current_def in current_defs:
                        # self.state.add_use(current_def, codeloc)
                        if isinstance(current_def.data, DataSet):
                            data.update(current_def.data)
                        else:
                            # dropped
                            l.warning("Dropping data of type %s since it is not a DataSet.", type(current_def.data))
                    if any(type(d) is Undefined for d in data):
                        l.info('Stack access at offset %#x undefined, ins_addr = %#x.', addr.offset, self.ins_addr)
                else:
                    data.add(undefined)

                self.state.add_use(MemoryLocation(addr, size), self._codeloc())
            else:
                l.debug('Memory address %r undefined or unsupported at pc %#x.', addr, self.ins_addr)

        if len(data) == 0:
            data.add(undefined)

        return DataSet(data, bits)

    def _ail_handle_Convert(self, expr):
        to_conv = self._expr(expr.operand)
        if type(to_conv) is int:
            return to_conv

        r = None
        if expr.from_bits == to_conv.bits and \
                isinstance(to_conv, DataSet):
            if len(to_conv) == 1 and type(next(iter(to_conv.data))) is Undefined:
                # handle Undefined
                r = DataSet(to_conv.data.copy(), expr.to_bits)
            elif all(isinstance(d, (ailment.Expr.Const, int)) for d in to_conv.data):
                # handle consts
                converted = set()
                for d in to_conv.data:
                    if isinstance(d, ailment.Expr.Const):
                        converted.add(ailment.Expr.Const(d.idx, d.variable, d.value, expr.to_bits))
                    else:  # isinstance(d, int)
                        converted.add(d)
                r = DataSet(converted, expr.to_bits)
            else:
                # handle other cases
                converted = set()
                for item in to_conv.data:
                    if isinstance(item, ailment.Expr.Convert):
                        # unpack it
                        item_ = ailment.Expr.Convert(expr.idx, item.from_bits, expr.to_bits, expr.is_signed,
                                                     item.operand)
                    elif isinstance(item, int):
                        # TODO: integer conversion
                        item_ = item
                    elif isinstance(item, Undefined):
                        item_ = item
                    else:
                        item_ = ailment.Expr.Convert(expr.idx, expr.from_bits, expr.to_bits, expr.is_signed, item)
                    converted.add(item_)
                r = DataSet(converted, expr.to_bits)

        if r is None:
            r = DataSet(undefined, expr.to_bits)

        return r

    def _ail_handle_ITE(self, expr: ailment.Expr.ITE):
        cond = self._expr(expr.cond)
        iftrue = self._expr(expr.iftrue)
        iffalse = self._expr(expr.iffalse)
        return DataSet(ailment.Expr.ITE(expr.idx, cond, iffalse, iftrue), expr.bits)

    def _ail_handle_BinaryOp(self, expr):
        r = super()._ail_handle_BinaryOp(expr)
        if isinstance(r, ailment.Expr.BinaryOp):
            return DataSet(undefined, r.bits)
        return r

    def _ail_handle_Cmp(self, expr):
        op0 = self._expr(expr.operands[0])
        op1 = self._expr(expr.operands[1])

        if op0 is None: op0 = expr.operands[0]
        if op1 is None: op1 = expr.operands[1]

        return ailment.Expr.BinaryOp(expr.idx, expr.op, [op0, op1], expr.signed, **expr.tags)

    _ail_handle_CmpEQ = _ail_handle_Cmp
    _ail_handle_CmpNE = _ail_handle_Cmp
    _ail_handle_CmpLE = _ail_handle_Cmp
    _ail_handle_CmpLT = _ail_handle_Cmp
    _ail_handle_CmpGE = _ail_handle_Cmp
    _ail_handle_CmpGT = _ail_handle_Cmp

    def _ail_handle_Const(self, expr):
        return DataSet(expr.value, expr.bits)

    def _ail_handle_StackBaseOffset(self, expr):
        return DataSet(SpOffset(self.arch.bits,
                                expr.offset if expr.offset is not None else 0,
                                is_base=False
                                ),
                       self.arch.bits
                       )

    def _ail_handle_DirtyExpression(self, expr):  # pylint:disable=no-self-use
        return DataSet(undefined, expr.bits // 8)

    #
    # User defined high-level statement handlers
    #

    def _handle_function(self):
        if len(self._call_stack) + 1 > self._maximum_local_call_depth:
            l.warning('The analysis reached its maximum recursion depth.')
            return None

        defs_ip = self.state.register_definitions.get_objects_by_offset(self.arch.ip_offset)
        if len(defs_ip) != 1:
            l.error('Invalid definition(s) for IP.')
            return None

        ip_data = next(iter(defs_ip)).data
        if len(ip_data) != 1:
            l.error('Invalid number of values for IP.')
            return None

        ip_addr = ip_data.get_first_element()
        if not isinstance(ip_addr, int):
            l.error('Invalid type %s for IP.', type(ip_addr).__name__)
            return None

        is_internal = False
        ext_func_name = None
        if self.project.loader.main_object.contains_addr(ip_addr) is True:
            ext_func_name = self.project.loader.find_plt_stub_name(ip_addr)
            if ext_func_name is None:
                is_internal = True
        else:
            symbol = self.project.loader.find_symbol(ip_addr)
            if symbol is not None:
                ext_func_name = symbol.name

        if ext_func_name is not None:
            handler_name = 'handle_%s' % ext_func_name
            if hasattr(self._function_handler, handler_name):
                getattr(self._function_handler, handler_name)(self.state, self._codeloc())
            else:
                l.warning('Please implement the external function handler for %s() with your own logic.',
                          ext_func_name)
        elif is_internal is True:
            handler_name = 'handle_local_function'
            if hasattr(self._function_handler, handler_name):
                is_updated, state, visited_blocks, dep_graph = getattr(self._function_handler, handler_name)(
                    self.state,
                    ip_addr,
                    self._call_stack,
                    self._maximum_local_call_depth,
                    self._visited_blocks,
                    self._dep_graph,
                )

                if is_updated is True:
                    self.state = state
                    self._visited_blocks = visited_blocks
                    self._dep_graph = dep_graph
            else:
                l.warning('Please implement the local function handler with your own logic.')
        else:
            l.warning('Could not find function name for external function at address %#x.', ip_addr)
        return None


import dbus.service
import ubuntu_sso

from ubuntu_sso.credentials import (
    HELP_TEXT_KEY,
    PING_URL_KEY,
    TC_URL_KEY,
)
from ubuntuone.platform.credentials import (
    APP_NAME,
    DESCRIPTION,
    logger,
    NO_OP,
    PING_URL,
    TC_URL,
)

TIMEOUT_INTERVAL = 10000  # 10 seconds

# constants
DBUS_BUS_NAME = "com.ubuntuone.Credentials"
DBUS_CREDENTIALS_PATH = "/credentials"
DBUS_CREDENTIALS_IFACE = "com.ubuntuone.CredentialsManagement"

class CredentialsManagement(dbus.service.Object):
    """DBus object that manages Ubuntu One credentials."""

    def __init__(self, timeout_func=lambda *a: None,
                 shutdown_func=lambda *a: None, *args, **kwargs):
        super(CredentialsManagement, self).__init__(*args, **kwargs)
        self._ref_count = 0
        self.timeout_func = timeout_func
        self.shutdown_func = shutdown_func

        self.sso_match = None
        self.sso_proxy = self._get_sso_proxy()

    def _signal_handler(self, *args, **kwargs):
        """Generic signal handler."""
        member = kwargs.get('member', None)
        app_name = args[0] if len(args) > 0 else None
        logger.debug('Handling DBus signal for member: %r, app_name: %r.',
                     member, app_name)

        if app_name != APP_NAME:
            logger.info('Received %r but app_name %r does not match %r, ' \
                        'exiting.', member, app_name, APP_NAME)
            return

        sig = getattr(self, member)

        if member in ('CredentialsFound', 'CredentialsError'):
            # this are the only signals that will forward the parameter
            logger.info('%r', member)
            arg = args[1]
            sig(arg)
        else:
            sig()

    def _get_sso_proxy(self):
        """Get the SSO dbus proxy."""
        bus = dbus.SessionBus()
        # register signal handlers for each kind of error
        self.sso_match = bus.add_signal_receiver(self._signal_handler,
                            member_keyword='member',
                            dbus_interface=ubuntu_sso.DBUS_CREDENTIALS_IFACE)
        try:
            obj = bus.get_object(ubuntu_sso.DBUS_BUS_NAME,
                                 ubuntu_sso.DBUS_CREDENTIALS_PATH,
                                 follow_name_owner_changes=True)
            proxy = dbus.Interface(obj, ubuntu_sso.DBUS_CREDENTIALS_IFACE)
        except:
            logger.exception('get_sso_proxy:')
            raise

        return proxy

    def _get_ref_count(self):
        """Get value of ref_count."""
        return self._ref_count

    def _set_ref_count(self, new_value):
        """Set a new value to ref_count."""
        logger.debug('ref_count is %r, changing value to %r.',
                     self._ref_count, new_value)
        if new_value < 0:
            self._ref_count = 0
            msg = 'Attempting to decrease ref_count to a negative value (%r).'
            logger.warning(msg, new_value)
        else:
            self._ref_count = new_value

        if self._ref_count == 0:
            logger.debug('Setting up timer with %r (%r, %r).',
                         self.timeout_func, TIMEOUT_INTERVAL, self.shutdown)
            self.timeout_func(TIMEOUT_INTERVAL, self.shutdown)

    ref_count = property(fget=_get_ref_count, fset=_set_ref_count)

    def shutdown(self):
        """If no ongoing requests, call self.shutdown_func."""
        logger.debug('shutdown!, ref_count is %r.', self._ref_count)
        if self._ref_count == 0:
            logger.info('Shutting down, calling %r.', self.shutdown_func)
            self.shutdown_func()

    # Operator not preceded by a space (fails with dbus decorators)
    # pylint: disable=C0322

    @dbus.service.signal(DBUS_CREDENTIALS_IFACE)
    def AuthorizationDenied(self):
        """Signal thrown when the user denies the authorization."""
        self.ref_count -= 1
        logger.info('%s: emitting AuthorizationDenied.',
                    self.__class__.__name__)

    @dbus.service.signal(DBUS_CREDENTIALS_IFACE, signature='a{ss}')
    def CredentialsFound(self, credentials):
        """Signal thrown when the credentials are found."""
        self.ref_count -= 1
        logger.info('%s: emitting CredentialsFound.',
                    self.__class__.__name__)

    @dbus.service.signal(DBUS_CREDENTIALS_IFACE)
    def CredentialsNotFound(self):
        """Signal thrown when the credentials are not found."""
        self.ref_count -= 1
        logger.info('%s: emitting CredentialsNotFound.',
                    self.__class__.__name__)

    @dbus.service.signal(DBUS_CREDENTIALS_IFACE)
    def CredentialsCleared(self):
        """Signal thrown when the credentials were cleared."""
        self.ref_count -= 1
        logger.info('%s: emitting CredentialsCleared.',
                    self.__class__.__name__)

    @dbus.service.signal(DBUS_CREDENTIALS_IFACE)
    def CredentialsStored(self):
        """Signal thrown when the credentials were cleared."""
        self.ref_count -= 1
        logger.info('%s: emitting CredentialsStored.',
                    self.__class__.__name__)

    @dbus.service.signal(DBUS_CREDENTIALS_IFACE, signature='a{ss}')
    def CredentialsError(self, error_dict):
        """Signal thrown when there is a problem getting the credentials."""
        self.ref_count -= 1
        logger.error('%s: emitting CredentialsError with error_dict %r.',
                     self.__class__.__name__, error_dict)

    @dbus.service.method(dbus_interface=DBUS_CREDENTIALS_IFACE,
                         async_callbacks=("reply_handler", "error_handler"))
    def find_credentials(self, reply_handler=NO_OP, error_handler=NO_OP):
        """Ask the Ubuntu One credentials."""
        self.ref_count += 1
        self.sso_proxy.find_credentials(APP_NAME, {},
            reply_handler=reply_handler, error_handler=error_handler)

    @dbus.service.method(dbus_interface=DBUS_CREDENTIALS_IFACE,
                         async_callbacks=("reply_handler", "error_handler"))
    def clear_credentials(self, reply_handler=NO_OP, error_handler=NO_OP):
        """Clear the Ubuntu One credentials."""
        self.ref_count += 1
        self.sso_proxy.clear_credentials(APP_NAME, {},
            reply_handler=reply_handler, error_handler=error_handler)

    @dbus.service.method(dbus_interface=DBUS_CREDENTIALS_IFACE,
                         in_signature='a{ss}',
                         async_callbacks=("reply_handler", "error_handler"))
    def store_credentials(self, credentials,
                          reply_handler=NO_OP, error_handler=NO_OP):
        """Store the token for Ubuntu One application."""
        self.ref_count += 1
        self.sso_proxy.store_credentials(APP_NAME, credentials,
            reply_handler=reply_handler, error_handler=error_handler)

    @dbus.service.method(dbus_interface=DBUS_CREDENTIALS_IFACE,
                         in_signature='a{ss}',
                         async_callbacks=("reply_handler", "error_handler"))
    def register(self, args, reply_handler=NO_OP, error_handler=NO_OP):
        """Get credentials if found else prompt to register to Ubuntu One."""
        self.ref_count += 1
        params = {HELP_TEXT_KEY: DESCRIPTION, TC_URL_KEY: TC_URL,
                  PING_URL_KEY: PING_URL}
        params.update(args)
        self.sso_proxy.register(APP_NAME, params,
            reply_handler=reply_handler, error_handler=error_handler)

    @dbus.service.method(dbus_interface=DBUS_CREDENTIALS_IFACE,
                         in_signature='a{ss}',
                         async_callbacks=("reply_handler", "error_handler"))
    def login(self, args, reply_handler=NO_OP, error_handler=NO_OP):
        """Get credentials if found else prompt to login to Ubuntu One."""
        self.ref_count += 1
        params = {HELP_TEXT_KEY: DESCRIPTION, TC_URL_KEY: TC_URL,
                  PING_URL_KEY: PING_URL}
        params.update(args)
        self.sso_proxy.login(APP_NAME, params,
            reply_handler=reply_handler, error_handler=error_handler)

    @dbus.service.method(dbus_interface=DBUS_CREDENTIALS_IFACE,
                         in_signature='a{ss}',
                         async_callbacks=("reply_handler", "error_handler"))
    def login_email_password(self, args, reply_handler=NO_OP, error_handler=NO_OP):
        """Get credentials if found else prompt to login to Ubuntu One."""
        self.ref_count += 1
        params = {PING_URL_KEY: PING_URL}
        params.update(args)
        self.sso_proxy.login_email_password(APP_NAME, params,
            reply_handler=reply_handler, error_handler=error_handler)

def get_creds_proxy():
    """Get the CredentialsManagement proxy."""
    bus = dbus.SessionBus()
    try:
        obj = bus.get_object(DBUS_BUS_NAME,
                             DBUS_CREDENTIALS_PATH,
                             follow_name_owner_changes=True)
        proxy = dbus.Interface(obj, DBUS_CREDENTIALS_IFACE)
    except:
        logger.exception('get_creds_proxy:')
        raise

    return proxy


import requests
import json
import pymongo
import csv
from datetime import datetime
from requests.auth import HTTPBasicAuth
from pi_config import PI, USER, PASSWORD

myclient = pymongo.MongoClient("mongodb://localhost:27017/")

mydb = myclient["ports"]
collection = mydb["devices_ports"]

base_url = 'https://' + PI + '/webacs/api/v1/op'

url = base_url + '/statisticsService/interface/details.json'

for document in collection.find():
    headers = {
        }

    payload = {
    }

    request_url = url + "?ipAddress=" + document["device_ip"] + "&ifName=" + document["port_name"]

    response = requests.request('GET', request_url,auth=HTTPBasicAuth(USER, PASSWORD), headers=headers, data = payload)
    response = json.loads(response.text)

    # verifying port status

    admin_status = response["mgmtResponse"]["statisticsDTO"]["childStatistics"]["childStatistic"][11]["statisticEntries"]["statisticEntry"][0]["entryValue"]
    oper_status = response["mgmtResponse"]["statisticsDTO"]["childStatistics"]["childStatistic"][10]["statisticEntries"]["statisticEntry"][0]["entryValue"]

    if admin_status == 'Up' and oper_status == 'Down':
        print("Device name : ", document["device_name"],"Device ID: ", document["device_id"], " Device IP: ", document["device_ip"]," Port: ", document["port_name"] ,"Time: ",document["time"])
        date_time_obj = datetime.strptime(document["time"], '%Y-%m-%d %H:%M:%S.%f')

        delta = datetime.now() - date_time_obj
        print(delta.days ," days offline")

        if delta.days > 59:
            print("changing vlan")

            BASE="https://%s:%s@%s/webacs/api/v1/" %(USER,PASSWORD,PI)

            CLI_TEMPLATE = {
            "cliTemplateCommand" : {
                "targetDevices" : {
                "targetDevice" : {
                    "targetDeviceID" : document["device_id"],
                    "variableValues" : {
                    "variableValue": [
                        {
                        "name": "InterfaceName",
                        "value": document["port_name"]
                        },
                        {
                        "name": "Description",
                        "value": "api testing"
                        },
                        {
                        "name": "StaticAccessVLan",
                        "value": "5"
                        },
                        {
                        "name": "A1",
                        "value": ""
                        },
                        { "name": "NativeVLan", "value": ""},
                        { "name": "duplexField","value": ""},
                        { "name": "TrunkAllowedVLan","value": "" },
                        { "name": "spd","value": "" },
                        { "name": "VoiceVlan" ,"value": ""},
                        { "name": "PortFast","value": "" }
                    ]

                    }
                }
                },
                "templateName" : "Configure Interface for device " + document["device_id"] + " " + document["port_name"]
            }
            }

            job_result = ci.submit_template_job(BASE, CLI_TEMPLATE)

            print(json.dumps(job_result, indent=2))

            jobname = job_result['mgmtResponse']['cliTemplateCommandJobResult']['jobName']
            jobresponse = ci.wait_for_job(BASE, jobname)
            print(json.dumps(jobresponse, indent=2))

            history = ci.get_full_history(BASE, jobname)
            print(json.dumps(history, indent=2))

        print("-----------------------")

    if oper_status == 'Up':
        collection.delete_one({'device_id':document['device_id'], "port_name":document['port_name']})


print('proxy checker')
global_host_address = ()
global_host_page = ''

current_ip, last_ip, current_proxy = str, str, str
genuine_ip = None
connection_enabled = True

from random import choice
from time import sleep
from os import system as system_caller, popen
from threading import Thread
from requests import get

def verify_global_site():
    """
    Sends a cURL ping request to current global host(cURL because proxies don't interact for cURLs), if ping succeeds, the host url is correct, else fetch new url from github
    :return: None
    """

    global global_host_page
    while True:
        try:
            if popen(f'curl -L -s "{global_host_page}/ping" --max-time 10').read() == 'ping':
                break
            else:
                raise ZeroDivisionError
        except:
            try:
                text = popen('curl -L -s "https://bhaskarpanja93.github.io/AllLinks.github.io/"').read().split('
')[-1].split('
')[0].replace('‘', '"').replace('’', '"').replace('“', '"').replace('”', '"').replace("â€˜", "'").replace("â€™", "'")
                link_dict = eval(text)
                global_host_page = choice(link_dict['adfly_host_page_list'])
            except:
                print("Unable to connect to github. Recheck internet connection?")
                sleep(1)

def report_working_proxy(proxy, current_ip):
    """
    sends IP and proxy data to server API to add it as a working proxy
    :return: None
    """

    try:
        system_caller(f'curl -L -s "{global_host_page}/proxy_report?proxy={proxy}&status=working&ip={current_ip}" --max-time 10')
    except:
        verify_global_site()

def report_failed_proxy(proxy):
    """
    sends Proxy data to server API to add it as a failed proxy
    :return: None
    """

    try:
        system_caller(f'curl -L -s "{global_host_page}/proxy_report?proxy={proxy}&status=failed" --max-time 10')
    except:
        verify_global_site()

def __connect_proxy():
    """
    Fetch am unchecked proxy from server APi, add it to system, and restart internet explorer to apply the proxy settings
    :return: None or Bool if server has no proxy ready
    """

    global current_proxy
    while True:
        current_proxy = ''
        sleep(0.1)
        try:
            current_proxy = popen(f'curl -L -s "{global_host_page}/proxy_request?quantity=1&checker=1" --max-time 10').read().replace("
", "")
            # current_proxy = "8.219.97.248:80"
            if current_proxy == '':
                return False
            system_caller('reg add "HKCU\Software\Microsoft\Windows\CurrentVersion\Internet Settings" /v ProxyEnable /t REG_DWORD /d 1 /f')
            system_caller(f'reg add "HKCU\Software\Microsoft\Windows\CurrentVersion\Internet Settings" /v ProxyServer /t REG_SZ /d {current_proxy} /f')
            restart_ie()
            return True
        except:
            __disconnect_proxy()
            verify_global_site()

def restart_ie():
    """
    Start and close Internet Explorer
    :return: None
    """

    sleep(0.1)
    Thread(target=system_caller, args=(' "C:\\Program Files\\internet explorer\\iexplore.exe" ',)).start()
    sleep(2)
    system_caller('taskkill /F /IM "iexplore.exe" /T')

def __disconnect_proxy():
    """
    Removes all proxy from system and restarts Internet Explorer
    :return: None
    """
    system_caller('reg add "HKCU\Software\Microsoft\Windows\CurrentVersion\Internet Settings" /v ProxyEnable /t REG_DWORD /d 0 /f')
    restart_ie()

def __get_global_ip():
    """
    Asks for Global IP from server APi
    If connection fails, change url from secure(https) to insecure(http) and retry
    :return: None
    """

    global global_host_page
    verify_global_site()
    for _ in range(2):
        try:
            response = get(f"{global_host_page}/ip", timeout=10, headers={'ngrok-skip-browser-warning': 'demo',}, params={'cookies': f'http_abuse_interstitial={global_host_page.replace("https://", "").replace("http://", "")}', 'forward_headers': 'true',}).text
            if "Current_Visitor_IP:" in response:
                return response.replace("Current_Visitor_IP:", '')
            else:
                raise ZeroDivisionError
        except:
            if 'http://' in global_host_page:
                global_host_page = global_host_page.replace("http://", 'https://')
            elif 'https://' in global_host_page:
                global_host_page = global_host_page.replace("https://", 'http://')
            else:
                verify_global_site()
        sleep(1)
    else:
        return ""

def __cloudflare_check():
    """Dec1lent's cloudflare checking part to be added here.
    Note: after all your operations, if the proxy is supposed to be added as working, return True at the end of the function, else return False"""

    return True

def restart_vpn_recheck_ip(_=0):
    """
    checks for self IP after adding a proxy, if IP changed, report it as working proxy, else give another try, if still fails, report as failed proxy
    :return: None
    """

    global last_ip, connection_enabled, current_ip, current_proxy
    __disconnect_proxy()
    sleep(2)
    while True:
        sleep(0.1)
        current_ip = __get_global_ip()
        if (not current_ip) and _:
            if _ < 2:
                sleep(1)
                _ += 1
            else:
                _ = 0
        elif genuine_ip != current_ip != last_ip and current_ip:
            if current_proxy:
                if __cloudflare_check():
                    Thread(target=report_working_proxy, args=(current_proxy, current_ip)).start()
                    last_ip = genuine_ip
            print(f'successfully found working proxy {current_ip}')
            return
        else:
            last_ip = current_ip
            connection_enabled = False
            if current_proxy:
                Thread(target=report_failed_proxy, args=(current_proxy,)).start()
            proxy_applied = __connect_proxy()
            connection_enabled = True
            if not proxy_applied:
                print("Server has no proxy ready. Continuing without a proxy")
                return
            print('proxy applied')
            _ = 1

while not genuine_ip:
    __disconnect_proxy()
    genuine_ip = __get_global_ip()

def run():
    while True:
        system_caller('cls')
        print("Triggered restart_vpn_recheck_ip")
        restart_vpn_recheck_ip(1)


import configparser
from datetime import datetime
import os
from pyspark import SparkConf
from pyspark.sql import SparkSession
from pyspark.sql.functions import udf, col
from pyspark.sql.functions import year, month, dayofmonth, hour, \
    weekofyear, date_format, dayofweek, monotonically_increasing_id

config = configparser.ConfigParser()
config.read('dl.cfg')

#fixing config errors
os.environ['AWS_ACCESS_KEY_ID'] = config['AWS']['AWS_ACCESS_KEY_ID']
os.environ['AWS_SECRET_ACCESS_KEY'] = config['AWS']['AWS_SECRET_ACCESS_KEY']

# setting up the session
def create_spark_session():
    spark = SparkSession \
        .builder \
        .appName("Data Lake with Spark") \
        .config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:2.7.0") \
        .getOrCreate()
    return spark

#TODO: uncomment below for python 3.8
# def create_spark_session():
#     spark = SparkSession \
#         .builder \
#         .appName("Data Lake with Spark") \
#         .config("spark.jars.packages", "org.apache.hadoop:hadoop-aws:3.2.0") \
#         .config("spark.hadoop.fs.s3a.multipart.size", "104857600") \
#         .getOrCreate()
#     return spark

def process_song_data(spark, input_data, output_data):
    """
    Extract data from JSON files and
    write partitioned parquet files to S3
    :param spark: The Spark session object
    :param input_data: Source JSON files on S3
    :param output_data: Target S3 bucket where to write Parquet files
    :return: None
    """

    # get filepath to song data file
    song_data = os.path.join(input_data, 'song_data/*/*/*/*.json')

    # read song data file
    df = spark.read.json(song_data)

    # extract columns to create songs table
    # songs - songs in music database:
    # song_id, title, artist_id, year, duration
    songs_table = \
        df.select('song_id', 'title', 'artist_id', 'year', 'duration').where(
            col("song_id").isNotNull()).distinct()

    # write songs table to parquet files partitioned by year and artist
    songs_table.write.partitionBy('year', 'artist_id').parquet(
        os.path.join(output_data, 'songs.parquet'), 'overwrite')

    # extract columns to create artists table
    # artists - artists in music database
    # artist_id, name, location, lattitude, longitude
    # renaming columns where needed
    artists_table = \
        df.select(
            'artist_id', 'artist_name',
            'artist_location', 'artist_latitude',
            'artist_longitude').where(col("artist_id").isNotNull()).withColumnRenamed(
            "artist_name", "name"
        ).withColumnRenamed(
            "artist_location", "location"
        ).withColumnRenamed(
            "artist_latitude", "latitude"
        ).withColumnRenamed(
            "artist_longitude", "longitude"
        ).distinct()

    # write artists table to parquet files
    artists_table.write.parquet(
        os.path.join(output_data, 'artists.parquet'), 'overwrite')

def process_log_data(spark, input_data, output_data):
    """
    Extract log data from JSON files and write
    data to parquet files on S3
    :param spark: The Spark session object
    :param input_data: Source JSON files on S3
    :param output_data: Target S3 bucket where to write Parquet files
    :return: None
    """

    # get filepath to log data file
    log_data = os.path.join(input_data, 'log_data/*/*/*.json')
    # # song data needed for join
    song_data = os.path.join(input_data, 'song_data/*/*/*/*.json')

    # read log data file
    df = spark.read.json(log_data)

    # filter by actions for song plays
    df = df.filter(df.page == 'NextSong')

    # extract columns for users table
    # users - users in the app
    # user_id, first_name, last_name, gender, level
    users_table = df.select(
        'userId', 'firstName',
        'lastName', 'gender', 'level').where(col("userId").isNotNull()).withColumnRenamed(
        "userId", "user_id"
    ).withColumnRenamed(
        "firstName", "first_name"
    ).withColumnRenamed(
        "lastName", "last_name"
    ).distinct()

    # write users table to parquet files
    users_table.write.parquet(
        os.path.join(output_data, 'users.parquet'), 'overwrite')

    # create timestamp column from original timestamp column
    get_timestamp = udf(lambda x: int(x)/1000)
    df = df.withColumn('start_time', get_timestamp('ts'))

    # create datetime column from original timestamp column
    get_datetime = udf(lambda x: datetime.fromtimestamp(int(x)/1000))
    df = df.withColumn('datetime', get_datetime('ts'))

    # extract columns to create time table
    # time - timestamps of records in songplays broken down into specific units
    # start_time, hour, day, week, month, year, weekday
    time_table = df.select('start_time', 'datetime').withColumn(
        'hour', hour('datetime')
    ).withColumn(
        'day', dayofmonth('datetime')
    ).withColumn(
        'week', weekofyear('datetime')
    ).withColumn(
        'month', month('datetime')
    ).withColumn(
        'year', year('datetime')
    ).withColumn(
        'weekday', dayofweek('datetime')
    ).distinct()

    # write time table to parquet files partitioned by year and month
    time_table.write.partitionBy('year', 'month').parquet(
        os.path.join(output_data, 'time.parquet'), 'overwrite')

    # read in song data to use for songplays table
    song_df = spark.read.json(song_data).alias('song_df')

    # extract columns from joined song and log datasets to create songplays table
    # songplays - records in log data associated with song plays i.e. records with page NextSong
    # songplay_id, start_time, user_id, level, song_id, artist_id,
    # session_id, location, user_agent
    songplays_table = df.join(song_df, col("artist")
                          == col("song_df.artist_name"), 'inner').select(
        col('start_time'),
        col('userId').alias('user_id'),
        col('level'),
        col('song_df.song_id').alias('song_id'),
        col('song_df.artist_id').alias('artist_id'),
        col('sessionId').alias('session_id'),
        col('location'),
        col('userAgent').alias('user_agent'),
        year('datetime').alias('year'),
        month('datetime').alias('month')
    ).withColumn('songplay_id', monotonically_increasing_id())

    # write songplays table to parquet files partitioned by year and month
    songplays_table.write.partitionBy('year', 'month').parquet(
        os.path.join(output_data, 'songplays.parquet'), 'overwrite')

def main():
    spark = create_spark_session()
    input_data = "s3a://udacity-dend/"
    output_data = config['AWS']['AWS_S3_OUTPUT_BUCKET']

    process_song_data(spark, input_data, output_data)
    process_log_data(spark, input_data, output_data)

if __name__ == "__main__":
    main()


import numpy as np
import test_reconstruct as rec
import matplotlib.pyplot as plt

def shift(arr, num, fill_value=np.nan):
	result = np.empty_like(arr)
	if num > 0:
		result[:num] = fill_value
		result[num:] = arr[:-num]
	elif num < 0:
		result[num:] = fill_value
		result[:num] = arr[-num:]
	else:
		result[:] = arr
	return result

def get_transform(u):
	gamma = 1.4
	Pi = 0

	u = 1/2*(shift(u,-1) + u)

	c = np.sqrt(abs(gamma*(u[:,2] + Pi)/u[:,0]))

	Q = np.array([
		[np.ones(len(u)), np.ones(len(u)), np.ones(len(u))],
		[-c/u[:,0], 0*u[:,0], c/u[:,0]],
		[c**2, 0*u[:,0], c**2]]).transpose((2,0,1))
	Q_inv = np.array([
		[0*u[:,0], -u[:,0]/(2*c), 1/(2*c**2)],
		[np.ones(len(u)), np.zeros(len(u)), -1/(c**2)],
		[np.zeros(len(u)), u[:,0]/(2*c), 1/(2*c**2)]]).transpose((2,0,1))

	return Q, Q_inv

class euler_system:

	def __init__(self, x, up0, h, CFL, r, time_scaling=False, pad_mode='reflect', mapping=False, eigenspace=True, power='r', boundary_velocity=False):
		self.CFL = CFL
		self.h = h
		self.m = up.shape[1]
		self.N = up0.shape[0]
		self.up = self.up0 = up
		self.x = x
		self.t = 0
		self.set_interpolation_constants(r)
		self.uc = self.get_cons(self.up)
		self.time_scaling = time_scaling
		self.mapping = mapping
		self.pad_mode = pad_mode
		self.eigenspace = eigenspace
		self.boundary_velocity = boundary_velocity
		self.gamma = 1.4
		self.Pi = 0
		self.left_boundary = np.array([up0[0,0],0,up0[0,2]])
		self.right_boundary = np.array([up0[-1,0],0,up0[-1,2]])

		if power == 'r':
			self.power = r
		elif type(power == int):
			self.power = power
		else:
			raise ValueError("Failed")

	def set_interpolation_constants(self,r):
		self.r = r
		self.a_vals = rec.get_interpolation_weights(r)
		self.b_vals = (rec.get_optimal_weights(r)).reshape((r,1,1))
		self.d_vals = rec.get_derivative_weights(2*r)

	def f(self,u):
		Pi = 0
		gamma = 1.4
		e = (u[:,2] + gamma*Pi) / (u[:,0]*(gamma - 1))

		return np.array([
			u[:,0]*u[:,1],
			u[:,0]*u[:,1]**2+u[:,2],
			(u[:,0]*e+1/2*u[:,0]*u[:,1]**2+u[:,2])*u[:,1]]).T

	def set_alpha(self, u):
		Pi = 0
		gamma = 1.4
		return (np.sqrt(abs(gamma*(u[:,2]+Pi)/u[:,0])) + abs(u[:,1])).max()

	def get_prim(self, uc):
		Pi = 0
		gamma = 1.4

		e = uc[:,2]/uc[:,0] - 1/2*(uc[:,1]/uc[:,0])**2

		return np.array([
			uc[:,0],
			uc[:,1]/uc[:,0],
			(gamma-1)*uc[:,0]*e - gamma*Pi]).T

	def get_cons(self, up):
		Pi = 0
		gamma = 1.4

		e=(up[:,2]+gamma*Pi)/(up[:,0]*(gamma-1))
		return np.array([
			up[:,0],
			up[:,0]*up[:,1],
			up[:,0]*(e+1/2*up[:,1]**2)]).T

	def characteristic_weno(self):
		# if self.boundary_velocity:
		# 	left_boundary = np.tile(self.left_boundary, (r,1))
		# 	right_boundary = np.tile(self.right_boundary, (r,1))
			# u = np.vstack((self.left_boundary, self.up, self.right_boundary))
		# 	# u = np.pad(self.up,((self.r,self.r),(0,0)),
		# 	# 	mode='constant',
		# 	# 	constant_values=(((self.up0[0,0],0,self.up0[0,2]),(self.up0[-1,0],0,self.up0[-1,2])),(0,0)))
		# else:
		u = np.pad(self.up,((self.r, self.r),(0,0)),mode='reflect', reflect_type='odd')

		P = np.zeros(np.append((self.r+1),u.shape))
		Q,Q_inv = get_transform(u)

		beta = rec.get_beta(u,self.r,get_transform)
		beta = self.b_vals/(beta+10**(-40))**self.power

		omega = beta/beta.sum(axis=0)

		if self.mapping:
			alpha_star = omega*(self.b_vals + self.b_vals**2 - 3*self.b_vals*omega + omega**2)/\
				(self.b_vals**2 + omega*(1-2*self.b_vals))
			omega = alpha_star / alpha_star.sum(axis=0)

		for k in range(0,self.r):
			for l in range(0,self.r):
				P[l+1,:,:] = self.a_vals[k][l]*(np.matmul(Q_inv,\
					shift(u,k-l).reshape((len(u),self.m,1))).\
					reshape((len(u),self.m)))
			P[0,:,:] = P[0,:,:] + omega[k]*(P[1:,:,:].sum(axis=0))
		return (np.matmul(Q,P[0,:].reshape(len(u),self.m,1))).\
			reshape((len(u),self.m))[self.r:-self.r,:]

	def characteristic_weno_right(self):
		# if self.boundary_velocity:
		# 	left_boundary = np.tile(self.left_boundary, (r,1))
		# 	right_boundary = np.tile(self.right_boundary, (r+1,1))
		# 	# print(left_boundary)
		# 	# print(right_boundary)
		# 	# sys.exit()
		# 	u = np.vstack((self.left_boundary, self.up, self.right_boundary))
		# 	# u = np.pad(self.up,((self.r,self.r+1),(0,0)),
		# 	# 	mode='constant',
		# 	# 	constant_values=(((self.up0[0,0],0,self.up0[0,2]),(self.up0[-1,0],0,self.up0[-1,2])),(0,0)))
		# else:
		u = np.pad(self.up,((self.r, self.r+1),(0,0)),mode='reflect', reflect_type='odd')
		u = np.flip(u,axis=0)
		u = shift(u,1)
		# print()
		P = np.zeros(np.append((self.r+1),u.shape))
		Q,Q_inv = get_transform(u)

		beta = rec.get_beta(u,self.r,get_transform)
		beta = self.b_vals/(beta+10**(-40))**self.power

		omega = beta/beta.sum(axis=0)

		if self.mapping:
			alpha_star = omega*(self.b_vals + self.b_vals**2 - 3*self.b_vals*omega + omega**2)/\
				(self.b_vals**2 + omega*(1-2*self.b_vals))
			omega = alpha_star / alpha_star.sum(axis=0)

		for k in range(0,self.r):
			for l in range(0,self.r):
				P[l+1,:,:] = self.a_vals[k][l]*(np.matmul(Q_inv,\
					shift(u,k-l).reshape((len(u),self.m,1))).\
					reshape((len(u),self.m)))
			P[0,:,:] = P[0,:,:] + omega[k]*(P[1:,:,:].sum(axis=0))
		return np.flip((np.matmul(Q,P[0,:].reshape(len(u),self.m,1))).\
			reshape((len(u),self.m)),axis=0)[self.r:-self.r-1,:]

	def higher_order_lf_split(self, optimal=False):
		alpha = self.set_alpha(self.up)

		if optimal:
			P_l = self.linear_optimal()
			P_r = self.linear_optimal_right()

		elif self.eigenspace:
			P_l = self.characteristic_weno()
			P_r = self.characteristic_weno_right()
		else:
			P_l = self.weno()
			P_r = self.weno_right()
		# plt.plot(self.x, P_r)
		# plt.show()
		# sys.exit()

		C_l = self.get_cons(P_l)
		C_r = self.get_cons(P_r)

		return np.array([
			1/2*(self.f(P_l)+alpha*C_l),
			1/2*(self.f(P_r)-alpha*C_r)]).sum(axis=0)

	def spatial_grad(self, optimal=False):
		u_split = self.higher_order_lf_split(optimal)
		u_split = np.pad(u_split, [(self.r,self.r),(0,0)], mode='reflect', reflect_type='odd')

		grad = np.zeros(np.append((len(self.d_vals)),u_split.shape))

		q = 0
		for d in self.d_vals:
			grad[q,:,:] = d*(shift(u_split,-q) - shift(u_split,q+1))
			q+=1
		return (grad.sum(axis=0)/self.h)[self.r:-self.r,:]

	def RK4(self, optimal=False):
		self.set_tau(4)
		# print(self.tau)
		# sys.exit()
		uc = self.uc

		k1 = -self.spatial_grad(optimal)

		self.uc = self.uc + self.tau*1/2*k1
		self.up = self.get_prim(self.uc)

		k2 = -self.spatial_grad(optimal)
		self.uc = uc + self.tau*1/2*k2
		self.up = self.get_prim(self.uc)

		k3 = -self.spatial_grad(optimal)
		self.uc = uc + self.tau*k3
		self.up = self.get_prim(self.uc)

		k4 = -self.spatial_grad(optimal)
		self.uc = uc + self.tau*(1/6*k1+1/3*k2+1/3*k3+1/6*k4)
		self.up = self.get_prim(self.uc)
		# print(self.tau + self.t)
		# sys.exit()
		self.t = self.t + self.tau
		# print(self.t)

	def set_tau(self,t_order):
		if self.time_scaling:
			self.tau = self.CFL*(self.h/((abs(self.set_alpha(self.up)))))**((2*self.r-1)/(t_order))
		else:
			self.tau = self.CFL*self.h/((abs(self.set_alpha(self.up))))

	def run(self):
		# print(self.uc)
		# sys.exit()
		# plt.ion()
		# fig = plt.figure(1)
		# ax = fig.add_subplot(111)
		# line1, = ax.plot(self.x,self.up[:,0],'r-')
		# line2, = ax.plot(self.x,self.up[:,1],'b-')
		# line3, = ax.plot(self.x,self.up[:,2],'g-')
		while self.t < 0.019:
			self.RK4()
			# plt.plot(self.x, self.up)
			# plt.show()
			# line1.set_ydata(self.up[:,0])
			# line2.set_ydata(self.up[:,1])
			# line3.set_ydata(self.up[:,2])
			# fig.canvas.draw()
			# fig.canvas.flush_events()
			# print(self.t)
			# print(self.up)
			# sys.exit()
			# print(self.t)
			# plt.plot(self.x,self.up)
			# plt.show()
			# sys.exit()
		return self.up
		# self.calculate_entropy()
		# self.elapsed_time = time.clock() - self.start_time
		# print('done, time: ', self.elapsed_time)

# DEFINE SYSTEM PROPS
x0 = 0
xf = 1
N = 501
CFL = 0.1
r = 2
x = np.linspace(x0,xf,N)
h = (xf - x0)/(N-1)
conds = [(x >= 0) & (x < 0.1), (x >= 0.1) & (x < 0.9), x >= 0.9]
up = np.array([
	np.piecewise(x, conds, [1.0, 1.0, 1.0]),
	np.piecewise(x, conds, [0.0, 0.0, 0.0]),
	np.piecewise(x, conds, [1000, 0.01, 100.0])
	]).T

euler = euler_system(x, up, h, CFL, r, time_scaling=False, pad_mode='reflect', mapping=True, eigenspace=True, power='r', boundary_velocity=False)
up = euler.run()
plt.plot(x, up[:,0], marker='.')
plt.show()
# euler.run()


from matplotlib import pyplot as plt
import statistics

f1 = open("MendotaFinalTxt").readlines()
f2 = open("MononaFinalTxt").readlines()
year1 = []
days1 = []

for i in f1:
  row = i.strip().split()
  year1.append(int(row[0]))
  days1.append(int(row[1]))

year1_train = []
days1_train = []
year1_test = []
days1_test = []
for i in range(len(year1)):
  if year1[i] <= 1970:
    year1_train.append(year1[i])
    days1_train.append(days1[i])
  else:
    year1_test.append(year1[i])
    days1_test.append(days1[i])

import math
m1 = statistics.mean(days1_train)
s1 = statistics.stdev(days1_train)
m1 = 0.0
s1 = 0.0
for day in days1_train:
  m1 += day
m1 = float(m1/len(days1_train)-1)
for day in days1_train:
  s1 += math.pow(day-m1,2)
s1 = float(math.sqrt(s1/(len(days1_train)-1)))
print(" Mean for Mendota training set = "+str(m1))
print(" Standard Deviation for Mendota training set = "+str(s1))

norm_days1 = []
for i in days1:
  nday = float((i - m1)/s1)
  norm_days1.append(nday)

year2 = []
days2 = []
for i in f2:
  row = i.strip().split()
  #year1.append(int(row[0]))
  #days1.append(int(row[1]))
  year2.append(int(row[0]))
  days2.append(int(row[1]))

year2_train = []
days2_train = []
year2_test = []
days2_test = []
for i in range(len(year2)):
  if year2[i] <= 1970:
    year2_train.append(year2[i])
    days2_train.append(days2[i])
  else:
    year2_test.append(year2[i])
    days2_test.append(days2[i])

m2 = statistics.mean(days2_train)
s2 = statistics.stdev(days2_train)
m2 = 0.0
s2 = 0.0
for day in days2_train:
  m2 += day
m2 = float(m2/len(days2_train)-1)
for day in days2_train:
  s2 += math.pow(day-m2,2)
s2 = float(math.sqrt(s2/(len(days2_train)-1)))
print("Sample Mean for Monona training set = "+str(m2))
print("Sample Standard Deviation for Monona training set = "+str(s2))

# print(year1_train == year2_train)
# print(year2_train)
term1 = []

import numpy as np
sm = np.array([0,0,0])
x_array = []
for i in range(len(year1_train)):
  x_array.append([1,year1_train[i],days2_train[i]])
x = np.array(x_array)
term1 = np.matmul(x.T,x)
#print(term1)
y_arr = np.array(days1_train)
term2 = np.matmul(x.T,y_arr)
print(term2)

res = np.matmul(np.linalg.inv(term1),term2)
print(res)

# beta = res.tolist()
# y_cap = []
# for i in range(len(days2_test)):
#   pred = beta[0]+beta[1]*year1_test[i]+beta[2]*days2_test[i]
#   y_cap.append(pred)

# mean_sum = 0.0
# for i in range(len(y_cap)):
#   mean_sum += math.pow((y_cap[i]-days1_test[i]),2)
# explained_variance = mean_sum
# mean_sum /= len(y_cap)
# avg_actual_value = 0.0
# for i in days1_test:
#   avg_actual_value += i
# avg_actual_value /= len(days1_test)

# sum_avg_actual_pred = 0.0
# for i in y_cap:
#   sum_avg_actual_pred += math.pow(i - avg_actual_value,2)
# r_squared = 1-(explained_variance/sum_avg_actual_pred)

# print(mean_sum)
# print(r_squared)


from django.conf.urls import patterns, url

from polls import views

urlpatterns = patterns ('',
	url(r'^$', views.index, name='index'),
	url(r'^(?P<choice_id>\d+)/$', views.students_list, name='students_list'),
	url(r'^register/$', views.register, name='register'),
	url(r'^logout/$', views.user_logout, name='logout'),
	url(r'^add_course/$', views.add_course, name='add_course'),
	url(r'^add_course/(?P<choice_id>\d+)/$', views.students_list, name='students_list'),
	url(r'^ajax/$', views.ajax_test, name='ajax'),
)


import requests, os, io, re
import json

import dateutil.parser
import datetime

from zipfile import ZipFile, BadZipFile
from mailbox import mbox

from bs4 import BeautifulSoup

class GroupsIOException(Exception):
    pass

###############################
# Functions for extracting
# email information from
# mailing list archives

def get_mbox_archives(groupsio_token):
    """
    Use the Groups.io API to obtain an mbox file
    for every subgroup. For each subgroup mbox file,
    parse it and add the email threads to a big
    final archive (dictionary).

    Returns the final archive.

    {
        <permalink> : {
                        'permalink': <permalink>,
                        'created_time': <date>,
                        'subject': <subject>,
                        'subgroup': <subgroup>
                        'sender_name': <name>,
                        'sender_email': <email>,
                        'content': <content>
                    }
    }
    """
    final_archive = {}

    subgroup_ids = get_all_subgroups(groupsio_token)

    for subgroup_id in subgroup_ids.keys():

        subgroup_name = subgroup_ids[subgroup_id]

        # This function call below will call
        # get_archive_zip for each subgroup,
        # and extract the mbox contents from
        # the zip file.
        html = extract_mbox_from_zip(subgroup_name, subgroup_id, groupsio_token)

        # Now extract each email thread and
        # add to final_archive dictionary
        subgroup_archive = extract_threads_from_mbox(html, subgroup_name)

        # keys = permalinks
        # values = dictionary of thread info

        # Merge subgroup archive into final archive
        merge_dicts(
                merge_from = subgroup_archive,
                merge_into = final_archive
        )

    return final_archive

def merge_dicts(merge_from,merge_into):
    """
    Utility function to merge two dictionaries.
    Strategy: don't overwrite any values in merge_into.
    """
    for k in merge_from.keys():
        if k not in merge_into.keys():
            merge_into[k] = merge_from[k]

def extract_threads_from_mbox(mbox_file, subgroup_name):
    """
    Extract threads from an mbox file (HTML format).
    This comes after you've already downloaded the
    zip file from the Groups.io API, extracted the
    HTML contents of the mbox file, and passed it
    here (mbox_file).
    """
    subgroup_archive = {}

    m = mbox(mbox_file)
    msgs = m.items()
    n_msgs = len(msgs)

    logging.info("=============================")
    logging.info("Processing mbox %s with %s messages"%(mbox_file,n_msgs))

    findall_email_pattern  = re.compile('.*<.*>')
    finditer_email_pattern = re.compile('"(.*)" <(.*)>')

    for i,msg in msgs:

        logging.info("Processing message %02d of %02d"%(i+1, n_msgs))

        #{
        #    <permalink> : {
        #                    'permalink': <permalink>,
        #                    'date': <date>,
        #                    'subject': <subject>,
        #                    '(sender)name': <name>,
        #                    '(sender)email': <email>,
        #                    'subgroup': <subgroup>
        #                    'content': <content>
        #                }
        #}

        archive_item = {}

        archive_item['permalink']   = permalink
        archive_item['date']        = msg['Date']
        archive_item['subject']     = msg['Subject']
        archive_item['subgroup']    = subgroup_name

        # process the from field
        try:
            if len(re.findall(findall_email_pattern,msg['From']))>0:
                # We have a From field in format
                # "Name" <email>
                for result in re.finditer(finditer_email_pattern,msg['From']):
                    (from_name, from_email) = result.groups()
                    archive_item['sender_name'] = from_name
                    archive_item['sender_email'] = from_email
            else:
                archive_item['sender_name'] = ''
                archive_item['sender_email'] = msg['From']
        except:
            raise Exception("Crashed on From extraction regular expressions")

        # process the email content
        if msg.is_multipart():
            for part in msg.walk():
                if part.is_multipart():
                    for subpart in part.walk():
                        if subpart.get_content_type() == 'text/plain':
                            body = subpart.get_payload(decode=True)
                elif part.get_content_type() == 'text/plain':
                    body = part.get_payload(decode=True)
        elif msg.get_content_type() == 'text/plain':
            body = msg.get_payload(decode=True)

        archive_item['content'] = body

        subgroup_archive[permalink] = archive_item

    return subgroup_archive

def extract_mbox_from_zip(subgroup_name, subgroup_id, groupsio_token):
    """
    Extract an mbox file from the zip file for this subgroup.
    The zip file stays in memory, and is not written to disk.
    """
    z = get_archive_zip(subgroup_name, subgroup_id, groupsio_token)
    if z is not None:
        file_contents = {name:z.read(name) for name in z.namelist()}
        html = file_contents['messages.mbox']
        return html

def get_all_subgroups(groupsio_token):
    """
    Returns a dictionary where keys are subgroup ids
    and values are subgroup names
    """
    MAX_GROUPS=100
    url = 'https://api.groups.io/v1/getsubgroups'

    key = groupsio_token

    data = [ ('group_name','dcppc'),
             ('limit',MAX_GROUPS)]

    response = requests.post(url,data=data,auth=(key,''))
    response = response.json()
    try:
        dat = response['data']
    except:
        print(response)
        raise Exception("Error getting subgroups")

    all_subgroups = {}
    for group in dat:
        all_subgroups[group['id']] = group['name']
    return all_subgroups

def get_archive_zip(group_name, group_id, groupsio_token):
    """
    Use the API to extract a zipped .mbox email archive
    for one subgroup, and return the contents as z.
    """
    url = "https://api.groups.io/v1/downloadarchives"

    key = groupsio_token

    data = [('group_id',group_id)]

    print("get_archive_zip(): getting .mbox archive for subgroup %s (%s)"%(group_name,group_id))
    r = requests.post(url,data=data,auth=(key,''),stream=True)

    try:
        z = ZipFile(io.BytesIO(r.content))
        z.extractall()
        print("SUCCESS: subgroup %s worked"%(group_name))
        print("")
        return z
    except BadZipFile:
        print("ABORTING: subgroup %s failed"%(group_name))
        print(r.content.decode('utf-8'))
        print("")
        return None


from enum import Enum
from abc import ABC, abstractmethod

class Relationship(Enum):
    PARENT = 1
    CHILD = 2

class Person:
    def __init__(self, name):
        self.name = name

class RelationshipGetter:
    @abstractmethod
    def get_children_of(self, name):
        pass

class RelationshipGraph(RelationshipGetter):  # Notice
    def __init__(self):
        self.relationships = []

    def create_relationship_graph(self, parent, child):
        self.relationships.append(
            (parent, Relationship.PARENT, child)
        )
        self.relationships.append(
            (child, Relationship.CHILD, parent)
        )

    def get_children_of(self, name):
        for p in self.relationships:
            if p[0].name == name and p[1] == Relationship.PARENT:
                yield p[2].name

class Search:
    # def __init__(self, relationship_graph): # Not need to pass relationship_graph
    #     self.relations_graph = relationship_graph

    """
    Here we have created dependency on the Low Level Classes(Tightly Coupled)
    Suppose in future if some data structure changes at low level classes then
    this code will also have to change accordingly. So we want to avoid this kind
    of breakage. Solution: Create a separate interface to handle this kind of problem
    """
    # def get_children_of(self, name):
    #     for p in self.relations_graph.relationships:
    #         if p[0].name == name and p[1] == Relationship.PARENT:
    #             yield p[2].name

    def __init__(self, getter):
        for p in getter.get_children_of('Jay Krishna Pandit'):
            print(p)

parent = Person('Jay Krishna Pandit')
child1 = Person('Ajay Pandit')
child2 = Person('Vijay Pandit')

rg_graph = RelationshipGraph()
rg_graph.create_relationship_graph(parent, child1)
rg_graph.create_relationship_graph(parent, child2)

s = Search(rg_graph)

"""
1. In object-oriented design, the dependency inversion principle is a specific
 methodology for loosely coupling software modules
2. High-level modules should not import anything from low-level modules. Both should
depend on abstractions (e.g., interfaces).
3. Abstractions should not depend on details. Details (concrete implementations) should
 depend on abstractions.
"""