import os

import numpy as np
import pandas as pd
import xarray as xr

import matplotlib.pyplot as plt

from dardar_plot import *
from find_overlap import *

main_dir = '/home/disk/eos3/blazg/SAT_DATA/ICDAR/2010/'

east_day_textfile = 'east_filenames_jja_day.txt'
east_night_textfile = 'east_filenames_jja_night.txt'
west_day_textfile = 'west_filenames_jja_day.txt'
west_night_textfile = 'west_filenames_jja_night.txt'

#lists to save filesnames to
east_day_filenames, west_day_filenames = [], []
east_night_filenames, west_night_filenames = [], []

#open text files, append elements to a list after stripping the newline character: '\n'
with open(east_day_textfile, 'r') as file:
    for element in file:
        east_day_filenames.append(element.rstrip())

with open(east_night_textfile, 'r') as file:
    for element in file:
        east_night_filenames.append(element.rstrip())

with open(west_day_textfile, 'r') as file:
    for element in file:
        west_day_filenames.append(element.rstrip())

with open(west_night_textfile, 'r') as file:
    for element in file:
        west_night_filenames.append(element.rstrip())

#create arrays to save sum and sum of squares to
#these should be the size of the height attribute
east_iwc_sum = np.zeros((436))
east_temp_sum = np.zeros((436))
east_reff_sum = np.zeros((436))

east_iwc_sqr = np.zeros((436))
east_temp_sqr = np.zeros((436))
east_reff_sqr = np.zeros((436))

west_iwc_sum = np.zeros((436))
west_temp_sum = np.zeros((436))
west_reff_sum = np.zeros((436))

west_iwc_sqr = np.zeros((436))
west_temp_sqr = np.zeros((436))
west_reff_sqr = np.zeros((436))

#keep track of how many samples we have for each variable
east_iwc_samples = 0
east_temp_samples = 0
east_reff_samples = 0

west_iwc_samples = 0
west_temp_samples = 0
west_reff_samples = 0

#change this variable to 'day' or 'night' depending on if you want to iterate over day or night orbits
#this variable should re-route the rest of the script automatically
nightday = 'day'

#iterate over day/night filenames depending on variable `nightday`
if nightday == 'day':
    east_filenames = east_day_filenames
    west_filenames = west_day_filenames
else:
    east_filenames = east_night_filenames
    west_filenames = west_night_filenames

#first, let's go through the EP files
for i, filename in enumerate(east_filenames):
    #open data
    data = xr.open_dataset(main_dir+filename)

    #also save the height array for plotting
    #we only need to do this once
    if i == 0:
        height = data.ta.height
        np.save('height_east_{}.npy'.format(nightday), height)

    #need to find overlap here, where the data is in the EP
    wherelon_east = set(np.where(np.logical_and(data.lon.data >=-150, data.lon.data <=-100))[0])
    wherelat_east = set(np.where(np.logical_and(data.lat.data >=5, data.lat.data <=15))[0])

    #find indices for where in our arrays both longitude and latitude booleans are satisfied
    overlap_idx_east = list(wherelon_east & wherelat_east)
    overlap_east = data.time.data[overlap_idx_east]

    #add samples, which is the number of profiles in the curtain
    east_iwc_samples += data.iwc[overlap_east, :].data.shape[0]
    east_temp_samples += data.ta[overlap_east, :].data.shape[0]
    east_reff_samples += data.reffcli[overlap_east, :].data.shape[0]

    #isolate variables of interest
    east_iwc = data.iwc[overlap_east, :]
    east_temp = data.ta[overlap_east, :]
    east_reff = data.reffcli[overlap_east, :]

    #now iterate over columns
    for column in range(data.ta[overlap_east, :].shape[0]):

        #add data to respective sums
        #shape=(436,)
        east_iwc_sum += east_iwc[column, :]
        east_temp_sum += east_temp[column, :]
        east_reff_sum += east_reff[column, :]

    #make sure to close dataset after we are done with it
    data.close()

    print('Done processing east sums for night/day flag: {}. File {}/{}.'.format(nightday, i+1, len(east_filenames)))

#next, let's go through the WP files
for i, filename in enumerate(west_filenames):
    #open data
    data = xr.open_dataset(main_dir+filename)

    #also save the height array for plotting
    #we only need to do this once
    if i == 0:
        height = data.ta.height
        np.save('height_west_{}.npy'.format(nightday), height)

    #need to find overlap here, where the data is in the EP
    wherelon_west = set(np.where(np.logical_and(data.lon.data >=120, data.lon.data <=160))[0])
    wherelat_west = set(np.where(np.logical_and(data.lat.data >=5, data.lat.data <=15))[0])

    #find indices for where in our arrays both longitude and latitude booleans are satisfied
    overlap_idx_west = list(wherelon_west & wherelat_west)
    overlap_west = data.time.data[overlap_idx_west]

    #add samples, which is the number of profiles in the curtain
    west_iwc_samples += data.iwc[overlap_west, :].data.shape[0]
    west_temp_samples += data.ta[overlap_west, :].data.shape[0]
    west_reff_samples += data.reffcli[overlap_west, :].data.shape[0]

    #isolate variables of interest, shape=(columns, rows)=(variable, 436)
    west_iwc = data.iwc[overlap_west, :]
    west_temp = data.ta[overlap_west, :]
    west_reff = data.reffcli[overlap_west, :]

    #now iterate over columns
    for column in range(data.ta[overlap_west, :].shape[0]):

        #add data to respective sum vectors
        #shape=(436,)
        west_iwc_sum += west_iwc[column, :]
        west_temp_sum += west_temp[column, :]
        west_reff_sum += west_reff[column, :]

    #make sure to close dataset after we are done with it
    data.close()

    print('Done processing west sums for night/day flag: {}. File {}/{}.'.format(nightday, i+1, len(west_day_filenames)))

#now create averages
east_iwc_avg = east_iwc_sum / east_iwc_samples
east_temp_avg = east_temp_sum / east_temp_samples
east_reff_avg = east_reff_sum / east_reff_samples

west_iwc_avg = west_iwc_sum / west_iwc_samples
west_temp_avg = west_temp_sum / west_temp_samples
west_reff_avg = west_reff_sum / west_reff_samples

#first, let's go through the EP files for the standard deviation calculation
for i, filename in enumerate(east_filenames):
    #open data
    data = xr.open_dataset(main_dir+filename)

    #need to find overlap here, where the data is in the EP
    wherelon_east = set(np.where(np.logical_and(data.lon.data >=-150, data.lon.data <=-100))[0])
    wherelat_east = set(np.where(np.logical_and(data.lat.data >=5, data.lat.data <=15))[0])

    #find indices for where in our arrays both longitude and latitude booleans are satisfied
    overlap_idx_east = list(wherelon_east & wherelat_east)
    overlap_east = data.time.data[overlap_idx_east]

    #isolate variables of interest
    east_iwc = data.iwc[overlap_east, :]
    east_temp = data.ta[overlap_east, :]
    east_reff = data.reffcli[overlap_east, :]

    #now iterate over columns
    for column in range(data.ta[overlap_east, :].shape[0]):

        #isolate variables of interest
        east_iwc_norm = east_iwc[column, :] - east_iwc_avg
        east_temp_norm = east_temp[column, :] - east_temp_avg
        east_reff_norm = east_reff[column, :] - east_reff_avg

        #add data to respective sums
        east_iwc_sqr += east_iwc_norm**2
        east_temp_sqr += east_temp_norm**2
        east_reff_sqr += east_reff_norm**2

    #make sure to close dataset after we are done with it
    data.close()

    print('Done processing east sum of squares for night/day flag: {}. File {}/{}.'.format(nightday, i+1, len(east_filenames)))

#now the WP files
for i, filename in enumerate(west_filenames):
    #open data
    data = xr.open_dataset(main_dir+filename)

    #need to find overlap here, where the data is in the EP
    wherelon_west = set(np.where(np.logical_and(data.lon.data >=120, data.lon.data <=160))[0])
    wherelat_west = set(np.where(np.logical_and(data.lat.data >=5, data.lat.data <=15))[0])

    #find indices for where in our arrays both longitude and latitude booleans are satisfied
    overlap_idx_west = list(wherelon_west & wherelat_west)
    overlap_west = data.time.data[overlap_idx_west]

    #isolate variables of interest
    west_iwc = data.iwc[overlap_west, :]
    west_temp = data.ta[overlap_west, :]
    west_reff = data.reffcli[overlap_west, :]

    #now iterate over columns
    for column in range(data.ta[overlap_east, :].shape[1]):

        #isolate variables of interest
        west_iwc_norm = west_iwc[column, :] - west_iwc_avg
        west_temp_norm = west_temp[column, :] - west_temp_avg
        west_reff_norm = west_reff[column, :] - west_reff_avg

        #add data to respective sums and standard deviation vectors
        west_iwc_sqr += west_iwc_norm**2
        west_temp_sqr += west_temp_norm**2
        west_reff_sqr += west_reff_norm**2

    #make sure to close dataset after we are done with it
    data.close()

    print('Done processing west sum of squares for night/day flag: {}. File {}/{}.'.format(nightday, i+1, len(west_filenames)))

#now create standard_deviations
east_iwc_std = np.sqrt(east_iwc_sqr / east_iwc_samples)
east_temp_std = np.sqrt(east_temp_sqr / east_temp_samples)
east_reff_std = np.sqrt(east_reff_sqr / east_reff_samples)

west_iwc_std = np.sqrt(west_iwc_sqr / west_iwc_samples)
west_temp_std = np.sqrt(west_temp_sqr / west_temp_samples)
west_reff_std = np.sqrt(west_reff_sqr / west_reff_samples)

#now create standard erros
east_iwc_ste = east_iwc_std / np.sqrt(east_iwc_samples)
east_temp_ste = east_iwc_std / np.sqrt(east_iwc_samples)
east_reff_ste = east_iwc_std / np.sqrt(east_iwc_samples)

west_iwc_ste = west_iwc_std / np.sqrt(west_iwc_samples)
west_temp_ste = west_temp_std / np.sqrt(west_temp_samples)
west_reff_ste = west_reff_std / np.sqrt(west_reff_samples)

#now save those files so we can use them later for graphing output
if nightday == 'day':
    np.save('east_iwc_avg_day.npy', east_iwc_avg)
    np.save('east_temp_avg_day.npy', east_temp_avg)
    np.save('east_reff_avg_day.npy', east_reff_avg)

    np.save('east_iwc_std_day.npy', east_iwc_std)
    np.save('east_temp_std_day.npy', east_temp_std)
    np.save('east_reff_std_day.npy', east_reff_std)

    np.save('east_iwc_ste_day.npy', east_iwc_ste)
    np.save('east_temp_ste_day.npy', east_temp_ste)
    np.save('east_reff_ste_day.npy', east_reff_ste)

    np.save('west_iwc_avg_day.npy', west_iwc_avg)
    np.save('west_temp_avg_day.npy', west_temp_avg)
    np.save('west_reff_avg_day.npy', west_reff_avg)

    np.save('west_iwc_std_day.npy', west_iwc_std)
    np.save('west_temp_std_day.npy', west_temp_std)
    np.save('west_reff_std_day.npy', west_reff_std)

    np.save('west_iwc_ste_day.npy', west_iwc_ste)
    np.save('west_temp_ste_day.npy', west_temp_ste)
    np.save('west_reff_ste_day.npy', west_reff_ste)

else:
    np.save('east_iwc_avg_night.npy', east_iwc_avg)
    np.save('east_temp_avg_night.npy', east_temp_avg)
    np.save('east_reff_avg_night.npy', east_reff_avg)

    np.save('east_iwc_std_night.npy', east_iwc_std)
    np.save('east_temp_std_night.npy', east_temp_std)
    np.save('east_reff_std_night.npy', east_reff_std)

    np.save('east_iwc_ste_night.npy', east_iwc_ste)
    np.save('east_temp_ste_night.npy', east_temp_ste)
    np.save('east_reff_ste_night.npy', east_reff_ste)

    np.save('west_iwc_avg_night.npy', west_iwc_avg)
    np.save('west_temp_avg_night.npy', west_temp_avg)
    np.save('west_reff_avg_night.npy', west_reff_avg)

    np.save('west_iwc_std_night.npy', west_iwc_std)
    np.save('west_temp_std_night.npy', west_temp_std)
    np.save('west_reff_std_night.npy', west_reff_std)

    np.save('west_iwc_ste_night.npy', west_iwc_ste)
    np.save('west_temp_ste_night.npy', west_temp_ste)
    np.save('west_reff_ste_night.npy', west_reff_ste)


# -*- coding: utf-8 -*-
"""
该代码用于生成发病前1天的数据集
"""

import pickle
import numpy as np

# == demo
def get_demo(input_data):
    Demo = np.array(input_data)
    # 将 demo类中数据对应到整个矩阵对应的位置 ，注意:【位置需要查映射表获取】
    for m in range(len(Demo)):
        demo_index = "demo"
        if m == 0:
            demo_index = demo_index + str(1)
        else:
            demo_index = demo_index + str(m + 1) + str(Demo[m])
            Demo[m] = 1

        indexNum = list(map_data).index(demo_index)
        valueAll[0, indexNum] = Demo[m]

# == vital
def get_vital(input_data, input_t_time):
    for m in range(len(input_data)):
        try:
            temp = input_data[m]
            temp1 = np.asarray(temp)
            temp2 = temp1[:, -1]
            temp3 = [x for x in temp2 if x <= input_t_time]
            temp4 = np.max(temp3)
            temp5 = list(temp2).index(temp4)

            if m == 3 or m == 4 or m == 5:
                # vital4,vital5,vital6为定性变量
                vitalIndex = 'vital' + str((m + 1) * 10) + str(int(temp[temp5][0]))
                indexNum = list(map_data).index(vitalIndex)  # 获取特征值对应下标索引
                valueAll[0, indexNum] = 1
                continue
            else:
                # 判断特征值在映射表中的位置。
                vitalIndex = 'vital' + str(m + 1)
                indexNum = list(map_data).index(vitalIndex)  # 获取特征值对应下标索引
                valueAll[0, indexNum] = temp[temp5][0]
        except:
            continue

# == lab
def get_lab_med(input_data, input_t_time):
    # value = np.zeros([1, len(input_data)], dtype=np.int)  # lab is 817
    for m in range(len(input_data)):
        try:
            labIndex = input_data[m][0][0][0]
            # 通过labIndex找到映射表中对应的位置
            indexNum = list(map_data).index(labIndex)  # 获取特征值对应下标索引
            temp = input_data[m][1]
            temp1 = np.asarray(temp)
            temp2 = temp1[:, -1]
            temp3 = [int(x) for x in temp2 if int(x) <= input_t_time]
            temp4 = np.max(temp3)
            temp5 = list(temp2).index(str(temp4))
            valueAll[0, indexNum] = temp[temp5][0]
        except:
            continue
    pass

# ==ccs
def get_ccs_px(input_data, input_t_time):
    for m in range(len(input_data)):
        try:
            ccsTime = int(input_data[m][1][-1])
            if ccsTime <= input_t_time:
                # 根据ccsIndex 确定映射位置
                ccsIndex = input_data[m][0][0]
                indexNum = list(map_data).index(ccsIndex)  # 获取特征值对应下标索引
                valueAll[0][indexNum] = 1
        except:
            continue
    pass

# ==label
def get_label(input_value, input_time):
    day_index = list(map_data).index("days")  # 获取特征值对应下标索引
    value_index = list(map_data).index("AKI_label")  # 获取特征值对应下标索引
    valueAll[0, day_index] = input_time
    valueAll[0, value_index] = input_value

# ====== input data
# cd='F:/2018_research_python/AKI_Python_rawData_process_20180517/result/'
cd = '/home/xzhang_sta/xyf/result/'
allFileName = ['ft_zip2010', 'ft_zip2011', 'ft_zip2012', 'ft_zip2013', 'ft_zip2014', 'ft_zip2015',
               'ft_zip2016', 'ft_zip2017', 'ft_zip2018']
# cd = 'D:/Pydataproject/xyf_data_algorithm/result/'
# allFileName = ['AKI_newdata', 'AKI_newdata1', 'AKI_newdata4']

for x in range(len(allFileName)):
    fileName = allFileName[x]  # AKI
    f = open(cd + fileName + '.pkl', 'rb')
    data = pickle.load(f)

    map_f = open(cd + 'feature_dict_BDAI_map.pkl', 'rb')
    map_data = pickle.load(map_f)

    # ====input parameter
    pre_day = 1
    Data = np.zeros([1, len(map_data)])
    # 遍历每一个样本
    for i in range(len(data)):

        #  feature category
        demo, vital, lab, ccs, px, med, label = data[i]
        # =======================首先解决：每个样本的AKI爆发时间的提前n天的对应时间
        # ==label 应该如何对应矩阵位置
        a = np.array(label)
        y_value = a[:, 0]
        t_time = a[:, 1] - pre_day
        for j in range(len(t_time)):
            # 每一个样本定义一个整的零矩阵
            valueAll = np.zeros([1, len(map_data)])  # all feature number is 28298

            y_value_sub = y_value[j]
            t_time_sub = t_time[j]

            # 测试上述代码
            # label
            get_label(y_value_sub, t_time_sub)

            # demo
            get_demo(demo)

            # vital
            get_vital(vital, t_time_sub)

            # lab && med
            get_lab_med(lab, t_time_sub)
            get_lab_med(med, t_time_sub)

            # ccs && px
            get_ccs_px(ccs, t_time_sub)
            get_ccs_px(px, t_time_sub)

            Data = np.row_stack((Data, valueAll))

    # '==========整合数据=============='
    # 删除数组中的第一行：
    final_Data = Data[1:, :]
    # print(final_Data)
    file = open(cd + fileName + '_day' + str(pre_day) + '.pkl', 'wb')
    pickle.dump(final_Data, file)
    file.close()


#!/usr/bin/python3
from signal import signal,SIGINT
import sys

def signal_handler(signal, frame):
    print(f'\nimpatient...')
    sys.exit(0)

def prime_seeker():
    result = []
    counter = 0
    try:
        for i in range(1,int(input("How long shall I search prime numbers? Above 10000 can be timeconsuming! Be careful. "))):
            for j in range(1,i + 1):
                if i % j == 0:
                    counter += 1
            if counter == 2:
                result.append(i)
            counter = 0
        print(f'{result}')
    except:
        print(f'Interrupted!')

signal(SIGINT,signal_handler)
prime_seeker()


# skip-if: '-x' in EXTRA_JIT_ARGS
# - too slow

# Note: CPython doesn't pass this test

import os
import sys
import multiprocessing

def worker():
    global done

    for i in xrange(1000):
        del sys.modules["pyc_import_target"]
        import pyc_import_target

    done = True

import pyc_import_target
path = os.path.join(os.path.dirname(__file__), "pyc_import_target.pyc")
assert os.path.exists(path)

TEST_THREADS = 3

l = []
for i in xrange(TEST_THREADS):
    p = multiprocessing.Process(target=worker)
    p.start()
    l.append(p)

idx = 0
while l:
    p = l.pop()
    while p.is_alive():
        for i in xrange(100):
            if os.path.exists(path):
                os.remove(path)
        for i in xrange(100):
            if os.path.exists(path):
                with open(path, "rw+") as f:
                    f.write(chr(i) * 100)
                    f.truncate(200)
    p.join()
    assert p.exitcode == 0, p.exitcode


#vim: set fileencoding=utf-8
from django.conf.urls.defaults import patterns, url, include
from django.views.generic.simple import direct_to_template
from app.ctlweb.views import *

urlpatterns = patterns('app.ctlweb.views',
        url(r'^$', 'index', name='index'),
        url(r'^admin_index/$', 'admin_index', name='admin_index'),
        url(r'^search/$', 'search', name='search'),
        url(r'^components/$', 'components', name='components'),
        url(r'^components/(?P<comp_id>\d+)/$', 'component_detail',
            name='component'),
        url(r'^components/add/(?P<token>\S{64})/$', 'receive_modules',
            name='component_receive'),
        url(r'^administration/$', 'index', name='administration'),
        url(r'^administration/easy/$', 'index', name='administration_easy'),
        url(r'^administration/advanced/$', 'index', name='administration_advanced'),
        url(r'^impressum/$', 'impressum', name='impressum'),
        url(r'^interface/(?P<int_id>\d+)/$', 'interface', name='interface'),
        url(r'^userkey/$', 'userkey_main', name='userkey'),
)

urlpatterns += patterns('',
        url(r'^', include('ctlweb.urls_auth')),
        url(r'^accounts/', include('registration.backends.default.urls')),
        )

# urlpatterns += patterns('django.contrib.auth.views',
#        url(r'^login/$', 'login', name='login'),
#        url(r'^logout/$', 'logout', {'next_page': '/'}, name='logout'),
#)


import torch
import torch.nn as nn
from functools import partial

from dparser.utils import gaussian_kernel_matrix, maximum_mean_discrepancy

class MMD(nn.Module):
    def __init__(self, encoder, configs):
        super().__init__()
        self.sigmas = torch.FloatTensor([
            1e-6, 1e-5, 1e-4, 1e-3, 1e-2, 1e-1,
            1, 5, 10, 15, 20, 25, 30, 35, 100,
            1e3, 1e4, 1e5, 1e6
        ])
        self.gaussian_kernel = partial(
            gaussian_kernel_matrix, sigmas=self.sigmas)

    def forward(self, hs, ht):
        loss_value = maximum_mean_discrepancy(
            hs, ht, kernel=self.gaussian_kernel)
        return torch.clamp(loss_value, min=1e-4)


'''
TCRpiper - a pipeline for TCR sequences treatment. Copyright (C) 2020  D. Malko
'''

from django.conf import settings

PYTHON_PATH = settings.PYTHON_PATH
PIPER_PATH = settings.TCR_PIPER_PATH
SITE_PATH = settings.SITE_PATH
MAX_MEMORY = '6G'
OUT_DIRNAME = 'output'
OUT_FILE = 'output.tar.gz'
LOG_FILE = 'pipeline.log'
PORT = 10000
THREADS = 1  # 'MAX_MEMORY' X 'THREADS' < SYSTEM MEMORY !!!


# coding=utf-8
# --------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
#
# Code generated by Microsoft (R) AutoRest Code Generator.
# Changes may cause incorrect behavior and will be lost if the code is
# regenerated.
# --------------------------------------------------------------------------

from msrest.serialization import Model

class RetentionPolicy(Model):
    """The retention policy for a container registry.

    Variables are only populated by the server, and will be ignored when
    sending a request.

    :param days: The number of days to retain manifest before it expires.
    :type days: int
    :ivar last_updated_time: The timestamp when the the policy was last
     updated.
    :vartype last_updated_time: datetime
    :param status: The value that indicates whether the policy is enabled or
     not. Possible values include: 'enabled', 'disabled'
    :type status: str or
     ~azure.mgmt.containerregistry.v2019_05_01.models.PolicyStatus
    """

    _validation = {
        'last_updated_time': {'readonly': True},
    }

    _attribute_map = {
        'days': {'key': 'days', 'type': 'int'},
        'last_updated_time': {'key': 'lastUpdatedTime', 'type': 'iso-8601'},
        'status': {'key': 'status', 'type': 'str'},
    }

    def __init__(self, **kwargs):
        super(RetentionPolicy, self).__init__(**kwargs)
        self.days = kwargs.get('days', None)
        self.last_updated_time = None
        self.status = kwargs.get('status', None)


from django.contrib import admin
from django.db import models

class CustomSlideshowAdmin(admin.ModelAdmin):
    # The forms to add and change events instances
    empty_value_display = "----"

    readonly_fields = ["created_at", "updated_at", "created_by", "updated_by"]

    # The fields to be used in displaying the Slideshow model.
    list_display = ("title", "image", "created_at", "updated_at")
    list_filter = ("title", "image", "created_at", "updated_at")

    # The fields to be used in updates on Slideshow model.
    fieldsets = (
        (
            "Informações básicas",
            {
                "classes": ("grp-collapse grp-open",),
                "fields": ("title", "resume", "image"),
            },
        ),
        (
            "Monitoramento",
            {
                "classes": ("grp-collapse grp-open",),
                "fields": ("created_at", "updated_at", "created_by", "updated_by"),
            },
        ),
    )

    # Search and ordering
    search_fields = ("title", "image", "created_at", "updated_at")
    ordering = ("title", "image", "created_at")

    def save_model(self, request, slideshow, form, change):
        if not slideshow.created_by:
            slideshow.created_by = request.user
        else:
            slideshow.updated_by = request.user

        super().save_model(request, slideshow, form, change)


#!/usr/bin/env python2.7
#
#  workdays.py
#  Report
#
#  Created by Greg Rogalski on 2017-08-03
#  Copyright 2017 Greg Rogalski, MD. All rights reserved.
#
#
#
# TODO:
# - adjust based on partial availability in the year
# - remove locums/NPs
# - calc average and variance and median
# - show header every 10 rows
# - adjust for Leave?
#
"""
CGI module for showing workdays in a certain period.
"""
import os
import sys
import cgi
import cgitb; cgitb.enable(display=1, logdir="logs")# se

# Fix import paths
import inspect
currentdir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parentdir = os.path.dirname(currentdir)
sys.path.insert(0,parentdir)

import numpy
import schedinc
from utils import isodate
from stats import stats
from stats import stats_utils
from utils import rotation
from utils import radiologist
from utils.holiday import HolidayList
from utils import schedule
from utils import style
from utils import schedutils
from admin.scratchpad import ScratchPad
from utils import reqs_utils

YEARLY_QUOTA = 189  # num of days per year to work for 1 FTE
VAC_YEARLY = 40.0
VAC_ADJUST_FACTOR = 1.0  # 4 out 5 days worked per week on average

NOTES = """

 Footnotes

 MMG rads (ALEE, SATRE, NEALON) will have a yearly deficit of around 5 days because of their cap of 10
    weekends per year.

 IR rads will have a surplus of several days, since they effectively take 2x weekend call, because
    their day and night shifts are separate now.

Flexers: their quota adjustment is based on how many Flex-X-OFF days they get, which automatically adjusts for
    start/stop to their flex schedule, even mid-year.

Sabbaticals are automatically adjusted based on their 'Vacation variance': excess vacation is added to their work
    days to compensate for working less. This also smooths out small variations for others.

For IRs, the IRDO are counted as a work day.  Weekend INT call is also counted as weekend call. (Should the latter count as half?)

"""

def calc_table(sgb, duration, end_date, par, include_scratchpad):
    """ Calculates table of workdays, par, adjustments, totals etc
    and returns as a list of lists:
    [ 'rad1', "VAC", "WEEK-DAYS","WEEK-ENDS","Total worked", "Quota", "Quota Adj", "HELP","Variance", "VAC adjust", "ADJUSTEDVariance"],]
    """
    ret_list = []
    # assume the inputs are already checked

    start_date = end_date.add_days(-duration+1)  # subtract!

    statobj = stats.RadStats()
    schedobj = schedule.Schedule(sgb)
    schedobj.import_from_db(start_date=start_date,
                            end_date=end_date)

    # get legacy part timers' extra days off per YEAR
    legacy_part_timers = reqs_utils.fetch_part_timers(sgb.get_dbhandle(), start_date)

    # If include_scratchpad is true, then only include
    # published schedule until the new schedule scratch
    # starts.
    # NB: the end of published schedule is not necessarily
    # the end of the actual schedule, e.g. I can publish
    # weekend call through the end of the block, but it
    # shouldn't be included in actual stats?
    if include_scratchpad:
        # breakpoint = schedobj.get_maxdate()  # last day of current schedule
        # scratch_start = breakpoint.add_days(1)
        scratch_start = schedobj.sgb.new_sched_start

        scratchpad = ScratchPad(sgb)
        scratchpad.import_from_db(start_date=scratch_start,
                                  end_date=end_date)

        scratch_matrix = statobj.summarize(scratchpad, scratch_start,
                                           end_date, lump_days_off=False,
                                           count_blanks=True, lump_other=False)

        # whole schedule stats -- just until the scratch begins
        matrix = statobj.summarize(schedobj, start_date,
            scratch_start.add_days(-1), lump_days_off=False, lump_other=False)
    else:
        matrix = statobj.summarize(schedobj, start_date, end_date, lump_days_off=False, lump_other=False)
    # print matrix
    # ----------------------------------------------------------------------

    # total_days = (end_date.subtract(start_date)+1)
    total_days = duration

    #print matrix

    for (radname, counts) in matrix.iteritems():
        if radiologist.is_locums(radname):
            continue  # skip over locums and NPs
        cols = []
        weekdays = counts.get("WORK",0) + counts.get("IRDO",0)
        weekends = counts.get("WEEKEND",0)
        scratch_weekdays = 0
        scratch_weekends = 0
        scratch_vacation = 0
        scratch_misc_workdays = 0

        # other stuff that counts: Sick days, Leave
        misc_workdays = counts.get("Sick",0) + counts.get("Leave",0) + counts.get("Jury",0)

        if include_scratchpad:
            scratch_weekdays = scratch_matrix.get(radname,{}).get("WORK",0) + scratch_matrix.get(radname,{}).get("IRDO",0)
            scratch_weekends = scratch_matrix.get(radname,{}).get("WEEKEND",0)
            scratch_vacation = scratch_matrix.get(radname,{}).get("VAC",0)
            scratch_misc_workdays = scratch_matrix.get(radname,{}).get("Leave",0)  # not planning on being sick in future :)

        vacation = counts.get("VAC",0)
        help_shifts = counts.get("HELP",0)
        radobj = radiologist.RadList.get_radobj(radname)
        FTE = radobj.FTE

        if FTE == 0:
            continue

        quota = round(par * total_days / 365.0,0)

        # Note: for legacy part-time employees, subtract their
        # prorated extra days from their quota
        # For flexers, use the Flex-*-OFF to adjust the quotas as well
        #
        # quota -= counts.get("PartTimeOff",0)
        part_time_offset = legacy_part_timers.get(radname,0)
        if part_time_offset:
            # prorate
            part_time_offset = round(part_time_offset * total_days / 365.0,1)

        #
        quota_adjustment = -part_time_offset
        quota_adjustment -= counts.get("Flex-Gain-OFF",0)
        quota_adjustment += counts.get("Flex-Lost-OFF",0)

        if include_scratchpad:
            # quota -= scratch_matrix.get(radname,{}).get("PartTimeOff",0)
            quota_adjustment -= scratch_matrix.get(radname,{}).get("Flex-Gain-OFF",0)
            quota_adjustment += scratch_matrix.get(radname,{}).get("Flex-Lost-OFF",0)

        quota += quota_adjustment

        cols.append(radname)
        # cols.append(FTE)
        if include_scratchpad:
            # cols.append(str(vacation+scratch_vacation) + " (" + str(scratch_vacation)+ ")")
            # cols.append(str(weekdays+scratch_weekdays) + " (" + str(scratch_weekdays)+ ")")
            # cols.append(str(weekends+scratch_weekends) + " (" + str(scratch_weekends)+ ")")

            cols.append(vacation+scratch_vacation)
            cols.append(weekdays+scratch_weekdays)
            cols.append(weekends+scratch_weekends)
            cols.append(misc_workdays + scratch_misc_workdays)

            Total_Work = weekdays + weekends + scratch_weekends + scratch_weekdays + misc_workdays + scratch_misc_workdays
            cols.append(Total_Work)  # total

            Total_Vaca = vacation + scratch_vacation
        else:
            cols.append(counts.get("VAC",0))
            cols.append(weekdays)
            cols.append(weekends)
            cols.append(misc_workdays)
            Total_Work = weekdays + weekends + misc_workdays
            cols.append(Total_Work)
            Total_Vaca = vacation

        cols.append(quota)
        cols.append(quota_adjustment)
        cols.append(help_shifts)
        variance = int(Total_Work - quota)
        cols.append(variance)

        # cols.append( schedutils.colorize_number(round((weekdays+weekends)*100/quota,0)-100.0))

        #
        # Calc vacation adjustment
        # (actual_vac - expected_vac) * adjust_factor
        vac_quota = round(VAC_YEARLY * total_days / 365,1)
        vac_adjustment = round((Total_Vaca - vac_quota) * VAC_ADJUST_FACTOR,1)
        cols.append(vac_adjustment)
        cols.append(round(variance + vac_adjustment,1))

        ret_list.append(cols)

    ret_list.sort()
    return ret_list

def index(sgb, duration=None, end_date=None, par=None, include_scratchpad=None):

    if not duration:
        duration = 365  # default is 90 days, i.e. 1 quarter -ish
    else:
        duration = int(duration)

    if par:
        par = int(par)
    else:
        par = YEARLY_QUOTA

    if end_date != None and end_date != "":
        end_date = isodate.Isodate(end_date)
    else:
        end_date=isodate.today()
    start_date = end_date.add_days(-duration+1)  # subtract!

    calc_matrix = calc_table(sgb, duration, end_date, par, include_scratchpad)

    stats_dict = {"weekdays":[],
                    "weekends":[],
                    "total":[]}

    sgb.output(style.header("Work Stats",sgb.get_media_URL(),
        stylesheet="stats_style.css"))
    sgb.output(style.menu(sgb))
    sgb.output("
Work statistics
")

    sgb.output(stats_utils.date_range_links(sgb, "report/workdays.py"))
    sgb.output("
")

    show_form(sgb, duration, end_date, par, include_scratchpad)
    sgb.output("
")

    if include_scratchpad:
        scratch_start = sgb.new_sched_start

    if include_scratchpad and scratch_start > sgb.new_sched_end:
        sgb.output("
WARNING: ScratchPad ends before the end of the published schedule

")

    if include_scratchpad and end_date <= scratch_start:
        sgb.output("
WARNING: ScratchPad starts AFTER the date window specified.

ScratchPad stats will NOT be shown.")
        include_scratchpad = False

    #
    sgb.output("
Include ScratchPad unpublished schedule: %s" % include_scratchpad)
    sgb.output("
Start date: %s" % start_date)
    sgb.output("
End date: %s" % end_date)
    if include_scratchpad:
        # sgb.output("
End date of published schedule: %s" % breakpoint)
        sgb.output("
Start date of ScratchPad schedule: %s" % scratch_start)
        sgb.output("
End of ScratchPad schedule: %s" % sgb.new_sched_end)

    total_days = duration #(end_date.subtract(start_date))
    sgb.output("
Yearly Quota: %s" % par)
    sgb.output("
Total # of days: %s" % total_days)
    sgb.output("
# of WORK Weekdays: %s" % stats_utils.count_workdays(start_date, end_date, daytype="weekday"))
    sgb.output("
# of WEEKENDS/Holidays: %s" % stats_utils.count_workdays(start_date, end_date, daytype="weekend"))

    sgb.output("
")

    #print matrix
    header = [ ["RADIOLOGIST", "VAC", "WEEK-
DAYS","WEEK-
ENDS","Misc
Sick, Leave","Total
days
worked", "Quota", "Quota Adj", "Variance", "VAC
adjust", "ADJUSTED
Variance"]]
    rows = []

    for row in calc_matrix:
        cols = []
        cols.append(row[0])  # name
        cols.append(row[1])  # vac
        cols.append(row[2])  # weekdays
        cols.append(row[3])  # weekends
        cols.append(row[4])  # misc: sick, leave, etc
        cols.append(row[5])  # total worked
        cols.append(row[6])  # quota
        cols.append(schedutils.colorize_number(row[7]))  # quota adj
        # cols.append(row[7])  # help
        cols.append(schedutils.colorize_number(row[9]))  # variance
        cols.append(row[10])  # vac adjust
        cols.append(schedutils.colorize_number(row[11]))  # adjusted variance

        stats_dict.setdefault("total",[]).append(row[5])
        stats_dict.setdefault("weekends",[]).append(row[3])
        stats_dict.setdefault("weekdays",[]).append(row[2])

        if row[11] > 3.0:
            cols.append("+++")
        elif row[11] < -3.0:
            cols.append("---")
        else:
            cols.append("")

        cols.append(row[0])  # radname again at the end

        rows.append(cols)

    # rows.sort()
    rows.append(["**Median**",
        "",
        numpy.median(stats_dict['weekdays']),
        numpy.median(stats_dict['weekends']),
        numpy.median(stats_dict['total']),
        "",
        "",
        "",
        "",
        "",
        ""
        ])

    rows.append(["**Average**",
        "",
        round(numpy.mean(stats_dict['weekdays']),1),
        round(numpy.mean(stats_dict['weekends']),1),
        round(numpy.mean(stats_dict['total']),1),
        "",
        "",
        "",
        "",
        "",
        ""
        ])

    header.extend(rows)
    sgb.output(schedutils.htmlify_table(header))

    annualized_median = round(numpy.median(stats_dict['total']) * 365.0 / total_days, 1)
    sgb.output("
Annualized median = " + str(annualized_median) + " (Projected assuming same staffing for 12 months)
")

    sgb.output(NOTES)

    sgb.output(style.footer())

def show_form(sgb, duration, end_date, par, inc_scratch=False):
    "Puts up form with entry boxes"
    sgb.output("<FORM METHOD='GET' ACTION='workdays.py'>")
    sgb.output("
")
#    sgb.output("<INPUT TYPE='hidden' NAME='ra_id' VALUE='%s'>" % av_obj.ra_id)
#    sgb.output("<INPUT TYPE='hidden' NAME='radid' VALUE='%s'>" % av_obj.radid)
    sgb.output("
End date: (YYYY-MM-DD)	 <INPUT TYPE='text' NAME='end_date' VALUE='%s'>
" % end_date)
    sgb.output("
Duration: (days)	<INPUT TYPE='text' NAME='duration' VALUE='%s'>
" % duration)
    sgb.output("
Nut par: (days/year)	<INPUT TYPE='text' NAME='par' VALUE='%s'>
" % par)

    if inc_scratch:
        sgb.output("
Include unpublished sched?	 <INPUT TYPE='checkbox' NAME='scratch' CHECKED='checked'>
")
    else:
        sgb.output("
Include unpublished sched?	 <INPUT TYPE='checkbox' NAME='scratch'>
")
#    sgb.output("
ON/OFF:	 <INPUT TYPE='text' NAME='on_off' VALUE='%s'>
" % "ON")
    sgb.output("
<INPUT TYPE='submit' NAME='Save' VALUE='RELOAD'>
")
    sgb.output("
")
    sgb.output("</FORM>")

def main():
    sgb = schedinc.SchedGlob(None)
    rotation.Rotation.read_from_db(sgb.get_dbhandle())
    radiologist.RadList(sgb.get_dbhandle())
    HolidayList.load_holidays(sgb.get_dbhandle())

    form = cgi.FieldStorage()
    index(sgb, form.getfirst("duration", None),
          form.getfirst("end_date", None),
          form.getfirst("par", None),
          form.getfirst("scratch", None)
          )
    sgb.db_logout()

def main2():
    sgb = schedinc.SchedGlob(None)
    rotation.Rotation.read_from_db(sgb.get_dbhandle())
    radiologist.RadList(sgb.get_dbhandle())
    HolidayList.load_holidays(sgb.get_dbhandle())

    sgb.output(calc_table(sgb, 90, isodate.Isodate('2018-12-31'),
        187, True))
    sgb.db_logout()

if __name__ == '__main__':
    main()


from django.http import HttpResponse
from django.conf import settings
from .profiler import Profiler
import os

class ProfilerMiddleware(object):
    def __init__(self):
        self.profiler = None

    def process_request(self, request):
        if getattr(settings, 'PYINSTRUMENT_URL_ARGUMENT', 'profile') in request.GET:
            self.profiler = Profiler()
            self.profiler.start()

    def process_response(self, request, response):
        if self.profiler:
            try:
                self.profiler.stop()

                location = os.path.dirname(os.path.abspath(__file__))

                with open(os.path.join(location, 'style.css')) as f:
                    css = f.read()

                with open(os.path.join(location, 'profile.js')) as f:
                    js = f.read()

                response = '''
                    <html>
                    <head>
                        <style>{css}</style>
                        <script src="http://code.jquery.com/jquery-1.11.0.min.js"></script>
                    </head>
                    <body>
                        {body}
                        <script>{js}</script>
                    </body>
                    </html>'''.format(css=css, js=js, body=self.profiler.output_html())

                return HttpResponse(response)
            finally:
                self.profiler = None
        else:
            return response


import string

import wx

from dictator.Shared import *
import dictator.ParametersDialog
import dictator.ReaderModel

class Plugin(dictator.ReaderModel.ReaderModel):

    '''Dynamic reader model.  Word duration is determined by word length,
    associated pauses, sentence termination, paragraph termination, and whether
    or not the word is a connecting word.'''

    def __init__(self, text, reader):

        dictator.ReaderModel.ReaderModel.__init__(self, text, reader)
        self.parameters = {
            'length': [0.06, 0, 0.3, 0.06],
            'connector': [0.25, 0, 0.9, 0.25],
            'pause': [0.9, 0, 5, 0.9],
            'terminator': [2, 0, 5, 2],
            'paragraph': [1, 0, 5, 1],
            }

    def Duration(self):

        duration = 60.0 / self.reader.wpm
        word = self.text.GetWord()
        if word == '__paragraph_break__':
            return 5 * duration
        # Duration proportional to word length
        # Median word length in English is about 5 - increase rate for
        # short words, decrease for long words in attempt to maintain
        # approximately the expected WPM
        mod = duration * (len(word) - 5) * self.parameters['length'][0]
        # Long pause sentence breaks
        if self.text.IsSentenceEnd():
#            print 'eos'
            mod += duration * self.parameters['terminator'][0]
        # Short pause at delimiters
        elif self.text.IsPause():
#            print 'pause'
            mod += duration * self.parameters['pause'][0]
        # Increase rate at connecting words
        if self.text.IsConnector():
#            print 'con'
            mod -= duration * self.parameters['connector'][0]
        # Pause at paragraph end
        if self.text.IsParagraphEnd():
#            print 'eop'
            mod += duration * self.parameters['paragraph'][0]
        return max(0.0, duration + mod)

    def __str__(self):

        return '''
The dynamic reader model allows control over the duration of presentation of different words based on the following pseudo-formula (scaling factors are excluded for simplicity).

               d = 60 / WPM
Duration (s) = d
               + d x length x (word length - 5)
               + d x terminator\t\t[ if end of sentence ]
               + d x pause\t\t\t[ if adjacent pause ]
               - d x connector\t[ if connecting word ]
               + d x paragraph\t\t[ if end of paragraph ]
'''


import unittest

import TextWizard

class TestTextWizard(unittest.TestCase):

    def setUp(self):
        self.hello_world = 'aGVsbG8gd29ybGQh'
        self.abcd = 'YWJjZA=='
        self.json_str = '{"ret":1,"start":-1,"end":-1,"country":"\u4e2d\u56fd","province":"\u4e0a\u6d77","city":"\u4e0a\u6d77","district":"","isp":"","type":"","desc":""}'

    def test_get_encode_type(self):
        self.assertEqual('BASE64', TextWizard.get_encode_type(self.hello_world))
        self.assertEqual('BASE64', TextWizard.get_encode_type(self.abcd))
        self.assertEqual('JSON', TextWizard.get_encode_type(self.json_str))

    def test_auto_decode(self):
        self.assertEqual('hello world!', TextWizard.auto_decode(self.hello_world))
        self.assertEqual('abcd', TextWizard.auto_decode(self.abcd))

if __name__ == "__main__":
    unittest.main()


import json

import jsonschema
from jsonschema.validators import RefResolver

from pystac import (STAC_VERSION, STAC_IO, Catalog, Collection, Item, ItemCollection, Extensions)
from pystac.serialization import STACObjectType

class STACValidationError(Exception):
    pass

class SchemaValidator:
    REPO = 'https://raw.githubusercontent.com/radiantearth/stac-spec'
    TAG = 'v{}'.format(STAC_VERSION)
    SCHEMA_BASE_URI = '{}/{}'.format(REPO, TAG)

    core_schemas = {
        STACObjectType.CATALOG: 'catalog-spec/json-schema/catalog.json',
        STACObjectType.COLLECTION: 'collection-spec/json-schema/collection.json',
        STACObjectType.ITEM: 'item-spec/json-schema/item.json',
        STACObjectType.ITEMCOLLECTION: 'item-spec/json-schema/itemcollection.json'
    }

    extension_schemas = {
        Extensions.LABEL: {
            STACObjectType.ITEM: 'extensions/label/json-schema/schema.json'
        },
        Extensions.EO: {
            STACObjectType.ITEM: 'extensions/eo/json-schema/schema.json'
        },
        Extensions.SINGLE_FILE_STAC: {
            # TODO: Move off of custom schema if schema in spec was fixed
            # before this extension got removed.
            STACObjectType.ITEMCOLLECTION: ('https://raw.githubusercontent.com/lossyrob/stac-spec/'
                                            '0.9.0/fix-single-file-stac-schema/extensions/'
                                            'single-file-stac/json-schema/schema.json')
        }
    }

    # Schemas that need to be downloaded for caching that are not tied directly to STAC objects.
    aux_schemas = [
        'item-spec/json-schema/basics.json', 'item-spec/json-schema/datetimerange.json',
        'item-spec/json-schema/instrument.json', 'item-spec/json-schema/licensing.json',
        'item-spec/json-schema/metadata.json', 'item-spec/json-schema/provider.json',
        'https://geojson.org/schema/Feature.json',
        'https://geojson.org/schema/FeatureCollection.json'
    ]

    _schema_cache = {}

    @staticmethod
    def _abs_schema(uri):
        return ('{}/{}'.format(SchemaValidator.SCHEMA_BASE_URI, uri)
                if not uri.startswith('https') else uri)

    _is_setup = False

    @staticmethod
    def _setup():
        if not SchemaValidator._is_setup:
            for c in SchemaValidator.core_schemas:
                uri = SchemaValidator._abs_schema(SchemaValidator.core_schemas[c])
                SchemaValidator.core_schemas[c] = uri
                SchemaValidator._schema_cache[uri] = json.loads(STAC_IO.read_text(uri))

            for ext in SchemaValidator.extension_schemas:
                for stac_object_type in SchemaValidator.extension_schemas[ext]:
                    uri = SchemaValidator._abs_schema(
                        SchemaValidator.extension_schemas[ext][stac_object_type])
                    SchemaValidator.extension_schemas[ext][stac_object_type] = uri
                    SchemaValidator._schema_cache[uri] = json.loads(STAC_IO.read_text(uri))

            for uri in SchemaValidator.aux_schemas:
                abs_uri = SchemaValidator._abs_schema(uri)
                SchemaValidator._schema_cache[abs_uri] = json.loads(STAC_IO.read_text(abs_uri))

            SchemaValidator._is_setup = True

    def __init__(self):
        SchemaValidator._setup()

    @classmethod
    def get_schema_from_uri(cls, schema_uri):
        schema = cls._schema_cache.get(schema_uri)
        if schema is None:
            raise Exception('Could not read schema from {}'.format(schema_uri))

        resolver = RefResolver(base_uri=schema_uri,
                               referrer=schema,
                               store=SchemaValidator._schema_cache)

        return (schema, resolver)

    @classmethod
    def get_core_schema(cls, obj_class):
        schema_uri = cls.core_schemas.get(obj_class)

        if schema_uri is None:
            raise Exception('No schema for type {}'.format(obj_class))

        return cls.get_schema_from_uri(schema_uri)

    @classmethod
    def get_extension_schema(cls, obj_class, extension_id):
        schemas_by_class = cls.extension_schemas.get(extension_id)
        if schemas_by_class is None:
            return None

        schema_uri = schemas_by_class.get(obj_class)

        if schema_uri is None:
            return None

        return cls.get_schema_from_uri(schema_uri)

    # Validate methods are instance methods to ensure initialization

    def validate_object(self, obj, print_on_error=False):
        obj_type = type(obj)
        if issubclass(obj_type, Collection):
            stac_object_type = STACObjectType.COLLECTION
        elif issubclass(obj_type, Catalog):
            stac_object_type = STACObjectType.CATALOG
        elif issubclass(obj_type, Item):
            stac_object_type = STACObjectType.ITEM
        elif issubclass(obj_type, ItemCollection):
            stac_object_type = STACObjectType.ITEMCOLLECTION
        else:
            raise Exception("Unknown STAC Object type: {}".format(obj_type))

        return self.validate_dict(obj.to_dict(), stac_object_type, print_on_error=print_on_error)

    def validate_dict(self, d, stac_object_type, print_on_error=False):
        schema, resolver = SchemaValidator.get_core_schema(stac_object_type)

        try:
            jsonschema.validate(instance=d, schema=schema, resolver=resolver)
        except jsonschema.exceptions.ValidationError as e:
            if print_on_error:
                print(json.dumps(d, indent=2))
            raise STACValidationError(
                'Validation failed for STAC {}'.format(stac_object_type)) from e

        if 'stac_extensions' in d:
            for extension_id in d['stac_extensions']:
                ext_schema_result = SchemaValidator.get_extension_schema(
                    stac_object_type, extension_id)
                if ext_schema_result is not None:
                    ext_schema, ext_resolver = ext_schema_result
                    try:
                        jsonschema.validate(instance=d, schema=ext_schema, resolver=ext_resolver)
                    except jsonschema.exceptions.ValidationError as e:
                        if print_on_error:
                            print(json.dumps(d, indent=2))
                        raise STACValidationError(
                            'Validation failed for STAC {} extension {}'.format(
                                stac_object_type, extension_id)) from e


"""
Utility functions for data downloading.

**Note**: these methods are liable to change at any time.
"""
import datetime as dt
import ftplib
import io
import logging
import pathlib as path
import re
import shutil
import sys
import urllib.error as urlerror
import urllib.request as urlreq
import astropy.units as u
import sunpy.timeseries as ts
import warnings
import collections as coll
import cdflib

import numpy as np
import pandas as pd
import heliopy.data.helper as helper

from heliopy import config
use_hdf = config['use_hdf']
logger = logging.getLogger(__name__)

def process(dirs, fnames, extension, local_base_dir, remote_base_url,
            download_func, processing_func, starttime, endtime,
            try_download=True, units=None,
            processing_kwargs={}, download_info=[], remote_fnames=None,
            warn_missing_units=True):
    """
    The main utility method for systematically loading, downloading, and saving
    data.

    Parameters
    ----------
    dirs : list
        A list of directories relative to *local_base_dir*.
    fnames : list or str or regex
        A list of filenames **without** their extension. These are the
        filenames that will be downloaded from the remote source. Must be the
        same length as *dirs*. Each filename is saved in it's respective entry
        in *dirs*. Can also be a regular expression that is used to match
        the filename (e.g. for version numbers)
    extension : str
        File extension of the raw files. **Must include leading dot**.
    local_base_dir : str
        Local base directory. ``fname[i]`` will be stored in
        ``local_base_dir / dirs[i] / fname[i] + extension``.
    remote_base_url : str
        Remote base URL. ``fname[i]`` will be downloaded from
        ``Remote / dirs[i] / fname[i] + extension``.
    download_func
        Function that takes

        - The remote base url
        - The local base directory
        - The relative directory (relative to the base url)
        - The local filename to download to
        - The remote filename
        - A file extension

        and downloads the remote file. The signature must be::

            def download_func(remote_base_url, local_base_dir,
                              directory, fname, remote_fname, extension)

        The function can also return the path of the file it downloaded,
        if this is different to the filename it is given. *download_func*
        can either silently do nothing if a given file is not available, or
        raise a `NoDataError` with a descriptive error message that will be
        printed.

    processing_func
        Function that takes an open CDF file or open plain text file,
        and returns a pandas DataFrame. The signature must be::

            def processing_func(file, **processing_kwargs)

    starttime : ~datetime.datetime
        Start of requested interval.
    endtime : ~datetime.datetime
        End of requested interval.
    try_download : bool, optional
        If ``True``, try to download data. If ``False`` don't.
        Default is ``True``.

    units : ~collections.OrderedDict, optional
        Manually defined units to be attached to the data that will be
        returned.

        Must map column headers (strings) to :class:`~astropy.units.Quantity`
        objects. If units are present, then a TimeSeries object is returned,
        else a Pandas DataFrame.

    processing_kwargs : dict, optional
        Extra keyword arguments to be passed to the processing funciton.

    download_info : list, optional
        A list with the same length as *fnames*, which contains extra info
        that is handed to *download_func* for each file individually.
    remote_fnames : list of str, optional
        If the remote filenames are different from the desired downloaded
        filenames, this should be a list of length ``len(fnames)`` with the
        files to be downloaded. The ordering must be the same as *fnames*.
    warn_missing_units : bool, optional
        If ``True``, warnings will be shown for each variable that does not
        have associated units.

    Returns
    -------
    :class:`~pandas.DataFrame` or :class:`~sunpy.timeseries.TimeSeries`
        Requested data.
    """
    local_base_dir = path.Path(local_base_dir)
    data = []
    if download_info == []:
        download_info = [None] * len(dirs)
    if remote_fnames is None:
        remote_fnames = fnames.copy()

    if len(dirs) != len(fnames):
        raise ValueError(
            'Must have the same number of directories as filenames')
    if len(fnames) != len(remote_fnames):
        raise ValueError(
            'Must have the same number of remote filenames as filenames')

    zips = zip(dirs, fnames, remote_fnames, download_info)
    for directory, fname, remote_fname, dl_info in zips:
        local_dir = local_base_dir / directory
        local_file = local_dir / fname

        # Try to load hdf file
        hdf_fname = _file_match(local_dir, fname + '.hdf')
        if hdf_fname is not None:
            hdf_file_path = local_dir / hdf_fname
            raw_file_path = hdf_file_path.with_suffix(extension)
            logger.info('Loading {}'.format(hdf_file_path))
            data.append(pd.read_hdf(hdf_file_path))
            continue

        # Try to load raw file
        raw_fname = _file_match(local_dir, fname + extension)
        if raw_fname is not None:
            raw_file_path = local_dir / raw_fname
            logger.info('Loading {}'.format(raw_file_path))
            df = _load_raw_file(raw_file_path,
                                processing_func, processing_kwargs)
            if df is not None:
                data.append(df)
                continue

        # If we can't find local file, try downloading
        if try_download:
            _checkdir(local_dir)
            args = ()
            if dl_info is not None:
                args = (dl_info,)
            try:
                new_path = download_func(remote_base_url, local_base_dir,
                                         directory, fname, remote_fname,
                                         extension, *args)
            except NoDataError as e:
                print(str(e))
                continue
            if new_path is not None:
                shutil.copy(new_path, local_file.with_suffix(extension))

            raw_fname = _file_match(local_dir, fname + extension)
            # Print a message if file hasn't been downloaded
            if raw_fname is not None:
                raw_file_path = local_dir / raw_fname
                df = _load_raw_file(raw_file_path,
                                    processing_func, processing_kwargs)
                if df is not None:
                    data.append(df)
                continue
            else:
                logger.info('File {}{}/{}{} not available remotely\n'.format(
                            remote_base_url, directory, fname, extension))
                continue
        else:
            msg = ('File {a}/{b}{c} not available locally,\n'
                   'and "try_download" set to False')
            logger.info(msg.format(a=local_dir, b=fname, c=extension))

    # Loaded all the data, now filter between times
    data = timefilter(data, starttime, endtime)
    data = data.sort_index()

    # Attach units
    if extension == '.cdf':
        cdf = _load_local(raw_file_path)
        units = cdf_units(cdf, manual_units=units)
    return units_attach(data, units, warn_missing_units=warn_missing_units)

def _file_match(directory, fname_regex):
    """
    Check if a file in *directory* matchs the regular expression given by
    *fname_regex*. If it does, return the filename. Otherwise returns None.

    Parameters
    ----------
    directory : Path
    fname_regex : str
        Must include file extension.

    Returns
    -------
    fname : str
        Includes file extension.
    """
    if directory.exists():
        for f in directory.iterdir():
            if f.is_file():
                if re.match(fname_regex, f.name):
                    return f.name

def _save_hdf(df, raw_file):
    hdf_file = raw_file.with_suffix('.hdf')
    df.to_hdf(hdf_file, 'data', mode='w', format='f')

def _load_raw_file(raw_file, processing_func, processing_kwargs):
    if not raw_file.exists():
        return
    # Convert raw file to a dataframe
    logger.info('Loading {}'.format(raw_file))
    try:
        file = _load_local(raw_file)
        df = processing_func(file, **processing_kwargs)
        if use_hdf:
            _save_hdf(df, raw_file)
        if isinstance(file, io.IOBase) and not file.closed:
            file.close()
        return df
    except NoDataError:
        return

class NoDataError(RuntimeError):
    pass

def units_attach(data, units, warn_missing_units=True):
    """
    Takes the units defined by the user and attaches them to the TimeSeries.

    Parameters
    ----------
    data : :class:`pandas.DataFrame`
        Input data. Takes the DataFrame which needs to have units attached.
    units : :class:`collections.OrderedDict`
        The units manually defined by the user.

    Returns
    -------
    out : :class:`~sunpy.timeseries.TimeSeries`
        DataFrame converted into TimeSeries with units attached.
    """
    missing_msg = ('If you are trying to auomatically download data '
                   'with HelioPy this is a bug, please report it at '
                   'https://github.com/heliopython/heliopy/issues')
    unit_key = list(units.keys())
    for column_name in data.columns:
        if column_name not in unit_key:
            units[column_name] = u.dimensionless_unscaled
            if warn_missing_units:
                message = (f"{column_name} column has missing units."
                           f"\n{missing_msg}")
                warnings.warn(message, Warning)
    with warnings.catch_warnings():
        warnings.simplefilter(
            'ignore', 'Discarding nonzero nanoseconds in conversion')
        timeseries_data = ts.TimeSeries(data, units)
    return timeseries_data

def cdf_units(cdf_, manual_units=None, length=None):
    """
    Takes the CDF File and the required keys, and finds the units of the
    selected keys.

    Parameters
    ----------
    cdf_ : cdf
        Opened cdf file
    manual_units : ~collections.OrderedDict, optional
        Manually defined units to be attached to the data that will be
        returned.

    Returns
    -------
    out : :class:`collections.OrderedDict`
        Returns an OrderedDict with units of the selected keys.
    """
    units = coll.OrderedDict()

    # Get the list of all variables
    var_list = []
    # To figure out whether rVariable or zVariable needs to be taken
    for attr in list(cdf_.cdf_info().keys()):
        if 'variable' in attr.lower():
            if len(cdf_.cdf_info()[attr]) > 0:
                var_list += [attr]

    logger.info(f'Found the following variables in CDF: {var_list}')
    # Get a mapping from each key to any sub-keys
    key_dict = {}
    # Extract the list of valid keys in the zVar or rVar
    for attr in var_list:
        for key in cdf_.cdf_info()[attr]:
            try:
                y = cdf_.varget(key)
                ncols = y.shape
                if len(ncols) == 1:
                    key_dict[key] = key
                if len(ncols) > 1:
                    val = []
                    val.append(key)
                    for x in range(0, ncols[1]):
                        field = key + "{}".format('_' + str(x))
                        val.append(field)
                    key_dict[key] = val
            except Exception as e:
                print("{} - Variable {}".format(e, key))
                continue

    logger.info(f'Getting units for {key_dict}')
    # Assigning units to the keys
    for key, val in key_dict.items():
        unit_str = None
        temp_unit = None
        # Try to get a unit string from the CDF file
        try:
            unit_str = cdf_.varattsget(key)['UNITS']
        # Fallback on user provided units
        except KeyError:
            if manual_units and key in manual_units:
                temp_unit = manual_units[key]
            else:
                continue

        if temp_unit is None:
            try:
                temp_unit = u.Unit(unit_str)
            except (TypeError, ValueError):
                if manual_units and key in manual_units:
                    temp_unit = manual_units[key]
                elif helper.cdf_dict(unit_str):
                    temp_unit = helper.cdf_dict(unit_str)

                if temp_unit is None:
                    message = (f"The CDF provided units ('{unit_str}') for"
                               f" key '{key}' are unknown")
                    warnings.warn(message)
                    continue

        if temp_unit is not None:
            if isinstance(val, list):
                units.update(coll.OrderedDict.fromkeys(val, temp_unit))
            else:
                units[val] = temp_unit
    if manual_units:
        units.update(manual_units)
    logger.info(f'Extracted following units: {units}')
    return units

def timefilter(data, starttime, endtime):
    """
    Puts data in a single dataframe, and filters it between times.

    Parameters
    ----------
    data : :class:`pandas.DataFrame` or list
        Input data. If a list, ``pd.concat(data)`` will be run to put it in
        a DataFrame.
    starttime : datetime
        Start of interval.
    endtime : datetime
        End of interval.

    Returns
    -------
    out : :class:`pandas.DataFrame`
        Filtered data.
    """
    if len(data) == 0:
        raise RuntimeError(
            'No data available between {} and {}'.format(starttime, endtime))
    if isinstance(data, list):
        data = pd.concat(data)
    # Get time values
    if 'Time' in data.columns:
        time = data['Time']
    elif 'Time' in data.index.names:
        time = data.index.get_level_values('Time')
    else:
        raise KeyError('The label "Time" was not found in '
                       'the dataframe columns or index')

    data = data[(time > starttime) &
                (time < endtime)]
    # Assume if this fails we have a multi-index that already has time in it
    if ('Time' in data.columns) and (len(data.index.shape) == 1):
        data = data.set_index('Time', drop=True)

    return data

def pitchdist_cdf2df(cdf, distkeys, energykey, timekey, anglelabels):
    """
    Converts cdf file of a pitch angle distribution to a pandas dataframe.

    MultiIndexing is used as a pitch angle distribution is essentially a 3D
    dataset *f(time, energy, angle)*. See
    http://pandas.pydata.org/pandas-docs/stable/advanced.html#multiindex-advanced-indexing
    for more information.

    This has been constructed for importing wind swe pitch angle distributions,
    and might not generalise very well to other data sets.

    Assumes that each energy in the cdf has its own 2D array (time, angle). In
    the below description of the function there are

        - ``n`` time data points
        - ``m`` energy data points
        - ``l`` anglular data points

    Parameters
    ----------
    cdf : cdf
        Opened cdf file.
    distkeys : list
        A list of the cdf keys for a given energies. Each array accessed by
        distkeys is shape `(n, l)`, and there must be `m` distkeys.
    energykey : string
        The cdf key for the energy values. The array accessed by energykey
        must have shape `(m)` or `(a, m)` where `a` can be anything. If it
        has shape `(a, m)`, we assume energies measured don't change, and
        take the first row as the energies for all times.
    timekey : string
        The cdf key for the timestamps. The array access by timekey must
        have shape `(n)`
    anglelabels : list
        A list of the labels to give each anglular bin (eg. [0, 10, 20] in
        degrees). Must be of length `l`.

    Returns
    -------
    df : :class:`pandas.DataFrame`
        Data frame with read in data.
    """
    times_ = cdf.varget(timekey)[...]
    utc_comp = cdflib.cdfepoch.breakdown(times_)
    times = np.asarray([dt.datetime(*x) for x in utc_comp])
    ntimesteps = times.size
    energies = cdf.varget(energykey)[...]
    # If energies is 2D, just take first set of energies
    if len(energies.shape) == 2:
        energies = energies[0, :]

    # Empty lists. index[0] will be times, index[1] will be energies, index[2]
    # will be angles. data will be the pdf
    index = [[], [], []]
    data = []
    # Loop through energies
    for i, key in enumerate(distkeys):
        thisenergy = energies[i]
        this_e_data = cdf.varget(key)[...]
        # Loop through angles
        for j in range(0, this_e_data.shape[1]):
            # Time steps
            index[0] += list(times)
            # Current energy
            index[1] += [thisenergy] * ntimesteps
            # Current angle
            index[2] += [anglelabels[j]] * ntimesteps

            thisdata = this_e_data[:, j]
            thisdata[thisdata == -9.99999985e+30] *= np.nan
            data += list(thisdata)

    tuples = list(zip(*index))
    index = pd.MultiIndex.from_tuples(tuples,
                                      names=['Time', 'Energy', 'Angle'])
    data = pd.DataFrame(data, index=index, columns=['df'])
    data = data.sort_index()
    return data

def cdf2df(cdf, index_key, dtimeindex=True, badvalues=None, ignore=None):
    """
    Converts a cdf file to a pandas dataframe.

    Note that this only works for 1 dimensional data, other data such as
    distribution functions or pitch angles will not work properly.

    Parameters
    ----------
    cdf : cdf
        Opened CDF file.
    index_key : string
        The CDF key to use as the index in the output DataFrame.
    dtimeindex : bool, optional
        If ``True``, the DataFrame index is parsed as a datetime.
        Default is ``True``.
    badvalues : dict, list, optional
        A dictionary that maps the new DataFrame column keys to a list of bad
        values to replace with nans. Alternatively a list of numbers which are
        replaced with nans in all columns.
    ignore : list, optional
        In case a CDF file has columns that are unused / not required, then
        the column names can be passed as a list into the function.

    Returns
    -------
    df : :class:`pandas.DataFrame`
        Data frame with read in data.
    """
    # Extract index values
    try:
        index_ = cdf.varget(index_key)[...][:, 0]
    except IndexError:
        index_ = cdf.varget(index_key)[...]
    try:
        utc_comp = cdflib.cdfepoch.breakdown(index_, to_np=True)
        if utc_comp.shape[1] == 9:
            millis = utc_comp[:, 6] * (10**3)
            micros = utc_comp[:, 8] * (10**2)
            nanos = utc_comp[:, 7]
            utc_comp[:, 6] = millis + micros + nanos
            utc_comp = np.delete(utc_comp, np.s_[-2:], axis=1)
        try:
            index = np.asarray([dt.datetime(*x) for x in utc_comp])
        except ValueError:
            utc_comp[:, 6] -= micros
            index = np.asarray([dt.datetime(*x) for x in utc_comp])
    except Exception:
        index = index_
    if dtimeindex:
        index = pd.DatetimeIndex(index, name='Time')
    df = pd.DataFrame(index=index)
    npoints = cdf.varget(index_key).shape[0]

    var_list = []
    for attr in list(cdf.cdf_info().keys()):
        if 'variable' in attr.lower():
            if len(cdf.cdf_info()[attr]) > 0:
                var_list += [attr]

    keys = {}
    for attr in var_list:
        for cdf_key in cdf.cdf_info()[attr]:
            if ignore:
                if cdf_key in ignore:
                    continue
            if cdf_key == 'Epoch':
                keys[cdf_key] = 'Time'
            else:
                keys[cdf_key] = cdf_key
    # Remove index key, as we have already used it to create the index
    keys.pop(index_key)
    # Remove keys for data that doesn't have the right shape to load in CDF
    for cdf_key in keys.copy():
        if type(cdf.varget(cdf_key)) is np.ndarray:
            key_shape = cdf.varget(cdf_key).shape
            if len(key_shape) == 0 or key_shape[0] != npoints:
                keys.pop(cdf_key)
        else:
            keys.pop(cdf_key)

    # Loop through each key and put data into the dataframe
    for cdf_key in keys:
        df_key = keys[cdf_key]
        if isinstance(df_key, list):
            for i, subkey in enumerate(df_key):
                df[subkey] = cdf.varget(cdf_key)[...][:, i]
        else:
            # If ndims is 1, we just have a single column of data
            # If ndims is 2, have multiple columns of data under same key
            key_shape = cdf.varget(cdf_key).shape
            ndims = len(key_shape)
            if ndims == 1:
                df[df_key] = cdf.varget(cdf_key)[...]
            elif ndims == 2:
                for i in range(key_shape[1]):
                    df[df_key + '_' + str(i)] = cdf.varget(cdf_key)[...][:, i]

    # Replace bad values with nans
    if badvalues is not None:
        df = df.replace(badvalues, np.nan)
    return df

class RemoteFileNotPresentError(RuntimeError):
    pass

def load(filename, local_dir, remote_url,
         try_download=True, remote_error=False):
    """
    Try to load a file from *local_dir*.

    If file doesn't exist locally, try to download from *remtote_url* instead.

    Parameters
    ----------
    filename : string
        Name of file
    local_dir : string
        Local location of file
    remote_url : string
        Remote location of file
    try_download : bool
        If a file isn't available locally, try to downloaded it. Default is
        *True*.
    remote_error : bool
        If ``True``, raise an error if the requested file isn't present
        locally or remotely. If ``False``, the method returns ``None`` if
        the file can't be found.

    Returns
    -------
    file : CDF, open file, None
        If *filename* ends in *.cdf* the CDF file will be opened and
        returned.

        Otherwise it is assumed that the file is an ascii file, and
        *filename* will be opened using python's :func:`open` method.

        If the file can't be found locally or remotely, and *remote_errror* is
        ``False``, ``None`` is returned.
    """
    # Check if file is cdf
    if filename[-4:] == '.cdf':
        filetype = 'cdf'
    # If not a cdf file assume ascii file
    else:
        filetype = 'ascii'

    # Try to load locally
    if _checkdir(local_dir):
        local_dir = path.Path(local_dir)
        for f in local_dir.iterdir():
            if str(f) == filename or ((str(f)[:-6] == filename[:-6])):
                filename = str(f)
                return _load_local(local_dir / f, filetype)

    if try_download:
        try:
            return _load_remote(remote_url, filename, local_dir, filetype)
        except urlerror.URLError:
            if remote_error:
                raise RemoteFileNotFoundError(
                    'File {}/{} not available'.format(remote_url, filename))
            else:
                return None
    else:
        return None

def _get_remote_fname(remote_url, fname_regex):
    '''
    Return a remote filename that matches a regular expression.

    Parameters
    ----------
    remote_url : str
        FTP directory
    fname_regex : str
        Must include extension.

    Retruns
    -------
    fname : str
        Filename that matches including extension.
    '''
    remote_url = _fix_url(remote_url)
    # Split remote url into a server name and directory
    # Strip ftp:// from front of url
    remote_ftp = remote_url[6:]
    for i, c in enumerate(remote_ftp):
        if c == '/':
            server = remote_ftp[:i]
            server_dir = remote_ftp[i:]
            break
    # Login to remote server
    with ftplib.FTP(server) as ftp:
        ftp.login()
        ftp.cwd(server_dir)
        # Loop through and find files
        for (f, _) in ftp.mlsd():
            if re.match(fname_regex, f):
                return f

def _load_cdf(file_path):
    '''
    A function to handle loading cdflib, and printing a nice error if things
    go wrong.
    '''
    try:
        cdf = cdflib.CDF(str(file_path))
    except Exception as err:
        print('Error whilst trying to load {}\n'.format(file_path))
        raise err
    return cdf

def _is_cdf(file_path):
    file_path = path.Path(file_path)
    if file_path.suffix == '.cdf':
        return True
    return False

def _load_local(file_path, filetype=None):
    # Import local file
    if _is_cdf(file_path):
        return _load_cdf(file_path)
    else:
        f = open(str(file_path))
        return f

def _reporthook(blocknum, blocksize, totalsize):
    readsofar = blocknum * blocksize
    if totalsize > 0:
        percent = min(100, readsofar * 1e2 / totalsize)
        s = "\r%5.1f%% %*d / %d" % (
            percent, len(str(totalsize)), readsofar, totalsize)
        sys.stderr.write(s)
        # Near the end
        if readsofar >= totalsize:
            sys.stderr.write("\n")
    # Total size is unknown
    else:
        sys.stderr.write("\rRead %d" % (readsofar,))

def _download_remote_unknown_version(
    remote_base_url, local_base_dir, directory,
        fname, remote_fname, extension):
    """
    Generic donwload code that can be used by data methods to download
    data where the version is unknown and specified in regex.
    """
    remote_url_dir = remote_base_url + str(directory)
    fname = _get_remote_fname(remote_url_dir, fname)
    local_dir = local_base_dir / directory
    _download_remote(remote_url_dir, fname, local_dir)

def _download_remote(remote_url, filename, local_dir):
    local_dir = path.Path(local_dir)
    remote_url = _fix_url(remote_url)
    remote_url = remote_url + '/' + filename
    print('Downloading', remote_url)
    urlreq.urlretrieve(remote_url,
                       filename=str(local_dir / filename),
                       reporthook=_reporthook)
    print('\n')

def _load_remote(remote_url, filename, local_dir, filetype):
    local_dir = path.Path(local_dir)
    _download_remote(remote_url, filename, local_dir)
    return _load_local(local_dir / filename, filetype)

def _fix_url(url):
    '''
    Given a url possibly constructued using an os.path.join method,
    replace all backlslashes with forward slashes to make the url valid
    '''
    if url is not None:
        return url.replace('\\', '/')
    else:
        return url

def _checkdir(directory):
    """
    Checks if directory exists, if not creates directory.

    Parameters
    ----------
    directory : string
        Directory to check.

    Returns
    -------
    isdir : bool
        True if directory exists, False if directory didn't exist when
        function was called.
    """
    directory = path.Path(directory)
    if not directory.exists():
        print('Creating new directory', directory)
        directory.mkdir(parents=True)
        return False
    else:
        return True

def _daysplitinterval(starttime, endtime):
    """
    Splits an interval into a list of dates, start times and end times

    Parameters
    ----------
    starttime : datetime
        Start date/time of interval
    endtime : datetime
        End date/time of interval

    Returns
    -------
    intervals : list
        A list of lists. Each item in the sublists consists of the date,
        start time, and end time for the particular date.
    """
    assert starttime < endtime, 'Start datetime must be before end datetime'
    out = []
    starttime_orig = starttime
    while starttime.date() <= endtime.date():
        if starttime.date() == starttime_orig.date():
            stime = starttime.time()
        else:
            stime = dt.time.min
        if starttime.date() == endtime.date():
            etime = endtime.time()
        else:
            etime = dt.time.max

        out.append([starttime.date(), stime, etime])
        starttime += dt.timedelta(days=1)
    return out

def _cart2sph(x, y, z):
    """
    Given cartesian co-ordinates returns shperical co-ordinates.

    Parameters
    ----------
    x : array_like
        x values
    y : array_like
        y values
    z : array_like
        z values

    Returns
    -------
    r : array_like
        r values
    theta : array_like
        Elevation angles defined from the x-y plane towards the z-axis.
        Angles are in the range [-pi/2, pi/2].
    phi : array_like
        Azimuthal angles defined in the x-y plane, clockwise about the
        z-axis, from the x-axis. Angles are in the range [-pi, pi].
    """
    xy = x**2 + y**2
    r = np.sqrt(xy + z**2)
    theta = np.arctan2(z, np.sqrt(xy))
    phi = np.arctan2(y, x)
    return r, theta, phi

def _sph2cart(r, theta, phi):
    """
    Given spherical co-orinates, returns cartesian coordiantes.

    Parameters
    ----------
    r : array_like
        r values
    theta : array_like
        Elevation angles defined from the x-y plane towards the z-axis
    phi : array_like
        Azimuthal angles defined in the x-y plane, clockwise about the
        z-axis, from the x-axis.

    Returns
    -------
    x : array_like
        x values
    y : array_like
        y values
    z : array_like
        z values
    """
    x = r * np.cos(theta) * np.cos(phi)
    y = r * np.cos(theta) * np.sin(phi)
    z = r * np.sin(theta)
    return x, y, z

def doy2ymd(y, doy):
    """
    Converts day of year and year to year, month, day

    Parameters
    ----------
    y : int
        Year
    doy : int
        Day of year

    Returns
    -------
    year : int
        Year
    month : int
        Month
    day : int
        Day of month
    """
    d = dt.datetime.strptime(str(y) + ':' + str(doy), '%Y:%j')

    return d.year, d.month, d.day

def dtime2doy(dt):
    """
    Returns day of year of a datetime object.

    Parameters
    ----------
    dt : datetime

    Returns
    -------
    doy : int
        Day of year
    """
    return int(dt.strftime('%j'))


import pandas as pd;import os

b=os.listdir('/home/az2/Downloads/bb33/')

df=pd.DataFrame()
for x in b:
    m=pd.read_csv('/home/az2/Downloads/bb33/' + x)
    print(df.size)
    df=df.append(m)
    print(df.shape,x)
pd.set_option("display.max_rows",580)
pd.set_option("display.max_columns",580)
df2=df.iloc[0:300,1:2]
df3=df.iloc[0:100,1:2]
s=pd.merge(df2,df3, on=[1,1], how='outer')

print(df2.shape, df2)


from typing import List

from ...byte_io_mdl import ByteIO
from ...data_structures.mdl_data import SourceMdlFileData, SourceMdlBone, SourceMdlAttachment, SourceMdlBodyPart, \
    SourceMdlTexture, SourceMdlModel, StudioHDRFlags, SourceMdlMesh, SourceBase
from ...data_structures.source_shared import SourceVector
from ...mdl.mdl_readers.mdl_v49 import SourceMdlFile49
from ...utilities.progressbar import ProgressBar

class SourceMdlFile10(SourceMdlFile49):
    def __init__(self, reader: ByteIO):
        super().__init__(reader)
        self.reader = reader
        self.file_data = SourceMdlFileDataV10()

    def read_bones(self):
        if self.file_data.bone_count > 0:
            pb = ProgressBar(
                desc='Reading bones',
                max_=self.file_data.bone_count,
                len_=20)
            self.reader.seek(self.file_data.bone_offset, 0)
            for i in range(self.file_data.bone_count):
                pb.draw()
                SourceMdlBoneV10().read(self.reader, self.file_data)
                pb.increment(1)
                pb.draw()

    def read_attachments(self):
        if self.file_data.local_attachment_count > 0:
            self.reader.seek(self.file_data.local_attachment_offset, 0)
            pb = ProgressBar(
                desc='Reading attachments',
                max_=self.file_data.local_attachment_count,
                len_=20)
            for _ in range(self.file_data.local_attachment_count):
                pb.draw()
                SourceMdlAttachmentV10().read(self.reader, self.file_data)
                pb.increment(1)

    def read_body_parts(self):
        if self.file_data.body_part_count > 0:
            self.reader.seek(self.file_data.body_part_offset)
            pb = ProgressBar(
                desc='Reading body parts',
                max_=self.file_data.body_part_count,
                len_=20)
            for _ in range(self.file_data.body_part_count):
                pb.draw()
                SourceMdlBodyPartV10().read(self.reader, self.file_data)
                pb.increment(1)

    def read_textures(self):
        if self.file_data.texture_count < 1:
            return
        self.reader.seek(self.file_data.texture_offset)
        for _ in range(self.file_data.texture_count):
            SourceMdlTextureV10().read(self.reader, self.file_data)

class SourceMdlFileDataV10(SourceMdlFileData):

    def __init__(self):
        super().__init__()
        self.texture_data_offset = 0
        self.transition_count = 0
        self.transition_offset = 0

    def read_header00(self, reader: ByteIO):
        self.id = reader.read_fourcc()
        self.version = reader.read_int32()
        self.name = reader.read_ascii_string(64)
        self.file_size = reader.read_int32()
        self.actual_file_size = reader.size()

        self.eye_position.read(reader)

        self.hull_min_position.read(reader)
        self.hull_max_position.read(reader)

        self.view_bounding_box_min_position.read(reader)
        self.view_bounding_box_max_position.read(reader)

        self.flags = StudioHDRFlags(reader.read_int32())

        self.bone_count = reader.read_int32()
        self.bone_offset = reader.read_int32()

        self.bone_controller_count = reader.read_int32()
        self.bone_controller_offset = reader.read_int32()

        self.hitbox_set_count = reader.read_int32()
        self.hitbox_set_offset = reader.read_int32()

        self.local_sequence_count = reader.read_int32()
        self.local_sequence_offset = reader.read_int32()

        self.sequence_group_count = reader.read_int32()
        self.sequence_group_offset = reader.read_int32()

        self.texture_count = reader.read_int32()
        self.texture_offset = reader.read_int32()
        self.texture_data_offset = reader.read_int32()

        self.skin_reference_count = reader.read_int32()
        self.skin_family_count = reader.read_int32()
        self.skin_family_offset = reader.read_int32()

        self.body_part_count = reader.read_int32()
        self.body_part_offset = reader.read_int32()

        self.local_attachment_count = reader.read_int32()
        self.local_attachment_offset = reader.read_int32()

        self.sound_table = reader.read_int32()
        self.sound_index = reader.read_int32()
        self.sound_groups = reader.read_int32()
        self.sound_group_offset = reader.read_int32()

        self.transition_count = reader.read_int32()
        self.transition_offset = reader.read_int32()

    def read_header01(self, reader: ByteIO):
        return

    def read_header02(self, reader: ByteIO):
        return

class SourceMdlBoneV10(SourceMdlBone):

    def read(self, reader: ByteIO, mdl: SourceMdlFileData):
        self.name = reader.read_ascii_string(32)
        self.parentBoneIndex = reader.read_int32()
        self.flags = reader.read_int32()

        self.position.read(reader)
        self.rotation.read(reader)
        self.positionScale.read(reader)
        self.rotationScale.read(reader)
        mdl.bones.append(self)

class SourceMdlAttachmentV10(SourceMdlAttachment):

    def read(self, reader: ByteIO, mdl: SourceMdlFileData):
        self.name = reader.read_ascii_string(32)
        self.type = reader.read_int32()
        self.parent_bone = reader.read_int32()
        self.attachmentPoint.read(reader)
        for _ in range(2):
            self.vectors.append(SourceVector().read(reader))

        mdl.attachments.append(self)

class SourceMdlBodyPartV10(SourceMdlBodyPart):

    def read(self, reader: ByteIO, mdl: SourceMdlFileData):
        self.name = reader.read_ascii_string(64)
        self.model_count = reader.read_int32()
        self.base = reader.read_int32()
        self.model_offset = reader.read_int32()

        if self.model_count > 0:
            reader.seek(self.model_offset)
            for _ in range(self.model_count):
                SourceMdlModelV10().read(reader, self)

        mdl.body_parts.append(self)

class SourceMdlTextureV10(SourceMdlTexture):

    def __init__(self):
        super().__init__()
        self.width = 0
        self.height = 0
        self.data_offset = 0

    def read(self, reader: ByteIO, mdl: SourceMdlFileData):
        self.path_file_name = reader.read_ascii_string(64)
        self.flags = reader.read_int32()
        self.width = reader.read_int32()
        self.height = reader.read_int32()
        self.data_offset = reader.read_int32()

        mdl.textures.append(self)

class SourceMdlModelV10(SourceMdlModel):

    def __init__(self):
        super().__init__()

        self.vertexes = []

        self.vertex_bone_info_offset = 0
        self.vertex_bone_info = []

        self.normal_count = 0
        self.normal_bone_info_offset = 0
        self.normal_bone_info = []
        self.normal_offset = 0

        self.normals = []

        self.group_count = 0
        self.group_offset = 0

    def read(self, reader: ByteIO, body_part: SourceMdlBodyPart):
        self.name = reader.read_ascii_string(64)
        self.type = reader.read_int32()
        self.bounding_radius = reader.read_float()
        self.mesh_count = reader.read_int32()
        self.mesh_offset = reader.read_int32()
        with reader.save_current_pos():
            reader.seek(self.mesh_offset)
            for _ in range(self.mesh_count):
                mesh = SourceMdlMeshV10()
                mesh.read(reader, self)

        self.vertex_count = reader.read_int32()
        self.vertex_bone_info_offset = reader.read_int32()
        with reader.save_current_pos():
            reader.seek(self.vertex_bone_info_offset)
            for _ in range(self.vertex_count):
                self.vertex_bone_info.append(reader.read_uint8())
        self.vertex_offset = reader.read_int32()
        with reader.save_current_pos():
            reader.seek(self.vertex_offset)
            for _ in range(self.vertex_count):
                self.vertexes.append(SourceVector().read(reader))

        self.normal_count = reader.read_int32()
        self.normal_bone_info_offset = reader.read_int32()

        with reader.save_current_pos():
            reader.seek(self.normal_bone_info_offset)
            for _ in range(self.normal_count):
                self.normal_bone_info.append(reader.read_uint8())

        self.normal_offset = reader.read_int32()
        with reader.save_current_pos():
            reader.seek(self.normal_offset)
            for _ in range(self.normal_count):
                self.normals.append(SourceVector().read(reader))

        self.group_count = reader.read_int32()
        self.group_offset = reader.read_int32()

        body_part.models.append(self)

class SourceMdlMeshV10(SourceMdlMesh):

    def __init__(self):
        super().__init__()
        self.face_count = 0
        self.face_offset = 0
        self.skinref = 0
        self.normal_count = 0
        self.normal_offset = 0
        self.strips_fans = SourceMeshTriangleStripOrFan10()  # type: SourceMeshTriangleStripOrFan10
        self.faces = []  # type: List[SourceMeshTriangleStripOrFan10]

    def read(self, reader: ByteIO, model: SourceMdlModel):
        self.face_count = reader.read_int32()
        self.face_offset = reader.read_int32()
        self.skinref = reader.read_int32()
        self.normal_count = reader.read_int32()
        self.normal_offset = reader.read_int32()
        self.strips_fans.read(reader, self)
        ...

class SourceMeshTriangleStripOrFan10(SourceBase):

    def __init__(self):
        self.strip = False
        self.list_count = 0
        self.vertex_infos = []

    def read(self, reader: ByteIO, mesh: SourceMdlMeshV10):
        while 1:
            self.list_count = reader.read_int16()
            if self.list_count == 0:
                break
            if self.list_count < 0:
                self.strip = False
            else:
                self.strip = True
            for _ in range(abs(self.list_count)):
                vi = SourceMdlVertexInfo10()
                vi.read(reader)
                self.vertex_infos.append(vi)

            mesh.faces.append(self)

class SourceMdlVertexInfo10(SourceBase):
    def __init__(self):
        self.vertex_index = 0
        self.normal_index = 0
        self.s = 0
        self.t = 0

    def read(self, reader: ByteIO):
        self.vertex_index = reader.read_uint16()
        self.normal_index = reader.read_uint16()
        self.s = reader.read_int16()
        self.t = reader.read_int16()


import googlemaps
import json
from datetime import datetime
from pprint import pprint

### Geocoding and address
##geocode_result = gmaps.geocode('1600 Amphitheatre Parkway, Mountain View, CA')
##
### Look up an address with reverse geocoding
##reverse_geocode_result = gmaps.reverse_geocode((40.714224, -73.961452))

# Request directions via public transit
now = datetime.now()
directions_result = gmaps.directions("Larchmont, NY 10538",
                                     "40.657181, -74.288338",
                                     mode="transit",
                                     departure_time=now)

dictio = directions_result[0]

leg = (dictio['legs'])
a = leg[0]

distlist = a['distance']
distance = distlist['text']

depart = a['departure_time']
departtime = depart['text']

loc1 = departtime.index(":")
prefix = departtime[loc1+3]
loc2 = departtime.index(prefix)

dhour = int(departtime[:loc1])
dmin = int(departtime[(loc1+1):loc2])

arrive = a['arrival_time']
arrivetime = arrive['text']

loc1 = arrivetime.index(":")
prefix = arrivetime[loc1+3]
loc2 = arrivetime.index(prefix)

ahour = int(arrivetime[:loc1])
amin = int(arrivetime[(loc1+1):loc2])

# pprint(directions_result[0])

traveltime = 0

if dhour == ahour:
    traveltime = amin - dmin
else:
    traveltime = ((ahour - dhour)*60) + amin - dmin

print ("Departure Time is: " + str(departtime))
print ("Arrival Time is: " + str(arrivetime))
print ("Travel Time is: " + str(traveltime) + " minutes")
print ("Travel Distance is: " + str(distance))