# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# pylint: disable=unused-import, redefined-builtin

import shutil
import os
import sys
import re
import string
import datetime
from typer.testing import CliRunner

# Include this to avoid circular import errors
import prompt_toolkit
import sphinx_material

os.environ['MLTK_BUILD_DOCS'] = '1'

import mltk
from mltk import cli
from mltk import MLTK_DIR, MLTK_ROOT_DIR
from mltk.utils.path import (
    clean_directory,
    get_user_setting,
    recursive_listdir,
    fullpath,
    walk_with_depth
)

curdir = os.path.dirname(os.path.abspath(__file__))
sys.path.insert(0, os.path.normpath(f'{curdir}/../../../mltk'))
repo_project_name = os.environ.get('MLTK_REPO_PROJECT_NAME', 'siliconlabs')
repo_url = f'https://github.com/{repo_project_name}/mltk'
repo_project_name = get_user_setting('repo_project_name', 'siliconlabs')

# -- Project information -----------------------------------------------------

project = 'MLTK'
copyright = f'{datetime.date.today().year}, Silicon Labs'
author = 'Silicon Labs'
release = mltk.__version__

# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    "sphinx.ext.autodoc",
    'sphinx.ext.autosummary',
    'sphinx.ext.coverage',
    "numpydoc",
    "sphinx.ext.doctest",
    "sphinx.ext.extlinks",
    "sphinx.ext.intersphinx",
    "sphinx.ext.todo",
    "sphinx.ext.mathjax",
    "sphinx.ext.viewcode",
    'sphinx.ext.napoleon',
    #'sphinx.ext.autosectionlabel',
    "myst_nb",
     # "myst_parser", # this is not needed when myst_nb is used
    "sphinx_markdown_tables",
    "sphinx_copybutton",
    'sphinx_design',
    'sphinx_material',
    'sphinx_autodoc_typehints', # This must come last
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_static/templates']

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = [
    '_static',
    '*lib/site-packages',
    '*lib\\site-packages'
]

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'sphinx_material'
html_logo = '_static/logo.png'
html_favicon = "_static/favicon.ico"

html_show_sourcelink = True
html_use_index = True
html_domain_indices = True
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

html_js_files = [
    'js/custom.js',
    'js/apitoc.js'
]
html_css_files = [
    'css/custom.css',
]

# https://bashtage.github.io/sphinx-material/customization.html#customization
html_theme_options = {
    # Set the name of the project to appear in the navigation.
    'nav_title': 'Machine Learning Toolkit',

    # Set you GA account ID to enable tracking
    'google_analytics_account': 'G-HZ5MW943WF',

    # Set the color and the accent color
    'color_primary': 'red',
    'color_accent': 'light-blue',

    'base_url': False,
    'repo_url': repo_url,
    'repo_name': 'MLTK Github Repository',

    # Visible levels of the global TOC;
    'globaltoc_depth': 2,
    # If False, expand all TOC entries
    'globaltoc_collapse': True,
    # If True, show hidden TOC entries
    'globaltoc_includehidden': True,
    'heroes': {
        'index': 'A Python package to aid the development of machine learning models for Silicon Lab\'s embedded platforms',
        'docs/installation': 'Install the MLTK Python Package via PIP or in <a href="https://colab.research.google.com/notebooks/welcome.ipynb" target="_blank">Google Colab<a/>',
        'docs/command_line': 'Execute the MLTK modeling features from the command-line',
        'docs/guides/index': 'The MLTK provides numerous features to aid the development of ML models',
        'docs/guides/model_profiler': 'Profile a model to determine how efficiently it may execute on hardware',
        'docs/guides/model_visualizer' : 'Visualize a model using an interactive webpage',
        'docs/guides/model_specification': 'Define a model specification using a standard Python script',
        'docs/guides/model_training': 'Train an ML model using Google Tensorflow',
        'docs/guides/model_evaluation': 'Evaluate an ML model to determine how accurate it is',
        'docs/guides/model_quantization': 'Quantize a model to reduce its memory footprint',
        'docs/guides/model_parameters': 'Embed custom parameters into the generated model file',
        'docs/guides/model_summary': 'Generate a textual summary of a model',
        'docs/guides/model_archive': 'All model development files are store in a distributable archive',
        'docs/guides/model_search_path': 'Specify the model search path',
    },
    'nav_links': [
        {
            'href': 'https://docs.silabs.com/gecko-platform/latest/machine-learning/tensorflow/overview',
            'internal': False,
            'title': 'Gecko SDK Documentation',
        },
        {
            'href': 'https://github.com/tensorflow/tflite-micro',
            'internal': False,
            'title': 'Tensorflow-Lite Micro Repository',
        },
        {
            'href': 'https://www.tensorflow.org/learn',
            'internal': False,
            'title': 'Tensorflow Documentation',
        },
    ],

    'html_minify': False,
    'html_prettify': False,
    'css_minify': False,

    'version_dropdown': False,
    'version_json': None,
    'version_info': None
}

html_sidebars = {
    '**': ['logo-text.html', 'globaltoc.html', 'localtoc.html', 'searchbox.html']
}

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
source_suffix = ['.rst', '.md']

pygments_style = 'trac'
add_module_names = False
autosummary_generate = True
#autosummary_imported_members = True
numpydoc_class_members_toctree = False

numpydoc_show_class_members = False
typehints_fully_qualified = False
set_type_checking_flag  = True

autodoc_typehints = 'description'
autodoc_member_order = 'bysource'
autoclass_content = "class"
autosectionlabel_prefix_document = True
autodoc_class_signature = 'separated'
#autosectionlabel_maxdepth = 1
myst_heading_anchors = 3
language = "en"
html_last_updated_fmt = ""

todo_include_todos = True

jupyter_execute_notebooks = 'off'

build_dir = sys.argv[-1]

# Copy all the .md files in <mltk root>/docs
# to <mltk root>/docs/website_builder/source/docs
# This allows for sphinx to generate HTML for each .md
docs_src_dir = fullpath(f'{curdir}/../..')
docs_dst_dir = f'{curdir}/docs'
clean_directory(docs_dst_dir)
include_re = re.compile(r'```{include}\s+(.*)\s*```.*')
cpp_path_re = re.compile(r'.*\]\((\.\.\/[\.\.\/]*cpp)[\/\)]+.*')
mltk_path_re = re.compile(r'.*\]\((\.\.\/[\.\.\/]*mltk\/core)[\/\)]+.*')
for fn in recursive_listdir(docs_src_dir, return_relative_paths=True):
    if not fn.endswith(('.md', '.rst')):
        continue
    if 'website_builder' in fn:
        continue

    src_path = f'{docs_src_dir}/{fn}'
    dst_path = f'{docs_dst_dir}/{fn}'
    os.makedirs(os.path.dirname(dst_path), exist_ok=True)

    # First check if the file contains:
    # ```{include} <path>
    #
    # If it does, then update the source path to that path.
    # This way we can process the actual markdown file
    with open(src_path, 'r') as f:
        data = f.read().replace('\r', '').replace('\n', '')
        match = include_re.match(data)
        if match:
            relpath = match.group(1)
            src_path = f'{os.path.dirname(src_path)}/{relpath}'

    # Now process the markdown file
    with open(src_path, 'r') as f:
        data = ''
        for line in f:
            if 'Refer to the [online documentation](https://siliconlabs.github.io/mltk) to properly view this file' in line:
                continue

            # Update the relative links to C++ code to point to the github repo
            # https://github.com/siliconlabs/mltk/cpp
            match = cpp_path_re.match(line)
            if match:
                line = line.replace(match.group(1), 'https://github.com/siliconlabs/mltk/tree/master/cpp/')
            else:
                # Update the relative links to python code to point to the github repo
                # https://github.com/siliconlabs/mltk/mltk/core
                match = mltk_path_re.match(line)
                if match:
                    line = line.replace(match.group(1), 'https://github.com/siliconlabs/mltk/tree/master/mltk/core/')
            data += line

    with open(dst_path, 'w') as f:
        f.write(data)

# Generate a markdown file containing each command's CLI --help
runner = CliRunner()
cli.create_cli()
for cmd in (
    'profile', 'train', 'tensorboard', 'ssh',
    'evaluate', 'quantize', 'summarize', 'view',
    'update_params', 'view_audio', 'classify_audio',
    'classify_image', 'fingerprint_reader', 'commander'
):
    result = runner.invoke(cli.root_cli, [cmd, '--help'], color=False, prog_name='mltk')
    help_str = ''.join(c for c in result.stdout if c in string.printable)
    with open(f'{docs_dst_dir}/command_line/{cmd}_cli_help.md', 'w') as f:
        f.write('```text\n')
        f.write(help_str)
        f.write('```\n')

# Copy all the files in <mltk root>/docs/img
# to <mltk root>/docs/website_builder/source/docs/img
# This allows for relative referencing
img_src_dir = f'{curdir}/../../img'
img_dst_dir = f'{curdir}/docs/img'
os.makedirs(img_dst_dir, exist_ok=True)
clean_directory(img_dst_dir)
shutil.copytree(img_src_dir, img_dst_dir, dirs_exist_ok=True)

# Copy <mltk root>/docs/img/models -> <mltk root>/docs/_images/models
# We need to do this because the images are referenced as raw HTML in
# each model python script header comments which won't be processed by sphinx
img_src_dir = f'{curdir}/../../img/models'
img_dst_dir = f'{curdir}/../../_images/models'
os.makedirs(img_dst_dir, exist_ok=True)
clean_directory(img_dst_dir)
shutil.copytree(img_src_dir, img_dst_dir, dirs_exist_ok=True)

# Copy each .ipynb or .md file in <mltk root>/mltk/examples
# to <mltk root>/docs/website_builder/source/mltk/examples directory
# Also update the Github project name as necessary
EXAMPLE_SRC_DIS = [
    'mltk/examples',
    'mltk/tutorials',
    'cpp/shared/uart_stream'
]

url_re = re.compile(r'.*\((https:\/\/siliconlabs\.github\.io\/mltk\/).*', re.I)
url_re2 = re.compile(r'.*\((https:\/\/siliconlabs\.github\.io\/mltk\/).*(\.html).*', re.I)
docs_img_re = re.compile(r'.*\((https:\/\/github.com\/SiliconLabs\/mltk\/raw\/master\/docs\/img\/).*', re.I)
# If the line contains something like:
# (https://siliconlabs.github.io/mltk/docs/python_api/mltk_model.html#mltk.core.TrainMixin.tflite_converter)
# Then we cannot convert it to it's corresponding relative markdown URL, so it's not supported
not_supported_url_re = re.compile(r'.*\(https:\/\/siliconlabs\.github\.io\/mltk\/.*\.html#.*[\.\-].*\)', re.I)

for src_dir in EXAMPLE_SRC_DIS:
    for root, _, files in walk_with_depth(f'{MLTK_ROOT_DIR}/{src_dir}', depth=10):
        for fn in files:
            if not fn.endswith(('.ipynb', '.md')):
                continue
            rel_dir = os.path.relpath(root, MLTK_ROOT_DIR)
            dst_dir = f'{curdir}/{rel_dir}'
            os.makedirs(dst_dir, exist_ok=True)
            shutil.copy(f'{root}/{fn}', f'{dst_dir}/{fn}')
            data = ''
            with open(f'{dst_dir}/{fn}', 'r', encoding='utf-8') as fp:
                for line in fp:
                    # The notebooks use full URLs.
                    # Convert the URLs to a relative path when we're generating
                    # so we can generate HTML with relative links
                    match = url_re.match(line)
                    not_supported_match = not_supported_url_re.match(line)
                    rel_path_to_root = os.path.relpath(curdir, dst_dir).replace('\\', '/') + '/'
                    if not_supported_match is None and match:
                        match2 = url_re2.match(line)
                        line = line.replace(match.group(1), rel_path_to_root)
                        if match2:
                            line = line.replace(match2.group(2), '.md')

                    match = docs_img_re.match(line)
                    if match:
                        line = line.replace(match.group(1), f'{rel_path_to_root}docs/img/')

                    data += line + '\n'

            with open(f'{dst_dir}/{fn}', 'w', encoding='utf-8') as fp:
                fp.write(data)

# Copy the <mltk root>/mltk/core/tflite_model_parameters/schema/dictionary.fbs
# to the docs build directory
dictionary_fbs_src = f'{MLTK_DIR}/core/tflite_model_parameters/schema/dictionary.fbs'
dictionary_fbs_dst = f'{docs_dst_dir}/python_api/tflite_model/dictionary.fbs'
dictionary_fbs_build_dir = f'{build_dir}/docs/python_api/tflite_model/dictionary.fbs'
shutil.copy(dictionary_fbs_src, dictionary_fbs_dst)
os.makedirs(os.path.dirname(dictionary_fbs_build_dir), exist_ok=True)
shutil.copy(dictionary_fbs_src, dictionary_fbs_build_dir)

def autodoc_skip_member(app, what, name, obj, skip, opts):
    # we can document otherwise excluded entities here by returning False
    # or skip otherwise included entities by returning True
    qual_name = getattr(obj, '__qualname__', '')
    module = getattr(obj, '__module__', '')
    fullname = f'{module}.{qual_name}'

    if 'keras.preprocessing.image.ImageDataGenerator.flow_from_dataframe' in fullname:
        return True

    return None

def setup(app):
    app.connect('autodoc-skip-member', autodoc_skip_member)


import neuralnet as nn
import numpy as np
from sklearn import datasets
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, confusion_matrix, classification_report
import matplotlib.pyplot as plt
import seaborn as sns

#Load the sklearn handwritten digits dataset
digits = datasets.load_digits()
X = digits.images.reshape((len(digits.images), -1))
labels = digits.target

#Normalize the data between 0 and 1
normalizer = MinMaxScaler()
X = normalizer.fit_transform(X)

#One-Hot Encode the output
lr = np.arange(10)
labels = [(k == lr).astype(int) for k in labels]
labels = np.vstack(labels)

#Split data
X_train, X_test, y_train, y_test = train_test_split(X, labels)

#Create a simple MLP
net = nn.Net(input_size = 64, lr = 0.1, momentum = 0.1, loss = 'CategoricalCrossEntropy')
net.add_layer(512)
net.add_layer(256)
net.add_layer(10, activation = 'sigmoid')

#Train the network using the training data
net.fit(X_train, y_train, batch_size = 1, epochs = 200, min_delta = 0.01, patience = 3)

#Predict the class of the test images
pred = net.predict(X_test)

#Go from one-hot encoding back to integers between 0 and 9
y_pred = [np.argmax(r) for r in pred]
y_true = [np.argmax(r) for r in y_test]

#Compute the accuracy of the network on the test data
print(accuracy_score(y_true, y_pred))
conf_matrix = confusion_matrix(y_true, y_pred)
sns.heatmap(conf_matrix, annot = True)
plt.xlabel('Predicted')
plt.ylabel('True')


# coding: utf-8

from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
import pandas as pd

# The hotel_main method extracts the information from the hotels.com website. The method takes in the
# string of the city name, check in date, and check out date, and uses the selenium with chrome driver
# and beautifulsoup to extract information from the website.
def hotel_main(city, checkInDate, checkOutDate):
    url = "https://www.hotels.com/"
    urlsAndCities = set()
    completeUrl = url
    completeUrl += "search.do?q-destination=" + city + "&q-check-in=" + checkInDate + "&q-check-out=" + checkOutDate

    # Open the chrome driver, scroll down using selenium and get the whole html page; find all the hotel tags
    driver = webdriver.Chrome(executable_path="./chromedriver")
    driver.get(completeUrl)
    body = driver.find_element_by_css_selector('body')
    for num in range(400):
        body.send_keys(Keys.PAGE_DOWN)
    html = driver.execute_script("return document.documentElement.outerHTML")
    soup = BeautifulSoup(html, features="html.parser")
    hotels = soup.find_all(class_="hotel-wrap")
    result = {}

    # for each hotel html part, extract information from the block
    for hotel in hotels:
        name = hotel.find(class_="p-name").text.strip()
        street = hotel.find(class_="contact").text.strip()[:-14]
        phone = hotel.find(class_="contact").text.strip()[-12:]
        try:
            guest_reviews = hotel.find(class_="guest-reviews-badge").text.strip()[-3:]
            guest_reviews = str(float(guest_reviews) / 10 * 5)

        except:
            guest_reviews = None
        try:
            popularity = hotel.find(class_="small-view").text.strip()[:-8].replace(',', '')
        except:
            popularity = None
        try:
            price = hotel.find(class_="price").text.strip()
            digits = [int(s) for s in price.split("$") if s.isdigit()]
            price = digits[-1]
        except:
            price = None

        result[name] = [name, street, guest_reviews, popularity, price, "", "", "", phone, ""]

    # Construct the dataframe from the combined dictionary in the indexed order
    columns = ["Name", "address", "rating", "popularity", "price", "category", "url", "imageurl", "phone",
               "description"]
    df = pd.DataFrame.from_dict(data=result, orient="index", columns=columns)
    return df

if __name__ == '__main__':
    hotel_main("Pittsburgh", "2019-03-01", "2019-03-02")


from django.http import HttpResponseBadRequest, HttpResponseNotFound, HttpResponse,Http404
from django.shortcuts import render,redirect,get_object_or_404
from django.views import View
from .forms import *
from .models import *
from google.auth.transport.requests import Request
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from googleapiclient.http import MediaFileUpload , MediaIoBaseUpload
from googleapiclient.http import MediaIoBaseDownload
import os
import os.path
from io import BytesIO
import io
from .functions import *
from Members.models import User

# class FileUpload(View):
#     def get(self,request,*args,**kwargs):
#         form = FileForm()
#         context = {
#             'form' : form,
#         }
#         return render(request,"ChApp/file-upload.html",context)

#     def post(self,request,*args,**kwargs):
#         SCOPES=["https://www.googleapis.com/auth/drive"]
#         form = FileForm(request.POST,request.FILES)
#         if form.is_valid():
#             f = form.save(commit=False)
#             f.user = request.user
#             f.file_hash = "deeezzzzznuuuuttttssss"

#             file_name = form.cleaned_data["title"]
#             uploaded_file = request.FILES['file']
#             file_data = request.FILES['file'].read()

#             creds = None
#             if os.path.exists('token.json'):
#                 creds = Credentials.from_authorized_user_file('token.json', SCOPES)
#             if not creds or not creds.valid:
#                 if creds and creds.expired and creds.refresh_token:
#                     creds.refresh(Request())
#                 else:
#                     flow = InstalledAppFlow.from_client_secrets_file("ChApp/creds.json",SCOPES)
#                     creds = flow.run_local_server(port=0)
#                 with open('token.json', 'w') as token:
#                     token.write(creds.to_json())

#             folder_name = request.user.username
#             try:
#                 service = build("drive","v3",credentials=creds)
#                 response = service.files().list(q=f"name='{folder_name}' and mimeType='application/vnd.google-apps.folder'",spaces='drive').execute()
#                 if not response['files']:
#                     file_metadata={
#                         "name" : folder_name,
#                         "mimeType" : "application/vnd.google-apps.folder"
#                     }
#                     file = service.files().create(body=file_metadata,fields = "id").execute()
#                     folder_id = file.get("id")
#                 else:
#                     folder_id = response['files'][0]["id"]
#                 file_metadata={
#                     "name" : file_name,
#                     "parents" : [folder_id]
#                 }
#                 media = MediaIoBaseUpload(io.BytesIO(file_data), mimetype=uploaded_file.content_type)
#                 upload_file=service.files().create(body = file_metadata,media_body=media,fields = "id").execute()
#                 print(f"FILE UPLOADED TO GOOGLE DRIVE {upload_file.get('id')}")
#                 f.file_id = upload_file.get("id")
#                 f.save()
#             except HttpError as ex:
#                 print(str(ex))

#             return redirect("home")

#         return render(request,"ChApp/file-upload.html")

class FileUpload(View):
    def get(self,request,*args,**kwargs):
        form = FileForm()
        context = {
            'form' : form,
        }
        return render(request,"ChApp/file-upload.html",context)

    def post(self,request,*args,**kwargs):

        form = FileForm(request.POST,request.FILES)
        if form.is_valid():
            f = form.save(commit=False)
            f.user = request.user

            file_name = form.cleaned_data["title"]
            uploaded_file = request.FILES['file']
            filename, file_extension = os.path.splitext(uploaded_file.name)
            a = file_name+file_extension
            file_data = request.FILES['file'].read()
            f.file_hash = HashFile(file_data)
            f.title = a
            upload_file = Upload(file_data,a,request.user.username,uploaded_file.content_type)

            f.file_id = upload_file
            f.save()

            return redirect("file_list")

        return render(request,"ChApp/file-upload.html")

# class FileList(View):
#     def get(self, request, *args, **kwargs):
#         # Get the user's credentials from the token.json file
#         SCOPES = ['https://www.googleapis.com/auth/drive']
#         creds = None
#         if os.path.exists('token.json'):
#             creds = Credentials.from_authorized_user_file('token.json', SCOPES)

#         # Check if the credentials are valid
#         if not creds or not creds.valid:
#             # Refresh the access token if it has expired
#             if creds and creds.expired and creds.refresh_token:
#                 creds.refresh(Request())
#             else:
#                 # Ask the user to authorize the app if this is the first time they're using it
#                 flow = InstalledAppFlow.from_client_secrets_file('client_secret.json', SCOPES)
#                 creds = flow.run_local_server(port=0)
#             # Save the credentials for future use
#             with open('token.json', 'w') as token:
#                 token.write(creds.to_json())

#         try:
#             # Build the Google Drive API client
#             service = build('drive', 'v3', credentials=creds)

#             # Get the list of files in the user's folder
#             folder_name = request.user.username
#             folder_query = "name='{}' and mimeType='application/vnd.google-apps.folder'".format(folder_name)
#             folder_results = service.files().list(q=folder_query, fields="nextPageToken, files(id, name)").execute()
#             folder_items = folder_results.get('files', [])
#             if not folder_items:
#                 message = 'No files found.'
#             else:
#                 # Get the list of files in the user's folder
#                 file_query = "'{}' in parents".format(folder_items[0]['id'])
#                 file_results = service.files().list(q=file_query, fields="nextPageToken, files(id, name, createdTime)").execute()
#                 file_items = file_results.get('files', [])
#                 if not file_items:
#                     message = 'No files found.'
#                 else:
#                     context = {'files': file_items}
#                     return render(request, 'ChApp/file-list.html', context)

#         except HttpError as error:
#             message = 'An error occurred: %s' % error
#             context = {'message': message}
#             return render(request, 'ChApp/file-list.html', context)

# class FileList(View):
#     def get(self,request,*args,**kwargs):
#         files = ListFile(request.user.username)
#         user = request.user
#         user_files = File.objects.filter(user = user)
#         if files and user_files :
#             context = {
#                 'files' : files,
#                 'user_files' : user_files,
#             }
#             return render(request, 'ChApp/file-list.html', context)

#         else:
#             return Http404('No Files Available')

class FileList(View):
    def get(self,request,*args,**kwargs):
        user = request.user
        user_files = File.objects.filter(user = user)

        context = {
                'files' : user_files,
        }
        return render(request, 'ChApp/file-list.html', context)

def download_file(request, file_id):
    SCOPES = ['https://www.googleapis.com/auth/drive.readonly']
    creds = None
    if os.path.exists('token.json'):
        creds = Credentials.from_authorized_user_file('token.json', SCOPES)
    if not creds or not creds.valid:
        if creds and creds.expired and creds.refresh_token:
            creds.refresh(Request())

        else:
            flow = InstalledAppFlow.from_client_secrets_file("ChApp/creds.json",SCOPES)
            creds = flow.run_local_server(port=0)
        with open('token.json', 'w') as token:
            token.write(creds.to_json())
    try:
        service = build('drive', 'v3', credentials=creds)
        file = service.files().get(fileId=file_id).execute()
        file_name = file['name']
        file_content = service.files().get_media(fileId=file_id).execute()
        response = HttpResponse(file_content, content_type=file['mimeType'])
        response['Content-Disposition'] = f'attachment; filename="{file_name}"'
        return response
    except HttpError as error:
        print(f'An error occurred: {error}')
        return Http404('File not found')

class ViewModifyFileInfo(View):
    def get(self,request,*args,**kwargs):
        file = get_object_or_404(File,file_id=self.kwargs.get('file_id'))
        form = FileModifyForm(instance=file)
        if file.user != request.user:
            return redirect('file_list')

        context = {
            'file' : file,
            'form' : form,
        }
        return render(request, 'ChApp/file-modify.html',context)

    def post(self,request,*args,**kwargs):
        file = get_object_or_404(File,file_id=self.kwargs.get('file_id'))
        form = FileModifyForm(request.POST,instance=file)
        if file.user != request.user:
            return redirect('file_list')
        if form.is_valid():
            form.save()
            return redirect('file_list')


from matplotlib.pyplot import imread
import numpy as np
from glob import glob
from os.path import join

import torch as th
from torch.utils.data import Dataset

import sys

'''
Currently this module holds Sintel dataset processor only.
To use it for another dataset, specific classes must be accomplished.
'''

# here must be a path to Sintel script
# for "get_depth" function in the "sintel_io" module
path2SintelModule = './../DATASETS/Sintel/depth_training_20150305/sdk/python/'
sys.path.append(path2SintelModule)
import sintel_io

from utils.functional import get_training_sample, get_full_sample

class Sintel(Dataset):
    '''
    Allows to work with raw files, RGB and Depth maps.
    '''
    def __init__(self,
        path2images='./../DATASETS/Sintel/training_images/training/final/',
        path2depths='./../DATASETS/Sintel/depth_training_20150305/training/depth/',
        m=1, num_images=None, get_name=False, size=(400,400),\
        mode=None, do_transforms=True, crop_size=256, sample_mode='ordinary'):
        '''
        "mode" can be: "train" - returns three images,
                       None - returns just numpy-like RGB and Depth images
        '''
        self.size=size
        self.m = m
        self.path2images = path2images
        self.path2depths = path2depths

        self.depth_max = 1e2

        self.image_paths = sorted(glob(path2images+'*/*.png'))
        self.depth_paths = sorted(glob(path2depths+'*/*.dpt'))
        self.get_name = get_name

        if not ((mode == 'train') or (mode == 'val')):
            raise Exception(
                'Incorrect mode type! Only "train" or "val" are available.')
        self.mode = mode
        self.sample_mode = sample_mode
        self.do_transforms = do_transforms
        self.crop_size = crop_size

        if num_images is not None:
            self.image_paths = self.image_paths[:num_images]
            self.depth_paths = self.depth_paths[:num_images]

    def __len__(self):
        return len(self.image_paths)

    def transforms(self, image, depth):

        assert self.crop_size < min(depth.shape), \
        'Inconsistent crop size, it is bigger than image dimensions!'
        y0 = np.random.randint(0,image.shape[0] - self.crop_size)
        x0 = np.random.randint(0,image.shape[1] - self.crop_size)

        image = image[y0:y0+self.crop_size, x0:x0+self.crop_size]
        depth = depth[y0:y0+self.crop_size, x0:x0+self.crop_size]
        return image, depth

    def __getitem__(self, idx):
        image = imread(self.image_paths[idx]).astype(np.float32)
        depth = sintel_io.depth_read(self.depth_paths[idx])

        image /= image.max()
        depth[depth>self.depth_max] = self.depth_max

        file_name = self.image_paths[idx][len(self.path2images):-4]
        file_name = file_name.replace('/', '_')

        if self.mode == 'train':
            if self.do_transforms:
                image, depth = self.transforms(image, depth)
            if self.sample_mode == 'ordinary':
                out = get_training_sample(image, depth, m=self.m)
                out = [th.FloatTensor(elem)[None,...] for elem in out]
            elif self.sample_mode == 'full':
                out = get_full_sample(image, depth)
        elif self.mode == 'val':
            if self.size is not None:
                image = image[:self.size[0],:self.size[1]]
                depth = depth[:self.size[0],:self.size[1]]
            out = [image, depth]

        if self.get_name:
            return out+[file_name]
        else:
            return out

class SintelCropped_hf(Dataset):
    '''
    Allows to construct necessary low/high-spectrum images.
    MSGNet training requires high-freq parts only.
    '''
    def __init__(self, path2folder='./../DATASETS/Sintel/CUSTOM_TRAINING_m3/',\
        mode='train', num_images=None, get_name=False):

        self.m = int(path2folder[-2:][0])
        self.mode = mode
        self.path2folder = path2folder+mode+'/'
        self.path2D_hr_hf_est = self.path2folder+'D_hr_hf_est/'
        self.path2D_lr_hf     = self.path2folder+'D_lr_hf/'
        self.path2Y_hr_hf     = self.path2folder+'Y_hr_hf/'

        self.D_hr_hf_est_paths = sorted(glob(self.path2D_hr_hf_est+'*.pth'))
        self.D_lr_hf_paths     = sorted(glob(self.path2D_lr_hf    +'*.pth'))
        self.Y_hr_hf_paths     = sorted(glob(self.path2Y_hr_hf    +'*.pth'))

        self.get_name = get_name

        if num_images is not None:
            self.D_hr_hf_est_paths = self.D_hr_hf_est_paths[:num_images]
            self.D_lr_hf_paths     = self.D_lr_hf_paths    [:num_images]
            self.Y_hr_hf_paths     = self.Y_hr_hf_paths    [:num_images]

    def __len__(self):
        return len(self.D_hr_hf_est_paths)

    def __getitem__(self, idx):
        D_hr_hf_est = th.load(self.D_hr_hf_est_paths[idx])
        D_lr_hf     = th.load(self.D_lr_hf_paths[idx])
        Y_hr_hf     = th.load(self.Y_hr_hf_paths[idx])

        file_name = self.D_hr_hf_est_paths[idx][len(self.path2D_hr_hf_est):-4]

        out = [D_hr_hf_est, D_lr_hf, Y_hr_hf]

        if self.get_name:
            return out+[file_name]
        else:
            return out

class Sintel_FullTraining(Dataset):
    '''
    Rewritten SintelCropped_hf. Now it allows to get any pack of spectra and scales
    (based on "training_type" choice).
    '''
    def __init__(self, mode='train', num_images=None,
        get_name=False, training_type='ordinary'):

        # here must be a path to folder with prepared dataset. Should be a parameter in this function.
        path2dataset='./../DATASETS/Sintel/all_scales_freqs_training_set/'
        self.mode = mode
        self.path2mode_folder = path2dataset+mode+'/'
        self.paths2folders = sorted(glob(self.path2mode_folder+'*'))
        self.folders = [folder[len(self.path2mode_folder):]
                            for folder in self.paths2folders]
        self.paths2files = {folder:sorted(glob(path2folder+'/*.pth'))
                                for folder, path2folder in zip(self.folders, self.paths2folders)}

        self.get_name = get_name
        self.training_type = training_type
        #Possible values: "full_set"|"ordinary"|"multiloss"|"full_freq"

        if self.training_type == 'ordinary': # as in paper, hf-to-hf training
            self.get_from_folders = ['Y_hf', 'Dx1_hf', 'Dx8_hf']
        elif self.training_type == 'multiloss': # hf-to-hfs training (at each scale of upsampling)
            self.get_from_folders = ['Y_hf', 'Dx1_hf', 'Dx2_hf', 'Dx4_hf', 'Dx8_hf']
        elif self.training_type == 'full_freq': # hf-to-(hf+lf) training,
                                                # it was used to be able to clip network output by depth_max value.
            self.get_from_folders = ['Y_hf', 'Dx1_hf', 'Dx8_hf', 'Dx8_lf']
        elif self.training_type == 'full_set': # independent from training, just full pack of all possible images.
            self.get_from_folders = self.folders.copy()
        else:
            assert False, 'Incorrect type of training.'

        if num_images is not None:
            for folder in self.folders:
                self.paths2files[folder] = self.paths2files[folder][:num_images]

    def __len__(self):
        return len(self.paths2files['Y_hf'])

    def __getitem__(self, idx):
        out = {}

        for folder in self.get_from_folders:
            path2file = self.paths2files[folder][idx]
            out[folder] = th.load(path2file)

        file_name = path2file[len(self.path2mode_folder+'/'+folder):-4]
        if self.get_name:
            return [out, file_name]
        else:
            return out


import os
from tempfile import TemporaryDirectory

from kaos_backend.clients.pachyderm import PachydermClient
from kaos_backend.services.job_service import JobService
from kaos_backend.util.tests import create_zip_file
from kaos_backend.util.validators import BundleValidator
from python_pachyderm import PpsClient, PfsClient

def t_any(cls):
    """
        Verifies if input is of same class
    """
    class AnyClass(cls):
        def __eq__(self, other):
            return True

    return AnyClass()

def create_job_service(mocker, workspaces=None):
    """
    Creates a Mock job Service

    Return:
         service: `kaos_backend.services.JobService`

    """
    client = PachydermClient(PpsClient(), PfsClient())
    service = JobService(client)

    def check_pipeline_exists_mock(pipeline_name):
        return pipeline_name.split('-')[-1] in workspaces

    client.check_pipeline_exists = mocker.Mock(side_effect=check_pipeline_exists_mock)

    return service

def create_test_file(dirname, filename):
    f = open(os.path.join(dirname, filename), "w")
    f.write("this is fake")
    f.close()
    return f

def create_bundle_zip(files):
    tempdir = TemporaryDirectory()
    root_dir = tempdir.name

    main_dir = os.path.join(root_dir, "test")
    os.mkdir(main_dir)

    model_dir = os.path.join(main_dir, BundleValidator.MODEL)
    os.mkdir(model_dir)

    filename = "Dockerfile"
    create_test_file(main_dir, filename)

    for f in files:
        create_test_file(model_dir, f)

    zip_filename = create_zip_file(root_dir)
    with open(zip_filename, 'rb') as file_data:
        bytes_content = file_data.read()

    return bytes_content, tempdir, zip_filename

def create_train_zip():
    return create_bundle_zip(BundleValidator.REQUIRED_TRAINING_FILES)

def create_inference_zip():
    return create_bundle_zip(BundleValidator.REQUIRED_INFERENCE_FILES)


import os

import torch
import torch.distributed as dist
import torch.multiprocessing as mp

def init_dist(launcher, backend='nccl', **kwargs):
    if mp.get_start_method(allow_none=True) is None:
        mp.set_start_method('spawn')
    if launcher == 'pytorch':
        _init_dist_pytorch(backend, **kwargs)
    else:
        raise ValueError('Invalid launcher type: {}'.format(launcher))

def _init_dist_pytorch(backend, **kwargs):
    rank = int(os.environ['RANK'])
    num_gpus = torch.cuda.device_count()
    torch.cuda.set_device(rank % num_gpus)
    dist.init_process_group(backend=backend, **kwargs)


#Autor : westboy31
#Tile Map Generator : https://westboy.itch.io/tilemapgenerator
import os, json, pygame

Map = {}
Map["images"] = {}
Map["tiles"] = {}
Map["items"] = {}
Map["collisions"] = []
tmg_json = {}

def loadFiles(jsonPath) :
    global Map
    tmg_jsonFile = open(jsonPath)
    tmg_folder = os.path.dirname(jsonPath) + "/"

    global tmg_json
    tmg_json = json.load(tmg_jsonFile)
    tmg_TilesList = list(dict.fromkeys(tmg_json["TilesList"]))

    for img in tmg_TilesList :
        tile = pygame.image.load(tmg_folder + img)
        Map["images"][img.split(".")[0]] = tile

    for img in tmg_json["ItemsList"] :
        item =  pygame.image.load(tmg_folder + img)
        Map["images"][img.split(".")[0]] = item

    return Map["images"]

def drawMap(screen) :

    for T in list(tmg_json["Tiles"]) :
        tile = Map["images"][T["Tile"]]
        tile = pygame.transform.flip(tile, T["FlipHorizontally"] , T["FlipVertically"] )
        tile = screen.blit(tile , ( T["X"] * tmg_json["TileSize"] , T["Y"] * tmg_json["TileSize"] ) )
        Map["tiles"][  "ix"+ str(T["X"]) + "iy" + str(T["Y"]) ] = tile

    for I in list(tmg_json["Items"]) :
        item = screen.blit(Map["images"][I["Item"].split(".")[0]], ( I["X"]  , I["Y"] ) )
        Map["items"][  "x"+ str(I["X"]) + "y" + str(I["Y"]) ] = item

    if tmg_json["Map"] == "PlatformerHorizontal" or tmg_json["Map"] == "PlatformerMultidirectional" :
        for C in list(tmg_json["Collisions"]) :
            collision = pygame.Rect( C["X"], C["Y"], C["Width"], C["Height"] )
            Map["collisions"].append(collision)

    return Map


import sys
import itertools

def _hamming_distance(a, b):
    assert len(a) == len(b)
    d = 0
    for i in range(len(a)):
        if a[i] != b[i]:
            d += 1
    return d

def median_string(consensus_kmer_dict, dnas, k):
    assert len(dnas) > 1

    for i, _ in enumerate(dnas):
        kmer_set = _convert_to_kmer_set(dnas[i], k)
        for compared_kmer in kmer_set:
            if compared_kmer in consensus_kmer_dict:
                consensus_kmer_dict[compared_kmer] += 1
                continue
            else:
                nearest_origin_kmer, nearest_hamming_dist = _find_minimal_hamming_distance(compared_kmer, consensus_kmer_dict)
                consensus_kmer_dict[nearest_origin_kmer] += 1

    median_string = ""
    max_point = 0
    for med_kmer, point in consensus_kmer_dict.items():
        if point > max_point:
            max_point = point
            median_string = med_kmer

    return median_string

def _find_minimal_hamming_distance(compared_kmer, origin_kmer_dict):
    minimal_hamming_dist = sys.maxsize
    minimal_hamming_kmer = ''
    for origin_kmer, origin_hamming_dist in origin_kmer_dict.items():
        result = _hamming_distance(compared_kmer, origin_kmer)
        if result < minimal_hamming_dist:
            minimal_hamming_dist = result
            minimal_hamming_kmer = origin_kmer
    return minimal_hamming_kmer, minimal_hamming_dist

def _convert_to_kmer_set(dna, k):
    kmer_set = set()
    dna_arr = list(dna)
    for i in range(len(dna_arr) - k + 1):
        kmer = dna_arr[i:i + k]
        kmer_str = ''.join(kmer)
        kmer_set.add(kmer_str)
    return kmer_set

if __name__ == '__main__':

    base = ['A', 'T', 'G', 'C']

    filename = "median_string_dataset.txt"
    content = list()
    k = 0
    dnas = list()
    with open(filename, 'r') as file:
        raw_content = file.read().rstrip()
        content = raw_content.split('\n')

    assert len(content) > 2

    k = int(content[0])
    dnas = content[1:]

    perms = itertools.product(base, repeat=k)
    ground_truth_set = set()
    ground_truth_dict = dict()
    for p in perms:
        origin_kmer = ''.join(p)
        ground_truth_set.add(origin_kmer)
        ground_truth_dict[origin_kmer] = 1

    result = median_string(ground_truth_dict, dnas, k)
    print(result)


import dash
import dash_core_components as dcc
import dash_html_components as html
from dash.dependencies import Input, Output
import pandas as pd
import numpy as np
import datetime
from datetime import timedelta
import plotly.figure_factory as ff
import dash_table
import itertools
import CreateSchedules
import os
import re

#Create the schedules...
CreateSchedules.CheckSchedules()
#extract all the csv file name in schedule folder
Schedules_avail = [ f.name for f in os.scandir('Schedules') if f.is_file() & bool(re.search('.*\.csv$', f.name))]

#Extract the name of the schedule
Schedules_avail = [x.replace('_Schedule.csv', '') for x in Schedules_avail]

#create options for dropdown
schedulesDict = [{'label' : i, 'value': i} for i in Schedules_avail]

app = dash.Dash(__name__)

app.layout = html.Div(
    children = [html.H1('Movie Schedules for I20 cinemas'),

                #dropdown
        html.P(children=[
        html.Label('Select a Schedule'),
        dcc.Dropdown(id='select', options= schedulesDict, multi = False, value = "TB_Original")],
            style = {'width':'400px'}),

        html.Div(children = [
            html.P("C: Cleaning time, T: Trailer time, P: Pre-show advertisement"),
            dcc.Graph(id = 'gantt'),
            html.P("Type in a movie or time slot to filter your choices:"),
            dash_table.DataTable( id='showtime',
            columns=[{'name' : 'Movie', 'id' : 'movie'},
                     {'name' : 'Theatre', 'id' : 'theatre'},
                     {'name' : 'Start Time', 'id' : 'startTimeDate'}],
            sort_action= "native",
            sort_mode="multi",
            filter_action="native",
            row_selectable="single"),
        html.A('Download Data',
        id='download-link',
        href="",
        target="_blank"
    )
                ]) #closes out div for graph and data table
    ]) #closes out outer div

@app.callback(
        [Output('gantt', 'figure'),
        Output('showtime','data')],
        [Input('select', 'value')])

def update_output(select):

    startTimesDF = pd.read_table('Schedules/' + select + '_Schedule.csv', delimiter = ',', header = 'infer')
    startTimesDF = startTimesDF.sort_values(by=['theatre'],ascending=False)

    #I will generate pre and post show time data, I manually compute/input time unit and duration for each of these actitives here.
    #generate cleaning data
    #cleaning starts when movie ends
    cleaning_data = startTimesDF[['theatre','endTimeDate']]
    #change the name so later I can concatenate the data
    cleaning_data.columns = ['theatre','startTimeDate']
    #create time unit, make it 0 so this column is not missing for cleaning time
    cleaning_data['timeUnit'] = 0
    #check to see if this block is a movie
    cleaning_data['check'] = 'N'
    #compute endtime for cleanings, which is 15 mins
    cleaning_data['endTimeDate'] = pd.to_datetime(cleaning_data['startTimeDate']) + timedelta(minutes=15)
    #create title for those time slots on Gantt chart
    cleaning_data['movie'] = 'C'

    #generate trailer data
    #Trailer ends when movies starts
    trailer_data = startTimesDF[['theatre','startTimeDate']]
    #change the name of columns
    trailer_data.columns = ['theatre','endTimeDate']
    #create time unit, same logic as cleaning time
    trailer_data['timeUnit'] = 0
    trailer_data['check'] = 'N'    #check to see if this block is a movie
    #compute start time for trailer, which is 20 min before the movie starts
    trailer_data['startTimeDate'] = pd.to_datetime(trailer_data['endTimeDate']) - timedelta(minutes=20)
    #create title for those time slots on Gantt chart
    trailer_data['movie'] = 'T'

    #generate preshow_data
    #preshow ends when trailer starts
    preshow_data = trailer_data[['theatre','startTimeDate']]
    #change the name of columns
    preshow_data.columns = ['theatre','endTimeDate']
    #create time unit same logic as above
    preshow_data['timeUnit'] = 0
    preshow_data['check'] = 'N'    #check to see if this block is a movie

    #compute start time for trailer, which is 30 min before the trailer starts
    preshow_data['startTimeDate'] = pd.to_datetime(preshow_data['endTimeDate']) - timedelta(minutes=30)
    #create title for those time slots on Gantt chart
    preshow_data['movie'] = 'P'

    startTimesDF_chart = pd.concat([startTimesDF,cleaning_data, trailer_data,preshow_data])

    #check column used to check if the time slot is a movie or not, if not a movie, the annotated time is 5 mins after start time, if it is a movie, then the annotation will be have x axis at 50 mins after the movie starts. I have to do this manually and have not found a way to to position the annotation nicely
        #create columns to position the label
    startTimesDF_chart['AnnotatedTime'] =  np.where(startTimesDF_chart['check'] == 'N', pd.to_datetime(startTimesDF_chart['startTimeDate']) + timedelta(minutes=5), pd.to_datetime(startTimesDF_chart['startTimeDate']) + timedelta(minutes=50))

    #copied from LP project, used to create dictionaries for plot

    showit = [dict(Task= "Theater" + str(row[1].theatre), Start= row[1].startTimeDate, Finish = row[1].endTimeDate, Resource = row[1].movie) for row in startTimesDF_chart.iterrows()]

    #colors = dict(T = 'red', C = 'blue', P = 'black')
    #unique_movie = np.unique(startTimesDF.movie)

    #colors2 = dict((movie, np.random.rand(3,)) for movie in unique_movie)
    #colors = colors.update(colors2),
    fig = ff.create_gantt(showit, index_col='Resource', group_tasks=True, show_hover_fill = True,show_colorbar=True,showgrid_x=True)

    #create dictionnary for info for the annotation
    annots = [dict(x=row[1].AnnotatedTime,y=row[1].theatre-1,text=row[1].movie, showarrow=False, font=dict(color='white')) for row in startTimesDF_chart.iterrows() ]
    #[dict(x=row.startTimeDate,y=row.theater,text=row.movie, showarrow=False, font=dict(color='white'))]

    fig['layout']['annotations'] = annots

    return fig, startTimesDF.to_dict('records')

@app.callback(
    Output('download-link', 'href'),
    [Input('select', 'value')])
def update_download_link(select):
    startTimesDF = pd.read_table('Schedules/' + select + '_Schedule.csv', delimiter = ',', header = 'infer')

    csv_string = startTimesDF.to_csv()
    return csv_string

if __name__ == "__main__":
    app.run_server(debug = True)


from django.shortcuts import render, redirect, get_object_or_404
from django.conf import settings
from orders.models import Order
from .tasks import payment_completed
import braintree

# Create your views here.

# 实例化braintree的网关.
gateway = braintree.BraintreeGateway(settings.BRAINTREE_CONF)

def payment_process(request):
    """
    支付流程.
    :param request:
    :return:
    """
    order_id = request.session.get('order_id')

    order = get_object_or_404(Order, id=order_id)
    total_cost = order.get_total_cost()

    if request.method == 'POST':
        # 取回付款方法.
        nonce = request.POST.get('payment_method_nonce', None)
        result = gateway.transaction.sale({
            'amount': f'{total_cost: .2f}',
            'payment_method_nonce': nonce,
            'options': {
                # 发送带有True的submit_for_settlement选项, 这样交易就会自动提交进行结算.
                'submit_for_settlement': True
            }
        })

        if result.is_success:
            # 更新订单支付状态.
            order.paid = True
            # 保存交易id.
            order.braintree_id = result.transaction.id
            order.save()

            # 发送异步任务.
            payment_completed.delay(order.id)

            return redirect('payment:done')
        else:
            return redirect('payment:canceled')
    else:
        client_token = gateway.client_token.generate()

        return render(request,
                      'payment/process.html',
                      {'order': order,
                       'client_token': client_token})

def payment_done(request):
    """
    支付完成.
    :param request:
    :return:
    """
    return render(request, 'payment/done.html')

def payment_canceled(request):
    """
    支付取消.
    :param request:
    :return:
    """
    return render(request, 'payment/canceled.html')


from flask import current_app
from flask import g, request
from flask_restful import abort
from sqlalchemy import and_
from sqlalchemy.orm import joinedload

from app import es, db
from app.global_search import constants as GS
from app.resources.accounts import constants as ACCT
from app.domain_resources.domains.helpers import (
    get_domain_info, get_domain_name)
from app.resources.roles.models import RoleMenuPermission, Role

from queueapp.elastic_search_tasks import (
    update_elastic_document, update_matched_docs)

def add_to_index(index, model, from_script=False, is_update=False):
    """
    Document will be added in elastic search , id will be prepared with the
    format : <table_class_name> + <_> + <row_id>
    e.g : Webinar_101
    If Document with same Id already present it will get updated with
    complete override

    :param index: string, this will be used for _index and _doc_type
    :param model: instance of a table row
    :param from_script: boolean flag to know whether reindexing
        as all data is populated in the db there will be no need of delay for
        indexing
    :param is_update: boolean flag to know if updating a record
    """
    try:
        payload = {}
        row_id = model.row_id
        account_id = None
        corporate_account_id = None
        user_id = None

        model_name = model.__class__.__name__
        if model_name == 'UserSettings':
            update_elastic_document.s(
                model.user_id, 'UserProfile', None,
                None, model.user_id, from_script).delay()
            return True
        for field in model.__global_searchable__:
            payload[field] = getattr(model, field)

        if model_name in ['UserProfile', 'CRMContact']:
            if model_name is 'UserProfile':
                # as changing user profile is done by passing user id and not
                # user profile id
                row_id = model.user_id
                user_id = row_id
            payload['full_name'] = model.first_name + ' ' + model.last_name
            payload['full_name_r'] = model.last_name + ' ' + model.first_name

        if model_name is 'Webinar' and not payload['open_to_account_types']:
            # if open_to_account_types is empty no need to index it
            payload.pop('open_to_account_types')

        if model_name == 'Account':
            # as changing account is done by account_profile_api
            model_name = 'AccountProfile'
            account_profile = get_act_dict(model)
            payload.update(account_profile)
            if is_update:
                state = db.inspect(model)
                for attr in state.attrs:
                    hist = attr.load_history()
                    if not hist.has_changes():
                        continue
                    if attr.key in account_profile:
                        update_matched_docs.s(row_id, account_profile).delay()
                        break

        if hasattr(model, 'account'):
            if model.account:
                account_profile = get_act_dict(model.account)
                payload['account'] = account_profile
            else:
                # account relationship is not yet populated in db
                # so add celery task to update account data
                account_id = model.account_id

        if hasattr(model, 'corporate_account'):
            if model.corporate_account:
                account_profile = get_act_dict(model.corporate_account)
                payload['corporate_account'] = account_profile
            else:
                # corporate_account relationship is not yet populated in db
                # so add celery task to update corporate_account data
                corporate_account_id = model.corporate_account_id

        if model_name == 'CorporateAccessEvent':
            payload['event_type'] = model.event_sub_type.name
        payload['module'] = model_name
        payload['priority'] = GS.MODULE_EXTRAS[model_name]['priority']
        doc_id = model_name + '_' + str(row_id)
        es.index(index=index, doc_type=index, id=doc_id, body=payload)
        if (model_name in ['CorporateAccessEvent', 'Webinar', 'Webcast']
                or any([account_id, corporate_account_id, user_id])):
            update_elastic_document.s(
                row_id, model_name, account_id,
                corporate_account_id, user_id, from_script).delay()

    except Exception as e:
        current_app.logger.exception(e)
        return False

def get_act_dict(account):
    """

    :param account: instance of account model
    :return: account profile dictionary
    """

    account_profile = {}
    sector_name = ''
    industry_name = ''
    if account.profile.sector:
        sector_name = account.profile.sector.name
    if account.profile.industry:
        industry_name = account.profile.industry.name
    account_profile['account_name'] = account.account_name
    account_profile['account_type'] = account.account_type
    account_profile['sector_name'] = sector_name
    account_profile['industry_name'] = industry_name
    account_profile['row_id'] = account.row_id
    account_profile['country'] = account.profile.country
    account_profile['blocked'] = account.blocked
    account_profile['domain_id'] = account.domain_id

    return account_profile

def remove_from_index(index, model):
    """
    Elastic Document Id is derived and the document is deleted from index

    :param index: string, this will be used for _index and _doc_type
    :param model: instance of a table row
    """
    try:
        class_name = model.__class__.__name__
        row_id = model.row_id
        if class_name is 'Account':
            class_name = 'AccountProfile'
        if class_name is 'UserProfile':
            row_id = model.user_id
        doc_id = class_name + '_' + str(row_id)
        es.delete(index=index, doc_type=index, id=doc_id, ignore=[404])

    except Exception as e:
        current_app.logger.exception(e)
        return False

def update_document(index, doc_id, document):
    """
    document will get merged with the existing
    document, i.e. existing key's value will get updated and new key
    will be added
    :param index: string, this will be used for _index and _doc_type
    :param doc_id: string, elastic document id
    :param document: dictionary
    """
    try:
        body = {"doc": document}
        es.update(index=index, doc_type=index, id=doc_id, body=body)

    except Exception as e:
        current_app.logger.exception(e)
        return False

def remove_index(index):
    """
    index will be removed from elastic search

    :param index: string
    """
    try:
        es.indices.delete(index=index, ignore=[400, 404])
    except Exception as e:
        current_app.logger.exception(e)
        return False

def query_index(index, query, main_filter, page=1, per_page=10):
    """
    query will be searched in all the documents with appropriate filter

    :param index: string
    :param query: string
    :param page: int
    :param per_page: int
    :return: dictionary
    """
    result = {
        'total': 0,
        'results': []
    }
    try:
        user_email = g.current_user['email']
        user_id = g.current_user['row_id']
        user_act_type = g.current_user['account_type']
        domain_name = get_domain_name()
        domain_id, domain_config = get_domain_info(domain_name)
        # clause for matching accounts
        account_clause = {
            'bool': {
                'must': {'match_phrase_prefix': {'account_name': query}},
                'filter': {
                    'bool': {
                        'must': [
                            {'term': {'module': 'AccountProfile'}},
                            {'term': {'domain_id': domain_id}},
                            {'term': {'blocked': False}}
                        ],
                        'must_not': [
                            {'term': {'account_type': ACCT.ACCT_ADMIN}},
                            {'term': {'account_type': ACCT.ACCT_GUEST}}
                        ]
                    }
                }

            }
        }

        # clause for matching corporate announcement
        announcement_clause = {
            'bool': {
                'should': [
                    {'match_phrase_prefix': {'subject': query}},
                    {'match_phrase_prefix': {'description': query}},
                    {'match_phrase_prefix': {'category': query}}
                ],
                'minimum_should_match': 1,
                'filter': [
                    {'term': {'account.domain_id': domain_id}},
                    {'term': {'module': 'CorporateAnnouncement'}},
                    {'term': {'account.blocked': False}}
                ],
            }
        }

        # creator - all matching events owned by a user are included
        # invited user - matching events which are not draft and cancelled
        # matching open events which are not draft and cancelled are included

        event_clause = {
            'bool': {
                'should': [
                    {'match_phrase_prefix': {'title': query}},
                    {'match_phrase_prefix': {'description': query}}
                ],
                'minimum_should_match': 1,
                'filter': {
                    'bool': {
                        'must': [
                            {'term': {'account.blocked': False}},
                            {'term': {'module': 'CorporateAccessEvent'}},
                            {'term': {'account.domain_id': domain_id}}
                        ],
                        'should': [
                            {'term': {'created_by': user_id}},
                            {'bool': {
                                'must': [
                                    {'term': {'cancelled': False}},
                                    {'term': {'is_draft': False}}
                                ],
                                'should': [
                                    {'term': {'user_emails': user_email}},
                                    {'bool': {
                                        'must': [
                                            {'term': {'open_to_all': True}},
                                            {'term': {
                                                'account_type_preference':
                                                    user_act_type}}]
                                    }},
                                ],
                                'minimum_should_match': 1
                            }},
                        ],
                        'minimum_should_match': 1
                    }
                }
            }
        }

        # matching user profiles (all) and

        user_profile_filters = [
            {'term': {'module': 'UserProfile'}},
            {'term': {'search_privacy': g.current_user['account_type']}},
            {'term': {'account.blocked': False}},
            {'term': {'account.domain_id': domain_id}}
        ]
        sector_id = g.current_user['account_model'].profile.sector_id
        if sector_id:
            user_profile_filters.append({
                'term': {'search_privacy_sector': sector_id}
            })

        industry_id = g.current_user['account_model'].profile.industry_id
        if industry_id:
            user_profile_filters.append({
                'term': {
                    'search_privacy_industry': industry_id}
            })

        market_cap = g.current_user['account_model'].profile.market_cap
        if market_cap:
            user_profile_filters.append({
                'range': {
                    'search_privacy_market_cap_min': {'lte': market_cap}
                }
            })
            user_profile_filters.append({
                'range': {
                    'search_privacy_market_cap_max': {'gte': market_cap}}
            })

        if g.current_user['profile']['designation_link']:
            designation_level = \
                g.current_user['profile']['designation_link']['designation_level']
            user_profile_filters.append({
                'term': {'search_privacy_designation_level': designation_level}
            })

        user_profile_clause = {
            'bool': {
                'should': [
                    {'match_phrase_prefix': {'first_name': query}},
                    {'match_phrase_prefix': {'last_name': query}},
                    {'match_phrase_prefix': {'full_name': query}},
                    {'match_phrase_prefix': {'full_name_r': query}},
                    {'match_phrase_prefix': {'designation': query}},
                    {'match_phrase_prefix': {'account.account_name': query}},
                ],
                'minimum_should_match': 1,
                'filter': {
                    'bool': {
                        'must': user_profile_filters,
                        'must_not': [
                            {'term':
                                {'_id': 'UserProfile_{}'.format(user_id)}},
                            {'terms':
                                {'account.account_type': [
                                    ACCT.ACCT_ADMIN,  ACCT.ACCT_GUEST]}}
                        ]
                    },
                }
            }
        }

        # matching crm contacts (created by user) are included
        crm_contacts_clause = {
            'bool': {
                'should': [
                    {'match_phrase_prefix': {'first_name': query}},
                    {'match_phrase_prefix': {'middle_name': query}},
                    {'match_phrase_prefix': {'last_name': query}},
                    {'match_phrase_prefix': {'full_name': query}},
                    {'match_phrase_prefix': {'full_name_r': query}},
                    {'match_phrase_prefix': {'company': query}},
                    {'match_phrase_prefix': {'designation': query}},
                ],
                'minimum_should_match': 1,
                'filter': {
                    'bool': {
                        'must': [
                            {'term': {'module': 'CRMContact'}},
                            {'term': {'created_by': user_id}}
                        ]
                    },
                }
            }
        }

        # creator - all matching webinars owned by a user are included
        # invited user - matching webinars which are not draft and cancelled
        # matching open_to_public webinars or open to account type
        # which are not draft and cancelled  are included
        webinar_clause = {
            'bool': {
                'should': [
                    {'match_phrase_prefix': {'title': query}},
                    {'match_phrase_prefix': {'description': query}}
                ],
                'minimum_should_match': 1,
                'filter': {
                    'bool': {
                        'must': [
                            {'term': {'account.blocked': False}},
                            {'term': {'module': 'Webinar'}},
                            {'term': {'account.domain_id': domain_id}}
                        ],
                        'should': [
                            {'term': {'created_by': user_id}},
                            {'bool': {
                                'must': [
                                    {'term': {'is_draft': False}},
                                    {'term': {'cancelled': False}},
                                ],
                                'should': [
                                    {'term': {'user_emails': user_email}},
                                    {'term': {'open_to_all': True}},
                                    {'term': {
                                        'open_to_account_types': user_act_type
                                    }},
                                ],
                                'minimum_should_match': 1
                            }},
                        ]
                    }
                }
            }
        }

        # creator - all matching webcasts owned by a user are included
        # invited user - matching webcasts which are not draft and cancelled

        webcast_clause = {
            'bool': {
                'should': [
                    {'match_phrase_prefix': {'title': query}},
                    {'match_phrase_prefix': {'description': query}}
                ],
                'minimum_should_match': 1,
                'filter': {
                    'bool': {
                        'must': [
                            {'term': {'account.blocked': False}},
                            {'term': {'module': 'Webcast'}},
                            {'term': {'account.domain_id': domain_id}}
                        ],
                        'should': [
                            {'term': {'created_by': user_id}},
                            {'bool': {
                                'must': [
                                    {'term': {'is_draft': False}},
                                    {'term': {'cancelled': False}},
                                    {'term': {'user_emails': user_email}}
                                ]
                            }}
                        ],
                        'minimum_should_match': 1
                    }
                }
            }
        }

        # creator- all matching distribution lists owned by a user are included
        dist_list_clause = {
            'bool': {
                'must': {'match_phrase_prefix': {'campaign_name': query}},
                'filter': {
                    'bool': {
                        'must': [
                            {'term': {'module': 'CRMDistributionList'}},
                            {'term': {'created_by': user_id}}
                        ]
                    }
                }
            }
        }

        # research report
        research_report_clause = {
            'bool': {
                'should': [
                    {'match_phrase_prefix': {'subject': query}},
                    # {'match_phrase_prefix': {'account.account_name': query}},
                ],
                'minimum_should_match': 1,
                'filter': {
                    'term': {'module': 'ResearchReport'}
                }
            }
        }

        filters = {
            GS.ALL: [user_profile_clause, account_clause, announcement_clause,
                     event_clause, crm_contacts_clause, webinar_clause,
                     webcast_clause, dist_list_clause, research_report_clause],
            GS.CAEVENT: [event_clause],
            GS.ACCOUNT: [account_clause],
            GS.ANNOUNCEMENT: [announcement_clause],
            GS.USER_PROFILE: [user_profile_clause],
            GS.CRM_CONTACT: [crm_contacts_clause],
            GS.WEBINAR: [webinar_clause],
            GS.WEBCAST: [webcast_clause],
            GS.CRM_DIST_LIST: [dist_list_clause],
            GS.RESEARCH_REPORT: [research_report_clause],
        }

        permitted_modules = []
        user_menu_codes = []
        perm_menus = RoleMenuPermission.query.filter(and_(
            RoleMenuPermission.role_id==g.current_user['role']['row_id'],
            )).options(joinedload(RoleMenuPermission.menu))
        for each in perm_menus:
            user_menu_codes.append(each.menu.code)
        for module in GS.ALL_MODULES:
            if (module in GS.MODULE_MENUCODE
                and GS.MODULE_MENUCODE[module] not in user_menu_codes):
                continue
            permitted_modules.append(module)

        query = {
            '_source': {
                "excludes": ["user_emails", "priority"],
            },
            'query': {
                'bool': {
                    'should': filters[main_filter],
                    'minimum_should_match': 1,
                    'filter': {
                        "terms": {"module": permitted_modules}
                    }
                },
            },
            'from': (page - 1) * per_page,
            'size': per_page,
            'sort': [{'priority': 'asc'}, '_score']
        }

        search = es.search(index=index, doc_type=index, body=query)
        result['total'] = search['hits']['total']

        for hit in search['hits']['hits']:
            splitted = hit['_id'].split('_')
            row_id = splitted.pop()
            obj = hit['_source']
            obj['row_id'] = int(row_id)
            obj['module_display'] = \
                GS.MODULE_EXTRAS[obj['module']]['display_name']
            result['results'].append(obj)

    except Exception as e:
        current_app.logger.exception(e)
        abort(500)
    return result


# -*- coding: utf-8 -*-
import pickle
from lib2to3.pgen2.token import OP
import os
from textwrap import shorten
import numpy as np
import matplotlib
from matplotlib import pyplot as plt
import pandas as pd
from datetime import datetime
from datetime import timedelta
import copy
from pyparsing import col
from sklearn.preprocessing import MinMaxScaler
from sklearn.cluster import KMeans
import matplotlib.dates as mdates
import matplotlib.colors as colors
from scipy.optimize import curve_fit
from water_vapor import water_vapor
import pysolar.solar as solar

def fit_func(x, a, b, c):
    # to use in fits
    return a*np.exp(x+b)+c

REPO_PATH = os.getcwd()
# 'mica_hourly'  # [str(i)+'0217' for i in range(1997,2013,1)] # Full dates to plot, set to None to plot all
MICAF = ['19990223'] # 'mica_calibration'  # 'mica_hourly' #'mica_vs_master' # ['19990222']
# other options are: 'mica_hourly' to plot the same day in all years
DEL_MICA_MONTHS = ['200507', '200508', '200509', '200510', '200511']  # months to delete
MICA_DIR = REPO_PATH + '/data/mica/Sky_Tes_1997_2012'
MASTER_DIR = REPO_PATH + '/data/master'
OPATH = REPO_PATH + '/output/mica_uncalibrated'
COL_NAMES = ['Date', 'Sky_T', 'Sun_T']
COL_UNITS = {'Date': '', 'Sky_T': '', 'Sun_T': '', 'Sky_T_over_Sun_T': '',
             'sky_class': '', 'date_diff': '[s]'}  # units incl a blank space
DEL_VAL = {'Sky_T': [4.91499996], 'Sun_T': [], 'Sky_T_over_Sun_T':[]}  # {'Sky_T': [4.91499996, 0.0], 'Sun_T': [0.0]}  # delete these values
MIN_VAL = {'Sky_T': [], 'Sun_T': [], 'Sky_T_over_Sun_T':[]}  # delet all values below these
NBINS = 50  # his num of bin
CLUSTERS = ['Sunny Good', 'Sunny Moderate', 'Sunny Bad', 'Cloudy']
CLUSTERING_METHOD = 'manual'  # 'kmeans' or 'manual'
N_CLUSTERS = len(CLUSTERS)  # Number of clusters to clasiffy the sky
CLUSTERS_COLOR = ['g', 'orange', 'r', 'k']
EMP_CLASS = {'Sky_T': [1, 2], 'Sun_T': [2, 2]}  # empirical limits for sky classes in mV
DO_NORM = False  # normalize the data
matplotlib.rc('font', size=12)  # font size
BWIDTH = 0.45
DPI = 300.  # image dpi
DATE_DIFF_LIM = [0, 50]  # date difference limit for the plot
RECOMPUTE_SCALER = True  # Set to recompute the data scaler
SCALER_FILE = REPO_PATH + '/output/mica/scaler.pckl'
MICA_CAL_DIR = '/media/sf_iglesias_data/cesco_sky_quality/MICA_processed/AvgGifs'

# get all mica files
mf = [os.path.join(MICA_DIR, f) for f in os.listdir(MICA_DIR) if f.endswith('.txt')]
if MICAF is not None:
    if MICAF == 'mica_hourly':
        allf = [str.split(i, '.')[0] for i in os.listdir(MICA_DIR)]
        allf = [i[4:8] for i in allf]
        # find daates that are in at least 14 years
        allf = [i for i in allf if allf.count(i) > 14]
        mf = [i for i in mf if i.split('/')[-1].split('.')[0][4:8] == allf[3]]
    elif MICAF == 'mica_vs_master':
        mf = [i for i in mf if str.split(os.path.basename(i), '.')[0][0:4] in ['2012']]
    elif MICAF == 'mica_calibration':
        mf_cal = os.listdir(MICA_CAL_DIR)
        mf = [i for i in mf if str.split(os.path.basename(i), '.')[0] in mf_cal]
    else:
        mf = [i for i in mf if str.split(os.path.basename(i), '.')[0] in MICAF]

# read the space separated files with pandas
df_all = []
print('Reading %s files...' % len(mf))
tnf = 0
for f in mf:
    yyyymmdd = f.split('/')[-1].split('.')[0]
    if yyyymmdd[0:6] not in DEL_MICA_MONTHS:
        df = pd.read_csv(f, delim_whitespace=True, skiprows=1, names=COL_NAMES)
        df['Date'] = [datetime(int(yyyymmdd[0:4]), int(yyyymmdd[4:6]), int(yyyymmdd[6:8])) +
                      timedelta(hours=h) for h in df['Date']]
        df['date_diff'] = df['Date'].diff().dt.total_seconds()
        df_all.append(df)
        tnf += 1

df_all = pd.concat(df_all, ignore_index=True)

# Eliminate wrong and sat data
for key in DEL_VAL:
    for val in DEL_VAL[key]:
        print('Deleting...', key, val)
        df_all = df_all.drop(df_all[df_all[key] == val].index)
for key in MIN_VAL:
    for val in MIN_VAL[key]:
        print('Deleting...<=', key, val)
        df_all = df_all.drop(df_all[df_all[key] <= val].index)

# Normalizes
if DO_NORM:
    if RECOMPUTE_SCALER and MICAF is None:
        print('Generating and saving normalization parameters for...', COL_NAMES[1:])
        scaler = MinMaxScaler()
        df_all[COL_NAMES[1:]] = scaler.fit_transform(df_all[COL_NAMES[1:]])
        with open(SCALER_FILE, 'wb') as handle:
            pickle.dump(scaler, handle, protocol=pickle.HIGHEST_PROTOCOL)
    else:
        print('Reading normalization parameters for...', COL_NAMES[1:])
        with open(SCALER_FILE, 'rb') as handle:
            scaler = pickle.load(handle)
        df_all[COL_NAMES[1:]] = scaler.transform(df_all[COL_NAMES[1:]])
    print('Data Max:', scaler.data_max_, 'Data Min:', scaler.data_min_,  'Scale:',
          scaler.scale_,  'Range:', scaler.data_range_, 'Min:', scaler.min_)

    lim = np.array([EMP_CLASS['Sky_T'], EMP_CLASS['Sun_T']])
    lim = scaler.transform(lim)
    EMP_CLASS['Sky_T'] = lim[:, 0]
    EMP_CLASS['Sun_T'] = lim[:, 1]
    print('Normalized empirical limits: ', EMP_CLASS)

# Clustering
if CLUSTERING_METHOD == 'kmeans':
    # clustering with KMeans
    km = KMeans(n_clusters=N_CLUSTERS)
    df_all['sky_class'] = km.fit_predict(df_all[['Sun_T', 'Sky_T']])
    nind = np.argsort(km.cluster_centers_[:, 0])
    for i in range(len(nind)):
        df_all['sky_class'] = df_all['sky_class'].replace(int(nind[i]), CLUSTERS[len(CLUSTERS)-1-i])
elif CLUSTERING_METHOD == 'manual':
    km = None
    # manual clustering
    df_all['sky_class'] = copy.deepcopy(df_all['Sun_T'])
    df_all['sky_class'] = 'None'
    print('Points with sky_class==None:', len(df_all[df_all['sky_class'] == 'None']))
    for c in CLUSTERS:
        if c == CLUSTERS[0]:  # good
            df_all.loc[(df_all['Sun_T'] > EMP_CLASS['Sun_T'][0]) & (
                df_all['Sky_T'] <= EMP_CLASS['Sky_T'][0]), 'sky_class'] = c
        elif c == CLUSTERS[1]:  # moderate
            df_all.loc[(df_all['Sun_T'] > EMP_CLASS['Sun_T'][0]) & (df_all['Sky_T'] > EMP_CLASS['Sky_T'][0]) &
                       (df_all['Sky_T'] <= EMP_CLASS['Sky_T'][1]), 'sky_class'] = c
        elif c == CLUSTERS[2]:  # bad
            df_all.loc[(df_all['Sun_T'] > EMP_CLASS['Sun_T'][0]) & (
                df_all['Sky_T'] > EMP_CLASS['Sky_T'][1]), 'sky_class'] = c
        elif c == CLUSTERS[3]:  # cloudy
            df_all.loc[(df_all['Sun_T'] <= EMP_CLASS['Sun_T'][0]), 'sky_class'] = c
COL_NAMES.append('sky_class')
print('Points with sky_class==None:', len(df_all[df_all['sky_class'] == 'None']))

# adds column with Sky_T_over_Sun_T
df_all['Sky_T_over_Sun_T'] = df_all['Sky_T']/df_all['Sun_T']
COL_NAMES.append('Sky_T_over_Sun_T')

# sort by date
df_all = df_all.sort_values(by='Date')
COL_NAMES.append('date_diff')

# prints some info
for var in np.array(COL_NAMES)[[1, 2, 4]]:
    print(var+'-------------:')
    print('Total number of files (days) read: %s' % tnf)
    print('Total number of data points: %s (%s days of net observation)' %
          (len(df_all[var]), len(df_all[var])*5./3600./24.))
    print('Mean: %s' % df_all[var].mean())
    print('Median: %s' % df_all[var].median())
    print('Std: %s' % df_all[var].std())
    print('Min: %s at date %s' % (df_all[var].min(), df_all.loc[df_all[var].idxmin()]['Date']))
    print('Max: %s at date %s' % (df_all[var].max(), df_all.loc[df_all[var].idxmin()]['Date']))
    print('p90: %s' % np.nanpercentile(df_all[var], 90))
    print('p99: %s' % np.nanpercentile(df_all[var], 99))
    print('p10: %s' % np.nanpercentile(df_all[var], 10))
    print('----------------------------------------------------')

# Plots
print('Plotting...')
os.makedirs(OPATH, exist_ok=True)
if MICAF == 'mica_vs_master':
    REMOVE = {"Sky_T": [-100], "Amb_T": [], "WindS": [], "Hum%": [], "DewP": [],
              "C": [], "W": [], "R": [], "Cloud_T": [], "Date_diff": []}  # Max val
    COL_NAMES_MASTER = ["DAY", "HOUR(UT)", "Sky_T", "Amb_T", "WindS", "Hum%",
                        "DewP", "C", "W", "R", "Cloud_T", "Date_diff"]
    mf = [os.path.join(MASTER_DIR, f) for f in os.listdir(MASTER_DIR) if f.endswith('_wea.txt')]
    mf = [i for i in mf if str.split(os.path.basename(i), '.')[0][0:4] in ['2012']]
    df_master = []
    for f in mf:
        df = pd.read_csv(f, delim_whitespace=True, skiprows=1, names=COL_NAMES_MASTER)
        df["DAY"] = pd.to_datetime(df["DAY"], format="%Y%m%d")
        df["HOUR(UT)"] = [timedelta(hours=h) for h in df['HOUR(UT)']]
        df["Date"] = df["DAY"]+df["HOUR(UT)"]
        df_master.append(df)
    df_master = pd.concat(df_master, ignore_index=True)
    df_master["Cloud_T"] = df_master["Sky_T"] - df_master["Amb_T"]
    df_master["water_vapor"] = water_vapor(df_master["Amb_T"], df_master["Hum%"])
    df_master["Date_diff"] = df_master["Date"].diff().dt.total_seconds()
    # remove values
    for var in COL_NAMES_MASTER[2:]:
        for i in REMOVE[var]:
            df_master = df_master.drop(df_master[df_master[var] < i].index)

    # normalizes
    scaler_master = MinMaxScaler()
    tonorm = ['Sky_T', "Amb_T", "WindS", "Hum%", "DewP", "Cloud_T", "water_vapor"]
    df_master[tonorm] = scaler_master.fit_transform(df_master[tonorm])

    for var in tonorm:
        # resamples master to mica times
        # df_all.loc[(df_all['sky_class'] != CLUSTERS[2]) & (df_all['sky_class'] != CLUSTERS[3]), "Date"] #  df_all['Date']#  # sunny good and moderate only
        interp_date = df_all.loc[(df_all['Date'].dt.month == 2), "Date"]  # df_all['Sky_T']# df_all['Date']
        # df_all.loc[(df_all['sky_class'] != CLUSTERS[2]) & (df_all['sky_class'] != CLUSTERS[3]), "Sky_T"] #
        interp_skyt = df_all.loc[(df_all['Date'].dt.month == 2), "Sun_T"]  # df_all['Sky_T'] #df_all['Sky_T']
        interp_cloudt = np.interp(interp_date, df_master["Date"], df_master[var])

        # # scatter
        # x = interp_cloudt
        # y = interp_skyt
        # plt.scatter(x, y, c='b', s=2)
        # plt.xlabel(var)
        # plt.ylabel('Sky_T')
        # plt.legend()
        # plt.tight_layout()
        # plt.show()

        # # 3d scatter
        # y = df_all.loc[(df_all['Date'].dt.month == 2), "Sky_T"]
        # z = interp_cloudt
        # x = df_all.loc[(df_all['Date'].dt.month == 2), "Sun_T"]
        # fig = plt.figure(figsize=[10, 10])
        # ax = fig.add_subplot(111, projection='3d')
        # ax.scatter3D(x, y, z, c='r', marker='.', s=2)
        # ax.set_xlabel(df_all['Sun_T'].name)
        # ax.set_ylabel(df_all['Sky_T'].name)
        # ax.set_zlabel(var)
        # plt.tight_layout()
        # plt.show()

        # # 2d histogram with fit
        if var in ['water_vapor']:
            # vs Date
            plt.figure(figsize=[10, 6])
            x = interp_date
            y = interp_cloudt
            plt.scatter(x, y, c='g', s=2, label='intp_'+var)
            y = interp_skyt
            plt.scatter(x, y, c='b', s=2, label='Sky_T')
            plt.xlim([min(x), max(x)])
            plt.legend()
            plt.tight_layout()
            plt.show()

            plt.figure(figsize=[8, 9])
            x = interp_cloudt
            y = interp_skyt
            plt.hist2d(x, y, bins=NBINS, cmap='Greys')  # , norm=matplotlib.colors.LogNorm())

            plt.xlabel('interp_'+var)
            plt.ylabel('Sky_T')
            plt.title('Correlation coef:' + str(np.corrcoef(x, y)))
            plt.tight_layout()
            plt.show()

else:
    for var in COL_NAMES:
        # vs date
        y = df_all[var]
        if var in ['Date', 'date_diff']:
            x = np.arange(len(y))
        else:
            x = df_all["Date"]
        plt.figure(figsize=[10, 6])
        plt.scatter(x, y, c='r', marker='.', s=2)
        # if var == 'date_diff':
        #    plt.ylim(DATE_DIFF_LIM)
        if var == 'Sky_T_over_Sun_T':
            ax = plt.gca()
            ax.set_yscale('log')

        plt.xlabel(df_all["Date"].name)
        plt.ylabel(df_all[var].name + COL_UNITS[var])
        plt.tight_layout()
        plt.savefig(OPATH+'/'+var, dpi=DPI)
        plt.close()

        # vs hour of that day
        if MICAF == 'mica_hourly':
            fig = plt.figure(figsize=[10, 6])
            for f in mf:
                yyyymmdd = f.split('/')[-1].split('.')[0]
                df = df_all[df_all['Date'].dt.strftime('%Y%m%d') == yyyymmdd]
                x = [datetime.combine(datetime.today(), datetime.time(i)) for i in pd.to_datetime(
                    df['Date'])]
                y = df[var]
                plt.scatter(x, y, marker='.', s=1, label=yyyymmdd)
                if var in EMP_CLASS:
                    plt.plot([min(x), max(x)], [EMP_CLASS[var][0], EMP_CLASS[var][0]], 'k-', linewidth=2)
                    plt.plot([min(x), max(x)], [EMP_CLASS[var][1], EMP_CLASS[var][1]], 'k-', linewidth=2)
            plt.xlabel('Hour of the day')
            plt.ylabel(df_all[var].name + COL_UNITS[var])
            plt.tight_layout()
            plt.grid(True)
            ax = plt.gca()
            monthyearFmt = mdates.DateFormatter('%H:%M')
            ax.xaxis.set_major_formatter(monthyearFmt)
            if var == 'Sun_T':
                #plt.ylim([0.4, 1])
                lgnd = plt.legend(loc='lower right')
                for handle in lgnd.legendHandles:
                    handle.set_sizes([40.0])
            plt.savefig(OPATH+'/'+var+'_hour', dpi=DPI)
            plt.close()

        # histograms
        plt.figure(figsize=[10, 6])
        if var == 'date_diff':
            plt.hist(y, log=True, bins=np.arange(DATE_DIFF_LIM[1]), color='k', histtype='step')
        elif var == 'Sky_T_over_Sun_T':
            plt.hist(y, log=True, bins=NBINS, range=(0,5), color='k', histtype='step')
        else:
            plt.hist(y, log=True, bins=NBINS, color='k', histtype='step')
        if var in EMP_CLASS:
            plt.plot([EMP_CLASS[var][0], EMP_CLASS[var][0]], [0, 4e6], 'k--', linewidth=2)
            plt.plot([EMP_CLASS[var][1], EMP_CLASS[var][1]], [0, 4e6],  'k--', linewidth=2)
        plt.grid()
        plt.ylabel('number of cases')
        plt.xlabel(df_all[var].name + COL_UNITS[var])
        plt.tight_layout()
        plt.savefig(OPATH+'/'+var+'_hist', dpi=DPI)
        plt.close()

    # 2d scatter
    plt.figure(figsize=[10, 10])
    for i in range(N_CLUSTERS):
        y = df_all[df_all['sky_class'] == CLUSTERS[i]]['Sky_T']
        x = df_all[df_all['sky_class'] == CLUSTERS[i]]['Sun_T']
        plt.scatter(x, y, c=CLUSTERS_COLOR[i], marker='.', s=0.1, label=CLUSTERS[i])
        plt.plot([EMP_CLASS['Sun_T'][0], max(x)], [EMP_CLASS['Sky_T'][0], EMP_CLASS['Sky_T'][0]], 'k--', linewidth=2)
        plt.plot([EMP_CLASS['Sun_T'][0], max(x)], [EMP_CLASS['Sky_T'][1], EMP_CLASS['Sky_T'][1]], 'k--', linewidth=2)
        plt.plot([EMP_CLASS['Sun_T'][0], EMP_CLASS['Sun_T'][1]], [min(y), max(y)], 'k--', linewidth=2)
    if km is not None:
        plt.scatter(km.cluster_centers_[:, 0], km.cluster_centers_[:, 1], color='b', marker='*', s=100)
    plt.xlabel(df_all['Sun_T'].name + COL_UNITS['Sun_T'])
    plt.ylabel(df_all['Sky_T'].name + COL_UNITS['Sky_T'])
    plt.tight_layout()
    plt.savefig(OPATH+'/Sky_T_vs_Sun_T', dpi=DPI)
    plt.close()

    # 2d histogram
    plt.figure(figsize=[8, 9])
    y = df_all['Sky_T']
    x = df_all['Sun_T']
    w = df_all['date_diff']/3600.
    hist, xbins, ybins = np.histogram2d(x, y, bins=NBINS, weights=w)
    plt.imshow(hist.T, extent=[min(xbins), max(xbins), min(ybins), max(ybins)],
               origin='lower', norm=colors.LogNorm(vmin=5./60., vmax=30*24), cmap='Greys')
    plt.colorbar(label='Total time [h]')
    if km is not None:
        plt.scatter(km.cluster_centers_[:, 0], km.cluster_centers_[:, 1], color='b', marker='*', s=100)
    plt.plot([EMP_CLASS['Sun_T'][0], max(x)], [EMP_CLASS['Sky_T'][0], EMP_CLASS['Sky_T'][0]], 'k--', linewidth=2)
    plt.plot([EMP_CLASS['Sun_T'][0], max(x)], [EMP_CLASS['Sky_T'][1], EMP_CLASS['Sky_T'][1]], 'k--', linewidth=2)
    plt.plot([EMP_CLASS['Sun_T'][0], EMP_CLASS['Sun_T'][1]], [min(y), max(y)], 'k--', linewidth=2)
    plt.xlabel(df_all['Sun_T'].name + COL_UNITS['Sun_T'])
    plt.ylabel(df_all['Sky_T'].name + COL_UNITS['Sky_T'])
    plt.tight_layout()
    plt.savefig(OPATH+'/Sky_T_vs_Sun_T_hist', dpi=DPI)
    plt.close()

    # plots bars of sky-class per year
    plt.figure(figsize=[10, 8])
    years = df_all['Date'].dt.year.unique()
    n0 = None
    print('sky_class_per_year:')
    print(years)
    for i in range(len(CLUSTERS)):
        n = []
        for y in years:
            # total time in all CLUSTERs per year
            tot = df_all.loc[df_all['Date'].dt.year == y, 'date_diff'].sum()
            # print(tot)
            if tot == 0:
                tot = 1
            n.append(df_all.loc[(df_all['sky_class'] == CLUSTERS[i]) & (
                df_all['Date'].dt.year == y), 'date_diff'].sum()/tot*100.)
        plt.bar(years, n, color=CLUSTERS_COLOR[i], label=CLUSTERS[i], zorder=3, bottom=n0)
        print(CLUSTERS[i], n)
        if n0 is None:
            n0 = np.array(n)
        else:
            n0 += np.array(n)
    plt.legend()
    plt.grid()
    plt.xlabel('year')
    plt.ylabel('$\%$ of observed time')
    plt.tight_layout()
    plt.savefig(OPATH+'/sky_class_per_year', dpi=DPI)
    plt.close()

    # plots bars of sky-class per month
    plt.figure(figsize=[10, 8])
    months = df_all['Date'].dt.month.unique()
    n0 = None
    print('sky_class_per_month:')
    print(months)
    for i in range(len(CLUSTERS)):
        n = []
        for m in months:
            # total time in all CLUSTERs per year
            tot = df_all.loc[df_all['Date'].dt.month == m, 'date_diff'].sum()
            # print(tot)
            if tot == 0:
                tot = 1
            n.append(df_all.loc[(df_all['sky_class'] == CLUSTERS[i]) & (
                df_all['Date'].dt.month == m), 'date_diff'].sum()/tot*100.)
        plt.bar(months, n, color=CLUSTERS_COLOR[i], label=CLUSTERS[i], zorder=3, bottom=n0)
        print(CLUSTERS[i], n)
        if n0 is None:
            n0 = np.array(n)
        else:
            n0 += np.array(n)
    plt.legend()
    plt.grid()
    plt.xlabel('month')
    plt.ylabel('$\%$ of observed time')
    plt.tight_layout()
    plt.savefig(OPATH+'/sky_class_per_month', dpi=DPI)
    plt.close()

    # plots bars of sky-class per year only of sunny time (no clouds)
    print('sky_class_per_year_sunny_time:')
    print(years)
    plt.figure(figsize=[10, 8])
    years = df_all['Date'].dt.year.unique()
    n0 = None
    for i in range(len(CLUSTERS)-1):
        n = []
        for y in years:
            # total number of elements in CLUSTER[0:2] per year
            tot = df_all.loc[(df_all['sky_class'] != CLUSTERS[3]) & (df_all['Date'].dt.year == y), 'date_diff'].sum()
            if tot == 0:
                tot = 1
            n.append(df_all.loc[(df_all['sky_class'] == CLUSTERS[i]) & (
                df_all['Date'].dt.year == y), 'date_diff'].sum()/tot*100.)
        plt.bar(years, n, color=CLUSTERS_COLOR[i], label=CLUSTERS[i], zorder=0, bottom=n0)
        # plt.plot(years,np.zeros(len(n))+np.mean(n),'--',color=CLUSTERS_COLOR[i] ,linewidth=4,zorder=1)
        if n0 is None:
            n0 = np.array(n)
        else:
            n0 += np.array(n)
        print(CLUSTERS[i], n)
    plt.legend()
    plt.grid()
    plt.xlabel('year')
    plt.ylabel('$\%$ of Sunny time')
    plt.tight_layout()
    plt.savefig(OPATH+'/sky_class_per_year_sunny_time', dpi=DPI)
    plt.close()

    # plots bars of sky-class per month only of sunny time (no clouds)
    print('sky_class_per_month_sunny_time:')
    print(months)
    plt.figure(figsize=[10, 8])
    months = df_all['Date'].dt.month.unique()
    n0 = None
    for i in range(len(CLUSTERS)-1):
        n = []
        for m in months:
            # total number of elements in CLUSTER[0:2] per month
            tot = df_all.loc[(df_all['sky_class'] != CLUSTERS[3]) & (df_all['Date'].dt.month == m), 'date_diff'].sum()
            if tot == 0:
                tot = 1
            n.append(df_all.loc[(df_all['sky_class'] == CLUSTERS[i]) & (
                df_all['Date'].dt.month == m), 'date_diff'].sum()/tot*100.)
        plt.bar(months, n, color=CLUSTERS_COLOR[i], label=CLUSTERS[i], zorder=0, bottom=n0)
        # plt.plot(months,np.zeros(len(n))+np.mean(n),'--',color=CLUSTERS_COLOR[i] ,linewidth=4,zorder=1)
        if n0 is None:
            n0 = np.array(n)
        else:
            n0 += np.array(n)
        print(CLUSTERS[i], n)
    plt.legend()
    plt.grid()
    plt.xlabel('month')
    plt.ylabel('$\%$ of Sunny time')
    plt.tight_layout()
    plt.savefig(OPATH+'/sky_class_per_month_sunny_time', dpi=DPI)
    plt.close()


# -*- coding: utf-8 -*-
"""
Created on Sat Jul 20 21:16:51 2019

@author: Mirac
"""

import torch
import pandas as pd
from torchvision import transforms, utils
from torch.utils.data import Dataset, DataLoader
from torchvision import transforms, utils
import sklearn
import sklearn.metrics as sklm
from torch.autograd import Variable
import numpy as np

def make_pred_multilabel(data_transforms, model, PATH_TO_MAIN_FOLDER, UNCERTAINTY, N_LABELS, epoch=0):
    """
    Gives predictions for test fold and calculates AUCs using previously trained model

    Args:
        data_transforms: torchvision transforms to preprocess raw images; same as validation transforms
        model: the model trained on chexpert images
        PATH_TO_MAIN_FOLDER: path where the extracted chexpert data is located
    Returns:
        pred_df: dataframe containing individual predictions and ground truth for each test image
        auc_df: dataframe containing aggregate AUCs by train/test tuples
    """

    # calc preds in batches of 32, can reduce if your GPU has less RAM
    BATCH_SIZE = 32

    # set model to eval mode; required for proper predictions given use of batchnorm
    model.train(False)

    # create dataloader
    dataset = CheXpertDataset(
        path_to_main_folder=PATH_TO_MAIN_FOLDER,
        fold="test",
        transform=data_transforms['valid'],
        uncertainty=UNCERTAINTY)
    dataloader = torch.utils.data.DataLoader(
        dataset, BATCH_SIZE, shuffle=False, num_workers=8)

    # create empty dfs
    pred_df = pd.DataFrame(columns=["Image Index"])
    true_df = pd.DataFrame(columns=["Image Index"])

    # iterate over dataloader
    for i, data in enumerate(dataloader):

        inputs, labels, _ = data
        inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())

        true_labels = labels.cpu().data.numpy()
        batch_size = true_labels.shape

        if UNCERTAINTY in ["multiclass", 'weighted_multiclass']:
          nn_outputs = model(inputs)
          multiclass_probs = torch.softmax(nn_outputs.view(-1, 3), dim=1)
          outputs = torch.softmax(multiclass_probs[:,[1,2]], dim=1)[:,1].view(-1, N_LABELS)
        else:
          outputs = model(inputs)
        probs = outputs.cpu().data.numpy()

        # get predictions and true values for each item in batch
        for j in range(0, batch_size[0]):
            thisrow = {}
            truerow = {}
            thisrow["Image Index"] = dataset.df.index[BATCH_SIZE * i + j]
            truerow["Image Index"] = dataset.df.index[BATCH_SIZE * i + j]

            # iterate over each entry in prediction vector; each corresponds to
            # individual label
            for k in range(len(dataset.PRED_LABEL)):
                thisrow["prob_" + dataset.PRED_LABEL[k]] = probs[j, k]
                truerow[dataset.PRED_LABEL[k]] = true_labels[j, k]

            pred_df = pred_df.append(thisrow, ignore_index=True)
            true_df = true_df.append(truerow, ignore_index=True)

        if(i % 10 == 0):
            print(str(i * BATCH_SIZE))

    auc_df = pd.DataFrame(columns=["label", "auc"])

    # calc AUCs
    for column in true_df:

        if column not in [
            'No Finding',
            'Enlarged Cardiomediastinum',
            'Cardiomegaly',
            'Lung Opacity',
            'Lung Lesion',
            'Edema',
            'Consolidation',
            'Pneumonia',
            'Atelectasis',
            'Pneumothorax',
            'Pleural Effusion',
            'Pleural Other',
            'Fracture',
            'Support Devices']:
                    continue
        actual = true_df[column]
        pred = pred_df["prob_" + column]
        thisrow = {}
        thisrow['label'] = column
        thisrow['auc'] = np.nan
        try:
            thisrow['auc'] = sklm.roc_auc_score(
                actual.as_matrix().astype(int), pred.as_matrix())
        except BaseException:
            print("can't calculate auc for " + str(column))
        auc_df = auc_df.append(thisrow, ignore_index=True)

    if epoch == 0:
      pred_df.to_csv(os.path.join('results',UNCERTAINTY,'preds.csv'), index=False)
      auc_df.to_csv(os.path.join('results',UNCERTAINTY,'aucs.csv'), index=False)

    else:
      pred_df.to_csv(os.path.join('results',UNCERTAINTY,'preds'+ str(epoch) + '.csv'), index=False)
      auc_df.to_csv(os.path.join('results',UNCERTAINTY,'aucs' + str(epoch) + '.csv'), index=False)

    return pred_df, auc_df