import signal
import logging
import argparse
import time
import os

"""custom logger"""
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)
file_handler = logging.FileHandler('dirwatcher.log')
formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(message)s')
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
searched_files = {}
exit_flag = True

def receive_signal(sig, stack):
    """Listener for SIGINT and SIGTERM"""
    logger.warning("Received signal: {}".format(sig))
    global exit_flag
    if sig == signal.SIGINT:
        exit_flag = False
    if sig == signal.SIGTERM:
        exit_flag = False

def find_magic(file, directory):
    """checks for magic word"""
    magic_word = 'beetlejuice'
    with open(directory + '/' + file) as f:
        for index, line in enumerate(f.readlines()):
            if magic_word in line and index not in searched_files[file]:
                logger.info('{} found at line {} in {}'
                            .format(magic_word, index + 1, file))
                searched_files[file].append(index)

def dir_watcher(args):
    directory = os.path.abspath(args.dir)
    log_files = [f for f in os.listdir(directory) if ".txt" in f]
    if len(log_files) > len(searched_files):
        for file in log_files:
            if file not in searched_files:
                logger.info(' {} found in {}'.format(file, args.dir))
                searched_files[file] = []
    elif len(log_files) < len(searched_files):
        for file in searched_files:
            if file not in log_files:
                logger.info(" {} removed from {}".format(file, args.dir))
                searched_files.pop(file, None)
    for file in log_files:
        find_magic(file, directory)

def main():
    start_time = time.time()
    parser = argparse.ArgumentParser(
        description="Checks a directory for modified files")
    parser.add_argument('--dir', help="Directory to be checked")

    args = parser.parse_args()

    logger.info('Searching in {}'.format(args.dir))

    signal.signal(signal.SIGINT, receive_signal)
    signal.signal(signal.SIGTERM, receive_signal)

    while exit_flag:
        try:
            dir_watcher(args)
        except IOError:
            logger.exception('There was an error opening the directory')
            time.sleep(5)
        except Exception:
            logger.exception('Unhandled exception')
            time.sleep(5)
    logger.info('Uptime: {} seconds'.format(time.time() - start_time))

if __name__ == '__main__':
        main()


# A Trading bot to be deployed on quantconnect

"""
@author: Niraj

"""
# This is the main script for the trading

from lstm_script import MyLSTM
from xgboost_script import XGBoost
from arima_script import MyARIMA
from datetime import timedelta, datetime

class MultidimensionalHorizontalFlange(QCAlgorithm):

    def Initialize(self):
        self.SetStartDate(2017, 1, 4)  # Set Start Date
        self.SetCash(100000)  # Set Strategy Cash
        self.SetEndDate(2021, 8, 1)

        """ Check for them """
        self.SetBrokerageModel(AlphaStreamsBrokerageModel())

        self.SetExecution(ImmediateExecutionModel())

        # self.SetPortfolioConstruction(EqualWeightingPortfolioConstructionModel())

        self.UniverseSettings.Resolution = Resolution.Daily
        self.SetUniverseSelection(LiquidETFUniverse())

        # Helper dictionaries
        self.models = {"Arima": None, "Arima_results": None, "XGBoost": None, "LSTM": None}

        # Initializing the expiry date
        self.lastest_expiry = datetime.min

        # Initializing the Securities
        equity = self.AddEquity('SPY')
        option = self.AddOption("SPY", Resolution.Minute)
        self.symbol_option = option.Symbol
        self.symbol_equity = equity.Symbol

        # Adding a consolidator as the options data is available in minute scale, and we need daily scale
        self.Consolidate(self.symbol_option, Resolution.Daily, self.OnOptionConsolidated)

        # Schedule prediction, and trading
        self.Schedule.On(self.DateRules.EveryDay('SPY'), self.TimeRules.At(6, 0), Action(self.Rebalance))
        self.Schedule.On(self.DateRules.EveryDay('SPY'), self.TimeRules.At(7, 0), Action(self.PredictModels))
        self.Schedule.On(self.DateRules.EveryDay('SPY'), self.TimeRules.AfterMarketOpen('SPY', 5), self.TradeOptions)

    def OnData(self,slice):

        # We wont use OnData, and rather used our rebalanced timeline
        pass

    def OnOptionConsolidated(self, consolidated):

        if self.lastest_expiry.date() == self.Time.date():
            self.symbol_consolidated = consolidated.Symbol
            self.optionchain = self.CurrentSlice.OptionChains

    def Rebalance(self):

        # Getting the Date for the nearest expiry
        calendar = self.TradingCalendar.GetDaysByType(TradingDayType.OptionExpiration, self.Time, self.EndDate)
        expiries = [i.Date for i in calendar]
        if len(expiries) == 0: return
        self.lastest_expiry = expiries[0]

    def PredictModels(self):

        if self.lastest_expiry.date() == self.Time.date():

            # Fetching the dataset, and performing training and prediction on it
            qb = self
            history = qb.History([self.symbol_equity], 1280, Resolution.Daily)
            data = history.loc[self.symbol_equity].close

            # (1) ... Creating the LSTM object and model
            lstm_obj = MyLSTM()
            self.features_set, self.labels, self.training_data_scaled = lstm_obj.ProcessData_LSTM(data)
            lstm_model = lstm_obj.Create_LSTM(self.features_set, self.labels)
            self.models["LSTM"] = lstm_model

            # (2) ... Creating the Arima object and model
            arima_obj = MyARIMA()
            arima_model = arima_obj.Create_ARIMA(data)
            self.models["Arima"] = arima_model

            # (3) ... Creating the XGBoost object and model
            xgboost_obj = XGBoost()
            self.train_data_X, self.train_data_Y = xgboost_obj.ProcessData_XGBoost(data)
            xgboost_model = xgboost_obj.Create_XGBoost()
            self.models['XGBoost'] = xgboost_model

            # Using self.Train as it avoids runtime error
            self.Train(self.TrainModels)

            # Gathering the predictions from models individually
            prediction_lstm =  lstm_obj.Forecast_LSTM(self.models["LSTM"], self.training_data_scaled)
            prediction_arima = arima_obj.Forecast_ARIMA(self.models["Arima_results"])
            prediction_xgboost = xgboost_obj.Forecast_XGBoost(self.models['XGBoost'], self.train_data_X, self.train_data_Y)

            # Storing the ensemble prediction
            self.prediction = 0.7*prediction_arima + 0.1*prediction_xgboost + 0.2*prediction_lstm
            self.prediction = prediction_xgboost
            self.change  = ((self.prediction - float(data[-1]))/self.prediction)*100

    def TrainModels(self):

        # Function to train models, and avoiding runtime error
        self.models['Arima'] = arima_obj.Fit_ARIMA(self.models['Arima'])
        self.models['LSTM'] = lstm_obj.Fit_LSTM(self.models['LSTM'], self.features_set, self.labels)
        self.models['XGBoost'] = xgboost_obj.Fit_XGBoost(self.models['XGBoost'], self.train_data_X, self.train_data_Y)

        # Function to train models, and avoiding runtime error
        self.model['Arima_results'] = self.model['Arima'].fit(disp=0)
        self.model['LSTM'].fit(self.features_set, self.labels, epochs = 50, batch_size = 32)
        self.model['XGBoost'].fit(self.train_data_X, self.train_data_Y)

    def TradeOptions(self):

        """ TradeOptions function when triggered, identifies the best spread based option trading stratergy
            using the underlying price, and market forecast and Executes it """

        if self.lastest_expiry.date() == self.Time.date():

            """ Bull Spread """

            if self.change > float(5):

                for i in self.optionchain:

                    # Using only the optionchain with symbol as SPY options Symbol
                    if i.Key != self.symbol_consolidated: continue
                    chain = i.Value

                    # Sorting the optionchain according to expiry, and choosing the nearest one (on latest_expiry)
                    expiry = sorted(chain,key = lambda x: x.Expiry, reverse=False)[0].Expiry

                    # Filtering the call options from the contracts that expires on that date
                    call = [i for i in chain if i.Expiry == expiry and i.Right == 0]

                    # Sorting the call options, with strike greater than prediction
                    call_contracts_buy = [i for i in call if i.Strike < self.prediction]
                    call_contratcs_sell = [i for i in call if i.Strike > self.prediction]

                    if len(call_contracts_buy) == 0: continue
                    if len(call_contracts_sell) == 0: continue

                    # getting the call option with the just low and higher strike price acc to prediction
                    self.call_buy = sorted(call_contracts_buy, key = lambda x: x.Strike, reverse=True)[0]
                    self.call_sell = sorted(call_contratcs_sell, key = lambda x: x.Strike, reverse=False)[0]

                    # Placing the orders according to bull spread
                    self.Buy(self.call_buy.Symbol, 10)
                    self.Sell(self.call_sell.Symbol ,10)

            elif self.change < float(5):

                """ Bear Spread """

                for i in self.optionchain:

                    # Using only the optionchain with symbol as SPY options Symbol
                    if i.Key != self.symbol_consolidated: continue
                    chain = i.Value

                    # Sorting the optionchain according to expiry, and choosing the nearest one (on latest_expiry)
                    expiry = sorted(chain,key = lambda x: x.Expiry, reverse=False)[0].Expiry

                    # Filtering the put options from the contracts that expires on that date
                    put = [i for i in chain if i.Expiry == expiry and i.Right == 1]

                    # Sorting the put options, with strike greater than prediction
                    put_contracts_sell = [i for i in put if i.Strike < self.prediction]
                    put_contratcs_buy = [i for i in put if i.Strike > self.prediction]

                    if len(put_contratcs_buy) == 0: continue
                    if len(put_contracts_sell) == 0: continue

                    # getting the put option with the just low and higher strike price a/c to prediction
                    self.put_buy = sorted(put_contratcs_buy, key = lambda x: x.Strike, reverse=False)[0]
                    self.put_sell = sorted(put_contracts_sell, key = lambda x: x.Strike, reverse=True)[0]

                    # Placing the orders according to bear spread
                    self.Buy(self.put_buy.Symbol, 10)
                    self.Sell(self.put_sell.Symbol ,10)

            else:

                """ Butterfly Spread """

                for i in self.optionchain:

                    # Using only the optionchain with symbol as SPY options Symbol
                    if i.Key != self.symbol_consolidated: continue
                    chain = i.Value

                    # Sorting the optionchain according to expiry, and choosing the nearest ones (on latest_expiry)
                    expiry = sorted(chain,key = lambda x: x.Expiry, reverse=True)[0].Expiry

                    # Filtering the call options from the contracts that expires on that date
                    call = [i for i in chain if i.Expiry == expiry and i.Right == 0]

                    # Sorting the call options using strike prices
                    call_contracts = sorted(call,key = lambda x: x.Strike, reverse=False)

                    if len(call_contracts) == 0: continue

                    # choose OTM call
                    self.otm_call = [i for i in call_contracts if i.Strike > self.prediction][0]
                    # choose ITM call
                    self.itm_call = [i for i in call_contracts if i.Strike < self.prediction][-1]
                    # choose ATM call
                    self.atm_call = sorted(call_contracts,key = lambda x: abs(chain.Underlying.Price - x.Strike))[0]

                    # Placing orders according to butterfly spread
                    self.Sell(self.atm_call.Symbol ,20)
                    self.Buy(self.itm_call.Symbol ,10)
                    self.Buy(self.otm_call.Symbol ,10)


import json
import requests
from bs4 import BeautifulSoup

city_code_name_pairs = {
    '1100': '서울특별시',
    '2600': '부산광역시',
    '2700': '대구광역시',
    '2800': '인천광역시',
    '2900': '광주광역시',
    '3000': '대전광역시',
    '3100': '울산광역시',
    '5100': '세종특별자치시',
    '4100': '경기도',
    '4200': '강원도',
    '4300': '충청북도',
    '4400': '충청남도',
    '4500': '전라북도',
    '4600': '전라남도',
    '4700': '경상북도',
    '4800': '경상남도',
    '4900': '제주특별자치도'
}

def fetch_election_district_info(city_code):
    url = ('http://info.nec.go.kr/electioninfo/electionInfo_report.xhtml?'
           'electionId=0020120411&requestURI=%%2Felectioninfo%%2F0020120411%%2Fbi%%2Fbigi05.jsp'
           '&statementId=BIGI05&electionCode=2&cityCode=%d&townCode=-1') % city_code
    html = requests.get(url).text
    soup = BeautifulSoup(html, 'html.parser')
    tbody = soup.find('tbody')
    trs = tbody.find_all('tr')

    district_list = []
    last_district_name = None
    for tr in trs:
        tds = tr.find_all('td')
        associated_towns = [x.strip() for x in tds[2].string.split(',')]
        district_name = tds[0].string
        if district_name is None:
            district_name = last_district_name
        local_name = tds[1].string

        last_district_name = district_name
        precint_info = {
            'name': district_name,
            'local': local_name,
            'associated_towns': associated_towns
        }
        district_list.append(precint_info)
    return district_list

def fetch_all_city_district_infos():
    city_infos = []
    for k, v in city_code_name_pairs.items():
        city_info = {
            'code': k,
            'name': v,
            'district_info': fetch_election_district_info(int(k))
        }
        city_infos.append(city_info)
    return city_infos

if __name__ == '__main__':
    data_json = json.dumps(fetch_all_city_district_infos(), ensure_ascii=False)
    with open('election-data.json', 'w', encoding='utf-8') as f:
        f.write(data_json)


#!/usr/bin/env python
from __future__ import absolute_import, division, print_function

# Make sure we can import stuff from util/
# This script needs to be run from the root of the DeepSpeech repository
import sys
import os
sys.path.insert(1, os.path.join(sys.path[0], '..'))
import argparse as ap
import codecs
import fnmatch
import pandas
import progressbar
import subprocess
import tarfile
import unicodedata
import json
from utils import update_progress
from sox import Transformer
from tensorflow.contrib.learn.python.learn.datasets import base
from tensorflow.python.platform import gfile

def _preprocess_data(data_dir,target_line):

    print("Converting trn and splitting transcriptions...")
    work_dir = data_dir
    phone_set = set()
    train_df, phone_set = _convert_audio_and_split_sentences(work_dir, "train", "train-wav", "train-"+str(target_line)+"-txt", phone_set,target_line)
    dev_df, phone_set = _convert_audio_and_split_sentences(work_dir, "dev", "dev-wav", "dev-"+str(target_line)+"-txt", phone_set,target_line)
    test_df, phone_set = _convert_audio_and_split_sentences(work_dir, "test", "test-wav", "test-"+str(target_line)+"-txt", phone_set,target_line)

    # Write sets to disk as CSV files
    train_df.to_csv(os.path.join(data_dir, "thchs30-train-"+str(target_line)+".csv"), index=False,header=False)
    dev_df.to_csv(os.path.join(data_dir, "thchs30-dev-"+str(target_line)+".csv"), index=False,header=False)
    test_df.to_csv(os.path.join(data_dir, "thchs30-test-"+str(target_line)+".csv"), index=False,header=False)
    json.dump(list(phone_set), open(os.path.join(data_dir,"thchs30_phone_"+str(target_line)+".json"),'w'), indent=4)

def _convert_audio_and_split_sentences(extracted_dir, data_set, dest_dir, dest_dir2, phone_set,target_line):
    source_dir = os.path.join(extracted_dir, data_set)
    target_dir = os.path.join(extracted_dir, dest_dir)
    target_txt_dir = os.path.join(extracted_dir, dest_dir2)

    if not os.path.exists(target_dir):
        os.makedirs(target_dir)
    if not os.path.exists(target_txt_dir):
        os.makedirs(target_txt_dir)

    files = []
    for root, dirnames, filenames in os.walk(source_dir):
        print(root, dirnames, len(filenames))
        for filename in fnmatch.filter(filenames, '*.trn'):
            tmp_filename = os.path.join(root, filename)
            with open(tmp_filename, "r") as fin:
                target_file = os.path.join(root, fin.readline().strip())
            with open(target_file, "r") as fin:
                curr_line = 0
                while curr_line < target_line:
                    line = fin.readline().strip()
                    curr_line += 1
                    # Parse each segment line
                transcript = line.split(' ')
                phone_set = phone_set | set(transcript)
                wav_file = os.path.join(target_dir,  filename[:-8]+'.wav')
                txt_file = os.path.join(target_txt_dir, filename[:-8]+".txt")
                txt_fid = open(txt_file,'w')
                for index, phone in enumerate(transcript):
                    txt_fid.write(phone+'\n')
                    if (index+1) % (target_line-1) == 0:
                        txt_fid.write('<space>\n')

                txt_fid.close()
                if not os.path.exists(wav_file):
                    os.system('cp '+target_file[:-4]+' '+wav_file)
                    # wav_filesize = os.path.getsize(wav_file)

                files.append((os.path.abspath(wav_file), os.path.abspath(txt_file)))

    return pandas.DataFrame(data=files, columns=["wav_filename", "txt_filename"]), phone_set

if __name__ == "__main__":
    parser = ap.ArgumentParser()
    parser.add_argument("--target_line",help="Path to input files", type=int, default=3)
    args = parser.parse_args()
    _preprocess_data('/data1/data_all/thchs30',args.target_line)

# thchs30
# |-- data
# |-- dev
# |-- lm_phone
# |-- lm_word
# |-- README.TXT
# |-- test
# `-- train


"""
terraform_plan provider

Error severity values explanation
Value = 1, When a resource is not found
Value = 2, When an attribute of a resource is not found
"""
# input->(list ["a.b","c", "d"],value of resource)
# returns->[any, any, any]
import pydash

from ..common import ProviderError

class PydashPathNotFound:
    pass

def _wrapper_get_exp_attribute(attribute, input_resource_change_attrs):
    splitted_attribute = attribute.split(".*.")
    return _get_exp_attribute(splitted_attribute, input_resource_change_attrs)

def _get_exp_attribute(split_expressions, input_data):
    # split_expressions=expression.split('*')
    final_data = []
    for i, expression in enumerate(split_expressions):
        intermediate_val = pydash.get(input_data, expression, default=PydashPathNotFound)
        if isinstance(intermediate_val, list) and i < len(split_expressions) - 1:
            for val in intermediate_val:
                final_attributes = _get_exp_attribute(split_expressions[1:], val)
                for final_attribute in final_attributes:
                    final_data.append(final_attribute)
        elif i == len(split_expressions) - 1 and intermediate_val is not PydashPathNotFound:
            final_data.append(intermediate_val)
        elif ".*" in expression:
            intermediate_exp = expression.split(".*")
            intermediate_data = pydash.get(input_data, intermediate_exp[0], default=PydashPathNotFound)
            if intermediate_data is not PydashPathNotFound and isinstance(intermediate_data, list):
                for val in intermediate_data:
                    final_data.append(val)
    return final_data

def provide(provider_inputs, input_data):
    # """Provides the value of the attribute from the input_data"""
    outputs = []
    input_resource_change_attrs = {}
    input_type = provider_inputs["operation_type"]
    resource_changes = input_data.get("resource_changes")
    if not resource_changes:
        outputs.append(
            {
                "value": ProviderError(severity_value=0),
                "err": f"No Terraform resources changes are found",
            }
        )
        return outputs
    # CASE 1
    # - Get value of an attribute for all instances of a resource
    # - resource_changes.*.change.after.<attr_name>
    if input_type == "attribute":
        attribute = provider_inputs["terraform_resource_attribute"]
        resource_type = provider_inputs["terraform_resource_type"]

        is_resource_found = False
        is_attribute_found = False

        for resource_change in resource_changes:
            if resource_change["type"] == resource_type:
                is_resource_found = True
                input_resource_change_attrs = resource_change["change"]["after"]
                if input_resource_change_attrs:
                    if attribute in input_resource_change_attrs:
                        is_attribute_found = True
                        outputs.append(
                            {
                                "value": input_resource_change_attrs[attribute],
                                "meta": resource_change,
                                "err": None,
                            }
                        )
                    elif "." in attribute or "*" in attribute:
                        evaluated_outputs = _wrapper_get_exp_attribute(attribute, input_resource_change_attrs)
                        if evaluated_outputs:
                            is_attribute_found = True
                        for evaluated_output in evaluated_outputs:
                            outputs.append({"value": evaluated_output, "meta": resource_change, "err": None})
                else:
                    outputs.append(
                        {
                            "value": ProviderError(severity_value=0),
                            "err": f"No Terraform changes found for resource type: '{resource_type}'",
                        }
                    )
        if not outputs:
            if not is_resource_found:
                outputs.append(
                    {
                        "value": ProviderError(severity_value=1),
                        "err": f"resource_type: '{resource_type}' is not found",
                    }
                )
                return outputs
            if not is_attribute_found:
                outputs.append(
                    {
                        "value": ProviderError(severity_value=2),
                        "err": f"attribute: '{attribute}' is not found",
                    }
                )

        return outputs
    # CASE 2
    # - Get actions performed on a resource
    # - resource_changes.*.change.actions
    elif input_type == "action":
        resource_type = provider_inputs["terraform_resource_type"]
        is_resource_type_found = False
        for resource_change in resource_changes:
            if resource_change["type"] == resource_type:
                is_resource_type_found = True
                for action in resource_change["change"]["actions"]:
                    outputs.append(
                        {
                            "value": action,
                            "meta": resource_change,
                            "err": None,
                        }
                    )
        if not is_resource_type_found:
            outputs.append(
                {
                    "value": ProviderError(severity_value=1),
                    "err": f"resource_type: '{resource_type}' is not found",
                }
            )
        return outputs
    # CASE 3
    # - Get count of a particular resource
    # - resource_changes.*.index
    elif input_type == "count":
        count = 0
        resource_meta = {}
        resource_type = provider_inputs["terraform_resource_type"]
        for resource_change in resource_changes:
            if resource_change["type"] == resource_type:
                # No need to check if the resource is not found
                # because the count of a resource can be zero
                resource_meta = resource_change
                count = +1

        outputs.append(
            {
                "value": count,
                "meta": resource_meta,
                "err": None,
            }
        )
        return outputs
    # CASE 4
    elif input_type == "direct_dependencies":
        direct_dependencies_operator(input_data, provider_inputs, outputs)
    elif input_type == "direct_references":
        direct_references_operator(input_data, provider_inputs, outputs)
    else:
        outputs.append(
            {
                "value": ProviderError(severity_value=99),
                "err": f"operation_type: '{input_type}' is not supported (severity_value: 99)",
            }
        )
    return outputs

def direct_dependencies_operator(input_data: dict, provider_inputs: dict, outputs: list):
    config_resources = input_data.get("configuration", {}).get("root_module", {}).get("resources", [])
    resource_type = provider_inputs.get("terraform_resource_type")

    if not resource_type:
        outputs.append(
            {
                "value": ProviderError(severity_value=99),
                "err": "`terraform_resource_type` must be provided in the provider input (severity_value: 99))",
            }
        )
        return

    is_resource_found = False

    for resource in config_resources:
        if resource.get("type") != resource_type:
            continue
        is_resource_found = True
        deps_resource_type = {resource_id.split(".")[0] for resource_id in resource.get("depends_on", [])}
        outputs.append({"value": list(deps_resource_type), "meta": config_resources})

    if not is_resource_found:
        outputs.append(
            {
                "value": ProviderError(severity_value=1),
                "err": f"resource_type: '{resource_type}' is not found",
                "meta": config_resources,
            }
        )

def direct_references_operator(input_data: dict, provider_inputs: dict, outputs: list):
    config_resources = input_data.get("configuration", {}).get("root_module", {}).get("resources", [])
    resource_type = provider_inputs.get("terraform_resource_type")

    if not resource_type:
        outputs.append(
            {
                "value": ProviderError(severity_value=99),
                "err": "`terraform_resource_type` must be provided in the provider input (severity_value: 99))",
            }
        )
        return

    is_resource_found = False

    for resource in config_resources:
        if resource.get("type") != resource_type:
            continue
        is_resource_found = True
        resource_references = set()
        for expressions_val in resource.get("expressions", {}).values():
            # Currently we don't support expressions other than dict
            # Might be needed in the future to search references in lists
            # Ideally we should do `*.references`
            if not isinstance(expressions_val, dict):
                continue
            for reference in expressions_val.get("references", []):
                # Only get the resource type
                resource_references.add(reference.split(".")[0])

        outputs.append({"value": list(resource_references), "meta": resource})

    if not is_resource_found:
        outputs.append(
            {
                "value": ProviderError(severity_value=1),
                "err": f"resource_type: '{resource_type}' is not found",
                "meta": config_resources,
            }
        )


from pyramid.config import Configurator
from p2.datashackle.core import init_datashackle_core

def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """

    config = Configurator(settings=settings)
    config.add_static_view('static', 'static', cache_max_age=3600)
    config.add_route('home', '/')
    config.add_route('media', '/media/{id}')
    config.scan()

    # This MUST happen after configuration scan
    init_datashackle_core(settings)

    return config.make_wsgi_app()


import pytz

from django.contrib.auth.decorators import login_required
from django.shortcuts import redirect, render
from django.views.decorators.cache import cache_page

from panopuppet.pano.puppetdb import puppetdb
from panopuppet.pano.puppetdb.puppetdb import set_server, get_server
from panopuppet.pano.settings import AVAILABLE_SOURCES, CACHE_TIME

__author__ = 'etaklar'

@login_required
@cache_page(CACHE_TIME)
def facts(request, certname=None):
    context = {'timezones': pytz.common_timezones,
               'SOURCES': AVAILABLE_SOURCES}
    if request.method == 'GET':
        if 'source' in request.GET:
            source = request.GET.get('source')
            set_server(request, source)
    if request.method == 'POST':
        request.session['django_timezone'] = request.POST['timezone']
        return redirect(request.POST['return_url'])

    source_url, source_certs, source_verify = get_server(request)
    facts_params = {
        'query':
            {
                1: '["=","certname","' + certname + '"]'
            },
    }
    facts_list = puppetdb.api_get(
        api_url=source_url,
        cert=source_certs,
        verify=source_verify,
        path='facts',
        params=puppetdb.mk_puppetdb_query(
            facts_params, request),
    )
    context['certname'] = certname
    context['facts_list'] = facts_list

    return render(request, 'pano/facts.html', context)


import torch
import numpy as np
import matplotlib.pyplot as plt
import os
from sklearn.model_selection import train_test_split

from network import Net

if torch.cuda.is_available():
    device = torch.device("cuda")
    print("Using GPU ", torch.cuda.get_device_name())
else:
    device = torch.device("cpu")
    print("Using CPU")

max_iters = 1000
g = 9.81
v_0 = np.pi

times = torch.linspace(0, 1, 100, device=device)
positions = -1./2. * g * times ** 2 + v_0 * times

# Create some noisy measurements:
amplitude = 0.2
np.random.seed(15)
noise = amplitude * torch.from_numpy(np.random.sample(times.shape[0])).to(device) - amplitude / 2.
noisy_positions = (positions + noise).float().to(device)

num_measurements = 100
np.random.seed(15)
idx = np.random.choice(np.arange(len(times)), num_measurements, replace=False)
measurement_times = times[idx]
measurements = noisy_positions[idx]

times_train, times_test, measurement_times_train, measurement_times_test, measurements_train, \
    measurement_times_test = train_test_split(times, measurement_times, measurements, test_size=0.2)

NN = Net(num_hidden_units=16, num_hidden_layers=2, inputs=1).to(device)

# Initialize with bad guess:
g_init = g / 2
g_param = torch.tensor(g_init, device=device, dtype=torch.float)
g_param = torch.nn.Parameter(g_param)
g_param = g_param.to(device)

params = list(NN.parameters()) + [g_param]

loss_function=torch.nn.MSELoss(reduction="mean")

lbfgs_optim = torch.optim.LBFGS(params,
                                max_iter=max_iters,
                                tolerance_grad=1e-7,
                                tolerance_change=1e-10,
                                line_search_fn="strong_wolfe")

def pde_loss(nn, residual_points, g):
    residual_points.requires_grad = True
    y = nn(residual_points)
    dy_dt, = torch.autograd.grad(outputs=y,
                             inputs=residual_points,
                             grad_outputs=torch.ones_like(residual_points),
                             create_graph=True)

    ddy_dtt, = torch.autograd.grad(outputs=dy_dt,
                                 inputs=residual_points,
                                 grad_outputs=torch.ones_like(residual_points),
                                 create_graph=True)

    # The ODE:
    residual = ddy_dtt + g

    return loss_function(residual, torch.zeros_like(residual))

def boundary_loss(nn, boundary_points, boundary_values):

    # Evaluate the NN at the boundary:
    predictions = nn(boundary_points)

    return loss_function(predictions, boundary_values)

losses_train = []
losses_test = []

g_values_train = []

def closure():
    lbfgs_optim.zero_grad()

    data_loss_value = boundary_loss(NN, measurement_times_train, measurements_train)

    pde_loss_value = pde_loss(NN, times_train, g_param)

    loss_train = data_loss_value + pde_loss_value

    if loss_train.requires_grad:
        loss_train.backward()

    # Log both the loss and g during training:
    losses_train.append(loss_train.item())
    g_values_train.append(g_param.item())

    return loss_train

lbfgs_optim.step(closure)

prediction = NN(times_test)

os.makedirs('results', exist_ok=True)

plt.rcParams['figure.figsize'] = (12,9)

plt.rc('axes', facecolor='whitesmoke', edgecolor='none',
    axisbelow=True, grid=True)
plt.rc('grid', color='w', linestyle='solid')
plt.rc('lines', linewidth=2)

plt.rcParams.update({'font.size': 20})
os.makedirs('results_gravity_test', exist_ok=True)

plt.figure(dpi = 200)
plt.scatter(times_test.tolist(), prediction.tolist(), label='Test set prediction')
plt.title('PINN prediction', pad=15)
plt.xlabel("Time", fontsize=20 , labelpad=10)
plt.ylabel("Pinn prediction of $y$", fontsize=20 , labelpad=10)
plt.legend(framealpha=0.9, facecolor=(1, 1, 1, 1))
plt.tight_layout()
plt.savefig(f'results_gravity_test/prediction.png')
plt.clf()

plt.figure(dpi = 200)
plt.plot(times.tolist(), positions.tolist(), label="True $y(t)$")
plt.plot(measurement_times.tolist(), measurements.tolist(), "x", label= str(num_measurements) + " Measurements")
plt.title('True $y(t)$ and noisy data measurements', pad=15)
plt.xlabel("Time", fontsize=20 , labelpad=10)
plt.ylabel("Height $y(t)$", fontsize=20 , labelpad=10)
plt.legend(framealpha=0.9, facecolor=(1, 1, 1, 1))
plt.tight_layout()
plt.savefig(f'results_gravity_test/data.png')
plt.clf()

plt.figure(dpi = 200)
plt.semilogy(losses_train, label='Traning loss')
plt.title('Train loss', pad=15)
plt.xlabel("Iterations", fontsize=20 , labelpad=10)
plt.ylabel("Loss", fontsize=20 , labelpad=10)
plt.legend(framealpha=0.9, facecolor=(1, 1, 1, 1))
plt.tight_layout()
plt.savefig(f'results_gravity_test/loss_plot.png')
plt.clf()

plt.figure(dpi = 200)
plt.title(f"Gracitational acceleration g during training, final g = {g_param.item():.2} ms^(-1)", pad=15)
plt.plot(g_values_train, label="Gracitational acceleration g")
plt.plot(np.zeros(len(g_values_train)) + g, label="True")
plt.xlabel("Iterations", fontsize=20 , labelpad=10)
plt.ylabel("Gracitational acceleration g", fontsize=20 , labelpad=10)
plt.legend(framealpha=0.9, facecolor=(1, 1, 1, 1))
plt.tight_layout()
plt.savefig(f'results_gravity_test/g_during_training.png')
plt.clf()

plt.figure(dpi = 200)
plt.plot(times.tolist(), positions.tolist(), marker="o", markevery=14, label="True")
plt.plot(measurement_times.tolist(), measurements.tolist(), "x", label= str(num_measurements)+ " Measurements")
plt.plot(times.tolist(), NN(times).tolist(), "-", marker="s", markevery=10, label="NN")
plt.xlabel("Time", fontsize=20 , labelpad=10)
plt.ylabel("Height", fontsize=20 , labelpad=10)
plt.legend(framealpha=0.9, facecolor=(1, 1, 1, 1))
plt.tight_layout()
plt.savefig(f'results_gravity_test/network_predict.png')
plt.clf()

Project 3 - Inverse Physics Informed Neural Network Problem/src/gravity_demo.py


import sys
import json

# Get arguments
semanticLabelPath = sys.argv[1]
changelogPath = sys.argv[2]
if len(sys.argv)>3:
  version = sys.argv[3]
else:
  version = None

# read semantic labels
with open(semanticLabelPath, 'r') as semanticLabelFile:
  semanticLabels = json.load(semanticLabelFile)

# print version changelog function
def printVersionChangelog(log: dict, version: str):
  sys.stdout.write("## Version " + version + "\n\n")
  for label in (semanticLabels["major"] + semanticLabels["minor"] + semanticLabels["patch"] + ["other"]):
    if label in log[version]:
      sys.stdout.write("### " + label + "\n\n")
      for change in log[version][label]:
        sys.stdout.write("* " + change + "\n")
      sys.stdout.write("\n")

# read changelog json
with open(changelogPath, 'r') as changelogFile:
  changelog = json.load(changelogFile)

sys.stdout.write("# CHANGELOG\n\n")
if version is not None:
  printVersionChangelog(changelog, version)
else:
  for ver in list(changelog.keys())[::-1]:
    printVersionChangelog(changelog, ver)


#!/usr/bin/env python

# -*- coding: utf-8 -*-
"""
udocker unit tests: CurlHeader
"""

from unittest import TestCase, main
from unittest.mock import patch
from io import StringIO
from udocker.utils.curl import CurlHeader
import collections

collections.Callable = collections.abc.Callable
BUILTINS = "builtins"

class CurlHeaderTestCase(TestCase):
    """Test CurlHeader() http header parser."""

    def test_01_init(self):
        """Test01 CurlHeader() constructor."""
        curl_header = CurlHeader()
        self.assertFalse(curl_header.sizeonly)
        self.assertIsInstance(curl_header.data, dict)
        self.assertEqual("", curl_header.data["X-ND-HTTPSTATUS"])
        self.assertEqual("", curl_header.data["X-ND-CURLSTATUS"])

    def test_02_write(self):
        """Test02 CurlHeader().write()."""
        buff = ["HTTP/1.1 200 OK",
                "Content-Type: application/octet-stream",
                "Content-Length: 32", ]
        curl_header = CurlHeader()
        for line in buff:
            curl_header.write(line)
        status = curl_header.data["content-type"]
        self.assertEqual(status, "application/octet-stream")
        status = curl_header.data["X-ND-HTTPSTATUS"]
        self.assertEqual(status, "HTTP/1.1 200 OK")

        curl_header = CurlHeader()
        for line in buff:
            curl_header.write(line)
        buff_out = curl_header.getvalue()
        self.assertTrue("HTTP/1.1 200 OK" in buff_out)

        line = ""
        curl_header = CurlHeader()
        curl_header.sizeonly = True
        self.assertEqual(-1, curl_header.write(line))

    @patch.object(CurlHeader, 'write')
    def test_03_setvalue_from_file(self, mock_write):
        """Test03 CurlHeader().setvalue_from_file()."""
        fakedata = StringIO('XXXX')
        with patch(BUILTINS + '.open') as mopen:
            mopen.return_value.__iter__ = (
                lambda self: iter(fakedata.readline, ''))
            curl_header = CurlHeader()
            self.assertTrue(curl_header.setvalue_from_file("filename"))
            self.assertTrue(mock_write.called_with('XXXX'))

    def test_04_getvalue(self):
        """Test04 CurlHeader().getvalue()."""
        curl_header = CurlHeader()
        curl_header.data = "XXXX"
        status = curl_header.getvalue()
        self.assertEqual(status, curl_header.data)

if __name__ == '__main__':
    main()


from backend import Backend
import antlr4
from language.antlr.TemplateLexer import TemplateLexer
from language.antlr.TemplateParser import TemplateParser
import subprocess as sp
import six
from utils.utils import *

class Pyro(Backend):
    def __init__(self, file_dir, data_dict, prior_dict, distmap, constmap, templatefile, config):
        super(Pyro, self).__init__(file_dir)
        self.output_program = ""
        self.data_dict = data_dict
        self.prior_dict = prior_dict
        self.distmap = distmap
        self.constmap = constmap
        self.templatefile = templatefile
        self.output_program=""
        self.model = ""
        self.guide = ""
        self.module = ""
        self.prior_post_map = dict()
        self.tab = ""
        self.forward=""
        self.forward_ref_string = ""
        self.data_string = ""
        self.config = config
        self.posteriors = {}
        self.queries = ""

    def create_program(self):
        template = antlr4.FileStream(self.templatefile)
        lexer = TemplateLexer(template)
        stream = antlr4.CommonTokenStream(lexer)
        parser = TemplateParser(stream)
        self.visit(parser.template())
        self.output_program = self.output_program.replace("\t", "    ")
        with open(self._directory + '/pyro_prog.py', 'w') as pyrofile:
            pyrofile.write(self.output_program)

    def run(self, algorithm, timeout, prog_id, python_cmd):
        print("Running Pyro program " + str(prog_id) + " >>>>")
        process = sp.Popen("cd " + self._directory + "; timeout {0} {1} pyro_prog.py".format(timeout, python_cmd),
                           stdout=sp.PIPE, stderr=sp.PIPE, shell=True)
        dataout, dataerr = process.communicate()

        with open(self._directory + '/pyroout_' + str(prog_id), 'w') as outputfile:
            outputfile.write(dataerr)
            outputfile.write(dataout)

        print("Done Pyro program " + str(prog_id) + " >>>>")

    def visitTemplate(self, ctx):
        # imports
        self.output_program += "import pyro, numpy as np, torch, pyro.distributions as dist, torch.nn as nn\n" \
                               "from pyro.optim import Adam\n"\
                               "from pyro.infer import SVI\n"\
                                "if pyro.__version__=='0.2.1': from pyro.infer import Trace_ELBO\n"\
                                "from torch.autograd import Variable\n"\
                                "from torch.nn.parameter import Parameter\n"
        # fix seeds
        self.output_program += "torch.manual_seed(77349895)\n"\
                              "np.random.seed(77349895)\n"

        for child in ctx.children:
            code = self.visit(child)

        self.output_program += self.createModule()
        self.output_program += "def model(" + self.data_string[:-1] + "):\n"+"\t"+self.model.replace("\n", "\n\t")
        self.output_program = self.output_program[:-1]

        # create map for guide and wrap up
        map = "priors = {"
        for p in self.prior_post_map:
            if p == 'p':
                continue
            map += "'{0}' : {1},".format(p, self.prior_post_map[p])
        self.guide += map[:-1] + "}\n"
        self.guide += "lifted_module = pyro.random_module('module', module, priors)\nreturn lifted_module()\n"

        self.output_program += "def guide(" + self.data_string[:-1] + "):\n"+"\t"+self.guide.replace("\n", "\n\t")

        self.output_program = self.output_program[:-1]
        self.output_program += self.writeInference()
        self.output_program += self.queries

    def visitData(self, ctx=TemplateParser.DataContext):
        name = ctx.ID().getText()
        data_item = self.data_dict[name]
        code=""
        if isinstance(data_item, np.ndarray):
            if data_item.ndim == 1:
                # just extending the dimension to 2d
                code += name + "= np.array(" + str(data_item.tolist()) + \
                                       ", dtype=np.float32).reshape({0},1)\n".format(str(data_item.shape[0]))
            else:
                code += name + "= np.array(" + str(data_item.tolist()) + ", dtype=np.float32)\n"
            code += name + "=Variable(torch.Tensor(" + name + "))\n"
        else:
            code += name + "=" + str(data_item)+"\n"
            code += name + "=Variable(torch.Tensor([" + name + "]))\n"

        self.output_program += code
        return code

    def visitPrior(self, ctx):
        name = ctx.ID().getText()
        prior = self.prior_dict[name]
        # add prior to model
        if prior.has_key('dims'):
            dims = prior['dims']
            if dims.count(',') == 0:
                dims+=",1"
        else:
            dims ="1"
        argslist = ''
        for arg in prior['args']:
            argslist+="Variable("+str(arg)+"*torch.ones(("+dims+"))),"

        self.model += "{0} = {1}({2})\n".format(name, prior['prior']['pyro'], argslist[:-1])

        # update guide
        # get posterior dist with matching support
        posterior_candidates = getSupportedDistributions(prior["prior"]["support"], pps="pyro")
        posterior = np.random.choice([p for p in posterior_candidates if p['type'] == 'C' and p["name"] in ["normal", "gamma", "beta"]])
        arglist = ''
        args = []
        for arg in posterior["args"]:
            argname = get_new_var_name("arg_")
            self.guide += "{0} = Variable(torch.randn(({1})), requires_grad=True)\n".format(argname, dims)
            if is_positive(arg["type"]):
                arglist+="torch.nn.Softplus()(pyro.param('{0}', {0})),".format(argname)
            else:
                arglist += "pyro.param('{0}', {0}),".format(argname)
            args.append(argname)

        param_var = name + "_dist"
        if name == 'p':
            self.guide += "{0}_dist = pyro.sample('{1}', {2}({3}))\n".format(name,
                                                                             name,
                                                                             posterior["pyro"],
                                                                             arglist[:-1])
        else:
            self.guide += param_var + "={0}({1})\n".format(posterior["pyro"], arglist[:-1])
        self.posteriors[name] = {"dist": posterior, "args": args}
        self.prior_post_map[name] = param_var

    def visitLrmodel(self, ctx):
        name = ctx.ID().getText()
        # fill the forward function
        model = self.visit(ctx.distexpr())
        paramlist = "self,"+self.forward_ref_string
        self.forward = "\tdef forward("+paramlist[:-1]+"):\n"
        if ctx.distexpr().dims() is not None:
            dims = ctx.distexpr().dims().getText()
        else:
            dims = "1"

        for priorname in self.prior_dict:
            self.forward += "\t\t{0} = self.{0}\n".format(priorname)

        self.forward += "\t\toutput = " + self.visit(ctx.distexpr().params().param(0)) + "*Variable(torch.ones((" + dims +")))\n"
        self.forward += "\t\treturn output\n"

        # get the priors map
        self.model += "priors = {"
        for priorname in self.prior_dict:
            if priorname == 'p':
                continue
            self.model += "'{0}' : {0},".format(priorname)
        self.model += "}\n"

        # lift module
        lifted_module_name = get_new_var_name("lifted_module")
        self.model += lifted_module_name+"=pyro.random_module('module', module, priors)\n"
        lifted_reg_name = get_new_var_name("lifted_reg")
        self.model += lifted_reg_name + "="+lifted_module_name+"()\n"
        self.data_string = name + "," + self.forward_ref_string

        self.model += "prediction="+lifted_reg_name+"(" + self.forward_ref_string[:-1]+").squeeze(-1)\n"

        if len(model[1]["args"]) == 1:
            self.model += "pyro.sample('obs', " + model[1]["pyro"] + "(prediction), obs=" + name + \
                              ".squeeze(-1))\n"
        else:
            self.model += "pyro.sample('obs', " + model[1]["pyro"] + "(prediction, " + self.visit(ctx.distexpr().params().param(1)) + ".sample()*Variable(torch.ones("+name+".squeeze(-1).size()))), obs=" + name + ".squeeze(-1))\n"
        return ""

    def visitCondmodel(self, ctx):
        code = "if torch.equal({0}, Variable(torch.Tensor([1.0]))):\n".format(self.visit(ctx.expr()))
        truebranch = ctx.model(0).lrmodel()
        falsebranch = ctx.model(1).lrmodel()
        model = self.visit(truebranch.distexpr())
        paramlist = "self," + self.forward_ref_string

        self.forward = "\tdef forward(" + paramlist[:-1] + "):\n"+self.forward
        if truebranch.distexpr().dims() is not None:
            dims = truebranch.distexpr().dims().getText()
        else:
            dims = "1"

        for priorname in self.prior_dict:
            self.forward += "\t\t{0} = self.{0}\n".format(priorname)

        self.forward += "\t\tif torch.equal({0}, Variable(torch.Tensor([1.0]))):\n".format(self.visit(ctx.expr()))

        # true branch
        self.forward += "\t\t\toutput = " + self.visit(truebranch.distexpr().params().param(0)) + "*Variable(torch.ones((" + dims + ")))\n"
        self.forward+= "\t\telse:\n"

        # false branch
        self.forward += "\t\t\toutput = " + self.visit(
            falsebranch.distexpr().params().param(0)) + "*Variable(torch.ones((" + dims + ")))\n"
        self.forward += "\t\treturn output\n"

        # get the priors map
        self.model += "priors = {"
        for priorname in self.prior_dict:
            if priorname == 'p':
                continue
            self.model += "'{0}' : {0},".format(priorname)
        self.model += "}\n"

        # lift module
        lifted_module_name = get_new_var_name("lifted_module")
        self.model += lifted_module_name + "=pyro.random_module('module', module, priors)\n"
        lifted_reg_name = get_new_var_name("lifted_reg")
        self.model += lifted_reg_name + "=" + lifted_module_name + "()\n"

        name = truebranch.ID().getText()
        self.data_string = name + "," + self.forward_ref_string
        self.model += "prediction=" + lifted_reg_name + "(" + self.forward_ref_string[:-1] + ").squeeze(-1)\n"

        self.model += "pyro.sample('obs', " + model[1]["pyro"] + "(prediction, " + self.visit(
            truebranch.distexpr().params().param(1)) + ".sample()*Variable(torch.ones(" + name + ".squeeze(-1).size()))), obs=" + name + \
                      ".squeeze(-1))\n"

        return ""

    def visitAssign(self, ctx):
        name = ctx.ID().getText()
        distexpr = ctx.distexpr()
        if distexpr.DISTHOLE() is not None:
            dist = self.distmap[distexpr.DISTHOLE().getSymbol().tokenIndex]
        else:
            dist = self.distmap[distexpr.DISTRIBUTION().getSymbol().tokenIndex]

        self.forward += "\t\t"+name+"="+self.visit(distexpr)[0]+".sample()\n"
        return ""

    def visitQuery(self, ctx):
        name = ctx.ID().getText()
        output = ""
        try:
            posterior = self.posteriors[name]
            if len(posterior["args"]) == 2:
                output += "print('{0}_mean', {1}({2}, {3}).mean if pyro.__version__=='0.2.1' else {1}({2}, {3}).analytic_mean())\n".\
                    format(name, posterior["dist"]["pyro"],
                           self._cast(posterior["dist"]["args"][0], posterior["args"][0]),
                           self._cast(posterior["dist"]["args"][1], posterior["args"][1]))
            else:
                output += "print('{0}_mean', {1}({2}).mean if pyro.__version__=='0.2.1' else {1}({2}).analytic_mean())\n".\
                    format(name,
                           posterior["dist"]["pyro"],
                           self._cast(posterior["dist"]["args"][0], posterior["args"][0]))
        except:
            print(name + " not found")
        self.queries += output

    @staticmethod
    def _cast(arg, val):
        if is_positive(arg["type"]):
            return "torch.nn.Softplus()(pyro.param('{0}'))".format(val)
        else:
            return "pyro.param('{0}')".format(val)

    def visitDistexpr(self, ctx):
        if ctx.DISTHOLE() is not None:
            dist = self.distmap[ctx.DISTHOLE().getSymbol().tokenIndex]
        else:
            dist = self.distmap[ctx.DISTRIBUTION().getSymbol().tokenIndex]
        params = self.visit(ctx.params())
        expr = dist['pyro'] + '(' + params + ')'
        return expr, dist, params

    def visitParams(self, ctx):
        params = ""
        dims = ctx.parentCtx.dims()
        if dims is not None:
            dim_string = 'Variable(torch.ones((' + dims.getText() + ')))'
        else:
            dim_string = 'Variable(torch.ones(1))'
        for child in ctx.param():
            params += self.visit(child) + "*" + dim_string + ","

        return params[:-1]

    def visitParam(self, ctx):
        if ctx.expr() is not None:
            return self.visit(ctx.expr())
        else:
            return str(self.constmap[ctx.CONSTHOLE().getSymbol().tokenIndex])

    def visitRef(self, ctx):
        ref = ctx.getText()
        if ref in self.data_dict and ref+"," not in self.forward_ref_string:
            self.forward_ref_string += ref + ","

        return ref

    def visitVal(self, ctx):
        if ctx.number().CONSTHOLE() is not None:
            return str(self.constmap[ctx.number().CONSTHOLE().getSymbol().tokenIndex])
        else:
            return ctx.getText()

    def visitArith(self, ctx=TemplateParser.ArithContext):
        operation = ctx.AOP().getText()
        lop = ctx.getChild(0)
        rop = ctx.getChild(2)
        if operation == '*':
            # check if one of them is scalar
            if self.isScalar(lop) or self.isScalar(rop):
                return self.visit(lop) + '*' + self.visit(rop)
            else:
                return self.visit(ctx.getChild(0)) + '.matmul(' + self.visit(ctx.getChild(2)) + ')'
        else:
            return self.visit(ctx.getChild(0)) + ctx.AOP().getText() + self.visit(ctx.getChild(2))

    def isScalar(self, ctx):
        if isinstance(ctx, TemplateParser.RefContext):
            ref = ctx.getText()
            if (ref in self.data_dict and not isinstance(self.data_dict[ref], np.ndarray)) or (ref in self.prior_dict and not 'dims' in self.prior_dict[ref]):
                return True
        elif isinstance(ctx, TemplateParser.ValContext):
            return True

        return False

    def createModule(self):
        initfunc="\t\tsuper(Module, self).__init__()\n"
        for priorname in six.iterkeys(self.prior_dict):
            prior = self.prior_dict[priorname]
            if prior.has_key('dims'):
                dims = prior['dims']
                if dims.count(',') == 0:
                    dims += ",1"
            else:
                dims = "1"

            initfunc+="\t\tself."+priorname+" = Parameter(torch.Tensor(({0})))\n".format(dims)

        module = "class Module(nn.Module):\n"+"\tdef __init__(self):\n"+initfunc+self.forward

        module += "module = Module()\n"
        return module

    def writeInference(self):

        try:
            optim = self.config["pyro"]["optimizers"][0]
            params = ""
            assert len(optim["params"]) > 0
            for param in optim["params"]:
                if "size" not in param:
                    params += "\"{0}\" : {1},".format(param["name"], generate_primitives(param["type"], 1, param["special"])[0])
                else:
                    params += "\"{0}\" : {1},".format(param["name"], tuple(generate_primitives(param["type"], param["size"], param["special"])))

            code = "optim = {0}({{ {1} }})\n".format(optim["name"], params[:-1])
        except Exception:
            code = "optim = Adam({'lr': 0.05})\n"
        code += "svi = SVI(model, guide, optim, loss=Trace_ELBO() if pyro.__version__=='0.2.1' else 'ELBO')\n"
        code += "for i in range(1000):\n"
        code += "\tloss = svi.step("+self.data_string[:-1]+")\n"
        code += "\tif ((i % 1000) == 0):\n"
        code += "\t\tprint(loss)\n"
        code += "for name in pyro.get_param_store().get_all_param_names():\n"
        code += "\tprint(('{0} : {1}'.format(name, pyro.param(name).data.numpy())))\n"
        return code


# -*- coding: utf-8 -*-
"""
Movielens Recommender system
(using knn-ish method)

@author: David André Rodríguez Méndez (AndreRdz7)
"""
# Import libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import sklearn
from sklearn.metrics import mean_squared_error
from sklearn.cross_validation import train_test_split
from sklearn.neighbors import NearestNeighbors
# Get dataset
df = pd.read_csv("./u.data", sep='\t', header=None)
df.columns = ["UserID","ItemID","Rating","TimeStamp"]
print(df.head())
# Data visualization
plt.hist(df.Rating)
plt.show()
plt.hist(df.TimeStamp)
plt.show()
print(df.groupby(["Rating"])["UserID"].count())
plt.hist(df.groupby(["ItemID"])["ItemID"].count())
plt.show()
# Matrix representation
n_users = df.UserID.unique().shape[0]
n_items = df.ItemID.unique().shape[0]
ratings = np.zeros((n_users,n_items))
# Sparse matrix
for row in df.itertuples():
    ratings[row[1]-1,row[2]-1] = row[3]
sparsity = float(len(ratings.nonzero()[0]))
sparsity /= (ratings.shape[0]*ratings.shape[1])
sparsity *= 100
print("Sparsity: {:4.2f}%".format(sparsity))
# Splitting data
ratings_train, ratings_test = train_test_split(ratings,test_size=0.3,random_state=42)
print(ratings_train.shape)
# User filter
sim_matrix = 1 - sklearn.metrics.pairwise.cosine_distances(ratings_train)
print(sim_matrix.shape)
users_predictions = sim_matrix.dot(ratings_train) / np.array([np.abs(sim_matrix).sum(axis=1)]).T
def get_mse(preds, actuals):
    if preds.shape[0] != actuals.shape[1]:
        actuals = actuals.T
    preds = preds[actuals.nonzero()].flatten()
    actuals = actuals[actuals.nonzero()].flatten()
    return mean_squared_error(preds,actuals)

get_mse(users_predictions,ratings_train)
get_mse(users_predictions,ratings_test)
k = 10
neighbors = NearestNeighbors(k,'cosine')
neighbors.fit(ratings_train)
top_k_distances, top_k_users = neighbors.kneighbors(ratings_train,return_distance=True)
users_predicts_k = np.zeros(ratings_train.shape)
for i in range(ratings_train.shape[0]):
    users_predicts_k[i,:] = top_k_distances[i].T.dot(ratings_train[top_k_users][i]) / np.array([np.abs(top_k_distances[i].T).sum(axis=0)]).T
get_mse(users_predicts_k, ratings_train)
get_mse(users_predicts_k,ratings_test)
# Item filter
n_movies = ratings_train.shape[1]
neighbors = NearestNeighbors(n_movies,'cosine')
neighbors.fit(ratings_train.T)
top_k_distances,top_k_items = neighbors.kneighbors(ratings_train.T,return_distance=True)
item_prediction = ratings_train.dot(top_k_distances) / np.array([np.abs(top_k_distances).sum(axis=1)])
get_mse(item_prediction,ratings_train)
get_mse(item_prediction,ratings_test)
# KNN filter
k = 30
neighbors = NearestNeighbors(k,'cosine')
neighbors.fit(ratings_train.T)
top_k_distances,top_k_items = neighbors.kneighbors(ratings_train.T,return_distance=True)
preds = np.zeros(ratings_train.T.shape)
for i in range(ratings_train.T.shape[0]):
    den = 1
    if(np.abs(top_k_distances[i]).sum(axis=0)>0):
        den = np.abs(top_k_distances[i]).sum(axis=0)
    preds[i,:] = top_k_distances[i].dot(ratings_train.T[top_k_items][i]) / np.array([den]).T
get_mse(preds,ratings_train)
get_mse(preds,ratings_test)


# -*- coding: utf-8 -*-
# author: wzpan
# 闹钟

import logging
import os
import pickle
import time

from robot import config, constants, utils
from robot.sdk.AbstractPlugin import AbstractPlugin

logger = logging.getLogger(__name__)

LOCAL_REMINDER = os.path.join(constants.TEMP_PATH, "reminder.pkl")

class Plugin(AbstractPlugin):
    def __init__(self, con):
        super(Plugin, self).__init__(con)

    def _dump_reminders(self):
        logger.info("写入日程提醒信息")
        with open(LOCAL_REMINDER, "wb") as f:
            pickle.dump(self.con.scheduler.get_jobs(), f)

    def alarm(self, remind_time, content, job_id):
        self.con.player.stop()  # 停止所有音频
        content = utils.stripPunctuation(content)
        self.say(
            f"现在是{time.strftime('%H:%M:%S', time.localtime(time.time()))}，该{content}了。"
            * int(config.get("/reminder/repeat", 3))
        )
        # 非周期性提醒，提醒完即删除
        if "repeat" not in remind_time:
            self.con.scheduler.del_job_by_id(job_id)
            self._dump_reminders()

    def list_reminder(self, parsed):
        """
        列举所有的日程
        """
        logger.info("list_reminder")
        _jobs = self.con.scheduler.get_jobs()
        if len(_jobs) == 0:
            self.say(f"您当前没有提醒。", cache=True)
        elif len(_jobs) > 1:
            self.say(f"您当前有{len(_jobs)}个提醒。", cache=True)
            index = 0
            for job in _jobs:
                self.say(f"第{index+1}个提醒内容是{job.describe}")
                logger.info(f"index: {index}, job.job_id: {job.job_id}")
                index += 1
        elif len(_jobs) == 1:
            self.say(f"您当前有1个提醒。", cache=True)
            self.say(f"提醒内容是{_jobs[0].describe}")

    def add_reminder(self, parsed):
        logger.info("add_reminder")
        remind_times = self.nlu.getSlotWords(parsed, "SET_REMIND", "user_remind_time")
        original_times = self.nlu.getSlotOriginalWords(
            parsed, "SET_REMIND", "user_remind_time"
        )
        contents = self.nlu.getSlotWords(parsed, "SET_REMIND", "user_wild_content")
        if len(remind_times) < 0 or "|" not in remind_times[0]:
            self.say("添加提醒失败。请说明需要我提醒的时间", cache=True)
            return
        if len(contents) < 0:
            self.say("添加提醒失败。请说明需要我提醒做什么", cache=True)
            return
        remind_time, original_time, content = (
            remind_times[0],
            original_times[0],
            contents[0],
        )
        job_id = utils.getTimemStap()
        job = self.con.scheduler.add_job(
            remind_time,
            original_time,
            content,
            lambda: self.alarm(remind_time, content, job_id),
            job_id=job_id,
        )
        if job:
            self._dump_reminders()
            logger.info(f"added reminder: {job.describe}, job_id: {job_id}")
            self.say(f"好的，已为您添加提醒：{job.describe}")
        else:
            self.say("抱歉，添加提醒失败了")

    def _assure(self):
        pick = self.activeListen()
        if "不" in pick:
            self.say("好的。取消删除", cache=True)
            return False
        elif any(yes in pick for yes in ("是", "要", "删除")):
            return True
        else:
            self.say("取消删除", cache=True)
            return False

    def _ask_which(self):
        self.say(f"要删除哪一个提醒呢", cache=True)
        pick = self.activeListen()
        parsed = self.parse(pick)
        if self.nlu.hasIntent(parsed, "HASS_INDEX"):
            _jobs = self.con.scheduler.get_jobs()
            index = int(
                float(
                    self.nlu.getSlotWords(parsed, "HASS_INDEX", "user_index")[0].split(
                        "|"
                    )[0]
                )
            )
            logger.info(f"用户选择了第{index}个")
            if index < 0 or index > len(_jobs):
                self.say("没有找到符合条件的提醒，删除失败", cache=True)
                return -1
            job = _jobs[index - 1]
            return job.job_id
        else:
            self.say("没有找到符合条件的提醒，删除失败", cache=True)
            return ""

    def del_reminder(self, parsed):
        logger.info("del_reminder")
        self.list_reminder(parsed)
        _jobs = self.con.scheduler.get_jobs()
        if len(_jobs) == 1:
            self.say("要删除这个提醒吗", cache=True)
            if self._assure():
                try:
                    self.con.scheduler.del_job_by_id(_jobs[0].job_id)
                    self._dump_reminders()
                    self.say("好的，已删除该提醒")
                except Exception as e:
                    logger.error(f"删除失败: {e}")
                    self.say("删除提醒失败")
        elif len(_jobs) > 1:
            job_id = self._ask_which()
            if job_id:
                try:
                    self.con.scheduler.del_job_by_id(job_id)
                    self._dump_reminders()
                    self.say("好的，已删除该提醒")
                except Exception as e:
                    logger.error(f"删除失败: {e}")
                    self.say("删除提醒失败")

    def handle(self, text, parsed):
        logger.info("Reminder handle")
        if self.nlu.hasIntent(parsed, "CHECK_REMIND"):
            # 查询当前设置的提醒
            self.list_reminder(parsed)
        elif self.nlu.hasIntent(parsed, "DELETE_REMIND"):
            # 删除指定的提醒
            self.del_reminder(parsed)
        elif self.nlu.hasIntent(parsed, "SET_REMIND"):
            # 设置提醒
            self.add_reminder(parsed)

    def isValid(self, text, parsed):
        return any(
            self.nlu.hasIntent(parsed, intent)
            for intent in ["CHECK_REMIND", "DELETE_REMIND", "SET_REMIND"]
        )


import logging
from copy import deepcopy

"""
INFORMACOES OBRIGATORIAS NOS LOGS
client, id, service
"""

from aws_xray_sdk.core import patch_all

FORMAT = "[%(levelname)s] | %(message)s"

class RequiredParametersException(Exception):
    pass

class Logger(object):

    def __init__(self, logger_name=None, logger_level='ERROR', enable_xray=False):
        """
        Cria uma nova instância de Logger

        Parameters:
        -----------
        logger_name: str
            Nome do logger que será criado

        logger_level: str
            Nível mínimo de log que será impresso na tela. Os níveis de log seguem uma hierarquia fixa e são ordenados por criticidade:
            (OBS: o nível NOTSET é um pseudo-nível que indica que não há restrições de log, podendo ser impressos até mesmo logs de sistema)
            (OBS2: O nível OFF é outro pseudo-nível que desabilita a impressão dos logs por meio desta classe)

            OFF < NOTSET < DEBUG < INFO < WARNING (ou WARN) < ERROR < CRITICAL (ou FATAL)

        enable_xray: boolean
            Flag que habilita o xray para lambdas
        """
        self.logger_name = logger_name
        self.logger_level = logger_level
        self.logger = self.__create_logger(enable_xray)
        self.__extra_fields = {}

    @property
    def extra_fields(self):
        return self.__extra_fields

    @extra_fields.setter
    def extra_fields(self, extra_fields={}):
        self.__extra_fields = extra_fields

    def critical(self, message, **kwargs):
        self.__print_log(message, logging.CRITICAL, **kwargs)

    def error(self, message, **kwargs):
        self.__print_log(message, logging.ERROR, **kwargs)

    def warn(self, message, **kwargs):
        self.__print_log(message, logging.WARNING, **kwargs)

    def info(self, message, **kwargs):
        self.__print_log(message, logging.INFO, **kwargs)

    def debug(self, message, **kwargs):
        self.__print_log(message, logging.DEBUG, **kwargs)

    def __create_logger(self, enable_xray=False):
        logger = logging.getLogger(self.logger_name or __name__)
        logger.propagate = False

        if enable_xray:
            patch_all()

        if not logger.handlers and self.logger_level != 'OFF':
            logger.setLevel(self.logger_level)
            formatter = logging.Formatter(FORMAT)
            ch = logging.StreamHandler()
            ch.setLevel(self.logger_level)
            ch.setFormatter(formatter)
            logger.addHandler(ch)

        return logger

    def __print_log(self, message, logger_level, **kwargs):
        local_extra_fields = deepcopy(self.extra_fields)
        local_extra_fields.update(kwargs)

        extra_fields_message = self.__dict_to_string(local_extra_fields)
        log_message = f'{message} | {extra_fields_message}' if extra_fields_message != '' else message

        if self.logger_level != "OFF":

            if logger_level == logging.CRITICAL:
                self.logger.critical(log_message)
            elif logger_level == logging.ERROR:
                self.logger.error(log_message)
            elif logger_level == logging.WARNING:
                self.logger.warning(log_message)
            elif logger_level == logging.INFO:
                self.logger.info(log_message)
            elif logger_level == logging.DEBUG:
                self.logger.debug(log_message)

    def __dict_to_string(self, dictionary):
        return ' | '.join([f'{key} = {value}' for key, value in dictionary.items()])


import logging

from pathlib import Path

from embeddingCreator.EmbeddingContainer import RequirementEmbeddingContainer
from embeddingCreator.EmbeddingCreator import EmbeddingCreator
from utility import Util

logging.basicConfig(level=logging.INFO)
log = logging.getLogger(__name__)

PREPROCESSED_REQ_OUTPUT_DIR = Path(__file__).parent.parent / "output/Preprocessed_Req"

class RequirementEmbeddingCreator(EmbeddingCreator):

    def __init__(self, requirements_word_chooser, preprocessor, wordemb_creator,
                  tokenizer, preprocessed_token_output_directory=PREPROCESSED_REQ_OUTPUT_DIR):
        super().__init__(preprocessor, wordemb_creator, tokenizer, preprocessed_token_output_directory)
        self._requirements_word_chooser = requirements_word_chooser

class UCEmbeddingCreator(RequirementEmbeddingCreator):

    def _create_embeddings(self, file_representation):
        chosen_word_groups = self._requirements_word_chooser.choose_words_from(file_representation)
        chosen_word_groups_embeddings = []
        requirement_element_vectors = []
        for word_group in chosen_word_groups:
            word_embeddings = self._create_word_embeddings_from_word_list(word_group)
            chosen_word_groups_embeddings.append(word_embeddings)
            requirement_element_vectors.append(Util.create_averaged_vector(word_embeddings))

        file_vector = Util.create_averaged_vector([word for word_group in chosen_word_groups for word in word_group])  # flat average over all (nested) word embeddings in chosen_word_groups

        return RequirementEmbeddingContainer(file_representation.file_path, file_vector, requirement_element_vectors)

class MockUCEmbeddingCreator(RequirementEmbeddingCreator):

    def _create_embeddings(self, file_representation):
        chosen_word_groups = self._requirements_word_chooser.choose_words_from(file_representation)
        file_vector = [word for word_group in chosen_word_groups for word in word_group]  # flat list over all (nested) words in chosen_word_groups

        return RequirementEmbeddingContainer(file_representation.file_path, file_vector, chosen_word_groups)


from kivy.uix.textinput import TextInput
from kivy.clock import Clock
from kivy.base import EventLoop

class TxtInput(TextInput):
	# Enable right click
	# Enforce single active selection / copy-paste menu
	# ---- BEGIN ----

	last_selected= dict(instance=None, hidden=True) # Indicates if a selection exists

	# Disable long touch -- for like, when you drag to cancel menu from showing
	def long_touch(self, dt): pass

	# Enable copy-paste menu on right click. Also disable selection when click occurs outside input.
	def on_touch_down(self, touch):
		if self.last_selected['instance'] and \
		not self.last_selected['instance'].collide_point(*touch.pos):
				self.hide_old()

		if self.collide_point(*touch.pos):
			if touch.button == "right":
				return self.show_copy_paste(touch)
		return super().on_touch_down(touch)

	# Override to enforce single active selection / copy-paste menu
	# Note that showing the context menu causes a loss in focus
	def _on_textinput_focused(self, instance, value, *largs):
		from kivy.base import EventLoop
		from kivy import Config
		_is_desktop = Config.getboolean('kivy', 'desktop')

		win = EventLoop.window
		# self.cancel_selection() # changed
		# self._hide_cut_copy_paste(win) # changed

		if value:
			self.hide_old() # changed
			self.assign_last(instance=self, hidden=False) # changed

			if (not (self.readonly or self.disabled) or _is_desktop and
					self._keyboard_mode == 'system'):
				self._trigger_cursor_reset()
				self._editable = True
			else:
				self._editable = False
		else:
			self._do_blink_cursor_ev.cancel()
			self._hide_handles(win)

	# Helper function
	@classmethod
	def hide_old(cls):
		win = EventLoop.window
		if cls.last_selected['hidden'] is False:
			cls.last_selected['instance']._hide_cut_copy_paste(win)
			cls.last_selected['instance'].cancel_selection()
			cls.assign_last(hidden=True)

	# Helper function
	@classmethod
	def assign_last(cls, instance=None, hidden=None):
		tmp= cls.last_selected
		if instance is not None:
			tmp['instance']= instance
		if hidden is not None:
			tmp['hidden']= hidden

	# Helper function
	def show_copy_paste(self, touch):
		if self.focus and self.selection_text:
			pos= self.to_local(*touch.pos, relative=False)
			self._show_cut_copy_paste(pos, EventLoop.window)
			self.assign_last(instance=self, hidden=False)
			return False
		return False

	# ---- END ----

	# Bugfix for focus_previous not working: https://github.com/kivy/kivy/issues/6560
	def keyboard_on_key_down(self, window, keycode, text, modifiers):
		from kivy.base import EventLoop
		from kivy.clock import Clock
		import sys
		_is_osx = sys.platform == 'darwin'

		# Keycodes on OS X:
		ctrl, cmd = 64, 1024
		key, key_str = keycode
		win = EventLoop.window

		# This allows *either* ctrl *or* cmd, but not both.
		modifiers = set(modifiers) - {'capslock', 'numlock'}
		is_shortcut = (modifiers == {'ctrl'} or (
		_is_osx and modifiers == {'meta'}))
		is_interesting_key = key in (list(self.interesting_keys.keys()) + [27])

		modifiers= list(modifiers) # changed
		if not self.write_tab and super(TextInput,
			self).keyboard_on_key_down(window, keycode, text, modifiers):
			return True

		if not self._editable:
			# duplicated but faster testing for non-editable keys
			if text and not is_interesting_key:
				if is_shortcut and key == ord('c'):
					self.copy()
			elif key == 27:
				self.focus = False
			return True

		if text and not is_interesting_key:

			self._hide_handles(win)
			self._hide_cut_copy_paste(win)
			win.remove_widget(self._handle_middle)

			# check for command modes
			# we use \x01INFO\x02 to get info from IME on mobiles
			# pygame seems to pass \x01 as the unicode for ctrl+a
			# checking for modifiers ensures conflict resolution.

			first_char = ord(text[0])
			if not modifiers and first_char == 1:
				self._command_mode = True
				self._command = ''
			if not modifiers and first_char == 2:
				self._command_mode = False
				self._command = self._command[1:]

			if self._command_mode:
				self._command += text
				return

			_command = self._command
			if _command and first_char == 2:
				from_undo = True
				_command, data = _command.split(':')
				self._command = ''
				if self._selection:
					self.delete_selection()
				if _command == 'DEL':
					count = int(data)
					if not count:
						self.delete_selection(from_undo=True)
					end = self.cursor_index()
					self._selection_from = max(end - count, 0)
					self._selection_to = end
					self._selection = True
					self.delete_selection(from_undo=True)
					return
				elif _command == 'INSERT':
					self.insert_text(data, from_undo)
				elif _command == 'INSERTN':
					from_undo = False
					self.insert_text(data, from_undo)
				elif _command == 'SELWORD':
					self.dispatch('on_double_tap')
				elif _command == 'SEL':
					if data == '0':
						Clock.schedule_once(lambda dt: self.cancel_selection())
				elif _command == 'CURCOL':
					self.cursor = int(data), self.cursor_row
				return

			if is_shortcut:
				if key == ord('x'):  # cut selection
					self._cut(self.selection_text)
				elif key == ord('c'):  # copy selection
					self.copy()
				elif key == ord('v'):  # paste clipboard content
					self.paste()
				elif key == ord('a'):  # select all
					self.select_all()
				elif key == ord('z'):  # undo
					self.do_undo()
				elif key == ord('r'):  # redo
					self.do_redo()
			else:
				is_sdl2 = (EventLoop.window.__class__.__module__ ==
						   'kivy.core.window.window_sdl2')
				if is_sdl2:
					# we expect to get managed key input via on_textinput
					return
				if self._selection:
					self.delete_selection()
				self.insert_text(text)
			# self._recalc_size()
			return

		if is_interesting_key:
			self._hide_cut_copy_paste(win)
			self._hide_handles(win)

		if key == 27:  # escape
			self.focus = False
			return True
		elif key == 9:  # tab
			self.insert_text(u'\t')
			return True

		k = self.interesting_keys.get(key)
		if k:
			key = (None, None, k, 1)
			self._key_down(key)


import requests
import time
import execjs
import pandas as pd

folderCode = './data/'

class FundInfoCrawler:
    def __init__(self, plateCode):
        self.fileFundTrain = folderCode + plateCode + "/fundinfo.csv"

        fundTrainTemp = pd.read_csv(folderCode + plateCode + "/code.csv",header=None,dtype=object).values.tolist()[0]
        self.fundTrainCode = fundTrainTemp

    def getUrl(self, fscode):
        head = 'http://fund.eastmoney.com/pingzhongdata/'
        tail = '.js?v=' + time.strftime("%Y%m%d%H%M%S", time.localtime())
        return head+fscode+tail

    # 根据基金代码获取净值
    def getWorth(self, fscode):
        content = requests.get(self.getUrl(fscode))
        jsContent = execjs.compile(content.text)
        # 累计净值走势
        ACWorthTrend = jsContent.eval('Data_ACWorthTrend')
        ACWorth = []
        for dayACWorth in ACWorthTrend:
            ACWorth.append(dayACWorth[1])
        return ACWorth

    def getFundTrain(self):
        trainFundFile = open(self.fileFundTrain, 'w')
        for code in self.fundTrainCode:
            try:
                trainFund = self.getWorth(code)
            except:
                continue
            if len(trainFund) > 0:
                trainFundFile.write(code + ",")
                trainFundFile.write(",".join(list(map(str, trainFund))))
                trainFundFile.write("\n")
                print('{} data downloaded'.format(code))
        trainFundFile.close()


import json
import argparse
import sys

parser = argparse.ArgumentParser()
parser.add_argument('--input', type=str, required=True)
parser.add_argument('--baseline', type=str, required=True)
parser.add_argument('--keys', type=str, required=True, help='Comma-separated list of keys to compare')
parser.add_argument('--tolerance', type=float, default=0.9)

args = parser.parse_args()

args.keys = args.keys.split(',')

input = json.load(open(args.input))
baseline = json.load(open(args.baseline))

fail = False
for key in args.keys:
    print('Checking key:', key)
    input_value = input['run'][key]
    baseline_value = baseline['run'][key]
    print("input value: ", input_value)
    print("baseline value: ", baseline_value)

    # only 'higher is better'-type scenarios currently supported
    if input_value < 0.9 * baseline_value:
        print('FAILED\n')
        fail = True
    else:
        print('PASSED\n')

if fail:
    sys.exit(1)
else:
    sys.exit(0)


import torch
import numpy as np
import random
from data.Dataset import SingleImageDataset
from models.model import Model
from util.losses import LossG
from util.util import get_scheduler, get_optimizer, save_result
import yaml
from argparse import ArgumentParser
from tqdm import tqdm

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

def train_model(dataroot, callback=None):
    with open("conf/default/config.yaml", "r") as f:
        config = yaml.safe_load(f)

    cfg = config

    if dataroot is not None:
        cfg['dataroot'] = dataroot

    # set seed
    seed = cfg['seed']
    if seed == -1:
        seed = np.random.randint(2 ** 32 - 1, dtype=np.int64)
    random.seed(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    print(f'running with seed: {seed}.')

    # create dataset, loader
    dataset = SingleImageDataset(cfg)

    # define model
    model = Model(cfg)

    # define loss function
    criterion = LossG(cfg)

    # define optimizer, scheduler
    optimizer = get_optimizer(cfg, model.netG.parameters())

    scheduler = get_scheduler(optimizer,
                              lr_policy=cfg['scheduler_policy'],
                              n_epochs=cfg['n_epochs'],
                              n_epochs_decay=cfg['scheduler_n_epochs_decay'],
                              lr_decay_iters=cfg['scheduler_lr_decay_iters'])

    with tqdm(range(1, cfg['n_epochs'] + 1)) as tepoch:
        for epoch in tepoch:
            inputs = dataset[0]
            for key in inputs:
                inputs[key] = inputs[key].to(device)
            optimizer.zero_grad()
            outputs = model(inputs)
            losses = criterion(outputs, inputs)
            loss_G = losses['loss']
            log_data = losses
            log_data['epoch'] = epoch

            # update learning rate
            lr = optimizer.param_groups[0]['lr']
            log_data["lr"] = lr
            tepoch.set_description(f"Epoch {log_data['epoch']}")
            tepoch.set_postfix(loss=log_data["loss"].item(), lr=log_data["lr"])

            # log current generated entire image
            if epoch % cfg['log_images_freq'] == 0:
                img_A = dataset.get_A().to(device)
                with torch.no_grad():
                    output = model.netG(img_A)
                save_result(output[0], cfg['dataroot'])
                if callback is not None:
                    callback(output[0])

            loss_G.backward()
            optimizer.step()
            scheduler.step()

if __name__ == '__main__':
    parser = ArgumentParser()
    parser.add_argument("--dataroot", type=str)
    args = parser.parse_args()
    dataroot = args.dataroot

    train_model(dataroot)


import logging
from mongolog.handlers import MongoHandler
log = logging.getLogger('demo')
log.setLevel(logging.DEBUG)
log.addHandler(MongoHandler.to(db='mongolog', collection='log'))

log.debug("1 - debug message")
log.info("2 - info message")
log.warn("3 - warn message")
log.error("4 - error message")
log.critical("5 - critical message")


import random
import argparse
import string
import math
import time

# Genetic Algorithm Solution

# Location class used for first domain
# Nodes encoded as A-T starting in bottom left corner and
# moving left-right while moving up a row
class Location:
	def __init__(self, label, x, y):
		self.label = label
		self.x = int(x)
		self.y = int(y)
	def get_coords(self):
		return (self.x, self.y)

	def get_label(self):
		return self.label

	def print_loc(self):
		print("Label: " + self.label + ", x: " + str(self.x) + ", y: " + str(self.y) )

	def has_label(self, label):
		return self.label == label

# Town class used for second domain
# Each town will have a label A-G
# Each town tracks the distance to another town with a dictionary
class Town:
	def __init__(self, town, label):
		self.town = town
		self.label = label
		self.distances = dict()

	def get_label(self):
		return self.label

	def has_label(self, label):
		return self.label == label

	def set_dist(self, label, dist):
		self.distances[label] = dist

	def get_dist(self, label):
		return self.distances[label]

# Main genetic algorithm
# Need to expand to handle second domain
def genetic(inputs, term_criteria, mutation_rate, init_pop, crossover_method, mutation_method, domain):
	best_path = None
	# STEP 1: Initialize population
	population = get_initial_population(inputs, init_pop)
	start_time = time.time()
	# Will loop until termination criteria is met
	while(True):
		distances = []
		# STEP 2: Calculate fitness value for each state
		for p in population:
			distances.append(calc_distance(p, inputs, domain))
		max_path = max(distances)
		# Scale fitness so smaller paths are more fit
		fitness_vals = [max_path-x for x in distances]
		# Zip fitness value with respective population
		pop = [ (y,x) for x, y in zip(population, fitness_vals)]
		# STEP 3: Select parents for crossover
		# Maybe want to pick 40% of population to be parents
		num_parents = init_pop * 0.4
		# Want even number of parents
		if num_parents % 2 == 1:
			num_parents -= 1
		parents = select_parents(pop, num_parents)
		# STEP 4: Generate child from crosssover of parents
		# maybe want 20% of population to be children in next itertion
		# STEP 5: Mutation of child will happen in crossover function at random
		children = crossover(parents, mutation_rate, crossover_method)
		# Check if algorithm should be terminated
		# Termination criteria is runtime of algorithm surpasses given time
		# If terminated will return the most fit (i.e. shortest path) state
		print("Time elapsed: " + str(time.time() - start_time))
		if(time.time() - start_time >= term_criteria):
			pop.sort(reverse=True)
			best_path = list(pop[0])
			best_path[0] = max_path - best_path[0]
			break
		# Prune population for next iteration
		population = prune_pop(pop, children)
		# print("Time elapsed: ", time.time()-start_time)
	return tuple(best_path)

# Calculate the fitness of a state
# Calculated by total distance of state path
def calc_distance(state, inputs, domain):
	distance = 0
	for i in range(len(state)):
		loc1 = get_location(state[i-1], inputs)
		loc2 = get_location(state[i], inputs)
		distance += get_distance(loc1, loc2, domain)
	return distance

# Get random states for initial population
# Ensures that locations are not visited twice
def get_initial_population(inputs, init_pop):
	pop = []
	while len(pop) < init_pop:
		state = ''
		unadded_chars = string.ascii_uppercase[:len(inputs)]
		while len(state) < len(inputs):
			label = unadded_chars[random.randint(0, len(unadded_chars)-1)]
			if label not in state:
				state += label
				unadded_chars.replace(label, "")
		if(state not in pop):
			pop.append(state)
	return pop

# Get location object with given label value
def get_location(label, inputs):
	for i in inputs:
		if i.has_label(label):
			return i
	return None

# Mutate child by swapping two locations at random
# First mutation shuffles each half of the state independently
# Second mutation swaps two random indices of the state
def mutate(state, mutation=1):
	mutant = None
	if mutation == 1:
		i, j = 0, 0
		while i == j:
			i = random.randint(0, len(state)-1)
			j = random.randint(0, len(state)-1)
		mutant = list(state)
		mutant[i] = state[j]
		mutant[j] = state[i]
	else:
		mid = len(state)/2
		mutant = list(state)
		mutant_1 = mutant[:mid]
		random.shuffle(mutant1)
		mutant_2 = mutant[mid:]
		random.shuffle(mutant_2)
		mutant = []
		mutant.extend(mutant_1)
		mutant.extend(mutant_2)
	return ''.join(mutant)

# Crossover two parent states to create child state.
# Child will randomly mutate at rate mutation_rate
# 1 child created per pair of parents
# Should add edge recombination crossover for method #2, as its the most accurate
def crossover(parents, mutation_rate, method=1):
	children = []
	while len(parents) > 0:
		i, j = 0, 0
		while i == j:
			i, j = random.randint(0, len(parents)-1), random.randint(0, len(parents)-1)
		parent1 = parents[i]
		parent2 = parents[j]
		child= ''
		if method == 1:
			pos = random.randint(0, len(parent1) -1)
			child += parent1[:pos]
			child += parent2[pos:]
			child = fix_child(child)

			if(random.randint(0, 99) < mutation_rate):
				child = mutate(child)

			child = fix_child(child)
			children.append(child)
			parents.remove(parent1)
			parents.remove(parent2)
		else:
			neighbours = dict()
			for c in string.ascii_uppercase[:len(parent1)]:
				n_set = set()
				i = parent1.index(c)
				n_set.add(parent1[i-1])
				if(i+1 >= len(parent1)):
					i = -1
				n_set.add(parent1[i+1])
				i = parent2.index(c)
				n_set.add(parent2[i-1])
				if(i+1 >= len(parent2)):
					i = -1
				n_set.add(parent2[i+1])
				neighbours[c] = n_set
			r = random.randint(0,1)
			if r == 0:
				child += parent1[0]
			else:
				child += parent2[0]
			neighbours = remove_element(neighbours, child[0])
			while len(child) < len(parent1):
				least = get_least_neighbours(neighbours)
				neighbours.pop(least, None)
				child+= least
				neighbours = remove_element(neighbours, least)
			child = fix_child(child)
			if(random.randint(0, 99) < mutation_rate):
				child = mutate(child)

			children.append(child)
			parents.remove(parent1)
			parents.remove(parent2)
	return children

# Find the neighbour set with the fewest elements
def get_least_neighbours(neighbours):
	least_neighbours = []
	least = 999
	for key in neighbours.keys():
		if len(neighbours[key]) < least:
			least_neighbours = []
			least = len(neighbours[key])
			least_neighbours.append(key)
		elif len(neighbours[key]) == least:
			least_neighbours.append(key)
	if(len(least_neighbours) == 1):
		return least_neighbours[0]
	else:
		i = random.randint(0, len(least_neighbours)-1)
		return least_neighbours[i]

# Removes the given element from each set in neighbours, if the element
# exists in the set.
def remove_element(neighbours, element):
	for key in neighbours.keys():
		s = neighbours[key]
		s.discard(element)
		neighbours[key] = s
	return neighbours

# Remove duplicate values/add in missing values from crossover
def fix_child(child):
	multiples = []
	missing = []
	fixed = False
	while(not fixed):
		fixed = True
		for c in string.ascii_uppercase[:len(child)]:
			if child.count(c) == 0:
				fixed = False
				missing.append(c)
			if child.count(c) > 1:
				fixed = False
				multiples.append(c)
		child = list(child)
		for i in range(len(child)):
			if child[i] in multiples:
				multiples.remove(child[i])
				child[i] = missing[random.randint(0, len(missing)-1)]
				missing = missing[1:]
	return ''.join(child)

# Select states to act as parents
def select_parents(pop, num_parents):
	fitness_sum = 0
	parents = []
	rand_pop = pop.copy()
	random.shuffle(rand_pop)
	for p in pop:
		fitness_sum +=  p[0]
	threshold = random.randint(0, int(fitness_sum))
	while len(parents) < num_parents:
		s = 0
		for p in rand_pop:
			s+= p[0]
			if s > threshold:
				parents.append(p[1])
				fitness_sum -= p[0]
				rand_pop.remove(p)
				random.shuffle(rand_pop)
				break
		threshold = random.randint(0, int(fitness_sum))
	return parents

# Prune population
def prune_pop(pop, children):
	new_pop = []
	pop.sort(reverse=True)
	for p in pop:
		new_pop.append(p[1])
	new_pop.extend(children)
	return new_pop

# Get distance between two locations
def get_distance(loc1, loc2, domain):
	if domain == 1:
		x1, y1 = loc1.get_coords()
		x2, y2 = loc2.get_coords()
		return (math.sqrt((x1-x2)**2 + (y1-y2)**2))
	else:
		return loc1.get_dist(loc2.get_label())

# Read in file
def read_data(fname, domain):
	if domain == 1:
		locations = []
		with open(fname, 'r') as f:
			for l in f.readlines():
				l = l.split(" ")
				locations.append(Location(l[0], l[1], l[2]))
		f.close()
		return locations
	else:
		towns = []
		with open(fname, 'r') as f:
			for i, l in enumerate(f.readlines()):
				vals = l.split(" ")
				t = Town(vals[0], string.ascii_uppercase[i])
				vals = vals[1:]
				for j, v in enumerate(vals):
					t.set_dist(string.ascii_uppercase[j], int(v))
				towns.append(t)
		f.close()
		return towns

# Write output file for use in graph generation
def write_path_to_file(path, locs, domain, dist):
	fname = "output.txt"
	with open(fname, 'w+') as f:
		f.write(str(domain) + "\n")
		f.write(str(dist) + "\n")
		if domain == 1:
			for i in range(len(path)):
				loc = get_location(path[i], locs)
				x, y = loc.get_coords()
				f.write(str(x) + "," + str(y) + "\n")
		else:
			f.write(path + "\n")
	f.close()

def main(args):
	fname = args.filename
	term_criteria = args.terminate
	domain = args.domain
	mutation_rate = args.mutation_rate
	init_pop = args.init_pop
	crossover_method = args.crossover_method
	mutation_method = args.mutation_method

	print("Finding best path for domain " + str(domain) + " with parameters:")
	print("Teminates after " + str(term_criteria) + " seconds")
	print("With an initial population of " + str(init_pop))
	print("Mutates at a rate of " + str(mutation_rate) + "%")
	print("Using mutation method " + str(mutation_method))
	print("Using crossover method " + str(crossover_method))

	inputs = read_data(fname, domain)
	best_path = genetic(inputs, term_criteria, mutation_rate, init_pop, crossover_method, mutation_method, domain)
	print("Best path is: " + best_path[1] + " with distance " + str(best_path[0]))
	write_path_to_file(best_path[1], inputs, domain, best_path[0])

if __name__ == '__main__':
	parser = argparse.ArgumentParser()
	parser.add_argument('-t', '--terminate', default=0, type=int)
	parser.add_argument('-m', '--mutation_rate', default=10, type=int)
	parser.add_argument('-c', '--crossover_method', default=1, type=int)
	parser.add_argument('-mm', '--mutation_method', default=1, type=int)
	parser.add_argument('-i', '--init_pop', default=10, type=int)
	parser.add_argument('-d', '--domain', default=1, type=int)
	parser.add_argument('filename')
	main(parser.parse_args())


import pandas as pd
import plotly.express as px
import plotly.graph_objs as go
import dash
import dash_core_components as dcc
import dash_html_components as html

def cargarArchivo(ruta):
    return pd.read_csv(ruta)

emisiones = cargarArchivo("DATOS/Emisiones_limpio.csv")

print(emisiones.head())

entidades = emisiones["ENTIDAD"].unique()

lista_entidades = []

for i in entidades:
    lista_entidades.append(["Femenino", i, emisiones[(emisiones["SEXO"] == "Femenino") & (emisiones["ENTIDAD"] == i)]["NUMERO DE ASEGURADOS"].sum()])
    lista_entidades.append(["Masculino", i, emisiones[(emisiones["SEXO"] == "Masculino") & (emisiones["ENTIDAD"] == i)]["NUMERO DE ASEGURADOS"].sum()])

df_entidades = pd.DataFrame(lista_entidades, columns=["SEXO", "ENTIDAD", "OCURRENCIAS"])

fig_entidades = px.bar(df_entidades, x="ENTIDAD", y="OCURRENCIAS", color="SEXO", barmode="group",
             title="Número de asegurados por entidad y sexo")
fig_entidades.update_layout(xaxis_title="", yaxis_title="Número de asegurados", legend_title="")

comisiones = cargarArchivo("DATOS/Comisiones_limpio.csv")

polizas = comisiones["MODALIDAD DE LA POLIZA"].unique()

lista_polizas = []

for i in polizas:
    lista_polizas.append(["Femenino", i, comisiones[(comisiones["SEXO"] == "Femenino") & (comisiones["MODALIDAD DE LA POLIZA"] == i)]["NUMERO DE ASEGURADOS"].sum()])
    lista_polizas.append(["Masculino", i, comisiones[(comisiones["SEXO"] == "Masculino") & (comisiones["MODALIDAD DE LA POLIZA"] == i)]["NUMERO DE ASEGURADOS"].sum()])

df_polizas = pd.DataFrame(lista_polizas, columns=["SEXO", "MODALIDAD DE LA POLIZA", "OCURRENCIAS"])

fig_polizas = go.Figure

import pandas as pd
import plotly.express as px
import dash
import dash_core_components as dcc
import dash_html_components as html

# Cargar los datos
emisiones = pd.read_csv("DATOS/Emisiones_limpio.csv")
comisiones = pd.read_csv("DATOS/Comisiones_limpio.csv")

# Crear la aplicación de Dash
app = dash.Dash(__name__)

# Crear el layout
app.layout = html.Div(children=[
    html.H1(children='Número de asegurados por entidad y sexo'),
    dcc.Graph(
        id='entidades',
        figure=px.bar(df_entidades, x="ENTIDAD", y="OCURRENCIAS", color="SEXO", barmode="group",
                      title="Número de asegurados por entidad y sexo")
    ),
    html.H1(children='Número de asegurados por modalidad de la póliza y sexo'),
    dcc.Graph(
        id='polizas',
        figure=px.bar(df_polizas, x="MODALIDAD DE LA POLIZA", y="OCURRENCIAS", color="SEXO", barmode="group",
                      title="Número de asegurados por modalidad de la póliza y sexo")
    )
])

# Iniciar la aplicación de Dash
if __name__ == '__main__':
    app.run_server(debug=True)