# coding=utf-8
# 此文件定义和房屋有关的api
import json
from datetime import datetime
from . import api

from iHome import db, constants, redis_store
from iHome.models import Area, Order
from iHome.response_code import RET
from iHome.models import House, Facility, HouseImage
from iHome.utils.commons import login_required
from iHome.utils.image_storage import image_storage

from flask import current_app, jsonify, request, g, session

@api.route('/users/houses')
@login_required
def get_user_houses():
    """
    获取用户发布的房屋信息:
    1. 根据登录id查询用户的所有房屋信息
    2. 组织数据，返回应答
    """
    # 1. 根据登录id查询用户的所有房屋信息
    user_id = g.user_id
    try:
        houses = House.query.filter(House.user_id == user_id).all()
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg='查询房屋信息失败')

    # 2. 组织数据，返回应答
    # 组织数据
    houses_dict_li = []
    for house in houses:
        houses_dict_li.append(house.to_basic_dict())

    return jsonify(errno=RET.OK, errmsg='OK', data=houses_dict_li)

@api.route('/houses')
def get_house_list():
    """
    搜索房屋信息:
    1. 接收参数并进行参数校验（城区id，起始时间，结束时间，排序方式，页码）
    2. 根据城区的id过滤房屋数据
    3. 根据搜索的起始时间和结束时间排序冲突的房屋信息
    4. 根据排序方式进行排序
    5. 对搜索结果的进行分页操作
    6. 组织数据(将房屋信息转换为字典数据)并返回应答
    """
    # print request.args
    area_id = request.args.get('aid')
    # new: 最新上线 booking: 入住最多 price-inc: 价格低->高 price-des: 价格高->低
    sort_key = request.args.get('sk', 'new')
    page = request.args.get('p')
    sd = request.args.get('sd')
    ed = request.args.get('ed')

    start_date = None
    end_date = None
    try:
        if area_id:
            area_id = int(area_id)

        page = int(page)

        if sd:
            start_date = datetime.strptime(sd, '%Y-%m-%d')

        if ed:
            end_date = datetime.strptime(ed, '%Y-%m-%d')

        if start_date and end_date:
            assert start_date < end_date, Exception('起始时间大于结束时间')
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.PARAMERR, errmsg='参数错误')

    # 尝试从缓存中获取搜索结果
    try:
        key = 'house:%s:%s:%s:%s' % (area_id, sd, ed, sort_key)
        res_json_str = redis_store.hget(key, page)
        if res_json_str:
            resp = json.loads(res_json_str)
            return jsonify(errno=RET.OK, errmsg='OK', data=resp)
    except Exception as e:
        current_app.logger.error(e)

    # 获取所有房屋的信息
    try:
        # houses = House.query.all()
        houses_query = House.query

        # 根据城区的id过滤房屋的信息，返回查询
        if area_id:
            houses_query = houses_query.filter(House.area_id == area_id)

        # 获取冲突订单
        try:
            conflict_orders_li = []
            if start_date and end_date:
                conflict_orders_li = Order.query.filter(end_date > Order.begin_date, start_date < Order.end_date).all()
            elif start_date:
                conflict_orders_li = Order.query.filter(start_date < Order.end_date).all()
            elif end_date:
                conflict_orders_li = Order.query.filter(end_date > Order.begin_date).all()

            if conflict_orders_li:
                # 获取冲突订单对应的房屋id
                conflict_houses_id = [order.house_id for order in conflict_orders_li]
                houses_query = houses_query.filter(House.id.notin_(conflict_houses_id))

        except Exception as e:
            current_app.logger.error(e)
            return jsonify(errno=RET.DBERR, errmsg='查询冲突订单失败')

        # 进行排序
        if sort_key == 'booking':
            houses_query = houses_query.order_by(House.order_count.desc())
        elif sort_key == 'price-inc':
            houses_query = houses_query.order_by(House.price)
        elif sort_key == 'price-des':
            houses_query = houses_query.order_by(House.price.desc())
        else:
            houses_query = houses_query.order_by(House.create_time.desc())

        # 分页操作
        house_paginate = houses_query.paginate(page, constants.HOUSE_LIST_PAGE_CAPACITY, False)
        # 获取当前页的结果列表
        houses = house_paginate.items
        # 获取分页之后的总页数
        total_page = house_paginate.pages

    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg='查询房屋信息失败')

    # 组织数据
    houses_dict_li = []
    for house in houses:
        houses_dict_li.append(house.to_basic_dict())

    # 在redis中缓存数据
    resp = {'houses': houses_dict_li, 'total_page': total_page}
    try:
        key = 'house:%s:%s:%s:%s' % (area_id, sd, ed, sort_key)

        # 获取管道对象
        pipeline = redis_store.pipeline()
        # 开启redis事务
        pipeline.multi()
        # 向管道中添加执行命令
        pipeline.hset(key, page, json.dumps(resp))
        pipeline.expire(key, constants.HOUSE_LIST_REDIS_EXPIRES)
        # 提交redis事务
        pipeline.execute()
    except Exception as e:
        current_app.logger.error(e)

    # 返回应答
    return jsonify(errno=RET.OK, errmsg='OK', data=resp)

@api.route('/houses/index')
def get_houses_index():
    """
    获取首页展示房屋信息：
    1. 获取房屋的信息，按照创建时间进行降序排序，默认取前5个
    2. 组织数据，返回应答
    """
    # 1. 获取房屋的信息，按照创建时间进行降序排序，默认取前5个
    try:
        houses = House.query.order_by(House.create_time.desc()).limit(constants.HOME_PAGE_MAX_HOUSES).all()
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg='获取房屋信息失败')

    # 2. 组织数据，返回应答
    houses_dict_li = []
    for house in houses:
        houses_dict_li.append(house.to_basic_dict())

    return jsonify(errno=RET.OK, errmsg='OK', data=houses_dict_li)

@api.route('/house/<int:house_id>')
def get_house_info(house_id):
    """
    获取房屋的详情信息:
    1. 根据房屋id获取房屋信息（如果查不到，代表房屋不存在)
    2. 组织数据，返回应答
    """
    # 1. 根据房屋id获取房屋信息（如果查不到，代表房屋不存在)
    try:
        house = House.query.get(house_id)
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg='查询房屋信息失败')

    if not house:
        return jsonify(errno=RET.NODATA, errmsg='房屋不存在')

    # 2. 组织数据，返回应答
    # 尝试从session获取user_id, 如果取不到，返回-1
    user_id = session.get('user_id', -1)

    return jsonify(errno=RET.OK, errmsg='OK', data={'house': house.to_full_dict(), 'user_id': user_id})

@api.route('/houses/image', methods=["POST"])
@login_required
def save_house_image():
    """
    上传房屋的图片:
    1. 接收房屋的id 和 房屋图片文件 并进行参数校验
    2. 上传房屋的图片到七牛云
    3. 创建HouseImage对象并保存房屋图片信息
    4. 添加房屋图片信息到数据库
    5. 返回应答
    """
    # 1. 接收房屋的id 和 房屋图片文件 并进行参数校验
    house_id = request.form.get('house_id')

    if not house_id:
        return jsonify(errno=RET.PARAMERR, errmsg='缺少参数')

    file = request.files.get('house_image')

    if not file:
        return jsonify(errno=RET.PARAMERR, errmsg='缺少参数')

    try:
        house = House.query.get(house_id)
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg='查询房屋信息失败')

    if not house:
        return jsonify(errno=RET.NODATA, errmsg='房屋不存在')

    # 2. 上传房屋的图片到七牛云
    try:
        key = image_storage(file.read())
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.THIRDERR, errmsg='上传房屋图片失败')

    # 3. 创建HouseImage对象并保存房屋图片信息
    house_image = HouseImage()
    house_image.house_id = house_id
    house_image.url = key

    # 判断当前房屋是否有默认的图片，如果没有，进行添加
    if not house.index_image_url:
        house.index_image_url = key

    # 4. 添加房屋图片信息到数据库
    try:
        db.session.add(house_image)
        db.session.commit()
    except Exception as e:
        db.session.rollback()
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg='保存房屋图片信息失败')

    # 5. 返回应答
    img_url = constants.QINIU_DOMIN_PREFIX + key
    return jsonify(errno=RET.OK, errmsg='OK', data={'img_url': img_url})

@api.route('/houses', methods=["POST"])
@login_required
def save_new_house():
    """
    发布新房源:
    1. 接收房屋的基本信息并进行参数校验
    2. 创建House对象并保存房屋基本信息
    3. 将房屋的基本信息添加进数据库
    4. 返回应答
    """
    # 1. 接收房屋的基本信息并进行参数校验
    req_dict = request.json

    title = req_dict.get('title')
    price = req_dict.get('price') # 房屋价格
    address = req_dict.get('address')
    area_id = req_dict.get('area_id')
    room_count = req_dict.get('room_count')
    acreage = req_dict.get('acreage')
    unit = req_dict.get('unit')
    capacity = req_dict.get('capacity')
    beds = req_dict.get('beds')
    deposit = req_dict.get('deposit') # 房屋押金
    min_days = req_dict.get('min_days')
    max_days = req_dict.get('max_days')

    if not all([title, price, address, area_id, room_count, acreage, unit, capacity, beds, deposit, min_days, max_days]):
        return jsonify(errno=RET.PARAMERR, errmsg='参数缺失')

    try:
        # 数据库中房屋的价格和押金以 分 保存
        price = float(price) * 100
        deposit = float(deposit) * 100
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.PARAMERR, errmsg='参数错误')

    # 2. 创建House对象并保存房屋基本信息
    house = House()
    house.user_id = g.user_id
    house.area_id = area_id
    house.title = title
    house.price = price
    house.address = address
    house.room_count = room_count
    house.acreage = acreage
    house.unit = unit
    house.capacity = capacity
    house.beds = beds
    house.deposit = deposit
    house.min_days = min_days
    house.max_days = max_days

    # 获取房屋的设施信息
    facility = req_dict.get('facility') # [1, 3, 4]

    try:
        facilities = Facility.query.filter(Facility.id.in_(facility)).all()
        if facilities:
            house.facilities = facilities
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg='获取房屋设施信息失败')

    # 3. 将房屋的基本信息添加进数据库
    try:
        db.session.add(house)
        db.session.commit()
    except Exception as e:
        db.session.rollback()
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg='保存房屋信息失败')

    # 4. 返回应答
    return jsonify(errno=RET.OK, errmsg='OK', data={'house_id': house.id})

@api.route('/areas')
def get_areas():
    """
    获取城区信息:
    1. 从数据库中获取所有的城区信息
    2. 组织数据，返回应答
    """
    # 先尝试从缓存中获取城区的信息，如果获取到，直接返回，如果获取不到，再去查询数据库
    try:
        area_json_str = redis_store.get("areas")
        if area_json_str:
            areas_dict_li = json.loads(area_json_str)
            return jsonify(errno=RET.OK, errmsg='OK', data=areas_dict_li)
    except Exception as e:
        current_app.logger.error(e)

    # 1. 从数据库中获取所有的城区信息
    try:
        areas = Area.query.all()
    except Exception as e:
        current_app.logger.error(e)
        return jsonify(errno=RET.DBERR, errmsg='查询城区信息失败')

    # 2. 组织数据，返回应答
    areas_dict_li = []
    for area in areas:
        areas_dict_li.append(area.to_dict())

    # redis设置缓存
    try:
        redis_store.set('areas', json.dumps(areas_dict_li), constants.AREA_INFO_REDIS_EXPIRES)
    except Exception as e:
        current_app.logger.error(e)

    return jsonify(errno=RET.OK, errmsg='OK', data=areas_dict_li)


#####################################################################

# Example : Neural Network learning
# basic illustrative python script

# For use with test / training datasets : optdigits.{train | test}

# Author : Toby Breckon, toby.breckon@durham.ac.uk

# Copyright (c) 2014 / 16 School of Engineering & Computing Sciences,
#                    Durham University, UK
# License : LGPL - http://www.gnu.org/licenses/lgpl.html

#####################################################################

import csv
import cv2
import numpy as np
import os

########### Define classes

### classes as integers {0 ... 9}

#####################################################################

########### construct output layer

# expand training responses defined as class labels {0,1...,N} to output layer
# responses for the OpenCV MLP (Neural Network) implementation such that class
# label c becomes {0,0,0, ... 1, ...0} where the c-th entry is the only non-zero
# entry (equal to "value", conventionally = 1) in the N-length vector

# labels : a row vector of class label transformed to {0,0,0, ... 1, ...0}
# max_classes : maximum class label
# value: value use to label the class response in the output layer vector
# sigmoid : {true | false} - return {-value,....value,....-value} instead for
#           optimal use with OpenCV sigmoid function

def class_label_to_nn_output(label, max_classes, is_sigmoid, value):
    if (is_sigmoid):
        output = np.ones(max_classes).astype(np.float32) * (-1 * value)
        output[int(label)] = value
    else:
        output = np.zeros(max_classes).astype(np.float32)
        output[int(label)] = value

    return output

#####################################################################

########### Load Training and Testing Data Sets

# load training data set

# Training data - as currenrtly split
def loadTrainingData():
    attribute_list = []
    label_list = []
    nn_outputs_list = []
    counter = 0
    reader=csv.reader(open(os.path.join("x_vals_train_all_1.txt"),"rt", encoding='ascii'),delimiter=' ')
    for row in reader:
            counter = counter+1
          #  print(counter)
            # attributes in columns 0-561
            lis = []
            for i in range(0,36):
                rowAmend = ''.join(filter(str.isdigit, row[i]))
                lis.append(rowAmend)
            attribute_list.append(lis)

    reader=csv.reader(open(os.path.join("y_vals_train_all_1.txt"),"rt", encoding='ascii'),delimiter=' ')
    for row in reader:
            # attributes in column 1
            label_list.append(row[0])
            nn_outputs_list.append(class_label_to_nn_output(row[0],13,True,1))

    training_attributes=np.array(attribute_list).astype(np.float32)

    training_class_labels=np.array(label_list).astype(np.float32)
    training_nn_outputs=np.array(nn_outputs_list).astype(np.float32)

    return training_attributes, training_nn_outputs, training_class_labels

# Testing data - as currently split
def testingData():
    attribute_list = []
    label_list = []

    counter = 0

    reader=csv.reader(open(os.path.join("x_vals_test_all_1.txt"),"rt", encoding='ascii'),delimiter=' ')
    for row in reader:
        counter = counter + 1
        #  print(counter)
        # attributes in columns 0-561
        lis = []
        for i in range(0, 36):
            rowAmend = ''.join(filter(str.isdigit, row[i]))
            lis.append(rowAmend)
        attribute_list.append(lis)

    counter = 0

    reader=csv.reader(open(os.path.join("y_vals_test_all_1.txt"),"rt", encoding='ascii'),delimiter=' ')
    for row in reader:
            counter = counter+1
          #  print(counter)
            # attributes in column 1
            label_list.append(row[0])

    testing_attributes=np.array(attribute_list).astype(np.float32)
    testing_labels=np.array(label_list).astype(np.float32)
    return testing_attributes, testing_labels

###create validation set for testing
def createValidationSet(version):
    attribute_list = []
    label_list = []

    counter = 0

    reader = csv.reader(open(os.path.join("x_valid"+str(version)+".txt"), "rt", encoding='ascii'), delimiter=' ')
    for row in reader:
            counter = counter+1
          #  print(counter)
            # attributes in columns 0-561
            lis = []
            for i in range(0,36):
                rowAmend = ''.join(filter(str.isdigit, row[i]))
                lis.append(rowAmend)
            attribute_list.append(lis)

    counter = 0

    reader = csv.reader(open(os.path.join("y_valid"+str(version)+".txt"), "rt", encoding='ascii'), delimiter=' ')
    for row in reader:
        counter = counter + 1
        #  print(counter)
        # attributes in column 1
        label_list.append(row[0])

    validation_attributes = np.array(attribute_list).astype(np.float32)
    validation_lables = np.array(label_list).astype(np.float32)
    return (validation_attributes, validation_lables)
############ Perform Training -- Neural Network
# create the network object

# define number of layers, sizes of layers and train neural network
# neural networks only support numerical inputs (convert any categorical inputs)

# set the network to be 2 layer 64->8->10
# - one input node per attribute in a sample
# - 8 hidden nodes
# - one output node per class
# defined by the column vector layer_sizes
def setParams(nnetwork, layers, momentum, weight, termCriteria1, termCriteria2, train_method, activation_function):
    layer_sizes = np.int32([36, layers, 13]) # format = [inputs, hidden layer n ..., output]
    print(layer_sizes)
    nnetwork.setLayerSizes(layer_sizes);

    # create the network using a sigmoid function with alpha and beta
    # parameters = 1 specified respectively (standard sigmoid)
    if (activation_function == "ANN_MLP_SIGMOID_SYM"):
        print("using sigmoid")
        nnetwork.setActivationFunction(cv2.ml.ANN_MLP_SIGMOID_SYM, 1, 1);
    elif (activation_function == "ANN_MLP_IDENTITY"):
        print("using identity")
        nnetwork.setActivationFunction(cv2.ml.ANN_MLP_IDENTITY, 0, 0);
    else:
        print("using gaussian")
        nnetwork.setActivationFunction(cv2.ml.ANN_MLP_GAUSSIAN, 1, 1);

    # available activation functions = (cv2.ml.ANN_MLP_SIGMOID_SYM or cv2.ml.ANN_MLP_IDENTITY, cv2.ml.ANN_MLP_GAUSSIAN)

    # specify stopping criteria and backpropogation for training

    if (train_method == "RPROP"):
        print("using RPROP")
        nnetwork.setTrainMethod(cv2.ml.ANN_MLP_RPROP);
        nnetwork.setRpropDW0(weight)
        nnetwork.setRpropDWMin(momentum)
    else: #(BACKPROP)
        print("using BACKPROP")
        nnetwork.setTrainMethod(cv2.ml.ANN_MLP_BACKPROP);
        nnetwork.setBackpropMomentumScale(momentum);
        nnetwork.setBackpropWeightScale(weight);

    nnetwork.setTermCriteria((cv2.TERM_CRITERIA_COUNT + cv2.TERM_CRITERIA_EPS, termCriteria1, termCriteria2))
    return nnetwork

        ## N.B. The OpenCV neural network (MLP) implementation does not
        ## support categorical variable output explicitly unlike the
        ## other OpenCV ML classes.
        ## Instead, following the traditional approach of neural networks,
        ## the output class label is formed by we a binary vector that
        ## corresponds the desired output layer result for a given class
        ## e.g. {0, 0 ... 1, 0, 0} components (one element by class) where
        ## an entry "1" in the i-th vector position correspondes to a class
        ## label for class i
        ## for optimal performance with the OpenCV intepretation of the sigmoid
        ## we use {-1, -1 ... 1, -1, -1}

        ## prior to training we must construct these output layer responses
        ## from our conventional class labels (carried out by class_label_to_nn_output()

# train the neural network (using training data)

############ Perform Testing -- Neural Network
def test(testing_attributes, testing_labels, nnetwork, layers, momentum, weight, termCriteria1, termCriteria2, train_method, activation_function, tempArray):
    correct = 0 # handwritten digit correctly identified
    wrong = 0   # handwritten digit wrongly identified

    # for each testing example
    y_true = []
    y_pred = []
    for i in range(0, len(testing_attributes[:,0])) :

        # perform neural network prediction (i.e. classification)

        # (to get around some kind of OpenCV python interface bug, vertically stack the
        #  example with a second row of zeros of the same size and type which is ignored).

        sample = np.vstack((testing_attributes[i,:],
                            np.zeros(len(testing_attributes[i,:])).astype(np.float32)))

        retrval,output_layer_responses = nnetwork.predict(sample)

        # the class label c (result) is the index of the most
        # +ve of the output layer responses (from the first of the two examples in the stack)

        result = np.argmax(output_layer_responses[0])

        #print("Test data example : " + str(i + 1) + " : result = " + str(int(result)))

        # record results as either being correct or wrong

        y_true.append(testing_labels[i])
        y_pred.append(result)

       # confusion = confusion_matrix(y_true, y_pred)
        # print(confusion)
       # prec = findPriors.precision(confusion)

      #  rec = findPriors.recall(confusion)
        print(result)
        print(testing_labels[i])

        if (result == testing_labels[i]) : correct+=1
        elif (result != testing_labels[i]) : wrong+=1

    # output summmary statistics

    total = wrong + correct

    print()
    print("Testing Data Set Performance Summary")
    print("Total Correct : "+ str(round((correct / float(total)) * 100, 2)) + "%")
    print("Total Wrong : "+ str(round((wrong / float(total)) * 100, 2)) + "%")

 #   print("layers: " + str(layers))
 #   print("momentum: " + str(momentum))
  #  print("weight: " + str(weight))
  #  print("termCriteria1: " + str(termCriteria1))
   # print("termCriteria2: " + str(termCriteria2))
   # print("train_method: " + str(train_method))
   # print("activation_function: " + str(activation_function))

    tempArray.append(round((correct / float(total)), 3))
    tempArray.append(round((wrong / float(total)), 3))
    tempArray.append(layers)
    tempArray.append(momentum)
    tempArray.append(weight)
    tempArray.append(termCriteria1)
    tempArray.append(termCriteria2)
    tempArray.append(train_method)
    tempArray.append(activation_function)
    return (correct / float(total))
#####################################################################

##run the neural network procedure
def run(layers, momentum, weight, termCriteria1, termCriteria2, train_method, activation_function, tempArray):
    nnetwork = cv2.ml.ANN_MLP_create();
    training_attributes, training_nn_outputs, training_class_labels = loadTrainingData()
    #validation_attributes1, validation_labels1 = createValidationSet(3)
   # validation_attributes2, validation_labels2 = createValidationSet(2)
    #validation_attributes3, validation_labels3 = createValidationSet(3)
    nnetwork = setParams(nnetwork, layers, momentum, weight, termCriteria1, termCriteria2, train_method, activation_function)
    nnetwork.train(training_attributes, cv2.ml.ROW_SAMPLE, training_nn_outputs);
    testing_attributes, testing_labels = testingData()
    #output = test(testing_attributes, testing_labels, nnetwork, layers, momentum, weight, termCriteria1, termCriteria2, train_method, activation_function, tempArray)
    #outputVal2 = test(validation_attributes2, validation_labels2, nnetwork, layers, momentum, weight, termCriteria1, termCriteria2,
        #          train_method, activation_function, tempArray)
    outputVal1 = test(testing_attributes, testing_labels, nnetwork, layers, momentum, weight, termCriteria1, termCriteria2,
                  train_method, activation_function, tempArray)
    #outputVal3 = test(validation_attributes3, validation_labels3, nnetwork, layers, momentum, weight, termCriteria1, termCriteria2,
           #       train_method, activation_function, tempArray)
    return outputVal1

if __name__ == '__main__':
    run(50, 0.1, 0.1, 100, 0.001, "BACKPROP", "ANN_MLP_SIGMOID_SYM", [])


import os
import io
import json
import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression
import traceback
import time
import sys
import requests
import traceback
import gc
from joblib import dump as joblib_dump, load as joblib_load
import pickle
#from sklearn.externals.joblib import dump as joblib_dump
#from sklearn.externals.joblib import load as joblib_load
import redis as saver_c
import base64

c_mambaml_load_model = 'mambaml_load_model('
# workarround to predict
# predict try to get model from redis memory
#   if not exist then download from s3 and update redis memory again for future processing
# workarround serialize
# binaryze the object and convert to b64 string
# save object to s3
# update redis memory

def mambaml_load_model(key):
    # this is a special function
    # all mambaml functions start with mambaml
    # this function locate the parent object
    # and get the model from memory instead of inputport
    # this function is usefule for predict endpoint
    # works in the same workarround
    # this method return a object getted from redis
    model_type = key.split('_')[0]
    print("MODEL_TYPE", model_type)
    if model_type == 'sklearn' or model_type == 'sklean':
        b64_object = saver_x.get(key)
        stream = b64_to_stream(b64_object)
        ret = joblib_load(stream)
        return joblib_load(stream)
    elif model_type == 'sklearnbestestimator':
        b64_object = saver_x.get(key)
        stream = b64_to_stream(b64_object)
        return joblib_load(stream)
    elif model_type == 'pickle':
        b64_object = saver_x.get(key)
        stream = b64_to_stream(b64_object)
        return pickle.load(stream)
    pass

# joblib.load(io.BytesIO(new_file.read()))
def b64_to_stream(b64data):
    stream = io.BytesIO(base64.b64decode(b64data))
    return stream

def file_to_b64(filename):
    with open(filename, "rb") as f:
        binfile=f.read()
    b64_file = base64.b64encode(binfile)
    return b64_file

def save_c_file(filename, key):
    b64_file = file_to_b64(filename)
    # one day
    # TODO  guardar en s3
    # save in cache
    saver_x.set(key, b64_file, ex=60*60*24)
    pass

def pickle_serialize_object_and_uplodad_to_s3(model, project_id, box_id, port_index):
    filename = 'pickle_{}_{}_{}.pickle'.format(str(project_id), box_id, port_index)
    with open(filename, 'wb') as pickle_file:
        pickle.dump(model, pickle_file)
    # TODO uplodad to s3 first
    save_c_file(filename, filename)
    print("Object saved correctly pickle")
    os.remove(filename)

def sk_learn_model_serialize_object_and_upload_to_s3(model, project_id, box_id, port_index):
    filename = 'sklearn_{}_{}_{}.joblib'.format(str(project_id), box_id, port_index)
    joblib_dump(model, filename, compress = False)
    # TODO upload to s3 first if we have problems with redis
    save_c_file(filename, filename)
    os.remove(filename)
    # upload direct to s3??

def sk_learn_best_estimator_model_serialize_object_and_upload_to_s3(model, project_id, box_id, port_index):
    filename = 'sklearnbestestimator_{}_{}_{}.joblib'.format(str(project_id), box_id, port_index)
    joblib_dump(model.best_estimator_, filename, compress = False)
    save_c_file(filename, filename)
    os.remove(filename)
    # TODO upload to s3 first if we have problems with redis
    # upload direct to s3??

def c_run_str_code(inputs, str_code, depend, params, input_boxes):
    global ret
    ret = []
    func_name = str_code.split('(')[0][4:].strip()

    sniped_code ="""
tmp_ret = FUNC_NAME(inputs)
for x in tmp_ret:
    ret.append(x)"""

    sniped_code = sniped_code.replace('FUNC_NAME',func_name,1)

    # input paramters
    parameters = None
    if not params is None and params != 'null':
        parameters = json.loads(params)

        for param in parameters:
            print("param-->", param)
            if param["type"] == "colselector":
                colsvalues = json.loads(param['value'])

                str_code = str_code.replace(param['name'], str(colsvalues["cols"]))
                print(str_code)
            else:
                str_code = str_code.replace(param['name'], param['value'])

    # if some saver information FOR NOW ONLY ONE mamab_load_model in box
    # cold be multiples but for now only 1 allowed
    # necessiamos parsear el codigo en busca de algo como mamba_load_model(
    line = str_code.find(c_mambaml_load_model)
    if line > 0:
        port_index = int(str_code[line+len(c_mambaml_load_model):line+len(c_mambaml_load_model)+1])
        save_key = input_boxes[port_index].parentBox.get_saver_key(input_boxes[port_index].numport-input_boxes[port_index].parentBox.n_inputs)
        print("RETRIVED KEY", port_index, save_key)
        str_code = str_code.replace(c_mambaml_load_model+str(port_index), c_mambaml_load_model+"'"+save_key+"'")
        print("RETRIVED KEY", port_index, save_key)

    LOC = ""

    if depend:
        LOC = str_code +("\r\n")+depend+("\r\n")+ sniped_code
    else:
        LOC = str_code+ "\r\n" + sniped_code

    print("NEWCODE", LOC)
    # debemos remplazar el mamba_load_model(index_port_id) por
    # mamba_load_model(BOX_PARENT_ID of index 0, index_port_id)
    exec(LOC)

    return ret

class InputPort():

    def __init__(self, name, numport, parentBox, input_port_num):
        self.name = name
        self.numport = numport
        self.parentBox = parentBox
        self.input_port_num = input_port_num

class BoxCode():

    def __init__(self, str_code, box_id, n_inputs,
                 n_outputs, json, depend, params, changed,
                 project_id, host, serialize_outputs, response=False):
        self.project_id = project_id
        self.host = host
        self.str_code = str_code
        self.depend  = depend
        self.n_inputs = n_inputs
        self.n_outputs = n_outputs
        self.inputs = []
        self.outputs = []
        self.box_id = box_id
        self.response = response
        self.setStatus('INIT')
        self.json = json
        self.params = params
        self.changed = changed
        self.serialize_outputs = serialize_outputs

    def get_saver_key(self, output_num):
        # necessitamos devolver algo como asi
        if self.serialize_outputs!=None and self.serialize_outputs!='null' and self.serialize_outputs != "[]":
            s_outputs = json.loads(self.serialize_outputs)
            for out in s_outputs:
                port_index = int(out['outputnum'])
                port_type = out['Outputtype']
                print("analize", port_index, output_num)
                if port_index == output_num and (port_type == 'sklearn_model' or port_type == 'sklean_model'):
                    return 'sklearn_{}_{}_{}.joblib'.format(str(self.project_id), self.box_id, str(port_index))
                elif port_index == output_num and port_type == 'GridSearch Model':
                    return 'sklearnbestestimator_{}_{}_{}.joblib'.format(str(self.project_id), self.box_id, str(port_index))
                elif (port_index == output_num and port_type == 'pickle'):
                    return 'pickle_{}_{}_{}.pickle'.format(str(self.project_id), self.box_id, str(port_index))
        return None

    def clear_inputs(self):
        for x in self.inputs:
            del x
        self.inputs = []

    def setChangedBox(self, str_code, box_id, n_inputs,
                      n_outputs, json, depend, params, changed, project_id, host, serialize_outputs):
        # to set trained false
        self.freespace()
        # if this box changed then result of preview run is empty
        self.json['nodes'][self.box_id]['properties']['payload']['result']=dict()
        # reset all parameter with new values
        self.__init__(str_code, box_id, n_inputs, n_outputs, json, depend,
                      params, changed, project_id, host, serialize_outputs, self.response)
        return None

    def isRunned(self):
        return self.status == 'RUNNED'

    def getStatus(self):
        return self.status

    def setStatus(self, status):
        # TODO update get project status
        self.status = status
        if self.response == False:
            requests.get(self.host+'/projects/set_status?id='+self.project_id+'&stat='+status+'&task='+self.box_id,)
        return self.status

    def freespace(self):
        for out in self.outputs:
            del out

        del self.outputs
        self.outputs = []

    def order_inputs(self):
        self.inputs.sort(key=lambda x: x.input_port_num, reverse=False)

    def run(self):
        try:
            if self.isRunned():
                return True

            self.setStatus('RUNNING')

            myinputs = []

            self.json['nodes'][self.box_id]['properties']['payload']['result']=dict()
            self.order_inputs()
            for i_input in self.inputs:
                print(i_input.parentBox.box_id, i_input.input_port_num)
                i_input.parentBox.run()
                myinputs.append(i_input.parentBox.outputs[i_input.numport-i_input.parentBox.n_inputs])

            out = c_run_str_code(myinputs, self.str_code, self.depend, self.params, self.inputs)

            for p_out in out:
                self.outputs.append(p_out)
        # after run
            index = 0
            # all box done ok
            self.json['nodes'][self.box_id]['properties']['payload']['result']['status']="DONE_OK"
            for out in self.outputs:
                self.json['nodes'][self.box_id]['properties']['payload']['result']['out'+str(index)]=dict()
                self.json['nodes'][self.box_id]['properties']['payload']['result']['out'+str(index)]['status'] = 'OK'
                if type(out) == type(pd.DataFrame()):
                    self.json['nodes'][self.box_id]['properties']['payload']['result']['out'+str(index)]['first100'] = out.head(1).to_json(force_ascii=True)
                    self.json['nodes'][self.box_id]['properties']['payload']['result']['out'+str(index)]['columns'] = pd.DataFrame(out.columns).to_json()
                index = index + 1
            # once trained ok then hasChange is False
            self.json['nodes'][self.box_id]['properties']['payload']['hasChange'] = False
            self.json['nodes'][self.box_id]['properties']['payload']['result']['error_message'] =''
            self.json['nodes'][self.box_id]['properties']['payload']['result']['error_args'] = ''
            self.json['nodes'][self.box_id]['properties']['payload']['result']['error_trace'] = ''

            if self.serialize_outputs!=None and self.serialize_outputs!='null' and self.serialize_outputs != "[]":
                # then they have a serialized outputs
                s_outputs = json.loads(self.serialize_outputs)

                for out in s_outputs:
                    print("OUT----->", out)
                    port_index = int(out['outputnum'])
                    port_type = out['Outputtype']
                    if port_type == 'sklearn_model' or port_type == 'sklean_model':
                       sk_learn_model_serialize_object_and_upload_to_s3(self.outputs[port_index],
                                                                        self.project_id,
                                                                        self.box_id,
                                                                        str(port_index))
                    elif port_type == 'GridSearch Model':
                       sk_learn_best_estimator_model_serialize_object_and_upload_to_s3(self.outputs[port_index],
                                                                                self.project_id,
                                                                                self.box_id,
                                                                                str(port_index))
                    elif port_type == 'pickle':
                        pickle_serialize_object_and_uplodad_to_s3(self.outputs[port_index],
                                                                  self.project_id,
                                                                  self.box_id,
                                                                  str(port_index))
                    else:
                        raise Exception(port_type + ' model type not allowed yet!')
            print("RUNNED")
            self.setStatus('RUNNED')
        except Exception as e:
            print("SOME ERROR")
            self.json['nodes'][self.box_id]['properties']['payload']['hasChange'] = True
            self.json['nodes'][self.box_id]['properties']['payload']['result']['status']="DONE_ERROR"
            self.json['nodes'][self.box_id]['properties']['payload']['result']['error_message'] = str(e)
            self.json['nodes'][self.box_id]['properties']['payload']['result']['error_args'] = ''
            self.json['nodes'][self.box_id]['properties']['payload']['result']['error_trace'] = traceback.format_exc()
            self.setStatus('ERROR')
            raise Exception('Run error check error message results!')
            return False
        return self.isRunned()

def run_celery_project(allboxes, project_id, task, host, saver_host, response):

    if saver_host != None:
        saver_x = saver_c.Redis(host=saver_host, port=6379, db=0)

    # TODO do it better
    def getboxby_name(box_name, boxes):
        for x in boxes:
            if x.box_id == box_name:
                return x
        return None

    # TODO do it better
    def getportid_to_index(portid):
        return portid.split('port')[1]

    # free space fot all boxes we retrain all model
    def freespace(boxes):
        for box in boxes:
            box.freespace()
        gc.collect()

    # clear all inputs ports
    def clear_all_input_ports(boxes):
        for x in boxes:
            x.clear_inputs()

        gc.collect()

    d_json = ""
    try:

        valid_boxes = []
        if allboxes is None or task == 'ALL':
            if allboxes is not None:
                freespace(allboxes)
            allboxes=[]
            r = requests.get(host+'/projects/get_internal?id='+project_id)
            d_json = json.loads(r.json()['json'])
            for x in d_json['nodes']:
                box_type = d_json['nodes'][x]['type']
                if (box_type[0:7] == 'Dataset'):
                    print("DATASET BOX")
                    node_name = x
                    import numpy as np
                    import pandas as pd
                    X = pd.DataFrame(np.array([[1, 1], [1, 2], [2, 2], [2, 3]]))
                    box = BoxCode("", node_name, 0, 1, d_json,"", "", False, project_id, host, "", response)
                    box.setStatus('RUNNED')
                    box.outputs.append(X)
                    # for now dummy data on dataset
                    allboxes.append(box)
                elif (box_type == ('Python Script')):
                    node_name = x
                    python_code = d_json['nodes'][x]['properties']['payload']['python_code']
                    n_inputs = d_json['nodes'][x]['properties']['payload']['n_input_ports']
                    n_outputs = d_json['nodes'][x]['properties']['payload']['n_output_ports']
                    depend = d_json['nodes'][x]['properties']['payload']['depen_code']
                    params = '' # input parameters
                    serialize_outputs = '' # output parameters to serialized
                    if 'parameters' in d_json['nodes'][x]['properties']['payload']:
                        params = d_json['nodes'][x]['properties']['payload']['parameters']

                    if 'outputs' in d_json['nodes'][x]['properties']['payload']:
                        serialize_outputs = d_json['nodes'][x]['properties']['payload']['outputs']

                    box = BoxCode(python_code, node_name, n_inputs, n_outputs,
                                  d_json, depend, params, False, project_id, host, serialize_outputs, response)
                    allboxes.append(box)

            for x in d_json['links']:
                orig_box_id = d_json['links'][x]['from']['nodeId']
                orig_input_port = d_json['links'][x]['from']['portId']

                dest_box_id = d_json['links'][x]['to']['nodeId']
                dest_input_port = d_json['links'][x]['to']['portId']

            # need locate orig_box
                orig_box = getboxby_name(orig_box_id, allboxes)
                orig_id = getportid_to_index(orig_input_port)

            # need locate dest_box
                dest_box = getboxby_name(dest_box_id, allboxes)
                dest_id = getportid_to_index(dest_input_port)
                if dest_box is not None and orig_box is not None and  dest_box.box_id != orig_box.box_id:
                    valid_boxes.append(dest_box)
                    valid_boxes.append(orig_box)

                    input_port = InputPort(orig_input_port, int(orig_id)-1, orig_box, int(dest_id) - 1)

                    dest_box.inputs.append(input_port)

        else:
            if allboxes is None:
                allboxes = []

            r = requests.get(host+'/projects/get_internal?id='+project_id)
            d_json = json.loads(r.json()['json'])
            # DO it easy first we start with changed boxes
            # and existing boxes

            # at this point we can clear all input ports and recreate again
            # we trust in hasChange recibed from fronted
            clear_all_input_ports(allboxes)

            changedBox = []
            # this loop only analize changed boxes
            for x in d_json['nodes']:
                box = getboxby_name(x, allboxes)
                if box:
                    if ((d_json['nodes'][x]['properties']['payload']['hasChange'] == 'True') or
                        (d_json['nodes'][x]['properties']['payload']['hasChange'] == 'true') or
                        (d_json['nodes'][x]['properties']['payload']['hasChange'] == True)) :
                        node_name = x
                        python_code = d_json['nodes'][x]['properties']['payload']['python_code']
                        n_inputs = d_json['nodes'][x]['properties']['payload']['n_input_ports']
                        n_outputs = d_json['nodes'][x]['properties']['payload']['n_output_ports']
                        depend = d_json['nodes'][x]['properties']['payload']['depen_code']
                        params = ''
                        serialize_utputs = ''
                        if 'parameters' in d_json['nodes'][x]['properties']['payload']:
                            params = d_json['nodes'][x]['properties']['payload']['parameters']

                        if 'outputs' in d_json['nodes'][x]['properties']['payload']:
                            serialize_outputs = d_json['nodes'][x]['properties']['payload']['outputs']

                        box.setChangedBox(python_code, node_name, n_inputs, n_outputs,
                                          d_json, depend, params, False, project_id, host, serialize_outputs)

                        # save changed box and existing maybe need latter
                        changedBox.append(box)
                        print('Caja cambiada')

            # this loop analize new boxes
            # existing boxes and NOT changed then nothing to do
            newboxes = []
            for x in d_json['nodes']:
                box = getboxby_name(x, changedBox)
                box_exit = getboxby_name(x, allboxes)
                if box is None and box_exit is None:
                    # we can continue
                    node_name = x
                    python_code = d_json['nodes'][x]['properties']['payload']['python_code']
                    n_inputs = d_json['nodes'][x]['properties']['payload']['n_input_ports']
                    n_outputs = d_json['nodes'][x]['properties']['payload']['n_output_ports']
                    depend = d_json['nodes'][x]['properties']['payload']['depen_code']
                    params = ''
                    serialize_outputs = ''
                    if 'parameters' in d_json['nodes'][x]['properties']['payload']:
                        params = d_json['nodes'][x]['properties']['payload']['parameters']

                    if 'outputs' in d_json['nodes'][x]['properties']['payload']:
                        serialize_outputs = d_json['nodes'][x]['properties']['payload']['outputs']

                    box = BoxCode(python_code, node_name, n_inputs, n_outputs,
                                  d_json, depend, params, False, project_id, host, serialize_outputs, response)
                    allboxes.append(box)
                    newboxes.append(box)
                else:
                    print("Nothing to do")

            # at this point we can regenerate all links again
            for x in d_json['links']:
                orig_box_id = d_json['links'][x]['from']['nodeId']
                orig_input_port = d_json['links'][x]['from']['portId']

                dest_box_id = d_json['links'][x]['to']['nodeId']
                dest_input_port = d_json['links'][x]['to']['portId']

            # need locate orig_box
                orig_box = getboxby_name(orig_box_id, allboxes)
                orig_id = getportid_to_index(orig_input_port)

            # need locate dest_box
                dest_box = getboxby_name(dest_box_id, allboxes)
                dest_id = getportid_to_index(dest_input_port)
                if dest_box is not None and orig_box is not None and  dest_box.box_id != orig_box.box_id:
                    valid_boxes.append(dest_box)
                    valid_boxes.append(orig_box)

                    input_port = InputPort(orig_input_port, int(orig_id)-1, orig_box)
                    dest_box.inputs.append(input_port)

            # now need analize changedboxes and set run status as 'init'
            # this maybe can do better algoritm
            hasmore = True
            analizedbox = []
            while hasmore:
                hasmore = False
                for box in allboxes:
                    for inputlink in box.inputs:
                        if ((getboxby_name(inputlink.parentBox.box_id, changedBox) or
                             getboxby_name(inputlink.parentBox.box_id, newboxes)) and
                            (getboxby_name(box.box_id, analizedbox) is None)):
                           # then this box need to be re-run
                           # this box not changed but need to be retrained some parent change
                           # or some parent is new box
                               box.setStatus('INIT')
                               changedBox.append(box)
                               hasmore = True
                               analizedbox.append(box)
                               break

        # we can remove all empty boxes not conected with input
        # and outputs
        for x in allboxes:
            if getboxby_name(x.box_id, valid_boxes) is None:
                x.setStatus('STAND-BY')

        if task == 'ALL':
            # if we retrain ALL then clean an retrain
            pendingTrain = True
            while pendingTrain:
                pendingTrain=False
                for x in allboxes:
                    if ((x.isRunned()==False) and
                        (not getboxby_name(x.box_id, valid_boxes) is None)):
                        x.run()
                        pendingTrain=True
        else:
            print("TRAY TO TRAIN TASK -->", task, file=sys.stderr)
            to_trainbox = getboxby_name(task, allboxes)
            to_trainbox.run()

            # set in standby all boxes init
            # means user changed but not need to run becausethey want to run another circuit
            for x in allboxes:
                if x.getStatus() == 'INIT':
                    x.setStatus('STAND-BY')

        if response == False:
            url = host+'/projects/set_status?id='+project_id+'&stat=OK&error=NONE'+'&data='+json.dumps(d_json)
            requests.get(url)
    except Exception as e:
        print("ERROR", e, file=sys.stderr)
        traceback.print_exc()
        if response == False:
            for x in allboxes:
                if x.getStatus() == 'INIT':
                    print("SET STAND-BY")
                    x.setStatus('STAND-BY')
            url = host+'/projects/set_status?id='+project_id+'&stat=ERROR'+'&error='+str(e)+'&data='+json.dumps(d_json)

            requests.get(url)
            print("END WITH ERROR", e)
            traceback.print_exc()

    return allboxes

def get_key_from_redis(project_id):
    r = requests.get(host+'/projects/get_status?id='+project_id)

    ret = r.json()
    print(ret['status'])
    d_json = json.loads(ret['status'])
    print(d_json)
    if d_json['project_stat'] == 'PENDING':
        return ret['task']

    return False

def run_str_code(func):
    global ret
    ret = []
    LOC1 = """def """+func+"""(num):"""
    LOC = """
    num*2
    print("hola")
    return num
"""
    exec(LOC1+LOC)

    return ret

def getOutput(allboxes, box_id, port):
    return "ok"

if __name__ == '__main__':
    allproject = None
    project_id = sys.argv[1]
    host = sys.argv[2]
    print(sys.argv[3])
    saver_host = sys.argv[3]
    saver_x = saver_c.Redis(host=saver_host, port=6379, db=0)

    print("Starting project", project_id, host)
    while True:
        print("ASK")
        try:
            task=get_key_from_redis(project_id)
            print("aaa", task)
            if task:
                print("NEW TASK", host)
                allproject = run_celery_project(allproject, project_id, task, host, None, False)
            else:
                print("NOTHING TO DO", task)
            time.sleep(1)
        except Exception as e:
            print('Error', e)
            time.sleep(1)
            pass


# coding:utf-8
import os
import sys
import time
import redis
from bob_modules import get_conf
from bob_modules.ptail import Tail
import pika

class Record_log(Tail):
    """
           overwrite ptail follw modules and it can be record logs
    """

    def __init__(self, tailed_file, pname):
        ''' Initiate a Tail instance.
            Check for file validity, assigns callback function to standard out.

            Arguments:
                tailed_file - File to be followed. '''
        self.check_file_exist(tailed_file)
        self.tailed_file = tailed_file
        self.callback = sys.stdout.write
        self.try_count = 0
        self.pname = pname

        try:
            self.file_ = open(self.tailed_file, "r")
            self.size = os.path.getsize(self.tailed_file)

            # Go to the end of file
            self.file_.seek(0, 2)
        except:
            raise

    def reload_tailed_file(self):
        """ Reload tailed file when it be empty be `echo "" > tailed file`, or
            segmentated by logrotate.
        """
        try:
            self.file_ = open(self.tailed_file, "r")
            self.size = os.path.getsize(self.tailed_file)

            # Go to the head of file
            self.file_.seek(0, 1)

            return True
        except:
            return False

    def follow(self, s=0.01):

        while True:
            _size = os.path.getsize(self.tailed_file)
            if _size < self.size:
                while self.try_count < 3:
                    if not self.reload_tailed_file():
                        self.try_count += 1
                    else:
                        self.try_count = 0
                        self.size = os.path.getsize(self.tailed_file)
                        break
                    time.sleep(0.1)

                if self.try_count == 3:
                    raise Exception("Open %s failed after try 3 times" % self.tailed_file)
            else:
                self.size = _size

            curr_position = self.file_.tell()
            line = self.file_.readline()

            if not line:
                self.file_.seek(curr_position)
            elif not line.endswith("\n"):
                self.file_.seek(curr_position)
            else:
                self._write_to_redis(line)
            time.sleep(s)

    def _write_to_redis(self, line):
        '''
        write logs to redis then search keywords from lines
        '''
        # print(line)
        log_config = get_conf.get_log_conf(self.pname)
        keywords = log_config.get('error_words_monitor', 'monitor_words')
        redis_conf = get_conf.get_conf().get('redis')
        pool = redis.ConnectionPool(**redis_conf)
        rclient = redis.Redis(connection_pool=pool)
        rclient.rpush(self.pname, line)
        line_num = rclient.llen(self.pname)
        self._search_keywords(keywords, line, line_num)

    def _search_keywords(self, keywords_list, line, line_num):
        '''
        :param keywords_list:one or more keywords from sets
        '''
        key_list = keywords_list.split(',')
        for keyword in key_list:
            status = line.find(keyword)
            if status != -1:
                # print line,line_num
                self._push_message_mq(line_num)

    def _push_message_mq(self, line_num):
        config = get_conf.get_conf().get('rabbitmq')
        user_info = pika.PlainCredentials(config['username'], config['pwd'])
        mq_conn = pika.BlockingConnection(pika.ConnectionParameters(config['host'], credentials=user_info))
        chan = mq_conn.channel()
        chan.queue_declare(queue=self.pname)
        chan.basic_publish(exchange='', routing_key=self.pname, body=str(line_num) + ',' + self.pname)
        chan.close()


from matcha.orm.data_access import DataAccess
from matcha.model.Topic import Topic
import csv
import logging

def populate():
    topics = []
    topic_dict = {}
    with open('../csv/topic.csv', newline='') as csvfile: # topic.csv Comprend une liste des topic avec une distribution difference entre hommes et femmes
        topicreader = csv.reader(csvfile, delimiter=',', quoting=csv.QUOTE_NONE)
        for row in topicreader:
            topic = row[0].split(' (')[0]
            both = int(row[0].split(' (')[1].rstrip(')'))
            man = int(row[1].split(' (')[0].rstrip(')'))
            for prefix in ['le ','la ', 'les ', "l'"]:
                if topic.startswith(prefix):
                    topic = topic[len(prefix):]
            topics.append(topic)
            topic_dict[topic] = (man, both - man)
    dataAccess = DataAccess()
    for tag in topics:
        topic = Topic()
        topic.tag = tag
        dataAccess.persist(topic, autocommit=False)
    dataAccess.commit()
    logging.info('End Topic populate')
    men_total = 0
    women_total = 0
    for (men, women) in topic_dict.values():
        men_total += men
        women_total += women
    men_topic_dict = {}
    women_topic_dict = {}
    for key, value in topic_dict.items():
        men_topic_dict[key] = float(value[0]/men_total)
        women_topic_dict[key] = float(value[1]/women_total)
    logging.info('Step 1  Users_topic populate')
    return (men_topic_dict, women_topic_dict)


import os
import json
import cv2

# Unfortunately COCO has some left out indices etc. that's why we hard-code the categories here
COCO_CATEGORIES = [{"supercategory": "person", "id": 1, "name": "person"},
                   {"supercategory": "vehicle", "id": 2, "name": "bicycle"},
                   {"supercategory": "vehicle", "id": 3, "name": "car"},
                   {"supercategory": "vehicle", "id": 4, "name": "motorcycle"},
                   {"supercategory": "vehicle", "id": 5, "name": "airplane"},
                   {"supercategory": "vehicle", "id": 6, "name": "bus"},
                   {"supercategory": "vehicle", "id": 7, "name": "train"},
                   {"supercategory": "vehicle", "id": 8, "name": "truck"},
                   {"supercategory": "vehicle", "id": 9, "name": "boat"},
                   {"supercategory": "outdoor", "id": 10, "name": "traffic light"},
                   {"supercategory": "outdoor", "id": 11, "name": "fire hydrant"},
                   {"supercategory": "outdoor", "id": 13, "name": "stop sign"},
                   {"supercategory": "outdoor", "id": 14, "name": "parking meter"},
                   {"supercategory": "outdoor", "id": 15, "name": "bench"},
                   {"supercategory": "animal", "id": 16, "name": "bird"},
                   {"supercategory": "animal", "id": 17, "name": "cat"},
                   {"supercategory": "animal", "id": 18, "name": "dog"},
                   {"supercategory": "animal", "id": 19, "name": "horse"},
                   {"supercategory": "animal", "id": 20, "name": "sheep"},
                   {"supercategory": "animal", "id": 21, "name": "cow"},
                   {"supercategory": "animal", "id": 22, "name": "elephant"},
                   {"supercategory": "animal", "id": 23, "name": "bear"},
                   {"supercategory": "animal", "id": 24, "name": "zebra"},
                   {"supercategory": "animal", "id": 25, "name": "giraffe"},
                   {"supercategory": "accessory", "id": 27, "name": "backpack"},
                   {"supercategory": "accessory", "id": 28, "name": "umbrella"},
                   {"supercategory": "accessory", "id": 31, "name": "handbag"},
                   {"supercategory": "accessory", "id": 32, "name": "tie"},
                   {"supercategory": "accessory", "id": 33, "name": "suitcase"},
                   {"supercategory": "sports", "id": 34, "name": "frisbee"},
                   {"supercategory": "sports", "id": 35, "name": "skis"},
                   {"supercategory": "sports", "id": 36, "name": "snowboard"},
                   {"supercategory": "sports", "id": 37, "name": "sports ball"},
                   {"supercategory": "sports", "id": 38, "name": "kite"},
                   {"supercategory": "sports", "id": 39, "name": "baseball bat"},
                   {"supercategory": "sports", "id": 40, "name": "baseball glove"},
                   {"supercategory": "sports", "id": 41, "name": "skateboard"},
                   {"supercategory": "sports", "id": 42, "name": "surfboard"},
                   {"supercategory": "sports", "id": 43, "name": "tennis racket"},
                   {"supercategory": "kitchen", "id": 44, "name": "bottle"},
                   {"supercategory": "kitchen", "id": 46, "name": "wine glass"},
                   {"supercategory": "kitchen", "id": 47, "name": "cup"},
                   {"supercategory": "kitchen", "id": 48, "name": "fork"},
                   {"supercategory": "kitchen", "id": 49, "name": "knife"},
                   {"supercategory": "kitchen", "id": 50, "name": "spoon"},
                   {"supercategory": "kitchen", "id": 51, "name": "bowl"},
                   {"supercategory": "food", "id": 52, "name": "banana"},
	                 {"supercategory": "food", "id": 53, "name": "apple"},
                   {"supercategory": "food", "id": 54, "name": "sandwich"},
                   {"supercategory": "food", "id": 55, "name": "orange"},
                   {"supercategory": "food", "id": 56, "name": "broccoli"},
                   {"supercategory": "food", "id": 57, "name": "carrot"},
                   {"supercategory": "food", "id": 58, "name": "hot dog"},
                   {"supercategory": "food", "id": 59, "name": "pizza"},
                   {"supercategory": "food", "id": 60, "name": "donut"},
                   {"supercategory": "food", "id": 61, "name": "cake"},
                   {"supercategory": "furniture", "id": 62, "name": "chair"},
                   {"supercategory": "furniture", "id": 63, "name": "couch"},
                   {"supercategory": "furniture", "id": 64, "name": "potted plant" },
                   {"supercategory": "furniture", "id": 65, "name": "bed"},
                   {"supercategory": "furniture", "id": 67, "name": "dining table"},
                   {"supercategory": "furniture", "id": 70, "name": "toilet"},
                   {"supercategory": "electronic", "id": 72, "name": "tv"},
                   {"supercategory": "electronic", "id": 73, "name": "laptop"},
                   {"supercategory": "electronic", "id": 74, "name": "mouse"},
                   {"supercategory": "electronic", "id": 75, "name": "remote"},
                   {"supercategory": "electronic", "id": 76, "name": "keyboard"},
                   {"supercategory": "electronic", "id": 77, "name": "cell phone"},
                   {"supercategory": "appliance", "id": 78, "name": "microwave"},
                   {"supercategory": "appliance", "id": 79, "name": "oven"},
                   {"supercategory": "appliance", "id": 80, "name": "toaster"},
                   {"supercategory": "appliance", "id": 81, "name": "sink"},
                   {"supercategory": "appliance", "id": 82, "name": "refrigerator"},
                   {"supercategory": "indoor", "id": 84, "name": "book"},
                   {"supercategory": "indoor", "id": 85, "name": "clock"},
                   {"supercategory": "indoor", "id": 86, "name": "vase"},
                   {"supercategory": "indoor", "id": 87, "name": "scissors"},
                   {"supercategory": "indoor", "id": 88, "name": "teddy bear"},
                   {"supercategory": "indoor", "id": 89, "name": "hair drier"},
                   {"supercategory": "indoor", "id": 90, "name": "toothbrush"}
]

def mrcnn_instance_detections_to_coco_format(data):
    """
    Receives a dictionary returned by Matterport's Mask RCNN implementation
    and returns a dictionary in COCO format.
    data: the dictionary of the network

    INFO: The conversion method in coco.py by Matterport requires a dataset which we do not have
    here, thus this conversion method.
    """

    cocoData = []
    for entry in data:
        for index in range(0, len(entry['rois'])):
            roi = entry['rois'][index]
            cocoData.append({
                    "image_id" : entry["image_id"],
                    "category_id" : int(entry["class_ids"][index]),
                    # Bounding box is in [y1, x1, y2, x2] but COCO in [x, y, width, height]
                    "bbox" : [int(roi[1]), int(roi[0]), int(roi[3] - roi[1]), int(roi[2] - roi[0])],
                    "score" : round(float(entry["scores"][index]), 3)
                })

    return cocoData

def via_data_to_coco_evaluation_format(imagesPath, annotationsPath):
    """
    Loads the VUFO data file at the given location, extracts the information
    and stores it in a COCO compatible file format at the specified output
    location
    """

    cocoData = {}
    # Description for the data
    cocoData["info"] = {"description" : "Frames of videos from VUFO",
                        "url" : "",
                        "version" : "1.0",
                        "year" : "2018",
                        "contributor" : "",
                        "date_created" : ""}
    # Add a default license for all images
    cocoData["licenses"] = [{"url" : "",
                            "id" : "1",
                            "name" : "license"}]
    cocoData["images"] = []
    cocoData["annotations"] = []
    cocoData["categories"] = COCO_CATEGORIES

    imageIndex = 0
    annotationIndex = 0

    with open(annotationsPath) as annotationData:
        data = json.load(annotationData)
        for entry in data:
            entryData = data[entry]
            imageFile = entryData["filename"]
            image = cv2.imread(os.path.join(imagesPath, imageFile))
            height, width, channels = image.shape
            cocoData["images"].append({"license" : "1",
                               "file_name" : imageFile,
                               "coco_url" : "",
                               "height" : height,
                               "width" : width,
                               "date_captured" : "",
                               "flickr_url" : "",
                               "id" : imageIndex})

            regions = entryData["regions"]
            for regionIndex in regions:
                region = regions[regionIndex]
                regionAttributes = region["region_attributes"]
                coco_class_name = regionAttributes["class"]
                category = next((_category for _category in COCO_CATEGORIES if _category["name"] == coco_class_name), None)

                if not category:
                  raise ValueError("A specified category was not found in the COCO categories. Please make sure to just use COCO categories.")

                # Process region values
                shapeAttributes = region["shape_attributes"]
                box_x = int(shapeAttributes["x"])
                box_y = int(shapeAttributes["y"])
                # VIA data can be negative numbers or larger than image size
                box_x = max(box_x, 0)
                box_y = max(box_y, 0)
                box_width = int(shapeAttributes["width"])
                box_height = int(shapeAttributes["height"])
                #print("{} {} {} {}".format(box_x, box_y, box_width, box_height))
                # Adjust to regions that go beyond the image
                if box_x + box_width > width:
                  box_width = width - box_x
                if box_y + box_height > height:
                  box_height = height - box_y
                if not box_width < 3 and not box_height < 3:
                  # filter out boxes that are too small
                  cocoData["annotations"].append({"segmentation" : [[box_x, box_y, \
                                                                    box_x + box_width, box_y, \
                                                                    box_x + box_width, box_y + box_width, \
                                                                    box_x, box_y + box_width]],
                                                  "area" : shapeAttributes["width"] * shapeAttributes["height"],
                                                  "iscrowd" : 0,
                                                  "image_id" : imageIndex,
                                                  "bbox" : [box_x,
                                                            box_y,
                                                            box_width,
                                                            box_height],
                                                  "category_id" : category["id"],
                                                  "id" : annotationIndex})
                annotationIndex += 1

            imageIndex += 1

    return cocoData

if __name__ == '__main__':
    import argparse

    parser = argparse.ArgumentParser(description="Perform inference on a set of images with pre-defined weights.")
    parser.add_argument("--imagesPath",
                        "-i",
                        required=True,
                        metavar="/path/to/images",
                        help="The path to the images.")
    parser.add_argument("--annotationsPath",
                        "-a",
                        required=True,
                        metavar="/path/to/annoations/",
                        help="The path to the annoations that are to be converted.")

    args = parser.parse_args()
    assert os.path.exists(args.imagesPath), "Images path {} does not exist.".format(args.imagesPath)
    assert os.path.exists(args.annotationsPath), "Annotations path {} does not exist.".format(args.annotationsPath)

    result = via_data_to_coco_evaluation_format(args.imagesPath, args.annotationsPath)

    annotationsFileName = os.path.splitext(args.annotationsPath)[0]
    convertedPath = annotationsFileName + "_converted.json"
    with open(convertedPath, "w") as jsonFile:
        json.dump(result, jsonFile)


import os
import traceback
import requests
import datetime

def log_event(name, params):
    _write_to_es(name, params, 'events')

def log_error(name, params):
    params['traceback'] = traceback.format_exc()
    _write_to_es(name, params, 'errors')

def _write_to_es(name, params, type):
    params = _curate(params)
    params['name'] = name
    params['app'] = os.environ['APP_NAME'] if 'APP_NAME' in os.environ else None
    params['timestamp'] = datetime.datetime.utcnow().isoformat()

    try:
        url = '{}/{}_{}_{}/{}'.format(os.environ['ES_URI'] if 'ES_URI' in os.environ else 'http://127.0.0.1:9200',
                                      os.environ['ES_INDEX_PREFIX'] if 'ES_INDEX_PREFIX' in os.environ else 'prod',
                                      type,
                                      datetime.datetime.utcnow().strftime("%Y_%m"),
                                      "_doc")
        r = requests.post(url, json = params)
        print('Logging to elasticsearch - {}, {}: {}'.format(r.status_code, r.reason, r.text))
    except:
        print('Failed to log {}: {}'.format(type, params))

def _curate(params):
    new_params = {}
    for key in params:
        new_params[key] = str(params[key])
    return new_params


# adjusted from rl-baselines-zoo:  https://github.com/araffin/rl-baselines-zoo

import os
import time
import uuid
import difflib
import argparse
import importlib
import warnings
from pprint import pprint
from collections import OrderedDict

# numpy warnings because of tensorflow
warnings.filterwarnings("ignore", category=FutureWarning, module='tensorflow')
warnings.filterwarnings("ignore", category=UserWarning, module='gym')

import gym
import numpy as np
import yaml
try:
    import mpi4py
    from mpi4py import MPI
except ImportError:
    mpi4py = None

from stable_baselines.common import set_global_seeds
from stable_baselines.common.cmd_util import make_atari_env
from stable_baselines.common.vec_env import VecFrameStack, SubprocVecEnv, VecNormalize, DummyVecEnv, VecEnv
from stable_baselines.common.noise import AdaptiveParamNoiseSpec, NormalActionNoise, OrnsteinUhlenbeckActionNoise
from stable_baselines.common.schedules import constfn
from stable_baselines.common.callbacks import CheckpointCallback, EvalCallback
from stable_baselines.her import HERGoalEnvWrapper
from stable_baselines.common.base_class import _UnvecWrapper

from utils import make_env, ALGOS, linear_schedule, get_latest_run_id, get_wrapper_class
from utils.utils import StoreDict

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--env', type=str, default="CartPole-v1", help='environment ID')
    parser.add_argument('-tb', '--tensorboard-log', help='Tensorboard log dir', default='', type=str)
    parser.add_argument('--algo', help='RL Algorithm', default='ppo2',
                        type=str, required=False, choices=list(ALGOS.keys()))
    parser.add_argument('--log-interval', help='Override log interval (default: -1, no change)', default=-1,
                        type=int)
    parser.add_argument('--eval-freq', help='Evaluate the agent every n steps (if negative, no evaluation)',
                        default=10000, type=int)
    parser.add_argument('--eval-episodes', help='Number of episodes to use for evaluation',
                        default=5, type=int)
    parser.add_argument('--save-freq', help='Save the model every n steps (if negative, no checkpoint)',
                        default=-1, type=int)
    parser.add_argument('-f', '--log-folder', help='Log folder', type=str, default='logs')
    parser.add_argument('--seed', help='Random generator seed', type=int, default=0)
    parser.add_argument('--verbose', help='Verbose mode (0: no output, 1: INFO)', default=1,
                        type=int)
    parser.add_argument('-params', '--hyperparams', type=str, nargs='+', action=StoreDict,
                        help='Overwrite hyperparameter (e.g. learning_rate:0.01 train_freq:10)')
    parser.add_argument('-uuid', '--uuid', action='store_true', default=False,
                        help='Ensure that the run has a unique ID')
    parser.add_argument('--env-kwargs', type=str, nargs='+', action=StoreDict,
                        help='Optional keyword argument to pass to the env constructor')
    args = parser.parse_args()

    env_id = args.env
    registered_envs = set(gym.envs.registry.env_specs.keys())

    # If the environment is not found, suggest the closest match
    if env_id not in registered_envs:
        try:
            closest_match = difflib.get_close_matches(env_id, registered_envs, n=1)[0]
        except IndexError:
            closest_match = "'no close match found...'"
        raise ValueError('{} not found in gym registry, you maybe meant {}?'.format(env_id, closest_match))

    # Unique id to ensure there is no race condition for the folder creation
    uuid_str = '_{}'.format(uuid.uuid4()) if args.uuid else ''
    if args.seed < 0:
        # Seed but with a random one
        args.seed = np.random.randint(2**32 - 1)

    set_global_seeds(args.seed)

    rank = 0
    if mpi4py is not None and MPI.COMM_WORLD.Get_size() > 1:
        print("Using MPI for multiprocessing with {} workers".format(MPI.COMM_WORLD.Get_size()))
        rank = MPI.COMM_WORLD.Get_rank()
        print("Worker rank: {}".format(rank))

        args.seed += rank
        if rank != 0:
            args.verbose = 0
            args.tensorboard_log = ''

    tensorboard_log = None if args.tensorboard_log == '' else os.path.join(args.tensorboard_log, env_id)

    print("=" * 10, env_id, "=" * 10)
    print("Seed: {}".format(args.seed))

    # Load hyperparameters from yaml file
    with open('script_experiment/rl_utils/{}.yml'.format(args.algo), 'r') as f:
        hyperparams_dict = yaml.safe_load(f)
        if env_id in list(hyperparams_dict.keys()):
            hyperparams = hyperparams_dict[env_id]
        else:
            raise ValueError("Hyperparameters not found for {}-{}".format(args.algo, env_id))

    if args.hyperparams is not None:
        # Overwrite hyperparams if needed
        hyperparams.update(args.hyperparams)

    # Sort hyperparams that will be saved
    saved_hyperparams = OrderedDict([(key, hyperparams[key]) for key in sorted(hyperparams.keys())])

    algo_ = args.algo
    if args.verbose > 0:
        pprint(saved_hyperparams)

    n_envs = hyperparams.get('n_envs', 1)

    if args.verbose > 0:
        print("Using {} environments".format(n_envs))

    # Create learning rate schedules for ppo2 and sac
    if algo_ in ["ppo2", "sac", "td3"]:
        for key in ['learning_rate', 'cliprange', 'cliprange_vf']:
            if key not in hyperparams:
                continue
            if isinstance(hyperparams[key], str):
                schedule, initial_value = hyperparams[key].split('_')
                initial_value = float(initial_value)
                hyperparams[key] = linear_schedule(initial_value)
            elif isinstance(hyperparams[key], (float, int)):
                # Negative value: ignore (ex: for clipping)
                if hyperparams[key] < 0:
                    continue
                hyperparams[key] = constfn(float(hyperparams[key]))
            else:
                raise ValueError('Invalid value for {}: {}'.format(key, hyperparams[key]))

    n_timesteps = int(hyperparams['n_timesteps'])

    # Convert to python object if needed
    if 'policy_kwargs' in hyperparams.keys() and isinstance(hyperparams['policy_kwargs'], str):
        hyperparams['policy_kwargs'] = eval(hyperparams['policy_kwargs'])

    # Delete keys so the dict can be pass to the model constructor
    if 'n_envs' in hyperparams.keys():
        del hyperparams['n_envs']
    del hyperparams['n_timesteps']

    # obtain a class object from a wrapper name string in hyperparams
    # and delete the entry
    env_wrapper = get_wrapper_class(hyperparams)
    if 'env_wrapper' in hyperparams.keys():
        del hyperparams['env_wrapper']

    log_path = "{}/{}/".format(args.log_folder, args.algo)
    save_path = os.path.join(log_path, "{}_{}{}".format(env_id, get_latest_run_id(log_path, env_id) + 1, uuid_str))
    params_path = "{}/{}".format(save_path, env_id)
    os.makedirs(params_path, exist_ok=True)

    callbacks = []
    if args.save_freq > 0:
        # Account for the number of parallel environments
        args.save_freq = max(args.save_freq // n_envs, 1)
        callbacks.append(CheckpointCallback(save_freq=args.save_freq,
                                            save_path=save_path, name_prefix='rl_model', verbose=1))

    env_kwargs = {} if args.env_kwargs is None else args.env_kwargs

    def create_env(n_envs, eval_env=False):
        """
        Create the environment and wrap it if necessary
        :param n_envs: (int)
        :param eval_env: (bool) Whether is it an environment used for evaluation or not
        :return: (Union[gym.Env, VecEnv])
        :return: (gym.Env)
        """
        global hyperparams
        global env_kwargs

        # Do not log eval env (issue with writing the same file)
        log_dir = None if eval_env else save_path

        env = DummyVecEnv([make_env(env_id, 0, args.seed, wrapper_class=env_wrapper, log_dir=log_dir, env_kwargs=env_kwargs)])

        return env

    env = create_env(n_envs)
    # Create test env if needed, do not normalize reward
    eval_env = None
    if args.eval_freq > 0:
        # Account for the number of parallel environments
        args.eval_freq = max(args.eval_freq // n_envs, 1)

        if args.verbose > 0:
            print("Creating test environment")

        eval_callback = EvalCallback(create_env(1, eval_env=True), callback_on_new_best=None,
                                     best_model_save_path=save_path, n_eval_episodes=args.eval_episodes,
                                     log_path=save_path, eval_freq=args.eval_freq)
        callbacks.append(eval_callback)

    if ALGOS[args.algo] is None:
        raise ValueError('{} requires MPI to be installed'.format(args.algo))

    # Train an agent from scratch
    model = ALGOS[args.algo](env=env, tensorboard_log=tensorboard_log, verbose=args.verbose, **hyperparams)

    kwargs = {}
    if args.log_interval > -1:
        kwargs = {'log_interval': args.log_interval}

    if len(callbacks) > 0:
        kwargs['callback'] = callbacks

    # Save hyperparams
    with open(os.path.join(params_path, 'config.yml'), 'w') as f:
        yaml.dump(saved_hyperparams, f)

    print("Log path: {}".format(save_path))

    try:
        model.learn(n_timesteps, **kwargs)
    except KeyboardInterrupt:
        pass

    # Only save worker of rank 0 when using mpi
    if rank == 0:
        print("Saving to {}".format(save_path))

        model.save("{}/{}".format(save_path, env_id))