# Author: Xiran Zhang, Pengfei Li
# training are implemented here

import os
import time
import argparse
import numpy as np
import cv2
from tqdm import tqdm
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.tensorboard import SummaryWriter

import dataloader.sf_listfile as lf
import dataloader.sf_loader as ld
from model import LossFunc
from model import SPAutoED as SPEDNet
from utils.disp_utils import depth_to_disp

parser = argparse.ArgumentParser(description='SPAutoED')
parser.add_argument('--datapath', default='/media/xiran_zhang/Crypto/sceneflow/')
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--loadmodel', default=None, help='path to the trained model, for continuous training')
parser.add_argument('--savemodel', default='./trained_model/', help='folder to the save the trained model')
parser.add_argument('--seed', type=int, default=1, metavar='S', help='seed for random (default:1)')

args = parser.parse_args()
writer = SummaryWriter()

# set train environment
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
print('torch cuda status:', torch.cuda.is_available())
if torch.cuda.is_available():
    cuda = True
else:
    EnvironmentError('cuda status error.')

# load all data
all_left_img, all_right_img, test_left_img, test_right_img = lf.dataloader(args.datapath)

trainImgLoader = torch.utils.data.DataLoader(
         ld.myImageFloder(all_left_img,all_right_img, True),
         batch_size= 10, shuffle= True, num_workers= 5, drop_last=False)

valImgLoader = torch.utils.data.DataLoader(
         ld.myImageFloder(test_left_img,test_right_img, False),
         batch_size= 15, shuffle= False, num_workers= 5, drop_last=False)

# create model
model = SPEDNet.SPAutoED()
if cuda:
    model = nn.DataParallel(model)
    model = model.cuda()
pasued_epoch = -1

# set criteria, optimizer and scheduler
# Option can be made here:
# optimizer: Adam, SGD
criteria = LossFunc.Loss_reonstruct().cuda()
optimizer = torch.optim.Adam(model.parameters(), lr = 1e-4)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 5, 0.7)

# load model
if args.loadmodel is not None:
    state_dict = torch.load(args.loadmodel)
    model.load_state_dict(state_dict['state_dict'])
    pasued_epoch = state_dict['epoch']
    train_loss = state_dict['train_loss']
    # val_loss = state_dict['val_loss']
    scheduler = state_dict['scheduler']
    optimizer = state_dict['optimizer']
    # print('train loss:', train_loss, 'val loss:', val_loss)

print('Number of model parameters: {}'.format(sum([p.data.nelement() for p in model.parameters()])))

# train function / validation function / test function
def train(imgL, imgR, model, optimizer):
    model.train()

    # plt.imshow(imgL[0,0])
    # plt.show()
    # plt.imshow(imgR[0,0])
    # plt.show()

    if cuda:
        imgL = imgL.cuda()
        imgR = imgR.cuda()

    optimizer.zero_grad()
    depthL = model(imgL)
    depthR = model(imgR)
    # depth to disparity
    # dispL, dispR = depth_to_disp(depthL, depthR, camera_para)

    dispL, dispR = depthL.squeeze(1), depthR.squeeze(1)

    # visualize
    # depthL_cpu = depthL.cpu()
    # depthR_cpu = depthR.cpu()
    # depthL_cpu.detach_()
    # depthR_cpu.detach_()
    # plt.imshow(depthL_cpu[0,0])
    # plt.show()
    # plt.imshow(depthR_cpu[0,0])
    # plt.show()

    left_ssim, right_ssim, left_l1, right_l1, lr_l1, rl_l1, smooth_left_disp, smooth_right_disp = criteria(imgL, imgR, dispL, dispR)

    start_plane = torch.ones_like(depthL)*10
    # reg = F.l1_loss(depthL, start_plane) + F.l1_loss(depthR,start_plane)

    print('left recons loss:', right_l1.data.item(), 'right recons loss:', left_l1.data.item(), \
        'left recons l1:', left_l1.data.item(), 'right recons l1:', right_l1.data.item(), \
        'lr consis loss:', lr_l1.data.item(), 'rl consis loss:', rl_l1.data.item(), \
         'smooth left disp:', smooth_left_disp, 'smooth right disp:', smooth_right_disp)

    loss = 0.85*(left_ssim + right_ssim) + 0.15*(left_l1 + right_l1) + 1.0 * (lr_l1 + rl_l1)/(960.0*0.2) + 0.1 * (smooth_left_disp + smooth_right_disp)/(960.0*0.2)
    # torch.autograd.set_detect_anomaly(True)
    loss.backward()
    optimizer.step()

    return loss.data.item()

def val(imgL, imgR, model, optimizer, epoch):
    model.eval()
    if cuda:
        imgL = imgL.cuda()
        imgR = imgR.cuda()

    with torch.no_grad():
        depthL = model(imgL)
        depthR = model(imgR)

        # visualization
        # fig, (ax1, ax2) = plt.subplots(1,2)
        # ax1.imshow(depthL)
        # ax2.imshow(dephtR)
        # plt.show()
        # dispL, dispR = depth_to_disp(depthL, depthR, camera_para)
        dispL, dispR = depthL.squeeze(1), depthR.squeeze(1)
        # reg = F.l1_loss(depthL) + F.l1_loss(depthR)
        left_ssim, right_ssim, left_l1, right_l1, lr_l1, rl_l1, smooth_left_disp, smooth_right_disp = criteria(imgL, imgR, dispL, dispR)
        loss = 0.85*(left_ssim + right_ssim) + 0.15*(left_l1 + right_l1) + 1.0 * (lr_l1 + rl_l1)/(960.0*0.2) + 0.1 * (smooth_left_disp + smooth_right_disp)/(960.0*0.2)

    return loss.data.item()

def main():
    start_epoch = pasued_epoch+1
    total_epochs = args.epochs
    print('Start training ...')

    total_iteration = 0
    for epoch in range(start_epoch, total_epochs):
        print('\nEPOCH ' + str(epoch + 1) + ' of ' + str(total_epochs) + '\n')
        # -----------------Training---------------
        train_batch_num = 0
        epoch_train_loss = 0

        for idx, (left, right) in enumerate(tqdm(trainImgLoader)):
            start_time = time.time()
            train_loss = train(left, right, model, optimizer)
            epoch_train_loss += train_loss
            train_batch_num += 1
            total_iteration += 1
            print('Iter %d training loss = %.3f, time =  %.2f' %(idx, train_loss, time.time() - start_time))
            writer.add_scalar('Iteration loss', train_loss, total_iteration+1)

        epoch_train_loss /= train_batch_num
        writer.add_scalar('Loss/train', epoch_train_loss, epoch+1)

        #----------------Validation--------------
        # val_batch_num = 0
        # epoch_val_loss = 0
        # for idx, (left, right) in enumerate(tqdm(valImgLoader)):
        #     start_time = time.time()
        #     val_loss = val(left, right, model, optimizer, epoch)
        #     epoch_val_loss += val_loss
        #     val_batch_num += 1
        #     print('Iter %d val loss = %.3f, time =  %.2f' %(idx, val_loss, time.time() - start_time))

        # epoch_val_loss /= val_batch_num
        # writer.add_scalar('Loss/val', epoch_val_loss, epoch+1)

        savefilename = args.savemodel + 'SPAutoED_' + str(epoch+1) + '.tar'
        torch.save({
            'epoch': epoch,
            'state_dict': model.state_dict(),
            'train_loss':epoch_train_loss,
            # 'val_loss':epoch_val_loss,
            'optimizer':optimizer,
            'scheduler':scheduler
        }, savefilename)

        scheduler.step()
        lr = scheduler.get_lr()

    writer.close()

if __name__ == '__main__':
    main()
    # test()


from OpenGL.GL import *
from OpenGL.GLUT import *
from OpenGL.GLU import *
import numpy as np
import cv2
from setuptools import glob
import math

class Arucadia:

    def __init__(self):

        # game board variables
        self.b_rows=10
        self.b_cols=5
        self.b_dist=2.0 # dist. bw. centers

        # globals for opencv
        self.cap = cv2.VideoCapture(1)
        self.dictionary = cv2.aruco.getPredefinedDictionary(cv2.aruco.DICT_4X4_50)
        self.board = cv2.aruco_GridBoard.create(self.b_cols, self.b_rows, 1.0, 1.0, self.dictionary)
        self.corners={}
        self.ids={}
        self.camPara={}
        self.distort={}
        self.rVec={}
        self.tVec={}

        # get image size by capturing the first frame
        _, frame = self.cap.read()
        self.width = frame.shape[1]
        self.height = frame.shape[0]

        # create opengl window
        glutInit()
        glutInitWindowSize(self.width, self.height)
        glutCreateWindow("Arucadia")
        glutDisplayFunc(self.draw)
        glutIdleFunc(self.draw)

        # define opengl settings
        glEnable(GL_DEPTH_TEST)
        glDisable(GL_CULL_FACE) # temporary

        # set camera parameters to a default set of values
        self.camPara = np.zeros((3, 3))
        self.camPara[0][0] = 534
        self.camPara[1][1] = 534
        self.camPara[2][2] = 1
        self.camPara[0][2] = 339
        self.camPara[1][2] = 233
        self.distort = np.zeros(5) # assuming no radial distortion
        self.rVec = np.zeros((3, 5)) # rot. vec. used later
        self.tVec = np.zeros((3, 5)) # transl. vec. used later

        # set view perspective (fov is set manually for best fit)
        glViewport(0, 0, self.width, self.height)
        glMatrixMode(GL_PROJECTION)
        glLoadIdentity()
        gluPerspective(48.5, float(self.width)/float(self.height), 1.0, 100.0)
        glMatrixMode(GL_MODELVIEW)

        # enter the main loop
        glutMainLoop()

    def draw(self):

        # capture image and look for markers
        _, frame = self.cap.read()
        parameters = cv2.aruco.DetectorParameters_create()
        self.corners, self.ids, _ = cv2.aruco.detectMarkers(frame, self.dictionary, parameters=parameters)

        # TEST
        # cv2.aruco.drawDetectedMarkers(frame, self.corners, self.ids)

        # Calculate rotations and translations for the camera
        if np.all(self.ids != None):
            _, self.rVec, self.tVec = cv2.aruco.estimatePoseBoard(self.corners, self.ids, self.board, self.camPara, self.distort)

        # flip the image vertically to display in opengl
        new_frame=frame.copy()
        cv2.flip(frame, 0, new_frame)

        # draw background
        glDrawPixels(self.width, self.height, GL_BGR, GL_UNSIGNED_BYTE, new_frame)

        # clear the depth buffer and reset position
        glClear(GL_DEPTH_BUFFER_BIT)
        glLoadIdentity()

        # draw board
        self.applyTransform()
        self.drawBoard()

        # draw example object for each field
        for i in range (self.b_cols):
            for j in range(self.b_rows):
                self.drawObject(i, j, (i+j)%3)

        # go to the next frame
        glutSwapBuffers()

    def applyTransform(self):

        # reset transform
        glLoadIdentity()

        # apply translation vector
        glTranslatef(self.tVec[0][0], -self.tVec[1][0], -self.tVec[2][0])

        # turn around the objects
        glRotatef(180.0, 1.0, 0.0, 0.0)

        # apply rodrigues rotation
        x = self.rVec[0][0]
        y = self.rVec[1][0]
        z = self.rVec[2][0]
        theta = math.sqrt(x * x + y * y + z * z) * 180.0 / 3.1416
        glRotatef(theta, x / theta, y / theta, z / theta)

    def drawBoard(self):

        # new transformation matrix
        glPushMatrix()

        # draw squares
        a=self.b_dist/2
        for i in range(self.b_cols):
            glPushMatrix()
            glTranslatef(self.b_dist*i, 0.0, 0.0)
            for j in range(self.b_rows):
                glPushMatrix()
                glTranslatef(0.0, self.b_dist*j, 0.0)
                glColor3f(1, 1, 1)
                glBegin(GL_QUADS)
                glVertex3f(-a, -a, 0); glVertex3f (-a, a, 0); glVertex3f(a, a, 0); glVertex3f(a, -a, 0)
                glEnd()
                glTranslatef(0.0, 0.0, 0.01)
                glColor3f(0, 0, 0)
                glBegin(GL_LINE_LOOP)
                glVertex3f(-a, -a, 0); glVertex3f(-a, a, 0); glVertex3f(a, a, 0); glVertex3f(a, -a, 0)
                glEnd()
                glPopMatrix()
            glPopMatrix()

        # reset transform
        glPopMatrix()

    # function to draw an object at specific board coordinates
    def drawObject(self, xpos, ypos, id):

        # apply transform
        self.applyTransform()

        # TEST: draw example object for each visible marker
        if np.all(self.ids != None):

            # this can be reset using pop matrix
            glPushMatrix()

            # set position to the matching field
            glTranslatef(xpos*self.b_dist, ypos*self.b_dist, 0.0)

            # set color
            glColor3f(0, 0, 1)

            # move a bit upwards (test only)
            glTranslatef(0, 0, 0.5)

            # draw example object on each square
            # id0: cube
            # id1: sphere
            # id2: cone

            if id==0:
                glutSolidCube(1.0); glColor3f(1, 1, 1); glutWireCube(1.0)
            if id==1:
                glutSolidSphere(0.5, 8, 8); glColor3f(1, 1, 1); glutWireSphere(0.5, 8, 8)
            if id==2:
                glTranslatef(0, 0, -0.5)
                glutSolidCone(0.5, 1.0, 4, 4); glColor3f(1, 1, 1); glutWireCone(0.5, 1.0, 4, 4)

            # end transform
            glPopMatrix()

arucadia=Arucadia()


import safety_gym
import gym
import random
import numpy as np
import pickle
from safety_gym.envs.engine import Engine
from tqdm import tqdm
from keras.models import load_model
import os
from copy import deepcopy
from collections import deque
from SafeRL.ModelLearning import NNModel
import matplotlib.pyplot as plt
"""
-ve rotation = clockwise torque
i.e. theta defined in the polar sense.
"""

config = {
        'robot_base': 'xmls/point.xml',
        'task': 'goal',
        'continue_goal': False,
        'observation_flatten': False,
        'observe_goal_dist': True,
        'observe_goal_comp': True,
        'observe_goal_lidar': False,
        'observe_hazards': True,
        'observe_vases': False,
        'observe_circle': True,
        'observe_vision': False,
        'observe_gremlins': False,
        'constrain_hazards': True,
        'constrain_vases': False,
        'constrain_gremlins': False,
        'lidar_max_dist': None,
        'lidar_num_bins': 32,
        'hazards_num': 10,
        'vases_num': 0,
        'gremlins_num': 0,
        'gremlins_travel': 0.5,
        'gremlins_keepout': 0.4,
    }

def sigmoid(x):
    return 2 / (1 + np.exp(-x)) - 1

def sat_exp(x, k):
    """Saturating exponential function."""
    return np.exp(-k*x)

def nn_policy(state, nn_model):
    modified_state = modify_state(state)
    stacked_state = np.expand_dims(np.hstack(list(modified_state.values())), axis=1).T
    return nn_model.predict(stacked_state)

def angle_invert(sin_angle, cos_angle):
    return np.arccos(cos_angle)*np.sign(sin_angle)

default_dist = 2

def compute_cost(position, hazards):
    sigma = 0.3
    displacements = hazards - position
    distances = np.einsum('ij,ji->i', displacements, displacements.T)
    return np.sum(np.exp(-distances / (sigma ** 2)))

def path_finder(robot_state, goal_pos, hazards):
    lookahead_dist = 0.4
    directions = 501
    goal_ind = (directions//2)
    sigma = 0.3
    sigma_a = np.pi
    safety_threshold = 0.3
    angles = np.linspace(-np.pi/2, np.pi/2, directions)
    xs = np.stack((np.sin(angles), np.cos(angles)))*lookahead_dist
    displacement_vector = -robot_state[:2]  # Vector from robot to goal
    displacement_vector /= np.linalg.norm(displacement_vector)
    sin, cos = displacement_vector
    rotation_matrix = np.array([[cos, sin], [-sin, cos]])
    robot_pos = (goal_pos+robot_state[:2]).reshape((1, 2, 1))
    trajectories = robot_pos + np.expand_dims(rotation_matrix@xs, axis=0)
    hazards = np.expand_dims(hazards, axis=-1)
    stacked_hazards = np.repeat(hazards, directions, axis=2)
    displacements = stacked_hazards-trajectories
    sq_distances = np.einsum('ij...,ji...->i...', displacements, np.swapaxes(displacements, 0, 1))
    costs = np.sum(np.exp(-sq_distances/(sigma**2)), axis=0)
    angle_penalties = 1-np.exp(-angles**2/(sigma_a**2))
    l2 = 0.5
    angle_values = angle_penalties+l2*costs
    path_ind = np.argmin(angle_values)
    delta_angle = angles[path_ind]*180/np.pi
    cost = costs[path_ind]
    return delta_angle, cost

REPLAY_MEMORY = []

def store_transition(robot_state, action, hazards, new_robot_state):
    stacked_transition = np.hstack((robot_state, action, hazards, new_robot_state))
    REPLAY_MEMORY.append(stacked_transition)
    return

def save_results(memory_buffer):
    pickle.dump(memory_buffer, open(f"policy_data/replay_memory", "wb"))
    return True

def get_forward(path_cost):
    return 0.01*(np.exp(-5*path_cost))

def safe_policy(robot_state, goal_pos, hazard_array):
    velocity = np.linalg.norm(robot_state[7:9])
    angular_velocity = robot_state[-1]
    k1, k2 = 0, 0
    return np.array([-k1*velocity, -k2*angular_velocity])

def human_policy(robot_state, goal_pos, hazard_array):
    """Hand crafted controller for problem."""
    cos_theta, sin_theta = np.clip(robot_state[3:5], -1, 1) #goal_compass
    goal_angle = angle_invert(sin_theta, cos_theta)*180/np.pi
    delta_angle, cost = path_finder(robot_state, goal_pos, hazard_array)
    new_target_angle = goal_angle - delta_angle
    forward = get_forward(cost)
    k = 0.1
    rotation = sigmoid(k*new_target_angle)
    if np.isnan(rotation) or np.isnan(forward):
        print(robot_state)
        print(sin_theta)
        print(cos_theta)
        raise ValueError
    return np.array([forward, rotation])

def modify_state(state_dict):
    modified_state_dict = deepcopy(state_dict)
    del modified_state_dict["magnetometer"]  # Useless state measurement.
    del modified_state_dict["gyro"]  # Useless state measurement.
    #modified_state_dict["gyro"] = modified_state_dict["gyro"][2]  # Just keep k value
    for key in ["accelerometer", "velocimeter"]:
        modified_state_dict[key] = modified_state_dict[key][:2]  # Remove z axis value
    print(state_dict)
    return modified_state_dict

def main(EPISODES, render=False, save=False, policy=human_policy):
    ep_rewards = []
    ep_costs = []
    actions = []

    for i in tqdm(range(EPISODES)):
        stored = 0
        total = 0
        episode_reward = 0
        episode_cost = 0
        state = env.reset()
        if render:
            env.render()
        done = False
        while not done:
            if np.random.random() < 0.3:
                action = np.random.uniform(-1, 1, size=(1,2))
            else:
                action = policy(state, env_state, env).reshape(1, 2)
            actions.append(action.T)
            print(state)
            new_state, reward, done, info = env.step(action)
            episode_reward += reward
            episode_cost += info["cost"]
            risky_state = np.min(-np.log(new_state["hazards_lidar"])) < 0.5
            if save:
                if risky_state or np.random.random() < 0.1:
                    stored += 1
                if np.random.random() < 0.3:
                    learner.train_models()
            total += 1
            state = new_state

            if render:
                env.render()

        print(f"Episode {i}, reward {episode_reward}, cost {episode_cost}, stored {stored}/{total}")
        if save:
            print(f"Buffer size: {len(learner.REPLAY_MEMORY)}")
        ep_rewards.append(episode_reward)
        ep_costs.append(episode_cost)

        if save and i != 0 and not i%(EPISODES//10):
            learner.save("jointmodel")

    if save:
        learner.save("jointmodel")
    return actions

lidar_bins = 32

if __name__ == "__main__":
    config = {
        'robot_base': 'xmls/point.xml',
        'task': 'goal',
        'observation_flatten': False,
        'observe_goal_dist': True,
        'observe_goal_comp': True,
        'observe_goal_lidar': False,
        'observe_hazards': True,
        'observe_vases': False,
        'observe_circle': True,
        'observe_vision': False,
        'observe_gremlins': False,
        'constrain_hazards': True,
        'constrain_vases': False,
        'constrain_gremlins': False,
        'lidar_max_dist': None,
        'lidar_num_bins': lidar_bins,
        'hazards_num': 10,
        'vases_num': 0,
        'gremlins_num': 0,
        'gremlins_travel': 0.5,
        'gremlins_keepout': 0.4,
    }
    env = Engine(config)
    #loaded_learner = pickle.load(open("jointmodel", "rb"))
    actions = main(EPISODES=10, render=True, policy=human_policy, save=False)
    plt.figure()
    actions = np.array(actions)
    print(actions.shape)
    plt.scatter(actions[:, 0], actions[:, 1])
    plt.xlabel("Forward")
    plt.ylabel("Rotation")
    plt.show()


from ..rman_utils.node_desc import NodeDesc
from ..rman_utils import filepath_utils
from ..rman_utils.filepath import FilePath
from ..rman_utils import texture_utils
from ..rman_utils import property_utils
from ..rfb_logger import rfb_log
from .rman_socket_utils import node_add_inputs
from .rman_socket_utils import node_add_outputs
from .. import rman_render
from .. import properties
from ..rman_properties import rman_properties_scene
from ..rman_properties import rman_properties_renderlayers
from ..rman_properties import rman_properties_world
from ..rman_properties import rman_properties_camera
from nodeitems_utils import NodeCategory, NodeItem
from collections import OrderedDict
from operator import attrgetter
from bpy.props import *
import bpy
import os
import sys
import traceback
import nodeitems_utils

# registers
from . import rman_bl_nodes_sockets
from . import rman_bl_nodes_shaders
from . import rman_bl_nodes_ops
from . import rman_bl_nodes_props

__RMAN_DISPLAY_NODES__ = []
__RMAN_BXDF_NODES__ = []
__RMAN_DISPLACE_NODES__ = []
__RMAN_INTEGRATOR_NODES__ = []
__RMAN_PROJECTION_NODES__ = []
__RMAN_DISPLAYFILTER_NODES__ = []
__RMAN_SAMPLEFILTER_NODES__ = []
__RMAN_PATTERN_NODES__ = []
__RMAN_LIGHT_NODES__ = []
__RMAN_LIGHTFILTER_NODES__ = []
__RMAN_NODE_TYPES__ = dict()
__RMAN_NODE_CATEGORIES__ = dict()

__RMAN_NODE_CATEGORIES__['bxdf'] = ('RenderMan Bxdfs', [])
__RMAN_NODE_CATEGORIES__['light'] = ('RenderMan Lights', [])
__RMAN_NODE_CATEGORIES__['patterns_misc'] = ('RenderMan Misc Patterns', [])
__RMAN_NODE_CATEGORIES__['displace'] = ('RenderMan Displacements', [])

__RMAN_NODES__ = {
    'displaydriver': __RMAN_DISPLAY_NODES__,
    'bxdf': __RMAN_BXDF_NODES__,
    'displace': __RMAN_DISPLACE_NODES__,
    'integrator': __RMAN_INTEGRATOR_NODES__,
    'projection': __RMAN_PROJECTION_NODES__,
    'displayfilter': __RMAN_DISPLAYFILTER_NODES__,
    'samplefilter': __RMAN_SAMPLEFILTER_NODES__,
    'pattern': __RMAN_PATTERN_NODES__,
    'light': __RMAN_LIGHT_NODES__,
    'lightfilter': __RMAN_LIGHTFILTER_NODES__
}

__RMAN_PLUGIN_MAPPING__ = {
    'integrator': rman_properties_scene.RendermanSceneSettings,
    'displaydriver': rman_properties_renderlayers.RendermanAOV,
    'projection': rman_properties_camera.RendermanCameraSettings,
    'light': rman_bl_nodes_props.RendermanLightSettings,
    'lightfilter': rman_bl_nodes_props.RendermanLightSettings,
    'displayfilter': rman_bl_nodes_props.RendermanDisplayFilterSettings,
    'samplefilter': rman_bl_nodes_props.RendermanSampleFilterSettings,
}

def update_conditional_visops(node):
    for param_name, prop_meta in getattr(node, 'prop_meta').items():
        if 'conditionalVisOp' in prop_meta:
            cond_expr = prop_meta['conditionalVisOp']['expr']
            try:
                hidden = not eval(cond_expr)
                prop_meta['hidden'] = hidden
                if hasattr(node, 'inputs') and param_name in node.inputs:
                    node.inputs[param_name].hide = hidden
            except:
                print("Error in conditional visop: %s" % (cond_expr))

def assetid_update_func(self, context):
    node = self.node if hasattr(self, 'node') else self
    light = None
    active = context.active_object
    if active.type == 'LIGHT':
        light = active.data

    texture_utils.update_texture(node, light=light)
    if context and hasattr(context, 'material'):
        mat = context.material
        if mat:
            node.update_mat(mat)
    elif context and hasattr(context, 'node'):
        mat = context.space_data.id
        if mat:
            node.update_mat(mat)

def update_func_with_inputs(self, context):
    # check if this prop is set on an input
    node = self.node if hasattr(self, 'node') else self

    if context and hasattr(context, 'material'):
        mat = context.material
        if mat:
            node.update_mat(mat)
    elif context and hasattr(context, 'node'):
        mat = context.space_data.id
        if mat:
            node.update_mat(mat)

    # update the conditional_vis_ops
    update_conditional_visops(node)

    if node.bl_idname in ['PxrLayerPatternNode', 'PxrSurfaceBxdfNode']:
        node_add_inputs(node, node.name, node.prop_names)
    else:
        update_inputs(node)

    # set any inputs that are visible and param is hidden to hidden
    prop_meta = getattr(node, 'prop_meta')
    if hasattr(node, 'inputs'):
        for input_name, socket in node.inputs.items():
            if 'hidden' in prop_meta[input_name]:
                socket.hide = prop_meta[input_name]['hidden']

def update_func(self, context):
    # check if this prop is set on an input
    node = self.node if hasattr(self, 'node') else self

    if context and hasattr(context, 'material'):
        mat = context.material
        if mat:
            node.update_mat(mat)
    elif context and hasattr(context, 'node'):
        mat = context.space_data.id
        if mat:
            node.update_mat(mat)

    # update the conditional_vis_ops
    update_conditional_visops(node)

    # set any inputs that are visible and param is hidden to hidden
    prop_meta = getattr(node, 'prop_meta')
    if hasattr(node, 'inputs'):
        for input_name, socket in node.inputs.items():
            if input_name not in prop_meta:
                continue
            if 'hidden' in prop_meta[input_name] \
                    and prop_meta[input_name]['hidden'] and not socket.hide:
                socket.hide = True

def update_integrator_func(self, context):
    rr = rman_render.RmanRender.get_rman_render()
    if rr.rman_interactive_running:
        rr.rman_scene.update_integrator(context)

def class_generate_properties(node, parent_name, node_desc):
    prop_names = []
    prop_meta = {}
    output_meta = OrderedDict()

    if "__annotations__" not in node.__dict__:
            setattr(node, "__annotations__", {})

    # pxr osl and seexpr need these to find the code
    if parent_name in ["PxrOSL", "PxrSeExpr"]:
        # Enum for internal, external type selection
        EnumName = "codetypeswitch"
        if parent_name == 'PxrOSL':
            EnumProp = EnumProperty(items=(('EXT', "External", ""),
                                           ('INT', "Internal", "")),
                                    name="Shader Location", default='INT')
        else:
            EnumProp = EnumProperty(items=(('NODE', "Node", ""),
                                           ('INT', "Internal", "")),
                                    name="Expr Location", default='NODE')

        EnumMeta = {'renderman_name': 'filename',
                    'name': 'codetypeswitch',
                    'renderman_type': 'string',
                    'default': '', 'label': 'codetypeswitch',
                    'type': 'enum', 'options': '',
                    'widget': 'mapper', '__noconnection': True}
        node.__annotations__[EnumName] = EnumProp
        prop_names.append(EnumName)
        prop_meta[EnumName] = EnumMeta
        # Internal file search prop
        InternalName = "internalSearch"
        InternalProp = StringProperty(name="Shader to use",
                                      description="Storage space for internal text data block",
                                      default="")
        InternalMeta = {'renderman_name': 'filename',
                        'name': 'internalSearch',
                        'renderman_type': 'string',
                        'default': '', 'label': 'internalSearch',
                        'type': 'string', 'options': '',
                        'widget': 'fileinput', '__noconnection': True}
        node.__annotations__[InternalName] = InternalProp
        prop_names.append(InternalName)
        prop_meta[InternalName] = InternalMeta
        # External file prop
        codeName = "shadercode"
        codeProp = StringProperty(name='External File', default='',
                                  subtype="FILE_PATH", description='')
        codeMeta = {'renderman_name': 'filename',
                    'name': 'ShaderCode', 'renderman_type': 'string',
                    'default': '', 'label': 'ShaderCode',
                    'type': 'string', 'options': '',
                    'widget': 'fileinput', '__noconnection': True}
        node.__annotations__[codeName] = codeProp
        prop_names.append(codeName)
        prop_meta[codeName] = codeMeta

    # inputs

    for node_desc_param in node_desc.params:

        update_function = None
        if node_desc.node_type == 'integrator':
            update_function = update_integrator_func
        else:
            param_widget = node_desc.widget.lower() if hasattr(node_desc,'widget') else 'default'
            if param_widget == 'fileinput' or param_widget == 'assetidinput' or (param_widget == 'default' and node_desc_param.name == 'filename'):
                update_fuction = assetid_update_func
            else:
                update_function = update_func_with_inputs if 'enable' in node_desc_param.name else update_func

        name, meta, prop = property_utils.generate_property(node_desc_param, update_function=update_function)
        if name is None:
            continue
        if hasattr(node_desc_param, 'page') and node_desc_param.page != '':
            page = node_desc_param.page
            tokens = page.split('|')
            sub_prop_names = prop_names
            page_name = tokens[0]

            if page_name not in prop_meta:
                sub_prop_names.append(page_name)
                prop_meta[page_name] = {'renderman_type': 'page'}
                ui_label = "%s_uio" % page_name
                node.__annotations__[ui_label] = BoolProperty(name=ui_label, default=False)
                setattr(node, page_name, [])

                if parent_name == 'PxrSurface' and 'Globals' not in page_name:
                    enable_param_name = 'enable' + page_name.replace(' ', '')
                    if enable_param_name not in prop_meta:
                        prop_meta[enable_param_name] = {
                            'renderman_type': 'enum', 'renderman_name': enable_param_name}
                        default = page_name == 'Diffuse'
                        enable_param_prop = BoolProperty(name="Enable " + page_name,
                                            default=bool(default),
                                            update=update_func_with_inputs)
                        node.__annotations__[enable_param_name] = enable_param_prop
                        page_prop_names = getattr(node, page_name)
                        if enable_param_name not in page_prop_names:
                            page_prop_names.append(enable_param_name)
                            setattr(node, page_name, page_prop_names)

            if len(tokens) > 1:

                for i in range(1, len(tokens)):
                    parent_page = page_name
                    page_name += '.' + tokens[i]
                    if page_name not in prop_meta:
                        prop_meta[page_name] = {'renderman_type': 'page'}
                        ui_label = "%s_uio" % page_name
                        node.__annotations__[ui_label] = BoolProperty(name=ui_label, default=False)
                        setattr(node, page_name, [])

                    sub_prop_names = getattr(node, parent_page)
                    if page_name not in sub_prop_names:
                        sub_prop_names.append(page_name)
                        setattr(node, parent_page, sub_prop_names)

            sub_prop_names = getattr(node, page_name)
            sub_prop_names.append(name)
            setattr(node, page_name, sub_prop_names)
            prop_meta[name] = meta
            node.__annotations__[name] = prop

        else:
            prop_names.append(name)
            prop_meta[name] = meta
            node.__annotations__[name] = prop

    # outputs
    for node_desc_param in node_desc.outputs:
        renderman_type = node_desc_param.type
        prop_name = node_desc_param.name

        output_prop_meta = dict()
        if hasattr(node_desc_param, 'vstructmember'):
            output_prop_meta['vstructmember'] = node_desc_param.vstructmember
        if hasattr(node_desc_param, 'vstructConditionalExpr'):
            output_prop_meta['vstructConditionalExpr'] = node_desc_param.vstructConditionalExpr
        output_prop_meta['name'] = node_desc_param.name
        output_meta[prop_name] = output_prop_meta #node_desc_param
        output_meta[prop_name]['renderman_type'] = renderman_type

    setattr(node, 'prop_names', prop_names)
    setattr(node, 'prop_meta', prop_meta)
    setattr(node, 'output_meta', output_meta)

def generate_node_type(node_desc):
    ''' Dynamically generate a node type from pattern '''

    name = node_desc.name
    nodeType = node_desc.node_type #args.find("shaderType/tag").attrib['value']
    typename = '%s%sNode' % (name, nodeType.capitalize())
    nodeDict = {'bxdf': rman_bl_nodes_shaders.RendermanBxdfNode,
                'pattern': rman_bl_nodes_shaders.RendermanPatternNode,
                'displace': rman_bl_nodes_shaders.RendermanDisplacementNode,
                'light': rman_bl_nodes_shaders.RendermanLightNode}

    if nodeType not in nodeDict.keys():
        return (None, None)
    ntype = type(typename, (nodeDict[nodeType],), {})
    ntype.bl_label = name
    ntype.typename = typename

    def init(self, context):
        if self.renderman_node_type == 'bxdf':
            self.outputs.new('RendermanShaderSocket', "Bxdf").type = 'SHADER'
            #socket_template = self.socket_templates.new(identifier='Bxdf', name='Bxdf', type='SHADER')
            node_add_inputs(self, name, self.prop_names)
            node_add_outputs(self)
            # if this is PxrLayerSurface set the diffusegain to 0.  The default
            # of 1 is unintuitive
            if self.plugin_name == 'PxrLayerSurface':
                self.diffuseGain = 0
        elif self.renderman_node_type == 'light':
            # only make a few sockets connectable
            node_add_inputs(self, name, self.prop_names)
            self.outputs.new('RendermanShaderSocket', "Light")
        elif self.renderman_node_type == 'displace':
            # only make the color connectable
            self.outputs.new('RendermanShaderSocket', "Displacement")
            node_add_inputs(self, name, self.prop_names)
        # else pattern
        elif name == "PxrOSL":
            self.outputs.clear()
        else:
            node_add_inputs(self, name, self.prop_names)
            node_add_outputs(self)

        if name == "PxrRamp":
            node_group = bpy.data.node_groups.new(
                'PxrRamp_nodegroup', 'ShaderNodeTree')
            node_group.nodes.new('ShaderNodeValToRGB')
            node_group.use_fake_user = True
            self.node_group = node_group.name
        update_conditional_visops(self)

    def free(self):
        if name == "PxrRamp":
            bpy.data.node_groups.remove(bpy.data.node_groups[self.node_group])

    ntype.init = init
    ntype.free = free

    if "__annotations__" not in ntype.__dict__:
            setattr(ntype, "__annotations__", {})

    if name == 'PxrRamp':
        ntype.__annotations__['node_group'] = StringProperty('color_ramp', default='')

    ntype.__annotations__['plugin_name'] = StringProperty(name='Plugin Name',
                                       default=name, options={'HIDDEN'})

    # lights cant connect to a node tree in 20.0
    class_generate_properties(ntype, name, node_desc)
    if nodeType == 'light':
        ntype.__annotations__['light_shading_rate'] = FloatProperty(
            name="Light Shading Rate",
            description="Shading Rate for this light.  \
                Leave this high unless detail is missing",
            default=100.0)
        ntype.__annotations__['light_primary_visibility'] = BoolProperty(
            name="Light Primary Visibility",
            description="Camera visibility for this light",
            default=True)

    bpy.utils.register_class(ntype)

    return (typename, ntype)

def register_plugin_to_parent(ntype, name, node_desc, plugin_type, parent):

    class_generate_properties(ntype, name, node_desc)
    setattr(ntype, 'renderman_node_type', plugin_type)

    if "__annotations__" not in parent.__dict__:
            setattr(parent, "__annotations__", {})

    # register and add to scene_settings
    bpy.utils.register_class(ntype)
    settings_name = "%s_settings" % name
    parent.__annotations__["%s_settings" % name] = PointerProperty(type=ntype, name="%s Settings" % name)

    if "__annotations__" not in rman_properties_world.RendermanWorldSettings.__dict__:
            setattr(rman_properties_world.RendermanWorldSettings, "__annotations__", {})

    # special case for world lights
    if plugin_type == 'light' and name in ['PxrDomeLight', 'PxrEnvDayLight']:
        rman_properties_world.RendermanWorldSettings.__annotations__["%s_settings" % name] = PointerProperty(type=ntype, name="%s Settings" % name)

def register_plugin_types(node_desc):

    items = []

    if node_desc.node_type not in __RMAN_PLUGIN_MAPPING__:
        return
    parent = __RMAN_PLUGIN_MAPPING__[node_desc.node_type]
    name = node_desc.name
    if node_desc.node_type == 'displaydriver':
        # remove the d_ prefix
        name = name.split('d_')[1]
    typename = name + node_desc.node_type.capitalize() + 'Settings'
    ntype = type(typename, (rman_bl_nodes_props.RendermanPluginSettings,), {})
    ntype.bl_label = name
    ntype.typename = typename
    ntype.bl_idname = typename
    ntype.plugin_name = name

    try:
        register_plugin_to_parent(ntype, name, node_desc, node_desc.node_type, parent)
    except Exception as e:
        rfb_log().error("Error registering plugin ", name)
        traceback.print_exc()

def get_path_list():
    paths = []
    rmantree = filepath_utils.guess_rmantree()
    paths.append(os.path.join(rmantree, 'lib', 'plugins'))
    paths.append(os.path.join(rmantree, 'lib', 'shaders'))
    paths.append(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                'Args'))

    if 'RMAN_RIXPLUGINPATH' in os.environ:
        RMAN_RIXPLUGINPATH = os.environ['RMAN_RIXPLUGINPATH']
        for p in RMAN_RIXPLUGINPATH.split(':'):
            paths.append(os.path.join(p, 'Args'))
    if 'RMAN_SHADERPATH' in os.environ:
        RMAN_SHADERPATH = os.environ['RMAN_SHADERPATH']
        for p in RMAN_SHADERPATH.split(':'):
            paths.append(p)

    return paths

class RendermanPatternNodeCategory(NodeCategory):

    @classmethod
    def poll(cls, context):
        return context.space_data.tree_type == 'ShaderNodeTree'

def register_rman_nodes():
    global __RMAN_NODE_CATEGORIES__

    rfb_log().debug("Registering RenderMan Plugin Nodes:")
    path_list = get_path_list()
    for path in path_list:
        for root, dirnames, filenames in os.walk(path):
            for filename in filenames:
                if filename.endswith(('.args', '.oso')):
                    node_desc = NodeDesc(FilePath(root).join(FilePath(filename)))
                    __RMAN_NODES__[node_desc.node_type].append(node_desc)
                    rfb_log().debug("\t%s" % node_desc.name)

                    # These plugin types are special. They are not actually shading
                    # nodes that can be used in Blender's shading editor, but
                    # we still create PropertyGroups for them so they can be inserted
                    # into the correct UI panel.
                    if node_desc.node_type in ['integrator', 'projection', 'displaydriver',
                                                'displayfilter', 'samplefilter',
                                                'light',
                                                'lightfilter']:
                        register_plugin_types(node_desc)
                        if node_desc.name != 'PxrMeshLight':
                            # for mesh light, we need to create a shader graph node
                            continue

                    typename, nodetype = generate_node_type(node_desc)
                    if not typename and not nodetype:
                        continue

                    if typename and nodetype:
                        __RMAN_NODE_TYPES__[typename] = nodetype

                    # categories
                    node_item = NodeItem(typename, label=nodetype.bl_label)
                    if node_desc.node_type == 'pattern':
                        if hasattr(node_desc, 'classification'):
                            try:
                                tokens = node_desc.classification.split('/')
                                category = tokens[-1].lower()
                                lst = __RMAN_NODE_CATEGORIES__.get('patterns_%s' % category, None)
                                if not lst:
                                    lst = ('RenderMan %s Patterns' % category.capitalize(), [])
                                lst[1].append(node_item)
                                __RMAN_NODE_CATEGORIES__['patterns_%s' % category] = lst
                            except:
                                pass
                        else:
                            __RMAN_NODE_CATEGORIES__['patterns_misc'][1].append(node_item)
                    elif node_desc.node_type == 'bxdf':
                        __RMAN_NODE_CATEGORIES__['bxdf'][1].append(node_item)
                    elif node_desc.node_type == 'displace':
                        __RMAN_NODE_CATEGORIES__['displace'][1].append(node_item)
                    elif node_desc.node_type == 'light':
                        __RMAN_NODE_CATEGORIES__['light'][1].append(node_item)

    rfb_log().debug("Finished Registering RenderMan Plugin Nodes.")

    # all categories in a list
    node_categories = [
        # identifier, label, items list
        RendermanPatternNodeCategory("PRMan_output_nodes", "RenderMan Outputs",
                                     items=[NodeItem('RendermanOutputNode', label=rman_bl_nodes_shaders.RendermanOutputNode.bl_label)]),
    ]

    for name, (desc, items) in __RMAN_NODE_CATEGORIES__.items():
        node_categories.append(RendermanPatternNodeCategory(name, desc,
                                                            items=sorted(items,
                                                                         key=attrgetter('_label'))))

    nodeitems_utils.register_node_categories("RENDERMANSHADERNODES",
                                             node_categories)

def register():
    register_rman_nodes()
    rman_bl_nodes_props.register()
    rman_bl_nodes_sockets.register()
    rman_bl_nodes_shaders.register()
    rman_bl_nodes_ops.register()

def unregister():
    nodeitems_utils.unregister_node_categories("RENDERMANSHADERNODES")

    rman_bl_nodes_props.unregister()
    rman_bl_nodes_sockets.unregister()
    rman_bl_nodes_shaders.unregister()
    rman_bl_nodes_ops.unregister()

    for cls in classes:
        bpy.utils.unregister_class(cls)


import numpy as np
import pytest
from sympy.abc import a, b, c, d, e
import time

from conftest import skipif
from devito.tools import toposort, filter_ordered

pytestmark = skipif(['yask', 'ops'])

@pytest.mark.parametrize('elements, expected', [
    ([[a, b, c], [c, d, e]], [a, b, c, d, e]),
    ([[e, d, c], [c, b, a]], [e, d, c, b, a]),
    ([[a, b, c], [b, d, e]], [a, b, d, c, e]),
    ([[a, b, c], [d, b, c]], [a, d, b, c]),
    ([[a, b, c], [c, d, b]], None),
])
def test_toposort(elements, expected):
    try:
        ordering = toposort(elements)
        assert ordering == expected
    except ValueError:
        assert expected is None

def test_sorting():
    key = lambda x: x
    array = np.random.rand(10000)
    t0 = time.time()
    sort_key = filter_ordered(array, key=key)
    t1 = time.time()
    sort_nokey = filter_ordered(array)
    t2 = time.time()
    # This one is slightly faster
    assert t2 - t1 < .5 * (t1 - t0)
    assert sort_key == sort_nokey


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed May 27 11:23:51 2020

@author: francois

PJ5 Openclassrooms : this script is prerequisites for Grid search validation and saves results
To be used as prerequisite of PJ5_GridSearch1.py

"""

#%matplotlib inline

#%load_ext autoreload  # Autoreload has a bug : when you modify function in source code and run again, python kernel hangs :(
#%autoreload 2

import datetime as dt

import sys, importlib

from functions import *
#importlib.reload(sys.modules['functions'])

import pandas as pd

pd.set_option('display.max_rows', 200)
pd.set_option('display.max_columns', 200)

import datetime as dt

import os
import zipfile
import urllib

import matplotlib.pyplot as plt
import pandas as pd
import numpy as np

import qgrid

import glob

from pandas.plotting import scatter_matrix

from sklearn.model_selection import StratifiedShuffleSplit

from sklearn.model_selection import GridSearchCV

from sklearn.manifold import TSNE
from sklearn.manifold import LocallyLinearEmbedding
from sklearn.manifold import Isomap

from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score
from sklearn.metrics import pairwise_distances
from sklearn.cluster import AgglomerativeClustering
from scipy.stats import entropy

from sklearn.feature_selection import RFE

DATA_PATH = os.path.join("datasets", "onlineretail")
DATA_PATH = os.path.join(DATA_PATH, "out")

DATA_PATH_FILE_INPUT = os.path.join(DATA_PATH, "OnlineRetail_transformed.csv")

ALL_FEATURES = []

#MODEL_FEATURES=['InvoiceNo', 'InvoiceDate', 'CustomerID', 'TotalPrice', 'DescriptionNormalized', 'InvoiceMonth', 'StockCode']
MODEL_CLIENT_FEATURES = ['TotalPricePerMonth', 'DescriptionNormalized', 'HasEverCancelled', 'BoughtTopValueProduct' ]

plt.rcParams["figure.figsize"] = [16,9] # Taille par défaut des figures de matplotlib

import seaborn as sns
sns.set()

#import common_functions

####### Paramètres pour sauver et restaurer les modèles :
import pickle
####### Paramètres à changer par l'utilisateur selon son besoin :

'''
RECOMPUTE_GRIDSEARCH = True  # CAUTION : computation is several hours long
SAVE_GRID_RESULTS = False # If True : grid results object will be saved to pickle files that have GRIDSEARCH_FILE_PREFIX
LOAD_GRID_RESULTS = False # If True : grid results object will be loaded from pickle files that have GRIDSEARCH_FILE_PREFIX
                          # Grid search results are loaded with full samples (SAMPLED_DATA must be False)
'''

RECOMPUTE_GRIDSEARCH = True  # CAUTION : computation is several hours long
SAVE_GRID_RESULTS = True # If True : grid results object will be saved to pickle files that have GRIDSEARCH_FILE_PREFIX
LOAD_GRID_RESULTS = False # If True : grid results object will be loaded from pickle files that have GRIDSEARCH_FILE_PREFIX

#GRIDSEARCH_CSV_FILE = 'grid_search_results.csv'

GRIDSEARCH_FILE_PREFIX = 'grid_search_results_'

EXECUTE_INTERMEDIATE_MODELS = True # If True: every intermediate model (which results are manually analyzed in the notebook) will be executed

# Necessary for predictors used in the notebook :
from sklearn.linear_model import LinearRegression
from sklearn.compose import TransformedTargetRegressor
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error

from sklearn.preprocessing import PolynomialFeatures

### For progress bar :
from tqdm import tqdm_notebook as tqdm

# Statsmodel :
import statsmodels.formula.api as smf

import statsmodels.api as sm
from scipy import stats

SAVE_API_MODEL = True # If True : API model ill be saved
API_MODEL_PICKLE_FILE = 'API_model_PJ5.pickle'

# Load data
print('Load data')

df = load_data(DATA_PATH_FILE_INPUT)

df, df_train, df_test = custom_train_test_split_sample(df, 'TotalPrice')
df_train.reset_index(inplace=True)
df_test.reset_index(inplace=True)
df_train_ori = df_train.copy(deep=True)
df_test_ori = df_test.copy(deep=True)

# Top value products (must be saved with the model, and passed to it)
print('Top value products (must be saved with the model, and passed to it)')
df_nocancel = df_train[df_train['InvoiceNo'].str.startswith('C') == False]
df_nocancel.reset_index(inplace=True)

df_gbproduct = df_nocancel[['StockCode', 'TotalPrice']].groupby('StockCode').sum()['TotalPrice']

TOP_VALUE_PRODUCT_THRESHOLD = 20
top_value_products = df_gbproduct.sort_values(ascending=False).head(TOP_VALUE_PRODUCT_THRESHOLD).index  # Get top value products

# Preparation pipeline : model with bow features + TotalPricePerMonth + BoughtTopValueProduct + HasEverCancelled
print('Preparation pipeline : model with bow features + TotalPricePerMonth + BoughtTopValueProduct + HasEverCancelled')

preparation_pipeline = Pipeline([
    #('features_selector', FeaturesSelector(features_toselect=MODEL_FEATURES)),
    ('bow_encoder', BowEncoder()),
    ('agregate_to_client_level', AgregateToClientLevel(top_value_products)),
    ('scaler', LogScalerMultiple(features_toscale=['TotalPricePerMonth'])),

    # Faire la réduction dimensionnelle à part pour les bag of words et pour les autres features

    ('minmaxscaler', MinMaxScalerMultiple(features_toscale=['TotalPricePerMonth'])),
    ('dimensionality_reductor', DimensionalityReductor(features_totransform=['DescriptionNormalized'], \
                                                        algorithm_to_use='PCA', n_dim=200)),
    ('minmaxscaler_final', MinMaxScalerMultiple(features_toscale='ALL_FEATURES')),
])

df_train = preparation_pipeline.fit_transform(df_train)
df_test = preparation_pipeline.transform(df_test)

series_total_price_per_month_train = df_train['TotalPricePerMonth']
series_total_price_per_month_test = df_test['TotalPricePerMonth']

series_hasevercancelled_train = df_train['HasEverCancelled']
series_hasevercancelled_test = df_test['HasEverCancelled']

series_boughttopvalueproduct_train = df_train['BoughtTopValueProduct']
series_boughttopvalueproduct_test = df_test['BoughtTopValueProduct']

# Generation of TotalPricePerMonth labels
pca = PCA(n_components=2, random_state=42)
X_transformed = pca.fit_transform(df_train)
X_test_transformed = pca.fit_transform(df_test)

print('Binarisation of color categories')
bins = [-np.inf,df_train['TotalPricePerMonth'].quantile(0.25),\
        df_train['TotalPricePerMonth'].quantile(0.50),\
        df_train['TotalPricePerMonth'].quantile(0.75),\
        df_train['TotalPricePerMonth'].quantile(1)]

labels = [0, 1, 2, 3]

df_score_cat_train = pd.cut(df_train['TotalPricePerMonth'], bins=bins, labels=labels)

bins = [-np.inf,df_test['TotalPricePerMonth'].quantile(0.25),\
        df_test['TotalPricePerMonth'].quantile(0.50),\
        df_test['TotalPricePerMonth'].quantile(0.75),\
        df_test['TotalPricePerMonth'].quantile(1)]

labels = [0, 1, 2, 3]

df_score_cat_test = pd.cut(df_test['TotalPricePerMonth'], bins=bins, labels=labels)

# Generation of Bag Of Words labels and columns
print('Generation of Bag Of Words labels and columns')

df_train = df_train_ori
df_test = df_test_ori

preparation_pipeline = Pipeline([
    #('features_selector', FeaturesSelector(features_toselect=MODEL_FEATURES)),
    ('bow_encoder', BowEncoder()),
    ('agregate_to_client_level', AgregateToClientLevel(top_value_products, compute_rfm=True)),
    ('features_selector', FeaturesSelector(features_toselect=['DescriptionNormalized', 'RfmScore'])),
    #('scaler', LogScalerMultiple(features_toscale=['TotalPricePerMonth', 'TotalQuantityPerMonth'])),

    # Faire la réduction dimensionnelle à part pour les bag of words et pour les autres features

    #('minmaxscaler', MinMaxScalerMultiple(features_toscale=['RfmScore'])),
    ('dimensionality_reductor', DimensionalityReductor(features_totransform=['DescriptionNormalized'], \
                                                        algorithm_to_use='NCA', n_dim=1, labels_featurename='RfmScore')),
    ('minmaxscaler_final', MinMaxScalerMultiple(features_toscale='ALL_FEATURES')),
])

df_train = preparation_pipeline.fit_transform(df_train)
df_test = preparation_pipeline.transform(df_test)

print('End of preparation pipeline transform')

kmeans_per_k = [KMeans(n_clusters=k, random_state=42).fit(df_train.loc[:, 0].to_numpy().reshape(-1,1))
                for k in range(1, 50)]

labels_test_per_k = [model.predict(df_test.loc[:, 0].to_numpy().reshape(-1,1)) for model in kmeans_per_k[1:]]

silhouette_scores = [silhouette_score(df_train.loc[:, 0].to_numpy().reshape(-1,1), model.labels_)
                     for model in kmeans_per_k[1:]]

silhouette_scores_test = [silhouette_score(df_test.loc[:, 0].to_numpy().reshape(-1,1), labels_test) for labels_test in labels_test_per_k]

entropy_mean_score_per_k_train = []

for model in kmeans_per_k[1:]:
    unique_labels = np.unique(model.labels_)

    entropy_sum = 0
    for unique_label in unique_labels:
        entropy_sum += entropy(df_train['RfmScore'][model.labels_ == unique_label].to_numpy())

    entropy_sum = entropy_sum / len(unique_labels)

    entropy_mean_score_per_k_train.append(entropy_sum)

entropy_mean_score_per_k_test = []

print('Next')

for labels_test in labels_test_per_k:
    unique_labels = np.unique(labels_test)

    entropy_sum = 0
    for unique_label in unique_labels:
        entropy_sum += entropy(df_test['RfmScore'][labels_test == unique_label].to_numpy())

    entropy_sum = entropy_sum / len(unique_labels)

    entropy_mean_score_per_k_test.append(entropy_sum)

# Cluster corresponding to kmeans_per_k[10] has the best silhouette score
bow_labels_train = kmeans_per_k[10].labels_  # sample : [4 7 8 ... 4 8 1]
bow_labels_test = kmeans_per_k[10].predict(df_test.loc[:, 0].to_numpy().reshape(-1,1))

series_bow_labels_train = pd.Series(bow_labels_train)
series_bow_labels_test = pd.Series(bow_labels_test)

series_bow_labels_train.set_axis(df_train.index, axis=0, inplace=True)
series_bow_labels_test.set_axis(df_test.index, axis=0, inplace=True)

rfm_score_train = df_train['RfmScore']
rfm_score_test = df_test['RfmScore']


"""
Chuhang Zou
07.2019

Code Revised from:

Finetuning Torchvision Models
=============================

**Author:** `Nathan Inkawhich <https://github.com/inkawhich>`__

"""

from __future__ import print_function
from __future__ import division
import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
from torchvision import models, transforms
import copy
print("PyTorch Version: ",torch.__version__)

import time
from torch.utils import data
import scipy.io as sio
from model_occ import *
from data_generator_occ import *
import re

# Top level data directory. Here we assume the format of the directory conforms
#   to the ImageFolder structure
data_dir = "./data/ShapeNet/"
model_path = "./model/PointCloud_Reconstruction_resnet50_Complete_Silhouette_Guidance.pth"

# use train-val since we evaluate on Pix3D but not ShapeNet
train_datapath = './data/trainval_list.txt'
val_datapath = './data/test_list.txt'

# Pre-trained models to choose from [resnet18, resnet34, resnet50]
model_name = "resnet50"

# load pretrained weights
Flag_loadweights = True
weight_path = './model/PointCloud_Reconstruction_resnet50_ShapeNet.pth'

# Number of classes in the dataset
num_classes = 1024

# grid size for refinement branch
grid_size = 2

# Batch size for training (change depending on how much memory you have)
batch_size = 32

# Number of epochs to train for
num_epochs = 100000
steps_per_epoch = 20

# Model Training and Validation Code
def train_model(model, train_generator, val_generator, optimizer, steps=100, num_epochs=25):
    since = time.time()

    val_acc_history = []

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = np.Inf

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
                dataloaders = train_generator
            else:
                model.eval()   # Set model to evaluate mode
                dataloaders = val_generator

            loss_3d_f_sum = 0.0
            loss_2d_f_sum = 0.0
            loss_2d_f_t_sum = 0.0
            loss_sum = 0.0
            step = 0

            # Iterate over data.
            for input in dataloaders:

                inputs = input[0]
                labels = input[1]
                masks = input[2]

                # gpu mode
                inputs = inputs.to(device)
                labels_f = labels.to(device)
                masks = masks.to(device)

                # zero the parameter gradients
                optimizer.zero_grad()

                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    # Get model outputs and calculate loss

                    # silhouette
                    masks = (masks>0.5).float()
                    inputs = torch.cat((inputs, masks),1)

                    _, outputs_f = model(inputs)

                    # loss
                    dist11, dist22 = distChamfer_bn(outputs_f, labels_f)

                    loss_2d_f = distChamfer_2d(outputs_f, masks)
                    loss_2d_f_t = distChamfer_2d_t(outputs_f, labels_f)

                    w1 = torch.mean(dist11)
                    w2 = torch.mean(dist22)
                    w1 = w1.data
                    w2 = w2.data

                    loss_3d_f = (torch.mean(dist11))+(torch.mean(dist22))
                    loss_2d_f = loss_2d_f*0.000000001
                    loss_2d_f_t = loss_2d_f_t*0.0000000001

                    loss = loss_3d_f + loss_2d_f + loss_2d_f_t

                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        torch.nn.utils.clip_grad_norm_(model.parameters(), 5.0)
                        optimizer.step()

                # statistics
                loss_3d_f_sum += loss_3d_f.item()*inputs.size(0)/steps
                loss_2d_f_sum += loss_2d_f.item()*inputs.size(0)/steps
                loss_2d_f_t_sum += loss_2d_f_t.item()*inputs.size(0)/steps
                loss_sum += (w1+w2)*inputs.size(0)/steps

                # clear cache
                #torch.cuda.empty_cache()

                # Break after 'steps' steps
                if step==steps-1:
                    break
                step += 1

            print('{} Loss: {:.6f}, Loss 3d: {:.6f}, Loss 2d: {:.6f},  Loss 2d t: {:.6f}'.format(phase, loss_sum, loss_3d_f_sum, loss_2d_f_sum*10000, loss_2d_f_t_sum*10000))

            # deep copy the model
            if phase == 'val' and best_acc > loss_sum:
                best_acc = loss_sum
                best_model_wts = copy.deepcopy(model.state_dict())
                # save model
                torch.save({'model_state_dict':best_model_wts, 'best_acc':best_acc}, model_path)
                print("Model saved ...")
            if phase == 'val':
                val_acc_history.append(loss_sum)

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:6f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model, val_acc_history

print("Load Models...")
# Define the encoder
encoder = initialize_encoder(model_name, num_classes,use_pretrained=True)

# Full model
model_ft = OccNet(encoder, num_classes, grid_size)

# Detect if we have a GPU available
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

# Send the model to GPU
model_ft = model_ft.to(device)

# Model initialization
set_parameter_requires_grad(model_ft)
# Print the model we just instantiated
#print(model_ft)
# if load weights
if Flag_loadweights:
    # pretrained header
    pretrained_dict = torch.load(weight_path)
    model_dict = model_ft.state_dict()
    pretrained_dict['resnet.conv1.weight'] = torch.cat((pretrained_dict['resnet.conv1.weight'], model_dict['resnet.conv1.weight'][:,3:,:,:]), 1)
    pretrained_dict = {k: v for k, v in pretrained_dict.items() if k in model_dict}
    model_dict.update(pretrained_dict)
    model_ft.load_state_dict(model_dict)

# update header
print("Params to learn:")
for name,param in model_ft.named_parameters():
#    # tune hearder only
#    if name != 'conv1.bias' and name != 'conv1.weight' and name != 'conv2.weight' and name != 'conv2.bias' and name != 'conv3.bias' and name != 'conv3.weight':
#        param.requires_grad = False

    if param.requires_grad == True:
        print("\t",name)

# Gather the parameters to be optimized/updated in this run.
params_to_update = [param for name, param in model_ft.named_parameters() if param.requires_grad]
#params_to_update = model_ft.parameters()
#time.sleep(500)

# Create the Optimizer
optimizer_ft = optim.Adam(params_to_update, lr = 1e-4, eps = 1e-6)

# Load Data
print("Initializing Datasets and Dataloaders...")
train_set = ShapeNetDataset(train_datapath, data_dir, 'train', grid_size=grid_size, transform=True)
train_generator = torch.utils.data.DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=4)
val_set = ShapeNetDataset(val_datapath, data_dir, 'val', grid_size=grid_size, transform=True)
val_generator = torch.utils.data.DataLoader(val_set, batch_size=batch_size, shuffle=True, num_workers=4)

# Train and evaluate
model_ft, hist = train_model(model_ft, train_generator, val_generator, optimizer_ft, steps_per_epoch, num_epochs=num_epochs)

print('training done')


from opentrons import labware, instruments, robot
import logging
logging.basicConfig(level=logging.DEBUG)

metadata = {
	'protocolName':'NAD kinetics assay',
	'author':'Alex Telfar <te1farac@gmail.com>',
	'source':'kinetics.py'
}

def load_labware():
    wells96 = labware.load('corning_96_wellplate_360ul_flat', '1')
    liquid_input = labware.load('opentrons_10_tuberack_nest_4x50ml_6x15ml_conical', '2')
    tiprack300 = labware.load('tiprack-200ul', '3')

    pipette300 = instruments.P300_Single(
        mount='left',
        tip_racks=[tiprack300])

    # pipette10 = instruments.P10_Single(
    #     mount='right',
    #     tip_racks=[tiprack10])

    return (pipette300,
            # pipette10,
            liquid_input['A3'],  # trash
            liquid_input['A4'],  # buffer source,
            liquid_input['A1'],  # nad source
            liquid_input['A2'],  # enzyme source
            liquid_input['B1'],  # substrate source
            wells96)  # 96 well plate

def serial_dilution(pipette, substrate_source, dilutant_source, output, trash,
    total_mixing_volume, num_of_dilutions, dilution_factor):
    transfer_volume = total_mixing_volume/dilution_factor
    diluent_volume = total_mixing_volume - transfer_volume

    # Distribute diluent across the plate to the the number of samples
    # And add diluent to one column after the number of samples for a blank
    for col in output.cols(*range(2, num_of_dilutions+1)):
        pipette.distribute(total_mixing_volume, dilutant_source, col)

    for row in output.rows():
        pipette.transfer(
            transfer_volume,
            row.wells(*range(1, num_of_dilutions)),
            row.wells(*range(2, num_of_dilutions+1)),
            mix_after=(3, total_mixing_volume / 2))

        pipette.transfer(
            transfer_volume,
            row.wells(num_of_dilutions),
            trash)

def run_custom_protocol(
        dilution_factor: float=2.0,
        num_of_dilutions: int=4,
        replicates: int=6,
        total_mixing_volume: float=200.0,
        substrate_conc: float=1,  # 1M conc
        nad_conc: float=20.0,  # 10mM of NAD
        v_enzyme: float=50.0, # 5uL of enzyme
        ):

    transfer_volume = total_mixing_volume/dilution_factor
    diluent_volume = total_mixing_volume - transfer_volume

    p300, trash, buffer_s, nad_s, enzyme_s, substrate_s, wells96 = load_labware()

    # serial dilution of the substrate
    serial_dilution(p300, substrate_s, buffer_s, wells96, trash, 100, 4, 0.5)

    # add 10mM of NAD to each well
    p300.distribute(nad_conc, nad_s, wells96.wells())

    # add 5uL of enzyme to each well
    p300.distribute(v_enzyme, enzyme_s, wells96.wells())

if __name__ == '__main__' or __name__ == 'builtins':
	run_custom_protocol()


from json import JSONDecodeError
from pprint import pprint

import jmespath

from attendees.session_control import MeetUpSession

def ack(session: MeetUpSession, method: str, *args, **kwargs):
    response = getattr(session, method)(*args, **kwargs)
    try:
        return response.json()
    except JSONDecodeError:
        return response.content

def get_attendees(session: MeetUpSession):
    response = ack(session, 'get')
    filter_query = "responses[0].value[].{updated: updated, id: member.id, name: member.name, " \
                   "pay_status: pay_status, response: response, bio: answers[0].answer}"
    return jmespath.search(filter_query, response)

def set_paid(session: MeetUpSession, member_id, amount):
    session.set_member_id(member_id)
    session.set_amount(amount)
    response = ack(session, 'post')
    pprint(response)

if __name__ == '__main__':
    # call attendees api
    event = MeetUpSession(267489117)
    attendees_data = get_attendees(event)
    pprint(attendees_data)

    # post
    '''
    {"responses":[{"ref":"markAsPaid","value":{"id":"43401196","confirm_code":"M43401196"},"meta":{"server":"ip-10-192-12-176","requestId":"210497bc-89a3-4712-9418-4e3de626d00c","endpoint":"awskrug/events/267489117/payments","statusCode":200}}]}
    '''
    # post_response = set_paid(event, 251549990, 5000)
    # pprint(post_response)


import json
import numpy as np
from scipy.spatial.distance import euclidean as distance_

dimensions_ingredients = 702
dimensions_nutrition_facts = 22
number_of_clusters=100

filename = "recipes.json"
with open(filename) as json_data:
    recipes = json.load(json_data)
    names = list(recipes.keys())
    X = np.zeros([len(names), dimensions_ingredients+dimensions_nutrition_facts])
    labels = {}
    index = 0
    for label, recipe in recipes.items():
        for ingredient in recipe["ingredients"]:
            X[index, ingredient[0]] = ingredient[1]
        for fact in recipe["nutrition"]:
            X[ index, dimensions_ingredients + fact[0] ] = fact[1]

        labels[index] = recipe["healthLabels"]
        index+=1
print("File "+filename+" was successfully read")
#names = [label for label, recipe in recipes.items()]

class IdssFood:
    # Carlos: Define the clusters, just a list
    CLUSTERS = {
        "starters": [],
        "mains": [],
        "desserts": []
    }
    NAMES = names
    RECIPES = recipes
    LABELS = labels
    X_ = X

    def __init__(self):
        self.liked = []
        self.disliked = []

        self.labels = None  # AND
        self.no_labels = []  # AND

        self.ingredients = None  # OR
        self.no_ingredients = []  # AND

    def set_liked(self, liked=[], disliked=[]):
        self.liked = self.label2index(self,liked)
        self.disliked = self.label2index(self,disliked)

    def set_labels(self, labels=None, no_labels=()):
        self.labels = labels
        self.no_labels = no_labels

    def set_ingredients(self, ingredients=None, no_ingredients=()):
        self.ingredients = ingredients
        self.no_ingredients = no_ingredients

    def get_closest_to_liked(self, cluster=None):
        # Marti: Use k-nearest neighbours having into account the valid clusters
        closest =list(range(20))
        return closest

    def filter(self, closest):
        # Nariman: Use labels, no_labels and ingredients and no_ingredients to filter out
        remove_list = []
        for index in closest:
            #labels filtering
            for label in  self.LABELS[index]:
                if label in self.no_labels:
                    remove_list += [index]
            for label in  self.labels:
                if label not in self.LABELS[index]:
                    remove_list += [index]
            #ingredients filtering            ###this is based on having X but it can easily changed to get from json file directly if needed.
            for ingredient in  self.no_ingredients:
                if self.X_[index,ingredient]:
                    remove_list += [index]
            for ingredient in  self.ingredients:
                if self.X_[index,ingredient]==0:
                    remove_list += [index]
        for item in set(remove_list):
            closest.remove(item)
        for item in self.disliked:
            if item in closest:
                closest.remove(item)
        return closest

    def rank(self, closest,utility = 0): # utility=0:OWA, utility=1:meanstd
        # Nariman: Use disliked dishes to rank
        #first approach: utility function => mean-std
        L = len(closest)
        X_ingredients = self.X_[:,0:dimensions_ingredients]

        if utility==1:
            dist_list = []
            for i in range(L):
                temp = []
                for dish in self.disliked:
                    temp.append( distance_(X_ingredients[int(dish),:],X_ingredients[int(closest[i]),:]) )
                dist_list.append(temp)
            rank_list = []
            for index in range(len(dist_list)):
                rank = 1*np.mean(dist_list[index])-1*np.std(dist_list[index])
                rank_list.append([rank, index])
            rank_list.sort(reverse=True)
            closest_temp = []
            for i in range(0,len(closest)):
                closest_temp.append(closest[int(rank_list[i][1])])
            closest = closest_temp
            return closest
        #second approach: utility function => OWA :  more weight for closer distances to dislike
        dist_list = []
        for i in range(L):
            temp = []
            for dish in self.disliked:
                temp.append( distance_(X_ingredients[int(dish),:],X_ingredients[int(closest[i]),:]) )
            dist_list.append([temp,i])
        L = len(closest)
        OWAweights = []
        if L != 1:
            for w in range(len(self.disliked)):
                OWAweights.append(2*(w+1)/((len(self.disliked))*((len(self.disliked))+1)))
        else: OWAweights = [1]
        rank_list_ = []
        dist_list.sort(reverse=True)
        for index in range(L):
            rank_ = sum(np.multiply(dist_list[index][0], OWAweights))
            rank_list_.append([rank_,dist_list[index][1]])
        rank_list_.sort(reverse=True)
        closest_temp_ = []
        for i in range(L):
            closest_temp_.append(closest[int(rank_list_[i][1])])
        closest = closest_temp_
        return closest

    def index2label(self, list):
        return [IdssFood.NAMES[index] for index in list]

    def label2index(self, list):
        return [IdssFood.NAMES.index(label) for label in list]

    def get_recommendation(self, clusters=("starters", "mains", "desserts")):
        recommendations = {"starters": None, "mains": None, "desserts": None}
        for cluster in clusters:
            recipes = self.get_closest_to_liked(cluster)
            recipes = self.filter(recipes)
            recipes = self.rank(recipes)
            recommendations[cluster] = self.index2label(recipes)
        return recommendations

TEST = IdssFood
TEST.set_labels(TEST, labels=['Gluten-Free'], no_labels=[])
TEST.set_ingredients(TEST, ingredients=[], no_ingredients=[])
TEST.set_liked(TEST, disliked=['Persimmon Tart','Cumin Lime Black Bean Quinoa Salad'])
x=TEST.get_closest_to_liked(TEST)
x=TEST.filter(TEST,x)
print(x)
x=TEST.rank(TEST,x)
y=TEST.index2label(TEST, x)
print(x,'\n',y)


import torch
from . import initialization as init

class SegmentationModel(torch.nn.Module):

    mean = torch.tensor([0.485, 0.456, 0.406])
    std = torch.tensor([0.229, 0.224, 0.225])

    def initialize(self):
        init.initialize_decoder(self.decoder)
        init.initialize_head(self.segmentation_head)
        if self.classification_head is not None:
            init.initialize_head(self.classification_head)

    def normalize(self, x):

        x = x / 255.

        r = (x[0][0] - self.mean[0]) / self.std[0]
        g = (x[0][1] - self.mean[1]) / self.std[1]
        b = (x[0][2] - self.mean[2]) / self.std[2]

        x_norm = torch.unsqueeze(torch.stack((r,g,b), 0), 0)

        return x_norm

    def forward(self, x):
        """Sequentially pass `x` trough model`s encoder, decoder and heads"""
        x = self.normalize(x)
        features = self.encoder(x)
        decoder_output = self.decoder(*features)

        masks = self.segmentation_head(decoder_output)
        masks = torch.unsqueeze(torch.flatten(torch.squeeze(masks)), -1)
        # masks = masks * 255.

        if self.classification_head is not None:
            labels = self.classification_head(features[-1])
            return masks, labels

        return masks

    def predict(self, x):
        """Inference method. Switch model to `eval` mode, call `.forward(x)` with `torch.no_grad()`

        Args:
            x: 4D torch tensor with shape (batch_size, channels, height, width)

        Return:
            prediction: 4D torch tensor with shape (batch_size, classes, height, width)

        """
        if self.training:
            self.eval()

        with torch.no_grad():
            x = self.forward(x)

        return x


import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import numpy as np
import cv2
import glob
import time
from sklearn.svm import LinearSVC
from sklearn.preprocessing import StandardScaler
from extract_feature import *
# NOTE: the next import is only valid for scikit-learn version <= 0.17
# for scikit-learn >= 0.18 use:
# from sklearn.model_selection import train_test_split
from sklearn.cross_validation import train_test_split
import pickle

# Read in cars and notcars
cars = glob.glob('./train_data/vehicles/*/*.png'); train_data_tpye = 'png'; notcars = glob.glob('./train_data/non-vehicles/*/*.png')
#cars = glob.glob('./hog_test_imgs/vehicles_smallset/*/*.jpeg'); train_data_tpye = 'jpeg'; #notcars = glob.glob('./hog_test_imgs/non-vehicles_smallset/*/*.jpeg')
sample_size = None
cars = cars[0:sample_size]
notcars = notcars[0:sample_size]

color_space = 'YCrCb' # Can be RGB, HSV, LUV, HLS, YUV, YCrCb
orient = 9  # HOG orientations
pix_per_cell = 8 # HOG pixels per cell
cell_per_block = 2 # HOG cells per block
hog_channel = "ALL" # Can be 0, 1, 2, or "ALL"
spatial_size = (32, 32) # Spatial binning dimensions
hist_bins = 32    # Number of histogram bins
spatial_feat = True # Spatial features on or off
hist_feat = True # Histogram features on or off
hog_feat = True # HOG features on or off
y_start_stop = [None, None] # Min and max in y to search in slide_window()

t = time.time()
print("start extract car_features")
car_features = extract_features(cars, train_data_tpye, color_space=color_space,
                        spatial_size=spatial_size, hist_bins=hist_bins,
                        orient=orient, pix_per_cell=pix_per_cell,
                        cell_per_block=cell_per_block,
                        hog_channel=hog_channel, spatial_feat=spatial_feat,
                        hist_feat=hist_feat, hog_feat=hog_feat)
print("start extract notcar_features")
notcar_features = extract_features(notcars, train_data_tpye, color_space=color_space,
                        spatial_size=spatial_size, hist_bins=hist_bins,
                        orient=orient, pix_per_cell=pix_per_cell,
                        cell_per_block=cell_per_block,
                        hog_channel=hog_channel, spatial_feat=spatial_feat,
                        hist_feat=hist_feat, hog_feat=hog_feat)

X = np.vstack((car_features, notcar_features)).astype(np.float64)

# Define the labels vector
y = np.hstack((np.ones(len(car_features)), np.zeros(len(notcar_features))))

# Split up data into randomized training and test sets
rand_state = np.random.randint(0, 100)
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=rand_state)

t2 = time.time()
print(round(t2-t, 2), 'Seconds to extract features SVC...')

print("X_train shape: {} \X_test shape: {}".format(X_train.shape, X_test.shape))

# Fit a per-column scaler
X_scaler = StandardScaler().fit(X_train)
# Apply the scaler to X
X_train = X_scaler.transform(X_train)
X_test = X_scaler.transform(X_test)

print('Using:',orient,'orientations',pix_per_cell,
    'pixels per cell and', cell_per_block,'cells per block')
print('Feature vector length:', len(X_train[0]))
# Use a linear SVC
svc = LinearSVC()
t=time.time()
svc.fit(X_train, y_train)
t2 = time.time()
print(round(t2-t, 2), 'Seconds to train SVC...')

# Check the score of the SVC
print('Test Accuracy of SVC = ', round(svc.score(X_test, y_test), 4))

# pickle SVC
pickle_file = 'svc_acc_%f.p'%round(svc.score(X_test, y_test), 4)
try:
    with open(pickle_file, 'wb') as pfile:
        pickle.dump(
                {
                    'svc': svc,
                    'scaler': X_scaler,
                    'color_space': color_space,
                    'orient': orient,
                    'pix_per_cell': pix_per_cell,
                    'cell_per_block': cell_per_block,
                    'spatial_size': spatial_size,
                    'hist_bins': hist_bins,
                    'spatial_feat': spatial_feat,
                    'hist_feat': hist_feat,
                },
                pfile, pickle.HIGHEST_PROTOCOL)
except Exception as e:
    print('Unable to save data to', pickle_file, ':', e)
    raise


import numpy as np
import cv2
import sys
import argparse
from skimage import morphology, measure
from matplotlib import pyplot as plt

class Transformations():
	"""docstring for Transformations"""
	def __init__(self):
		pass

	""" interval func converter : 0-255 -> 120->180 """
	@staticmethod
	def toBinary(image):
		gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
		_, binaryImage = cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV*cv2.THRESH_OTSU)

		return binaryImage
	@staticmethod
	def getBackground(image):
		kernel = np.ones((3,3),np.uint8)
		m_ = cv2.morphologyEx(image,cv2.MORPH_OPEN,kernel, iterations = 2)

		# sure background area
		backgroundImg = cv2.dilate(m_,kernel,iterations=3)

		return backgroundImg

	@staticmethod
	def getForeground(image,size_dist):
		dist_transform = cv2.distanceTransform(image,cv2.DIST_L2,size_dist)
		ret, foregroundImage = cv2.threshold(dist_transform,0.2*dist_transform.max(),255,0)

		return foregroundImage

def main():

	parser = argparse.ArgumentParser(description='Parser')

	parser.add_argument('--input', help='input image')
	#parser.add_argument('--output', help='output image')

	args = parser.parse_args()

	# checking
	assert (args.input.endswith('.png') or args.input.endswith('.jpg') )
	#assert (args.output.endswith('.png') or args.output.endswith('.jpg') )

	# reading input image
	img = cv2.imread(args.input)

	gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
	ret, thresh = cv2.threshold(gray,0,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)

	labels = morphology.label(thresh)
	properties = measure.regionprops(labels)

	# noise removal
	kernel = np.ones((3,3),np.uint8)
	opening = cv2.morphologyEx(thresh,cv2.MORPH_OPEN,kernel, iterations = 2)

	# sure background area
	sure_bg = cv2.dilate(opening,kernel,iterations=3)

	# Finding sure foreground area
	dist_transform = cv2.distanceTransform(opening,cv2.DIST_L2,5)
	ret, sure_fg = cv2.threshold(dist_transform,0.2*dist_transform.max(),255,0)

	# Finding unknown region
	sure_fg = np.uint8(sure_fg)
	unknown = cv2.subtract(sure_bg,sure_fg)

	# Marker labelling
	ret, markers = cv2.connectedComponents(sure_fg)

	# Add one to all labels so that sure background is not 0, but 1
	markers = markers+1

	# Now, mark the region of unknown with zero
	markers[unknown==255] = 0

	markers = cv2.watershed(img,markers)
	img[markers == -1] = [0,0,255]

	coin_areas = [float(prop.area) for prop in properties]
	coin_centroids = [prop.centroid for prop in properties]
	coin_perimeter = [prop.perimeter for prop in properties]

	font = cv2.FONT_HERSHEY_SIMPLEX
	for txt in range(len(coin_areas)):
		if txt == 0:
			continue
		fontScale = (( float(coin_areas[txt])-np.min(coin_areas))/(np.max(coin_areas)-np.min(coin_areas)))+0.21
		print ("região : {}       perimetro : {}     area: {}".format(txt,float(coin_perimeter[txt]),float(coin_areas[txt])))

		cv2.putText(img,str(txt),(int(round(coin_centroids[txt][1])),int(round(coin_centroids[txt][0]))), font, fontScale,(255,255,255),1,cv2.LINE_AA)

	#cv2.imshow('input', img)
	#cv2.waitKey(0)

if __name__ == '__main__':
	main()


from dateutil import rrule
from datetime import datetime, timedelta

import workout          # Entity
import term_io as io    # Interface class
import xlsx_io as file  # Storage

# Initalize which exercises to track, their current weights and planned increments
def initWorkout(wo):
    # Give the new program a name
    wo.workoutName = io.reqInput("Enter program name: ")
    setExercises(wo)
    setWeights(wo)
    setIncrements(wo)

def modifyWorkout(wo):
    choice = ''
    while(choice != '4'):
        io.printStatus(wo)
        options = {}
        options['1'] = "Set exercises"
        if(len(wo.ex) > 0):
            options['2'] = "Set weights"
            options['3'] = "Set increments"
        options['4'] = "Continue"
        choice = io.chooseAction(options)

        if(choice == '1'):
            setExercises(wo)
        elif(choice == '2'):
            setWeights(wo)
        elif(choice == '3'):
            setIncrements(wo)

# Name exercises that you want to track
def setExercises(wo):
    choice = ''
    while(choice != '3'):
        if(len(wo.ex) > 0):
            io.ssprint("Exercises:")
            io.printList(wo.ex)
        else:
            io.ssprint("No exercises set")
        options = {}
        options['1'] = "Add an exercise"
        options['2'] = "Remove an exercise"
        options['3'] = "Done"
        choice = io.chooseAction(options)

        if(choice == '1'):
            ex = io.reqInput("Exercise name: ")
            if(ex != ""):               # If anything was returned
               wo.ex.append(ex)
        elif(choice == '2'):
            i = 0
            exerciseOptions = {}
            for ex in wo.ex:
                i = i+1
                exerciseOptions[str(i)] = ex
            io.ssprint("Enter the number of the exercise you want to remove:")
            exerciseChoice = io.chooseAction(exerciseOptions)

            removee = exerciseOptions[exerciseChoice]
            if(io.confirmRemoval(removee)):
               wo.ex.remove(removee)

# Set the current weight of each exercise
def setWeights(wo):
    if(len(wo.ex) > 0):
        choice = ''
        while(choice != '-1'):       #Arbritrary number. Loop should exit on return command
            if(len(wo.weights) > 0):
                io.ssprint("Weights:")
                io.printDict(wo.weights)
            else:
                io.ssprint("No weights set")

            io.ssprint("Choose an exercise to set your working weight:")
            options = {}
            options['1'] = "All"
            i = 2                    # Start i as next available option
            for ex in wo.ex:
                options[str(i)] = ex # Add option
                i = i+1              # Increment i for next option

            doneOpNum = i            # The done option comes after the dynamic middle, so make sure we remember it's number
            options[str(doneOpNum)] = "Done"

            choice = io.chooseAction(options)

            if(choice == '1'):
                io.ssprint("Enter your current weight for each exercise: ")
                wo.weights = io.getDict(wo.ex)
            elif(choice == str(doneOpNum)):
                return
            else:
                exercise = options[choice]                  # Get the name of the exercise chosen
                newWeight = io.reqInput(exercise + ": ")    # Request the new weight
                wo.weights[exercise] = newWeight            # Set the new weight

# Set the planned increment of each exercise
def setIncrements(wo):
    if(len(wo.ex) > 0):
        choice = ''
        while(choice != '-1'):  #Arbritrary number. Loop should exit on return command
            if(len(wo.increments) > 0):
                io.ssprint("Increments:")
                io.printDict(wo.increments)
            else:
                io.ssprint("No increments set")
            io.ssprint("Choose an exercise to set your planned weight increments per workout:")
            options = {}
            options['1'] = "All"
            i = 2                    # Start i as next available option
            for ex in wo.ex:
                options[str(i)] = ex # Add option
                i = i+1              # Increment i for next option

            doneOpNum = i            # The done option comes after the dynamic middle, so make sure we remember it's number
            options[str(doneOpNum)] = "Done"

            choice = io.chooseAction(options)

            if(choice == '1'):
                io.ssprint("Enter your current per workout weight increment for each exercise: ")
                wo.increments = io.getDict(wo.ex)
            elif(choice == str(doneOpNum)):
                return
            else:
                exercise = options[choice]                  # Get the name of the exercise chosen
                newWeight = io.reqInput(exercise + ": ")    # Request the new increment
                wo.weights[exercise] = newWeight            # Set the new increment

def export_xlsx(wo):
    f = file.xlsx_io(wo.workoutName) # Create an instance of our file class

    columns = wo.ex[:]               # Copy wo.ex by value
    columns.insert(0, "Day")
    columns.insert(0, "Date")
    f.setColumns(columns)

    weights = wo.weights
    now = datetime.now()
    for d in rrule.rrule(rrule.DAILY, dtstart=now, until=now+timedelta(days=90)):   # Iterate through 30 days
        if(d.strftime('%a') in ["Mon", "Wed", "Fri"]):                              # If the day is in the list
            row = {}
            row["Day"] = d.strftime('%a')
            row["Date"] = d.strftime('%b %d')
            for ex in wo.ex:
                row[ex] = float(weights[ex])
                weights[ex] = float(weights[ex]) + float(wo.increments[ex])

            f.writeRow(row)

    f.generateXLSX()                # Write the file (Overwrites existing)

def main():
    wo = workout.Workout()
    # TODO: Attempt to load workout
    if(len(wo.ex) <= 0):
        initWorkout(wo)
    modifyWorkout(wo)

    export_xlsx(wo)

main()


import logging
import termios
import codecs
import struct
import fcntl
import sys
import os
import io

__all__ = ('ConsoleShell', 'TerminalShell')

class ConsoleShell():
    """ A shell appropriate for use with the TelnetClient protocol
        argument 'shell'. stream_out recieves bare uncode values.
        callbacks for window size, xdisploc, and speed return
        static default values.
    """
    def __init__(self, client, log=logging, stream_out=None):
        #: TelnetClient instance associated with console
        self.client = client
        self.log = log
        self.stream_out = stream_out or io.StringIO()

        #: default encoding 'errors' argument
        self.encoding_errors = 'replace'

        #: codecs.IncrementalDecoder for current CHARSET
        self.decoder = None

    def write(self, string=u''):
        """ Write string to console output stream.
        """
        self.stream_out.write(string)

    @property
    def terminal_type(self):
        """ Always returns 'unknown'.
        """
        return 'unknown'

    @property
    def xdisploc(self):
        """ Always returns ''
        """
        return ''

    @property
    def terminal_speed(self):
        """ Returns (38400, 38400).
        """
        return 38400, 38400

    @property
    def terminal_width(self):
        """ Returns 80.
        """
        return 80

    @property
    def terminal_height(self):
        """ Returns 24.
        """
        return 24

    def decode(self, input, final=False):
        """ Decode input string using preferred encoding.
        """
        enc = self.client.encoding(incoming=True)
        if (self.decoder is None or enc != self.decoder._encoding):
                self.decoder = codecs.getincrementaldecoder(enc)(
                    errors=self.encoding_errors)
                self.decoder._encoding = enc
        return self.decoder.decode(input, final)

    def feed_byte(self, byte):
        """ Receive byte from telnet server, decodes to client encoding
            and writes to output stream as unicode.
        """
        ucs = self.decode(byte)
        if ucs:
            self.write(ucs)

    def encode(self, buf, errors=None):
        """ Encode byte buffer ``buf`` using client-preferred encoding.

            If ``outbinary`` is not negotiated, ucs must be made of strictly
            7-bit ascii characters (valued less than 128), and any values
            outside of this range will be replaced with a python-like
            representation.
        """
        errors = errors if errors is not None else self.encoding_errors
        return bytes(buf, self.client.encoding(outgoing=True), errors)

    @property
    def will_echo(self):
        """ Returns wether to expect the server to display our input; if
            False, it is our own responsibility to write a copy to screen.
        """
        from .telopt import ECHO
        return self.client.stream.remote_option.enabled(ECHO)

    def __str__(self):
        """ Returns string describing state of stream encoding.
        """
        enc_in = self.client.encoding(incoming=True)
        enc_out = self.client.encoding(outgoing=True)
        return (enc_in if enc_in == enc_out else
                '{} in, {} out'.format(enc_in, enc_out))

class TerminalShell(ConsoleShell):
    """ A shell appropriate for use with the TelnetClient protocol
        argument 'shell' and a stream_out connected to a tty, by
        default, output stream is sys.stdout, and callbacks registered
        for send terminal window size, and speed query the
        connected terminal and environment values.
    """
    def __init__(self, client, log=logging, stream_out=None):
        self.stream_out = stream_out or sys.__stdout__
        super().__init__(client, log, self.stream_out)

    def write(self, string=u''):
        """ Write string to console output stream.
        """
        # XXX probably an out-of-order timing condition, here
        try:
            if string:
                self.stream_out.write(string)
        except BlockingIOError:
            # output is blocking, defer *write* for another 50ms,
            self.client._loop.call_later(0.05, self.write, string)
        try:
            self.stream_out.flush()
        except BlockingIOError:
            # output cannot flush, defer *flush* for another 50ms,
            self.client._loop.call_later(0.05, self.write)

    @property
    def terminal_type(self):
        """ The terminfo(5) terminal type name: the value found in the
        ``TERM`` environment variable, if exists; otherwise 'unknown'.
        """
        return (os.environ.get('TERM', '') or super().terminal_type)

    @property
    def xdisploc(self):
        """ The XDISPLAY value: the value found as os environ key ``DISPLAY``.
        """
        return (os.environ.get('DISPLAY', '') or super().xdisploc)

    @property
    def terminal_speed(self):
        """ The terminal speed is a legacy application of determining
        the bandwidth (bits per second) of the connecting terminal, esp.
        when that terminal is serial attached. This method retuns a tuple
        of receive and send speed (rx, tx).

        If the connecting output stream's terminal speed cannot be
        determined, a default value of (38400, 38400) is returned.
        """
        if (hasattr(self.stream_out, 'fileno')
                and os.isatty(self.stream_out.fileno())):
            return self._query_term_speed(self.stream_out)
        return super().terminal_speed

    @property
    def terminal_width(self):
        """ The terminal width in printable character columns as integer:
        if the stream_out file descriptor is a terminal, the terminal is
        queried for its size; otherwise the value found in the ``COLUMNS``
        environment variable is returned.
        """
        try:
            if (hasattr(self.stream_out, 'fileno')
                    and os.isatty(self.stream_out.fileno())):
                rows, cols, xpixels, ypixels = (
                    self._query_term_winsize(self.stream_out))
                return cols
        except io.UnsupportedOperation:
            self.log.warning('{}: stream_out {} does not support fileno()'.format(
                self.__class__.__name__, self.stream_out))
        try:
            cols = int(os.environ.get('COLUMNS', str(super().terminal_width)))
        except ValueError:
            cols = super().terminal_width
        return cols

    @property
    def terminal_height(self):
        """ The terminal height printable character columns as integer:
        if the stream_out file descriptor is a terminal, the terminal is
        queried for its size; otherwise the value found in the ``LINES``
        environment variable is returned.  """
        try:
            if (hasattr(self.stream_out, 'fileno')
                    and os.isatty(self.stream_out.fileno())):
                rows, cols, xpixels, ypixels = (
                    self._query_term_winsize(self.stream_out))
                return rows
        except io.UnsupportedOperation:
            self.log.warning('{}: stream_out {} does not support fileno()'.format(
                self.__class__.__name__, self.stream_out))
        try:
            rows = int(os.environ.get('LINES', str(super().terminal_height)))
        except ValueError:
            rows = super().terminal_height
        return rows

    @staticmethod
    def _query_term_speed(tty_fd):
        """ .. function:: _query_term_speed(int) -> type((int, int,))

            Returns the input and output speed of the terminal specified
            by argument ``tty_fd`` as two integers: (rx, tx).
        """
        iflag, oflag, cflag, lflag, ispeed, ospeed, cc = (
            termios.tcgetattr(tty_fd))
        return ispeed, ospeed

    @staticmethod
    def _query_term_winsize(tty_fd):
        """ .. function:: _query_term_winsize(int) -> type((int, int, int, int,))

            Returns the value of the `winsize' struct returned by ioctl of
            TIOCGWINSZ for the terminal specified by argument ``tty_fd``
            as its natural 4 unsigned short integers:
                (ws_rows, ws_cols, ws_xpixels, ws_ypixels).
        """
        #  struct winsize {
        #      unsigned short  ws_row;         /* rows, in characters */
        #      unsigned short  ws_col;         /* columns, in characters */
        #      unsigned short  ws_xpixel;      /* horizontal size, pixels */
        #      unsigned short  ws_ypixel;      /* vertical size, pixels */
        #  };
        val = fcntl.ioctl(tty_fd, termios.TIOCGWINSZ, b'\x00' * 8)
        return struct.unpack('hhhh', val)

#    def can_write(self, ucs):
#        """ Returns True if transport can receive ``ucs`` as a single-cell,
#            carriage-forwarding character, such as 'x' or ' '. Values outside
#            of 7-bit NVT ASCII range may only be written if server option
#            ``outbinary`` is True.
#
#            Otherwise, a return value of False indicates that a write of this
#            unicode character would be an encoding error on the transport
#            (may crash or corrupt client screen).
#        """
#        return ord(ucs) > 31 and (ord(ucs) < 127 or self.client.outbinary)
#