import hlt
import logging
import micro
from objective import Objective

# GAME START
# Here we define the bot's name as Settler and initialize the game, including communication with the Halite engine.
game = hlt.Game("Version2")
# Then we print our start message to the logs
logging.info("Starting my Version2 bot!")
'''
def attack_docked_ships(ship, planet, own_ships, game_map):
    docked_ships = planet.all_docked_ships()
    for opponent_ship in docked_ships:
        navigate_command = ship.navigate(ship.closest_point_to(opponent_ship, min_distance=0), game_map, speed=int(hlt.constants.MAX_SPEED), entities=own_ships)

        #if you can't attack nearest ship just return None
        return navigate_command

    return None

def attack_ships(ship, entities_by_distance, own_ships, game_map):
    for ship_distance in sorted(entities_by_distance):
        nearest_ship = next((nearest_entity for nearest_entity in entities_by_distance[ship_distance] if isinstance(nearest_entity, hlt.entity.Ship)), None)

        if nearest_ship == None:
            continue
        if nearest_ship.owner==game_map.get_me():
            continue
        navigate_command = ship.navigate(ship.closest_point_to(nearest_ship, min_distance=0), game_map, speed=int(hlt.constants.MAX_SPEED), entities=own_ships)
        return navigate_command
    return None
'''

# objtype = ['attack','defend','dock_owned', 'dock_unowned']
def updateObjectives(game_map, opponent_ships):
    objs = []
    for p in planets:
        en_list = sorted(opponent_ships, key=lambda x: p.calculate_distance_between(x))
        for i in range(len(en_list)):
            if p.calculate_distance_between(en_list[i]) > 40 - p.radius:
                break

        if not p.is_owned() and p.remaining_resources > 0:
            objs.append(Objective(p, 'dock_unowned'))
            objs[-1].addEnShip(en_list[:i])
        elif p.owner == me:
            objs.append(Objective(p, 'defend'))
            objs[-1].addEnShip(en_list[:i])
            if not p.is_full() and p.remaining_resources > 0:
                objs.append(Objective(p,'dock_owned'))
                objs[-1].addEnShip(en_list[:i])
        elif p.owner != me and p.is_owned():
            objs.append(Objective(p, 'attack'))
            objs[-1].addEnShip(en_list[:i])

    return objs

def assignObjectives(objectives, my_ships):
    if len(objectives) == 0:
        return objectives
    for ship in my_ships:
        if (ship.docking_status == 1 or ship.docking_status ==2) and ship.planet.remaining_resources > 0:
            continue
        bestObj = objectives[0]
        bestScore = -100000
        for obj in objectives:
            score = obj.priority - ship.calculate_distance_between(obj.entity) * 2
            if score > bestScore:
                bestScore = score
                bestObj = obj
        bestObj.addMyShip(ship)
        logging.info('i am at ' + str(ship.x) + ' ' + str(ship.y) + ' my obj is to ' + str(bestObj.objtype) + ' at ' + str(bestObj.entity.x) + ' ' + str(bestObj.entity.y) + ' against ' + str(len(bestObj.en_ships)) + ' ships')
    return objectives

#flee_pos = [hlt.entity.Position(0,0), hlt.entity.Position(game_map.width-1,0),
#                    hlt.entity.Position(0,game_map.height-1),
#                    hlt.entity.Position(game_map.width-1,game_map.height-1)]
def flee(game_map):
    for ship in my_ships:
        navigate_command = None
        if (ship.docking_status != ship.DockingStatus.UNDOCKED):
            navigate_command = ship.undock()
        else:
            if (ship.y <= 5) and (ship.x > 5):
                flee_pos = hlt.entity.Position(0,3)
            elif (ship.x <= 5) and (ship.y < game_map.height - 5):
                flee_pos = hlt.entity.Position(3,game_map.height)
            elif (ship.y >= game_map.height - 5) and (ship.x < game_map.width - 5):
                flee_pos = hlt.entity.Position(game_map.width, game_map.height - 3)
            elif (ship.x >= game_map.width - 5) and (ship.y > 5):
                flee_pos = hlt.entity.Position(game_map.width - 3, 0)
            else:
                flee_pos = min( hlt.entity.Position(ship.x, 0),
                                hlt.entity.Position(0, ship.y),
                                hlt.entity.Position(ship.x, game_map.height),
                                hlt.entity.Position(game_map.width, ship.y),
                                key = lambda p: ship.calculate_distance_between(p))

            navigate_command = ship.navigate(flee_pos,
                                            game_map,
                                            speed=int(hlt.constants.MAX_SPEED),
                                            entities=own_ships_nav)

        if navigate_command is not None:
            command_queue.append(navigate_command)

    return command_queue

me = None
turn = 0

while True:
    # TURN START
    # Update the map for the new turn and get the latest version
    turn+=1
    logging.info(str(turn))
    game_map = game.update_map()
    me = game_map.get_me()
    # Here we define the set of commands to be sent to the Halite engine at the end of the turn

    own_ships_nav = [] #list of our ships by position just for navigation purposes
    command_queue = []
    planets = game_map.all_planets()
    opponent_ships = []
    my_ships = game_map.get_me().all_ships()

    for player in game_map.all_players():
        if player.id != game_map.get_me().id:
            opponent_ships += player.all_ships()

    if (0.95*len(my_ships) < 0.1*len(opponent_ships)):
        command_queue = flee(game_map)
        game.send_command_queue(command_queue)
        continue

    objectives = updateObjectives(game_map, opponent_ships)

    objectives = assignObjectives(objectives, my_ships)

    for objective in objectives:
        command_queue += micro.getMovesForObjective(objective, game_map, own_ships_nav, opponent_ships)

    '''
    # For every ship that I control
    for ship in game_map.get_me().all_ships():
        # If the ship is docked
        if ship.docking_status != ship.DockingStatus.UNDOCKED:
            # Skip this ship
            continue

        entities_by_distance = game_map.nearby_entities_by_distance(ship)
        nearest_planet = None

        if len(game_map.all_players()) == 2:
            navigate_command = attack_ships(ship, entities_by_distance, own_ships, game_map)

            if (navigate_command != None):
                command_queue.append(navigate_command)
                continue

        for distance in sorted(entities_by_distance):
            nearest_planet = next((nearest_entity for nearest_entity in entities_by_distance[distance] if isinstance(nearest_entity, hlt.entity.Planet)), None)

            if nearest_planet == None:
                continue

            navigate_command = None
            #if nearest planet is opponents, attack the docked ships there
            if nearest_planet.is_owned() and nearest_planet.owner!=game_map.get_me():
                navigate_command = attack_docked_ships(ship, nearest_planet, own_ships, game_map)

            # If we can dock at a planet, let's (try to) dock. If two ships try to dock at once, neither will be able to.
            elif ship.can_dock(nearest_planet) and not (nearest_planet.is_full()):
                # We add the command by appending it to the command_queue
                command_queue.append(ship.dock(nearest_planet))

            elif nearest_planet.owner==game_map.get_me():
                #if we can't doc and it is our planet, just continue
                continue

            else:
                navigate_command = ship.navigate(
                    ship.closest_point_to(nearest_planet),
                    game_map,
                    speed=int(hlt.constants.MAX_SPEED), entities=own_ships_nav)

            if (navigate_command != None):
                command_queue.append(navigate_command)
            break

    '''
    # Send our set of commands to the Halite engine for this turn
    game.send_command_queue(command_queue)
    # TURN END

# GAME END


import copy
import itertools
from typing import List, Dict, Tuple

import torch

from modules.block import Block
from modules.cell import Cell
from modules.model import Model
from search_space import Operation

MAX_NUMBER_OF_BLOCKS_PER_CELL = 0

def _prediction_accuracy(inputs, targets) -> float:
    n = len(inputs)
    assert n == len(targets)

    mse = torch.nn.MSELoss()

    la_tensor = torch.FloatTensor(inputs)
    lb_tensor = torch.FloatTensor(targets)

    return mse(la_tensor, lb_tensor).mean().item()

def _get_input_word(input_num: int) -> str:
    """
    convert this input number into the input word in the dictionary
    :param input_num: the input num
    :return: the input word
    """
    return 'input-'.format(input_num)

def _get_op_word(op: Operation) -> str:
    """
    get the operation identifier in the dictionary
    :param op: the operation
    :return: the operation identifier
    """
    return 'op-{}'.format(op.__str__())

def _get_vocabulary_size() -> int:
    """
    get the vocabulary size
    :return: the size of the vocabulary for the embeddings
    """
    return len(_get_vocabulary())

def _get_vocabulary() -> Dict[str, int]:
    """
    get the full vocabulary
    :return: the vocabulary
    """
    vocabulary = [_get_input_word(i) for i in range(MAX_NUMBER_OF_BLOCKS_PER_CELL + 2)]
    vocabulary += [_get_op_word(op) for op in list(Operation)]

    return {w: i for i, w in enumerate(vocabulary)}

def _get_all_possible_blocks(current_cell_size: int) -> List[Block]:
    """
    get's all possible blocks
    :param current_cell_size: the current cell sizes from (1 to ...), it is necessary, to calculate all possible inputs
    :return: all possible blocks
    """
    possible_first_input_operations = list(Operation)
    possible_second_input_operations = possible_first_input_operations.copy()
    possible_first_input = list(range(current_cell_size))
    possible_second_input = possible_first_input.copy()

    combinations = itertools.product(*[possible_first_input, possible_first_input_operations, possible_second_input,
                                       possible_second_input_operations])

    blocks = []

    for c in combinations:
        b = Block(current_cell_size)
        b.first_input_block = c[0]
        b.first_input_op = c[1]

        b.second_input_block = c[2]
        b.second_input_op = c[3]

        blocks.append(b)

    return blocks

def _expand_cell(cell: Cell) -> List[Cell]:
    """
    Expand the cell by a single block
    :param cell: the cell with block-size b
    :return: all descendant cells with block-size b + 1
    """
    expanded_cells = []
    possible_blocks = _get_all_possible_blocks(len(cell.blocks) + 2)

    for block in possible_blocks:
        tmp_cell = copy.deepcopy(cell)
        tmp_cell.blocks.append(block)
        expanded_cells.append(tmp_cell)

    return expanded_cells

def _create_initial_cells() -> List[Cell]:
    """
    create all initial cells with size b = 1
    :return: the initially created cells
    """
    possible_blocks = _get_all_possible_blocks(2)
    cells = []

    for block in possible_blocks:
        c = Cell()
        c.blocks.append(block)

        cells.append(c)

    return cells

def _expand_cells(cells: List[Cell]) -> List[Cell]:
    """
    expand the cells by one more block
    :param cells: the cells that should be expanded
    :return: the expanded cells
    """
    if len(cells) == 0:
        return _create_initial_cells()

    return_cells = []

    for cell in cells:
        expanded_cells = _expand_cell(cell)
        return_cells += expanded_cells

    return return_cells

def _get_normal_and_reduction_cells(cells: List[Cell]) -> List[Tuple[Cell, Cell]]:
    """
    get a pairwise combination of normal and reduction cells
    :param cells: the cells
    :return: normal and reduction cell combination
    """
    return [(cell, cell) for cell in cells]

def _to_architecture_tensor(model: Model) -> List[int]:
    """

    :return:
    """
    normal_cell_tokens = model.normal_cell.__str__().split(' ')
    reduction_cell_tokens = model.normal_cell.__str__().split(' ')

    architecture_tokens = normal_cell_tokens + reduction_cell_tokens

    tokens = [int(t) for t in architecture_tokens]

    expected_block_count = 2 * MAX_NUMBER_OF_BLOCKS_PER_CELL * 4

    # apply padding
    if len(tokens) < expected_block_count:
        tokens += [-1] * (expected_block_count - len(tokens))

    return tokens

class PNASDataset(torch.utils.data.Dataset):
    def __init__(self, inputs, targets=None, train=True):
        super(PNASDataset, self).__init__()
        if targets is not None:
            assert len(inputs) == len(targets)
        self.inputs = copy.deepcopy(inputs)
        self.targets = copy.deepcopy(targets)
        self.train = train

    def __getitem__(self, index):
        surrogate_input = self.inputs[index]
        surrogate_target = None
        if self.targets is not None:
            surrogate_target = [self.targets[index]]
        if self.train:
            sample = {
                'surrogate_input': torch.LongTensor(surrogate_input),
                'surrogate_target': torch.FloatTensor(surrogate_target)
            }
        else:
            sample = {
                'surrogate_input': torch.LongTensor(surrogate_input)
            }
            if surrogate_target is not None:
                sample['surrogate_target'] = torch.FloatTensor(surrogate_target)
        return sample

    def __len__(self):
        return len(self.inputs)


from bin.common.AppConfigurations import mongo_host, mongo_port, db_name, cart_collection
from bin.core.utilities.mongo_utility import MongoUtility
from datetime import date, datetime

class CartFormation(object):
    def __init__(self):
        self.mongo_client = MongoUtility(_mongo_port=mongo_port, _mongo_host=mongo_host)

    def list_cart_item_from_collection(self):
        try:
            print("Fetching endpoint details from mongo")
            mongo_content = self.mongo_client.find_all(database_name=db_name,
                                                       collection_name=cart_collection)
            response = list()
            for content in mongo_content:
                del content["_id"]
                response.append(content)
            return response
        except Exception as e:
            import traceback
            print("EROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")

    def add_item_into_collection(self, input_json):
        try:
            if input_json.get("user_name") == "admin":
                mongo_content = self.mongo_client.insert_one(json_data=input_json, database_name=db_name,
                                                             collection_name=cart_collection)
                print("mongo_content : ", mongo_content)
                return True
            else:
                return False
        except Exception as e:
            import traceback
            print("EROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")

    def add_multiple_item_into_collection(self, input_json):
        try:
            print("Type InputJson : ", type(input_json))
            flag = False
            for data in input_json:
                print("data : ", data)
                if data.get("user_name", "") == "admin":
                    flag = True
                    if not data.get("id", ""):
                        data["timestamp"] = str(datetime.utcnow()).split('.')[0]
                        data["id"] = self.mongo_client.generating_ramdon_id(cart_collection)
                else:
                    flag = False
                    break
            print("FLAG : ", flag)
            print("INPUT JSON MULTI : ", input_json)
            if flag:
                mongo_content = self.mongo_client.insert_many(json_data=input_json, database_name=db_name,
                                                          collection_name=cart_collection)
                return True
            else:
                return False

        except Exception as e:
            import traceback
            print("ERROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")

    def update_cart_data(self, input_json):
        try:
            print("Type InputJson : ", type(input_json))
            input_json["timestamp"] = str(datetime.utcnow()).split('.')[0]
            condition = dict(id=input_json.get("id", ""))
            mongo_content = self.mongo_client.update_one(condition=condition, json_data=input_json,
                                                         _database_name=db_name,
                                                         collection_name=cart_collection)
            return True
        except Exception as e:
            import traceback
            print("EROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")

    def remove_cart(self, input_json):
        try:
            print("Type InputJson : ", type(input_json))
            # condition = dict(id=input_json.get("id", ""))
            mongo_content = self.mongo_client.remove(json_data=input_json,
                                                     _database_name=db_name,
                                                     collection_name=cart_collection)
            print("mongo_content : ", mongo_content)
            # flask.jsonify(message="success", insertedIds=todo_many.inserted_ids)
            response = list()
            print("REMOVE CART mongoCOntent : ", mongo_content)
            return mongo_content
        except Exception as e:
            import traceback
            print("EROR :", traceback.print_exc())
            print("Failure to fetch cart collection --> {}".format(str(e)))
            raise Exception("Error when fetching cart collection!")


import csv
import numpy as np
import itertools
import operator
import json
from multiprocessing import Process
import datetime

with open('/Users/nishant/Downloads/mea_ml_train.csv', 'r') as f:
    reader = csv.reader(f)
    mea_fml = list(reader)

## Can add as many parameters as you want to
param_1 = []
param_2 = []
param_3 = []
param_4 = []
# param_5 = []
# param_6 = []
# param_7 = []
# param_8 = []
# param_9 = []
# param_10 = []
target = []

for each in mea_fml[1:]:
    param_1.append(each[1])
    param_2.append(each[2])
    param_3.append(each[3])
    param_4.append(each[4])
    # param_5.append(each[5])
    # param_6.append(each[6])
    # param_7.append(each[7])
    # param_8.append(each[8])
    # param_9.append(each[9])
    # param_10.append(each[10])
    target.append(each[0])

param_list = [param_1, param_2, param_3, param_4]

comparator_dict = {}
measurements_match = {}

for i in range(3, 5):
    comb_list = list(itertools.combinations(param_list, i))
    for each in comb_list:
        measurements_match_list = []
        key = ""
        list_1 = []
        index_list = []
        for element in each:
            index_list.append(param_list.index(element))
            list_1.append(element)
        list_1.append(target)
        length = len(list_1)

        final_arr = np.array(list_1)
        final_arr_trans = np.transpose(final_arr)
        counter = 0
        counter_not = 0
        index_list_new = []

        for index, each in enumerate(final_arr_trans):
            for element in each:
                if float(element) < 10 or float(element) > 60:
                    index_list_new.append(index)
        final_arr_trans_clean = np.delete(final_arr_trans, index_list_new, 0)
        # print(final_arr_trans_clean)

        ## Clustering of profiles with same input set and a tolerance of 1 in output
        for i, each in enumerate(final_arr_trans_clean):
            if list(each) not in measurements_match_list:
                measurements_match_list.append(list(each))
            for element in final_arr_trans_clean[i + 1:, ...]:
                if (list(each)[0:-1] == list(element)[0:-1]) and (
                        abs(float(list(each)[-1]) - float(list(element)[-1])) < 1):
                    if list(element) not in measurements_match_list:
                        measurements_match_list.append(list(element))
                    counter += 1
                elif (list(each)[0:-1] == list(element)[0:-1]) and (
                        abs(float(list(each)[-1]) - float(list(element)[-1])) > 1):
                    counter_not += 1
                else:
                    pass

        # for i, each in enumerate(final_arr_trans_clean):
        #     for element in final_arr_trans_clean[i+1:,...]:
        #         if list(each)[1:] == list(element)[1:] and list(each)[0] != list(element)[0]:
        #             counter_not += 1

        # counter_row = 0
        # for j in range(0,length-1):
        #     if (each[j] == element[j]):
        #         counter_row += 1
        # if counter_row == length-1 and each[-1] != element[-1]:
        #     counter_not += 1

        ## User profile list for different combination of input parameters
        for each in index_list:
            key = key + str(each) + "-"
            comparator_dict[key] = [counter, counter_not]
            measurements_match[key] = measurements_match_list

sorted_comparator_dict = sorted(comparator_dict.items(), key=lambda t: t[1][0])
# print(sorted_comparator_dict)

with open("match.json", 'w') as outfile:
    json.dump(measurements_match, outfile)


# -*- coding: utf-8 -*-

import xmltodict

from django.core.urlresolvers import reverse

from xmodule.modulestore.tests.factories import CourseFactory
from xmodule.modulestore.tests.django_utils import ModuleStoreTestCase

class FeedTest(ModuleStoreTestCase):
    def setUp(self):
        super(FeedTest, self).setUp()
        self.unpublished_course = CourseFactory(org='fun', number='001', display_name=u"unpublished", ispublic=False)
        self.published_course = CourseFactory(org='fun', number='002', display_name=u"published", ispublic=True)
        self.url = reverse('fun-courses-feed')

    def test_feed(self):
        response = self.client.get(self.url)
        self.assertEqual(200, response.status_code)
        data = xmltodict.parse(response.content)
        self.assertEqual(u"Fun latest published courses", data['rss']['channel']['title'])
        self.assertEqual(1, len(data['rss']['channel']['item']))
        self.assertEqual(u"published", data['rss']['channel']['item'][0]['title'])


from pathlib import Path
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
from sklearn.decomposition import PCA

if __name__ == '__main__':

    # set paths
    dir_path = Path('./data')
    csv_path = dir_path / 'パラメーターテスト4.txt'
    out_dir_path = Path('./out')
    out_pc_path = out_dir_path / 'principal_components_100dim_to_6val.csv'
    out_dir_path.mkdir(parents=True, exist_ok=True)

    # set params
    j_time_frames = ['5M','15M','1H','4H','Daily']
    i_periods = ['10','20','40','80','200']
    indicators = ['wkRSI','wkBB','wkATR','wkSMA']
    y_label = 'wkFixPips'
    indicator_start_label = 'wkSMA[0][0]'

    # load csv
    df = pd.read_csv(str(csv_path))
    keys = df.keys().values
    st_indicator = list(keys).index(indicator_start_label)

    # set flags
    is_buy = df['strbuysell'].values == 'close_buy'

    pc = []
    for bs_i, buy_or_sell in enumerate([is_buy, ~is_buy]):

        if bs_i == 0:
            bs = 'buy'
        else:
            bs = 'sell'

        # pca to compress indicator value
        X = np.array([df.iloc[i,st_indicator:].values for i in range(len(df)) if buy_or_sell[i]])
        X = np.array([(X[:,i]-np.mean(X[:,i]))/np.std(X[:,i]) for i in range(len(X[0]))]).T
        pca = PCA(n_components=6)
        pca.fit(X)
        Xd = pca.transform(X)
        print('Cumulative contribution rate from 1st to 6th PC:')
        print(np.cumsum(pca.explained_variance_ratio_))

        pc.append(pca.components_)

        for xi in range(6):

            # sliding window
            w_gp_mean, w_gp_std, w_cent, w_num = [], [], [], []
            w_min, w_max = Xd[:,xi].min(), Xd[:,xi].max()
            w_stride = (w_max-w_min)/50
            w_width = w_stride*2
            for w_st in np.arange(w_min,w_max,w_stride):
                w_cent.append(w_st + (w_width/2))
                is_w = (Xd[:,xi] >= w_st) & (Xd[:,xi] < w_st+10)
                w_num.append(is_w.sum())
                w_gp_mean.append(np.mean(df[y_label].values[buy_or_sell][is_w]))
                w_gp_std.append(np.std(df[y_label].values[buy_or_sell][is_w]))
            w_cent = np.array(w_cent)
            w_gp_mean = np.array(w_gp_mean)
            w_gp_std = np.array(w_gp_std)
            w_num = np.array(w_num)
            err = np.array([w_gp_mean + w_gp_std, w_gp_mean-w_gp_std]).T

            # plot
            plt.close('all')
            plt.figure()
            plt.scatter(
                Xd[:,xi],
                df[y_label].values[buy_or_sell],
                label='samples',
                marker='+')
            plt.plot(w_cent,w_gp_mean*10,color='r',label=y_label+'_mean x 10')
            # plt.plot(w_cent,err*10,color='r',label='$\pm$std x 10',linestyle='dashed', alpha=0.5)
            plt.plot(w_cent, w_num, label='sample_num',linestyle='dashed',color='g',alpha=0.6)
            plt.grid()
            plt.xlabel('{}th-pcv-projected value'.format(xi+1))
            # plt.ylim([-100,100])
            plt.ylabel(y_label)
            plt.legend()
            plt.title('{}-th-pcv-projected value VS gain/loss ({})'.format(xi+1,bs))
            fname = str(out_dir_path/ 'pc_vs_gl_w_{}_pc{:02}.png'.format(bs,xi))
            print('saving: {}'.format(fname))
            plt.savefig(fname)

    # save principal components
    pc = np.array(pc).reshape(len(pc[0])*2,-1)
    df_pc= pd.DataFrame(pc.T,
        columns = ['buy_pc{:02}'.format(num+1) for num in range(6)]+['sell_pc{:02}'.format(num+1) for num in range(6)],
        index = keys[st_indicator:])
    print('saving: {}'.format(str(out_pc_path)))
    df_pc.to_csv(str(out_pc_path))


"""
Utility functions for pulse2percept
"""
import numpy as np
import multiprocessing
import random
import copy
import logging

from scipy import misc as spm
from scipy import interpolate as spi
from scipy import signal as sps

# Rather than trying to import these all over, try once and then remember
# by setting a flag.
try:
    import joblib
    joblib.Parallel
    has_joblib = True
except (ImportError, AttributeError):
    has_joblib = False

try:
    import dask
    import dask.multiprocessing
    dask.delayed
    has_dask = True
except (ImportError, AttributeError):
    has_dask = False

try:
    from numba import jit
    has_jit = True
except ImportError:
    has_jit = False

def deprecated(arg):
    """Decorator used to mark functions as deprecated

    This is a decorator which can be used to mark functions as deprecated.
    It will result in a warning being emitted when the function is used.

    Adapted from: http://www.artima.com/weblogs/viewpost.jsp?thread=240845

    Parameters
    ----------
    alt_func : str, optional
        The function to use instead of the deprecated one.
        Be sure to list a string, not a function object.
        Default: None.
    """
    def wrap(func):
        """The function performing the decoration

        This function is perfoming the actual decoration process. It can only
        have a single argument, which is the function object.

        Parameters
        ----------
        func : function object
            The function that is deprecated.
        """
        e_s = "Call to deprecated function %s." % func.__name__
        if alt_func:
            e_s += " Use %s instead." % alt_func
        logging.getLogger(__name__).warn(e_s)

        def wrapped_func(*args, **kwargs):
            return func(*args, **kwargs)
        return wrapped_func

    alt_func = None
    if callable(arg):
        # Direct decoration
        return wrap(arg)
    else:
        # Alternative function given
        alt_func = arg
        return wrap

class Parameters(object):
    """Container to wrap a MATLAB array in a Python dict

    This function is deprecated as of v0.2 and will be completely removed
    in v0.3.

    .. deprecated:: 0.2
    """

    @deprecated
    def __init__(self, **params):
        for k, v in params.items():
            self.__dict__[k] = v

    def __repr__(self):
        my_list = []
        for k, v in self.__dict__.items():
            my_list.append("%s : %s" % (k, v))
        my_list.sort()
        my_str = "\n".join(my_list)
        return my_str

    def __setattr(self, name, values):
        self.__dict__[name] = values

class TimeSeries(object):

    def __init__(self, tsample, data):
        """Container for time series data

        Provides a container for time series data. Every time series has a
        sampling step `tsample`, and some `data` sampled at that rate.

        Parameters
        ----------
        tsample : float
            Sampling time step (seconds).
        data : array_like
            Time series data sampled at every `tsample` seconds.
        """
        self.data = data
        self.tsample = tsample
        self.duration = self.data.shape[-1] * tsample
        self.shape = data.shape

    def __getitem__(self, y):
        return TimeSeries(self.tsample, self.data[y])

    def append(self, other):
        """Appends the data of another TimeSeries object (in time)

        This function concatenates the data of another TimeSeries object to
        the current object, along the last dimension (time). To make this work,
        all but the last dimension of the two objects must be the same.

        If the two objects have different time sampling steps, the other object
        is resampled to fit the current `tsample`.

        Parameters
        ----------
        other : p2p.utils.TimeSeries
            A TimeSeries object whose content should be appended.

        Examples
        --------
        >>> from pulse2percept import utils
        >>> pt = utils.TimeSeries(1.0, np.zeros((2, 2, 10)))
        >>> num_frames = pt.shape[-1]
        >>> pt.append(pt)
        >>> pt.shape[-1] == 2 * num_frames
        True
        """
        # Make sure type is correct
        if not isinstance(other, TimeSeries):
            raise TypeError("Other object must be of type "
                            "p2p.utils.TimeSeries.")

        # Make sure size is correct for all but the last dimension (number
        # of frames)
        if self.shape[:-1] != other.shape[:-1]:
            raise ValueError("Shape mismatch: ", self.shape[:-1], " vs. ",
                             other.shape[:-1])

        # Then resample the other to current `tsample`
        resampled = other.resample(self.tsample)

        # Then concatenate the two
        self.data = np.concatenate((self.data, resampled.data), axis=-1)
        self.duration = self.data.shape[-1] * self.tsample
        self.shape = self.data.shape

    def max(self):
        """Returns the time and value of the largest data point

        This function returns the the largest value in the TimeSeries data,
        as well as the time at which it occurred.

        Returns
        -------
        t : float
            time (s) at which max occurred
        val : float
            max value
        """
        # Find index and value of largest element
        idx = self.data.argmax()
        val = self.data.max()

        # Find frame that contains the brightest data point using `unravel`,
        # which maps the flat index `idx_px` onto the high-dimensional
        # indices (x,y,z).
        # What we want is index `z` (i.e., the frame index), given by the last
        # dimension in the return argument.
        idx_frame = np.unravel_index(idx, self.data.shape)[-1]

        # Convert index to time
        t = idx_frame * self.tsample

        return t, val

    def max_frame(self):
        """Returns the time frame that contains the largest data point

        This function returns the time frame in the TimeSeries data that
        contains the largest data point, as well as the time at which
        it occurred.

        Returns
        -------
        t : float
            time (s) at which max occurred
        frame : TimeSeries
            A TimeSeries object.
        """
        # Find index and value of largest element
        idx = self.data.argmax()

        # Find frame that contains the brightest data point using `unravel`,
        # which maps the flat index `idx_px` onto the high-dimensional
        # indices (x,y,z).
        # What we want is index `z` (i.e., the frame index), given by the last
        # dimension in the return argument.
        idx_frame = np.unravel_index(idx, self.data.shape)[-1]
        t = idx_frame * self.tsample
        frame = self.data[..., idx_frame]

        return t, TimeSeries(self.tsample, copy.deepcopy(frame))

    def resample(self, tsample_new):
        """Returns data sampled according to new time step

        This function returns a TimeSeries object whose data points were
        resampled according to a new time step `tsample_new`. New values
        are found using linear interpolation.

        Parameters
        ----------
        tsample_new : float
            New sampling time step (s)

        Returns
        -------
        TimeSeries object
        """
        if tsample_new is None or tsample_new == self.tsample:
            return TimeSeries(self.tsample, self.data)

        # Try to avoid rounding errors in arr size by making sure `t_old` is
        # too long at first, then cutting it to the right size
        y_old = self.data
        t_old = np.arange(0, self.duration + self.tsample, self.tsample)
        t_old = t_old[:y_old.shape[-1]]
        f = spi.interp1d(t_old, y_old, axis=-1, fill_value='extrapolate')

        t_new = np.arange(0, self.duration, tsample_new)
        y_new = f(t_new)

        return TimeSeries(tsample_new, y_new)

def _sparseconv(data, kernel, mode):
    """
    Returns the discrete, linear convolution of two one-dimensional sequences.
    output is of length len(kernel) + len(data) -1 (same as the default for
    numpy.convolve).

    Can run faster than numpy.convolve if:
    (1) a is much longer than v
    (2) a is sparse (has lots of zeros)
    """
    kernel_len = kernel.shape[-1]
    data_len = data.shape[-1]
    out = np.zeros(data_len + kernel_len - 1)

    pos = np.where(data != 0)[0]
    # add shifted and scaled copies of kernel only where data is nonzero
    for p in pos:
        out[p:p + kernel_len] = out[p:p + kernel_len] + kernel * data[p]

    if mode.lower() == 'full':
        return out
    elif mode.lower() == 'valid':
        return center_vector(out, data_len - kernel_len + 1)
    elif mode.lower() == 'same':
        return center_vector(out, data_len)
    else:
        raise ValueError("Acceptable mode flags are 'valid',"
                         " 'same', or 'full'.")

if has_jit:
    _sparseconvj = jit(_sparseconv)

def sparseconv(data, kernel, mode='full', dojit=True):
    """
    Returns the discrete, linear convolution of two one-dimensional sequences.

    Can run faster than numpy.convolve if:
    (1) `data` is much longer than `kernel`
    (2) `data` is sparse (has lots of zeros)

    Parameters
    ----------
    data : array_like
        First input, typically the data array.
    kernel : array_like
        Second input, typically the kernel.
    mode : str {'full', 'valid', 'same'}, optional
        A string indicating the size of the output:
        ``full``
        The output is the full discrete linear convolution of the inputs.
        (Default)
        ``valid``
        The output consists only of those elements that do not rely on
        zero-padding.
        ``same``
        The output is the same size as `data`, centered with respect to the
        'full' output.
    dojit : boolean
        A flag indicating whether to use numba's just-in-time compilation
        option.

    """
    if dojit and not has_jit:
        e_s = ("You do not have numba ",
               "please run sparseconv with dojit=False")
        raise ImportError(e_s)
    else:
        return _sparseconv(data, kernel, mode)

def conv(data, kernel, tsample, mode='full', method='fft', dojit=True):
    """Convoles data with a kernel using either FFT or sparseconv

    This function convolves data with a kernel, relying either on the
    fast Fourier transform (FFT) or a sparse convolution function.

    Parameters
    ----------
    data : array_like
        First input, typically the data array
    kernel : array_like
        Second input, typically the kernel
    mode : str {'full', 'valid', 'same'}, optional
        A string indicating the size of the output:
        ``full``
        The output is the full discrete linear convolution of the inputs.
        (Default)
        ``valid``
        The output consists only of those elements that do not rely on
        zero-padding.
        ``same``
        The output is the same size as `data`, centered with respect to the
        'full' output.
    method : str {'fft', 'sparse'}, optional
        A string indicating the convolution method:
        ``fft``
        Use the fast Fourier transform (FFT). (Default)
        ``sparse``
        Use the sparse convolution.
    dojit : bool, optional
        A flag indicating whether to use numba's just-in-time compilation
        option (only relevant for `method`=='sparse').
    """
    if method.lower() == 'fft':
        # Use FFT: faster on non-sparse data
        conved = tsample * sps.fftconvolve(data, kernel, mode)
    elif method.lower() == 'sparse':
        # Use sparseconv: faster on sparse data
        conved = tsample * sparseconv(data, kernel, mode, dojit)
    else:
        raise ValueError("Acceptable methods are: 'fft', 'sparse'.")
    return conved

def center_vector(vec, newlen):
    """
    Returns the center `newlen` portion of a vector.

    Adapted from scipy.signal.signaltools._centered:
    github.com/scipy/scipy/blob/v0.18.0/scipy/signal/signaltools.py#L236-L243

    """
    currlen = vec.shape[-1]
    startind = (currlen - newlen) // 2
    endind = startind + newlen
    return vec[startind:endind]

def parfor(func, in_list, out_shape=None, n_jobs=-1, engine='joblib',
           backend='threading', func_args=[], func_kwargs={}):
    """
    Parallel for loop for numpy arrays

    Parameters
    ----------
    func : callable
        The function to apply to each item in the array. Must have the form:
        func(arr, idx, *args, *kwargs) where arr is an ndarray and idx is an
        index into that array (a tuple). The Return of `func` needs to be one
        item (e.g. float, int) per input item.

    in_list : list
       All legitimate inputs to the function to operate over.

    n_jobs : integer, optional
        The number of jobs to perform in parallel. -1 to use all cpus
        Default: 1

    engine : str
        {"dask", "joblib", "serial"}
        The last one is useful for debugging -- runs the code without any
        parallelization.

    backend : str
        What dask backend to use. Irrelevant for other engines.

    func_args : list, optional
        Positional arguments to `func`

    func_kwargs : list, optional
        Keyword arguments to `func`

    Returns
    -------
    ndarray of identical shape to `arr`

    Notes
    -----
    Imported from pyAFQ (blob e20eaa0 from June 3, 2016):
    https://github.com/arokem/pyAFQ/blob/master/AFQ/utils/parallel.py

    Examples
    --------
    """
    if n_jobs == -1:
        n_jobs = multiprocessing.cpu_count()
        n_jobs = n_jobs - 1

    if engine.lower() == 'joblib':
        if not has_joblib:
            err = "You do not have `joblib` installed. Consider setting"
            err += "`engine` to 'serial' or 'dask'."
            raise ImportError(err)

        p = joblib.Parallel(n_jobs=n_jobs, backend=backend)
        d = joblib.delayed(func)
        d_l = []
        for in_element in in_list:
            d_l.append(d(in_element, *func_args, **func_kwargs))
        results = p(d_l)
    elif engine.lower() == 'dask':
        if not has_dask:
            err = "You do not have `dask` installed. Consider setting"
            err += "`engine` to 'serial' or 'joblib'."
            raise ImportError(err)

        def partial(func, *args, **keywords):
            def newfunc(in_arg):
                return func(in_arg, *args, **keywords)
            return newfunc
        p = partial(func, *func_args, **func_kwargs)
        d = [dask.delayed(p)(i) for i in in_list]
        if backend == 'multiprocessing':
            results = dask.compute(*d, get=dask.multiprocessing_get,
                                   workers=n_jobs)
        elif backend == 'threading':
            results = dask.compute(*d, get=dask.threaded.get, workers=n_jobs)
    elif engine.lower() == 'serial':
        results = []
        for in_element in in_list:
            results.append(func(in_element, *func_args, **func_kwargs))

    if out_shape is not None:
        return np.array(results).reshape(out_shape)
    else:
        return results

@deprecated
def mov2npy(movie_file, out_file):
    """Converts a movie file to a NumPy array

    This function is deprecated as of v0.2 and will be removed completely
    in v0.3.

    Parameters
    ----------
    movie_file : array
        The movie file to be read
    out_file: str
        Name of .npy file to be created.

    .. deprecated:: 0.2
    """
    # Don't import cv at module level. Instead we'll use this on python 2
    # sometimes...
    try:
        import cv
    except ImportError:
        e_s = "You do not have opencv installed. "
        e_s += "You probably want to run this in Python 2"
        raise ImportError(e_s)

    capture = cv.CaptureFromFile(movie_file)
    frames = []
    img = cv.QueryFrame(capture)
    while img is not None:
        tmp = cv.CreateImage(cv.GetSize(img), 8, 3)
        cv.CvtColor(img, tmp, cv.CV_BGR2RGB)
        frames.append(np.asarray(cv.GetMat(tmp)))
        img = cv.QueryFrame(capture)
    frames = np.fliplr(np.rot90(np.mean(frames, -1).T, -1))
    np.save(out_file, frames)

def gamma(n, tau, tsample, tol=0.01):
    """Returns the impulse response of `n` cascaded leaky integrators

    This function calculates the impulse response of `n` cascaded
    leaky integrators with constant of proportionality 1/`tau`:
    y = (t/theta).^(n-1).*exp(-t/theta)/(theta*factorial(n-1))

    Parameters
    ----------
    n : int
        Number of cascaded leaky integrators
    tau : float
        Decay constant of leaky integration (seconds).
        Equivalent to the inverse of the constant of proportionality.
    tsample : float
        Sampling time step (seconds).
    tol : float
        Cut the kernel to size by ignoring function values smaller
        than a fraction `tol` of the peak value.
    """
    n = int(n)

    # Allocate a time vector that is long enough for sure.
    # Trim vector later on.
    t = np.arange(0, 5 * n * tau, tsample)

    # Calculate gamma
    y = (t / tau) ** (n - 1) * np.exp(-t / tau)
    y /= (tau * spm.factorial(n - 1))

    # Normalize to unit area
    y /= np.trapz(np.abs(y), dx=tsample)

    # Cut off tail where values are smaller than `tol`.
    # Make sure to start search on the right-hand side of the peak.
    peak = y.argmax()
    small_vals = np.where(y[peak:] < tol * y.max())[0]
    t = t[:small_vals[0] + peak]
    y = y[:small_vals[0] + peak]

    return t, y

def traverse_randomly(seq):
    """Traverses a list in random order

    Parameters
    ----------
    seq : list
        An iterable list of elements.
    Returns
    -------
    A list iterator.

    Examples
    --------
    Shuffle a list:

    >>> list_ordered = [0, 1, 2, 3]
    >>> list_shuffled = [l for l in traverse_randomly(list_ordered)]

    Traverse a list in random order:

    >>> list_ordered = [0, 1, 2, 3]
    >>> for idx, val in traverse_randomly(enumerate(list_ordered)):
    ...     pass

    Notes
    -----
    From: http://stackoverflow.com/a/9253366
    Note that even for rather small `len(x)`, the total number of permutations
    of `seq` can quickly grow larger than the period of most random number
    generators. For example, a sequence of length 2080 is the largest that can
    fit within the period of the Mersenne Twister random number generator.
    """
    # Make sure we are dealing with a list
    shuffled = list(seq)

    # Import `random` at module level (for performance)
    random.shuffle(shuffled)

    return iter(shuffled)


## -*- coding: utf-8 -*-
import argparse
import os
import torch
import datetime
import logging
from pathlib import Path
import sys
import importlib
import shutil
from tqdm import tqdm
from utils import *
from ShapeNetDataLoader import PartNormalDataset
from concurrent.futures import ThreadPoolExecutor

BASE_DIR = os.path.dirname(os.path.abspath(__file__))
ROOT_DIR = BASE_DIR
sys.path.append(os.path.join(ROOT_DIR, 'models'))

def parse_args():
    parser = argparse.ArgumentParser('Model')
    parser.add_argument('--model', type=str, default='transformer', help='model name [default: transformer]')
    parser.add_argument('--batch_size', type=int, default=4, help='Batch Size during training [default: 8]')
    parser.add_argument('--epoch',  default=128, type=int, help='Epoch to run [default: 128]')
    parser.add_argument('--learning_rate', default=0.002, type=float, help='Initial learning rate [default: 0.002]')
    parser.add_argument('--gpu', type=str, default='1', help='GPU to use [default: GPU 0]')
    parser.add_argument('--optimizer', type=str, default='Adam', help='Adam or SGD [default: Adam]')
    parser.add_argument('--log_dir', type=str, default=None, help='Log path [default: None]')
    parser.add_argument('--decay_rate', type=float, default=1e-4, help='weight decay [default: 1e-4]')
    parser.add_argument('--npoint', type=int,  default=1024, help='Point Number [default: 2048]')
    parser.add_argument('--normal', action='store_true', default=False, help='Whether to use normal information [default: False]')
    parser.add_argument('--step_size', type=int,  default=20, help='Decay step for lr decay [default: every 20 epochs]')
    parser.add_argument('--lr_decay', type=float,  default=0.5, help='Decay rate for lr decay [default: 0.5]')
    return parser.parse_args()

def main(args):
    logger=logging.getLogger("Transformer")
    def log_string(str):
        logger.info(str)
        print(str)

    '''HYPER PARAMETER'''
    os.environ["CUDA_VISIBLE_DEVICES"] = args.gpu
    timestr = str(datetime.datetime.now().strftime('%Y-%m-%d_%H-%M'))
    experiment_dir = Path('./log/')
    experiment_dir.mkdir(exist_ok=True)
    experiment_dir = experiment_dir.joinpath('transformer')
    experiment_dir.mkdir(exist_ok=True)
    if args.log_dir is None:
        experiment_dir = experiment_dir.joinpath(timestr)
    else:
        experiment_dir = experiment_dir.joinpath(args.log_dir)
    experiment_dir.mkdir(exist_ok=True)
    visual_dir=experiment_dir.joinpath("visual")
    visual_dir.mkdir(exist_ok=True)
    checkpoints_dir = experiment_dir.joinpath('checkpoints/')
    checkpoints_dir.mkdir(exist_ok=True)
    log_dir = experiment_dir.joinpath('logs/')
    log_dir.mkdir(exist_ok=True)

    '''LOG'''
    args = parse_args()
    logger.setLevel(logging.INFO)
    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    file_handler = logging.FileHandler('%s/%s.txt' % (log_dir, args.model))
    file_handler.setLevel(logging.INFO)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    log_string('INIT PARAMETERS ...')
    log_string(args)
    root = 'data/shapenetcore_partanno_segmentation_benchmark_v0_normal/'

    TRAIN_DATASET = PartNormalDataset(root = root, npoints=args.npoint, split='trainval', normal_channel=args.normal,class_choice=["Bag","Cap","Earphone","Rocket"])
    trainDataLoader = torch.utils.data.DataLoader(TRAIN_DATASET, batch_size=args.batch_size,shuffle=True, num_workers=4,drop_last=True)
    TEST_DATASET = PartNormalDataset(root = root, npoints=args.npoint, split='test', normal_channel=args.normal,class_choice=["Bag","Cap","Earphone","Rocket"])
    testDataLoader = torch.utils.data.DataLoader(TEST_DATASET, batch_size=args.batch_size,shuffle=False, num_workers=4,drop_last=True)
    log_string("The number of training data is: %d" % len(TRAIN_DATASET))
    log_string("The number of test data is: %d" %  len(TEST_DATASET))

    MODEL = importlib.import_module(args.model)
    shutil.copy('models/%s.py' % args.model, str(experiment_dir))
    shutil.copy('utils.py', str(experiment_dir))
    channel=6 if args.normal else 3
    classifier = MODEL.get_model(d_model=128,channel=channel).cuda()
    def weights_init(m):
        classname = m.__class__.__name__
        if classname.find('Conv2d') != -1:
            torch.nn.init.xavier_normal_(m.weight.data)
            torch.nn.init.constant_(m.bias.data, 0.0)
        elif classname.find('Linear') != -1:
            torch.nn.init.xavier_normal_(m.weight.data)
            torch.nn.init.constant_(m.bias.data, 0.0)

    try:
        checkpoint = torch.load(str(experiment_dir) + '/checkpoints/best_model.pth')
        epoch = checkpoint['epoch']
        model_loss=checkpoint['loss']
        classifier.load_state_dict(checkpoint['model_state_dict'])
        log_string('Use pretrain model')
    except:
        log_string('No existing model, starting training from scratch...')
        epoch = 0
        model_loss=999.0
        classifier = classifier.apply(weights_init)

    if args.optimizer == 'Adam':
        optimizer = torch.optim.Adam(
            classifier.parameters(),
            lr=args.learning_rate,
            betas=(0.9, 0.999),
            eps=1e-08,
            weight_decay=args.decay_rate
        )
    else:
        optimizer = torch.optim.SGD(classifier.parameters(), lr=args.learning_rate, momentum=0.9)

    def bn_momentum_adjust(m, momentum):
        if isinstance(m, torch.nn.BatchNorm2d) or isinstance(m, torch.nn.BatchNorm1d):
            m.momentum = momentum

    LEARNING_RATE_CLIP = 1e-5
    MOMENTUM_ORIGINAL = 0.1
    MOMENTUM_DECCAY = 0.5
    MOMENTUM_DECCAY_STEP = args.step_size

    t = ThreadPoolExecutor(args.batch_size * 5)

###   仔细检查训练流程
    for epoch in range(epoch, args.epoch):
        log_string('Epoch %d (%d/%s):' % (epoch + 1, epoch + 1, args.epoch))
        '''Adjust learning rate and BN momentum'''
        lr = max(args.learning_rate * (args.lr_decay ** (epoch // args.step_size)), LEARNING_RATE_CLIP)
        log_string('Learning rate:%f' % lr)
        for param_group in optimizer.param_groups:
            param_group['lr'] = lr
        momentum = MOMENTUM_ORIGINAL * (MOMENTUM_DECCAY ** (epoch // MOMENTUM_DECCAY_STEP))
        if momentum < 0.01:
            momentum = 0.01
        print('BN momentum updated to: %f' % momentum)
        classifier = classifier.apply(lambda x: bn_momentum_adjust(x,momentum))

#  Train one epoch
        total_loss1 ,total_loss2,i=0,0,0
        for  data in tqdm(trainDataLoader, total=len(trainDataLoader), smoothing=0.9):
            i+=1
            points1, points2, _ = data
            points1,points2=points1.cuda(),points2.cuda()
            optimizer.zero_grad()
            classifier = classifier.train()
            warped,loss1,loss2 = classifier(points1, points2)
            if epoch <10:
                # loss = (loss1 + loss2).mean()
                loss=loss1.mean()
            else:
                # loss = (loss1 + 0.1*loss2).mean()
                loss=loss1.mean()
            total_loss1+=float(loss1.mean())
            total_loss2+=float(loss2.mean())
            loss.backward()
            optimizer.step()

        log_string('EPOCH %d train: loss1 is: %.5f   loss2 is: %.5f ' % (epoch+1,total_loss1/i,total_loss2/i))
        with torch.no_grad():
            total_loss1, total_loss2, i = 0, 0, 0
            for batch_id, (points1, points2, cls) in tqdm(enumerate(testDataLoader), total=len(testDataLoader), smoothing=0.9):
                i+=1
                points1, points2 = points1.cuda(), points2.cuda()
                classifier = classifier.eval()
                warped, loss1, loss2 = classifier(points1, points2)
                total_loss1 += float(loss1.mean())
                total_loss2 += float(loss2.mean())

            log_string('EPOCH %d test: loss1 is: %.5f   loss2 is: %.5f ' % (epoch+1, total_loss1 / i, total_loss2 / i))

            if (total_loss1/batch_id) < model_loss :
                model_loss=total_loss1/batch_id
                logger.info('Save model...')
                savepath = str(checkpoints_dir) + '/best_model.pth'
                log_string('Saving at %s'% savepath)
                state = {
                    'epoch': epoch,
                    'loss' :model_loss,
                    'model_state_dict': classifier.state_dict(),
                    'optimizer_state_dict': optimizer.state_dict(),
                }
                torch.save(state, savepath)

    #   save the last epoch
    model_loss = total_loss1 / batch_id
    logger.info('Save last model...')
    savepath = str(checkpoints_dir) + '/last_model.pth'
    log_string('Saving at %s' % savepath)
    state = {
        'epoch': epoch,
        'loss': model_loss,
        'model_state_dict': classifier.state_dict(),
        'optimizer_state_dict': optimizer.state_dict(),
    }
    torch.save(state, savepath)

if __name__ == '__main__':
    args = parse_args()
    main(args)


import scrapy
from locations.items import GeojsonPointItem
import json
import re

regex_times = r"\d{1,2}:\d{1,2}\s?[Aa]?[Mm]?[Pp]?[Mm]?\s?-\s?\d{1,2}:\d{1,2}" \
              r"\s?[Pp]?[Mm]?[Aa]?[Mm]?"
regex_am = r"\s?[Aa][Mm]"
regex_pm = r"\s?[Pp][Mm]"

#  Covers United States, Canada, UK, Puerto Rico, Bahamas with 500 mile radius
lats = ['32.806671', '31.054487',  '39.059811', '46.921925', '45.694454',
        '38.039119', '44.045876', '32.593106', '33.596319', '47.398349',
        '24.44715', '18.229351', '19.725342', '64.014496', '51.563412',
        '52.48278', '55.27911529201562', '55.17886766328199',
        '63.15435519659188', '52.96187505907603']

lons = ['-86.791130', '-97.563461', '-105.311104', '-110.454353', '-93.900192',
        '-87.618638', '-72.710686', '-82.342529', '-113.334961', '-121.289062',
        '-78.00293', '-65.830078', '-155.610352', '-153.28125' '-86.923828',
        '-65.126953', '-103.974609375', '-120.76171875', '-136.142578125',
        '-0.17578125']

class RegisSpider(scrapy.Spider):
    name = 'regis'
    item_attributes = { 'brand': "Regis Salon" }
    download_delay = 0
    allowed_domains = ['www.regissalons.com']
    start_urls = ['https://www.regissalons.com']

    def convert_hours(self, hours):
        if not hours:
            return ''
        for i in range(len(hours)):
            converted_times = ''
            if 'Closed' not in hours[i]:
                from_hr, to_hr = [hr.strip() for hr in hours[i].split('-')]
                if re.search(regex_am, from_hr):
                    from_hr = re.sub(regex_am, '', from_hr)
                    hour_min = from_hr.split(':')
                    if len(hour_min[0]) < 2:
                        hour_min[0] = hour_min[0].zfill(2)
                    converted_times += (":".join(hour_min)) + ' - '
                else:
                    from_hr = re.sub(regex_pm, '', from_hr)
                    hour_min = from_hr.split(':')
                    if int(hour_min[0]) < 12:
                        hour_min[0] = str(12 + int(hour_min[0]))
                    converted_times += (":".join(hour_min)) + ' - '

                if re.search(regex_am, to_hr):
                    to_hr = re.sub(regex_am, '', to_hr)
                    hour_min = to_hr.split(':')
                    if len(hour_min[0]) < 2:
                        hour_min[0] = hour_min[0].zfill(2)
                    if int(hour_min[0]) == 12:
                        hour_min[0] = '00'
                    converted_times += (":".join(hour_min))
                else:
                    to_hr = re.sub(regex_pm, '', to_hr)
                    hour_min = to_hr.split(':')
                    if int(hour_min[0]) < 12:
                        hour_min[0] = str(12 + int(hour_min[0]))
                    converted_times += (":".join(hour_min))
            else:
                converted_times += "off"
            hours[i] = converted_times

        days = ['Mo', 'Tu', 'We', 'Th', 'Fr', 'Sa', 'Su']
        hours = ''.join(' {} {} '.format(*t) for t in zip(days, hours))
        return hours

    def parse_store(self, response):
        results = json.loads(response.body_as_unicode())
        for i in results:
            store_id = i['id']
            name = i['address']
            website = i['permalink'].replace('\\', '')
            street = i['address2']
            city = i['city']
            state = i['state']
            postcode = i['zip']
            addr_full = "{} {}, {} {}".format(street, city, state, postcode)
            country = i['country']
            lat = i['lat']
            lon = i['lng']
            hours = re.findall(regex_times, i['hours'])
            hours = self.convert_hours(hours)

            yield GeojsonPointItem(
                ref=store_id,
                name=name,
                website=website,
                street=street,
                city=city,
                state=state,
                postcode=postcode,
                country=country,
                addr_full=addr_full,
                lat=lat,
                lon=lon,
                opening_hours=hours
            )

    def parse(self, response):
        for x, y in zip(lats, lons):
            base_url = 'https://www.regissalons.com/wp-admin/admin-ajax.php?action=store_search&max_results=100&search_radius=500&'
            base_url += "lat={}&lng={}".format(x, y)
            yield scrapy.Request(base_url, callback=self.parse_store)


# <Source: https://github.com/rtqichen/ffjord/blob/master/lib/toy_data.py >

import numpy as np
import sklearn
import torch
import sklearn.datasets
from PIL import Image
import os

# Dataset iterator
def inf_train_gen(data, rng=None, batch_size=200):
    if rng is None:
        rng = np.random.RandomState()
        #rng = np.random.RandomState(42)
        #print(rng)

    if data == "1Normal":
        z_dist = torch.distributions.Normal(loc=2, scale=.5)
        z = z_dist.sample((batch_size, 1))
        return z

    if data == "2spirals-8gaussians":
        data1 = inf_train_gen("2spirals", rng=rng, batch_size=batch_size)
        data2 = inf_train_gen("8gaussians", rng=rng, batch_size=batch_size)
        return np.concatenate([data1, data2], axis=1)

    if data == "4-2spirals-8gaussians":
        data1 = inf_train_gen("2spirals", rng=rng, batch_size=batch_size)
        data2 = inf_train_gen("8gaussians", rng=rng, batch_size=batch_size)
        data3 = inf_train_gen("2spirals", rng=rng, batch_size=batch_size)
        data4 = inf_train_gen("8gaussians", rng=rng, batch_size=batch_size)
        return np.concatenate([data1, data2, data3, data4], axis=1)

    if data == "8-2spirals-8gaussians":
        data1 = inf_train_gen("4-2spirals-8gaussians", rng=rng, batch_size=batch_size)
        data2 = inf_train_gen("4-2spirals-8gaussians", rng=rng, batch_size=batch_size)
        return np.concatenate([data1, data2], axis=1)

    if data == "3-MIX_DEP":
        data1 = inf_train_gen("2spirals", rng=rng, batch_size=batch_size)
        data2 = inf_train_gen("8gaussians", rng=rng, batch_size=batch_size)
        data3 = inf_train_gen("circles", rng=rng, batch_size=batch_size)
        p = rng.randint(0, 2, batch_size).reshape(-1, 1)
        p = np.concatenate([p, p], axis=1)
        dist = 2*((data1[:, 0]**2 + data1[:, 1]**2)**.5).reshape(-1, 1) - .5
        data3 = np.concatenate([dist, dist], axis=1) * data2*.5 + rng.randn(batch_size, 2) * .1
        return np.concatenate([data1, data2, data3], axis=1).astype(data2.dtype)

    if data == "7-pyramid":
        data1 = inf_train_gen("circles", rng=rng, batch_size=batch_size)
        data2 = inf_train_gen("8gaussians", rng=rng, batch_size=batch_size)
        data3 = rng.randn(batch_size, 1) + data1.max(1).reshape(-1, 1)
        data4 = rng.randn(batch_size, 1) + data2.min(1).reshape(-1, 1)
        p = rng.randint(0, 2, batch_size).reshape(-1, 1)
        data5 = p * np.sin(data3 + data4) + (1 - p) * np.cos(data3 + data4) + rng.randn(batch_size, 1)
        return np.concatenate([data1, data2, data3, data4, data5], axis=1).astype(data2.dtype)

    if data == "8-MIX":
        data1 = inf_train_gen("2spirals", rng=rng, batch_size=batch_size)
        data2 = inf_train_gen("8gaussians", rng=rng, batch_size=batch_size)
        data3 = inf_train_gen("swissroll", rng=rng, batch_size=batch_size)
        data4 = inf_train_gen("circles", rng=rng, batch_size=batch_size)
        data8 = inf_train_gen("moons", rng=rng, batch_size=batch_size)
        data6 = inf_train_gen("pinwheel", rng=rng, batch_size=batch_size)
        data7 = inf_train_gen("checkerboard", rng=rng, batch_size=batch_size)
        data5 = inf_train_gen("line", rng=rng, batch_size=batch_size)
        std = np.array([1.604934 , 1.584863 , 2.0310535, 2.0305095, 1.337718 , 1.4043778,  1.6944685, 1.6935346,
                        1.7434783, 1.0092416, 1.4860426, 1.485661 , 2.3067558, 2.311637 , 1.4430547, 1.4430547], dtype=np.float32)
        data = np.concatenate([data1, data2, data3, data4, data5, data6, data7, data8], axis=1)

        return data/std
    if data == "7-MIX":
        data1 = inf_train_gen("2spirals", rng=rng, batch_size=batch_size)
        data2 = inf_train_gen("8gaussians", rng=rng, batch_size=batch_size)
        data3 = inf_train_gen("swissroll", rng=rng, batch_size=batch_size)
        data4 = inf_train_gen("circles", rng=rng, batch_size=batch_size)
        data5 = inf_train_gen("moons", rng=rng, batch_size=batch_size)
        data6 = inf_train_gen("pinwheel", rng=rng, batch_size=batch_size)
        data7 = inf_train_gen("checkerboard", rng=rng, batch_size=batch_size)
        std = np.array([1.604934 , 1.584863 , 2.0310535, 2.0305095, 1.337718 , 1.4043778,  1.6944685, 1.6935346,
                        1.7434783, 1.0092416, 1.4860426, 1.485661 , 2.3067558, 2.311637], dtype=np.float32)
        data = np.concatenate([data1, data2, data3, data4, data5, data6, data7], axis=1)

        return data/std

    if data == "swissroll":
        data = sklearn.datasets.make_swiss_roll(n_samples=batch_size, noise=1.0)[0]
        data = data.astype("float32")[:, [0, 2]]
        data /= 5
        return data

    elif data == "circles":
        data = sklearn.datasets.make_circles(n_samples=batch_size, factor=.5, noise=0.08)[0]
        data = data.astype("float32")
        data *= 3
        return data

    elif data == "moons":
        data = sklearn.datasets.make_moons(n_samples=batch_size, noise=0.1)[0]
        data = data.astype("float32")
        data = data * 2 + np.array([-1, -0.2])
        data = data.astype("float32")
        return data

    elif data == "8gaussians":
        scale = 4.
        centers = [(1, 0), (-1, 0), (0, 1), (0, -1), (1. / np.sqrt(2), 1. / np.sqrt(2)),
                   (1. / np.sqrt(2), -1. / np.sqrt(2)), (-1. / np.sqrt(2),
                                                         1. / np.sqrt(2)), (-1. / np.sqrt(2), -1. / np.sqrt(2))]
        centers = [(scale * x, scale * y) for x, y in centers]

        dataset = []
        for i in range(batch_size):
            point = rng.randn(2) * 0.5
            idx = rng.randint(8)
            center = centers[idx]
            point[0] += center[0]
            point[1] += center[1]
            dataset.append(point)
        dataset = np.array(dataset, dtype="float32")
        dataset /= 1.414
        return dataset

    elif data == "2gaussians":
        scale = 4.
        centers = [(.5, -.5), (-.5, .5)]
        centers = [(scale * x, scale * y) for x, y in centers]

        dataset = []
        for i in range(batch_size):
            point = rng.randn(2) * .75
            idx = rng.randint(2)
            center = centers[idx]
            point[0] += center[0]
            point[1] += center[1]
            dataset.append(point)
        dataset = np.array(dataset, dtype="float32")
        #dataset /= 1.414
        return dataset

    elif data == "4gaussians":
        scale = 4.
        centers = [(.5, -.5), (-.5, .5), (.5, .5), (-.5, -.5)]
        centers = [(scale * x, scale * y) for x, y in centers]

        dataset = []
        for i in range(batch_size):
            point = rng.randn(2) * .75
            idx = rng.randint(4)
            center = centers[idx]
            point[0] += center[0]
            point[1] += center[1]
            dataset.append(point)
        dataset = np.array(dataset, dtype="float32")
        # dataset /= 1.414
        return dataset

    elif data == "2igaussians":
        scale = 4.
        centers = [(.5, 0.), (-.5, .0)]
        centers = [(scale * x, scale * y) for x, y in centers]

        dataset = []
        for i in range(batch_size):
            point = rng.randn(2) * .75
            idx = rng.randint(2)
            center = centers[idx]
            point[0] += center[0]
            point[1] += center[1]
            dataset.append(point)
        dataset = np.array(dataset, dtype="float32")
        # dataset /= 1.414
        return dataset

    elif data == "conditionnal8gaussians":
        scale = 4.
        centers = [(1, 0), (-1, 0), (0, 1), (0, -1), (1. / np.sqrt(2), 1. / np.sqrt(2)),
                   (1. / np.sqrt(2), -1. / np.sqrt(2)), (-1. / np.sqrt(2),
                                                         1. / np.sqrt(2)), (-1. / np.sqrt(2), -1. / np.sqrt(2))]
        centers = [(scale * x, scale * y) for x, y in centers]

        dataset = []
        context = np.zeros((batch_size, 8))
        for i in range(batch_size):
            point = rng.randn(2) * 0.5
            idx = rng.randint(8)
            context[i, idx] = 1
            center = centers[idx]
            point[0] += center[0]
            point[1] += center[1]
            dataset.append(point)
        dataset = np.array(dataset, dtype="float32")
        dataset /= 1.414
        return dataset, context

    elif data == "pinwheel":
        radial_std = 0.3
        tangential_std = 0.1
        num_classes = 5
        num_per_class = batch_size // 5
        rate = 0.25
        rads = np.linspace(0, 2 * np.pi, num_classes, endpoint=False)

        features = rng.randn(num_classes*num_per_class, 2) \
            * np.array([radial_std, tangential_std])
        features[:, 0] += 1.
        labels = np.repeat(np.arange(num_classes), num_per_class)

        angles = rads[labels] + rate * np.exp(features[:, 0])
        rotations = np.stack([np.cos(angles), -np.sin(angles), np.sin(angles), np.cos(angles)])
        rotations = np.reshape(rotations.T, (-1, 2, 2))

        return 2 * rng.permutation(np.einsum("ti,tij->tj", features, rotations)).astype("float32")

    elif data == "2spirals":
        n = np.sqrt(rng.rand(batch_size // 2, 1)) * 540 * (2 * np.pi) / 360
        d1x = -np.cos(n) * n + rng.rand(batch_size // 2, 1) * 0.5
        d1y = np.sin(n) * n + rng.rand(batch_size // 2, 1) * 0.5
        x = np.vstack((np.hstack((d1x, d1y)), np.hstack((-d1x, -d1y)))) / 3
        x += rng.randn(*x.shape) * 0.1
        return x.astype("float32")

    elif data == "checkerboard":
        x1 = np.random.rand(batch_size) * 4 - 2
        x2_ = np.random.rand(batch_size) - np.random.randint(0, 2, batch_size) * 2
        x2 = x2_ + (np.floor(x1) % 2)
        return np.concatenate([x1[:, None], x2[:, None]], 1).astype("float32") * 2

    elif data == "line":
        x = rng.rand(batch_size)
        #x = np.arange(0., 1., 1/batch_size)
        x = x * 5 - 2.5
        y = x #- x + rng.rand(batch_size)
        return np.stack((x, y), 1).astype("float32")
    elif data == "line-noisy":
        x = rng.rand(batch_size)
        x = x * 5 - 2.5
        y = x + rng.randn(batch_size)
        return np.stack((x, y), 1).astype("float32")
    elif data == "cos":
        x = rng.rand(batch_size) * 6 - 3
        y = np.sin(x*5) * 2.5 + np.random.randn(batch_size) * .3
        return np.stack((x, y), 1).astype("float32")
    elif data == "joint_gaussian":
        x2 = torch.distributions.Normal(0., 4.).sample((batch_size, 1))
        x1 = torch.distributions.Normal(0., 1.).sample((batch_size, 1)) + (x2**2)/4

        return torch.cat((x1, x2), 1)
    elif data == "woodStructural":
        log_lik = 0.
        z0 = torch.distributions.log_normal.LogNormal(5., 1., validate_args=None).sample((batch_size, 1))
        log_lik += torch.distributions.log_normal.LogNormal(5., 1., validate_args=None).log_prob(z0)

        z1 = torch.distributions.log_normal.LogNormal(-2., 1., validate_args=None).sample((batch_size, 1))
        log_lik += torch.distributions.log_normal.LogNormal(-2., 1., validate_args=None).log_prob(z1)

        z2 = torch.distributions.Normal(loc=torch.tanh(z0 + z1 - 2.8), scale=.1).sample()
        log_lik += torch.distributions.Normal(loc=torch.tanh(z0 + z1 - 2.8), scale=.1).log_prob(z2)

        z3 = torch.distributions.Normal(loc=z0 * z1, scale=.1).sample()
        log_lik += torch.distributions.Normal(loc=z0 * z1, scale=.1).log_prob(z3)

        z4 = torch.distributions.Normal(loc=7, scale=2.).sample((batch_size, 1))
        log_lik += torch.distributions.Normal(loc=7, scale=2.).log_prob(z4)

        z5 = torch.distributions.Normal(loc=torch.tanh(z3 + z4), scale=.1).sample()
        log_lik += torch.distributions.Normal(loc=torch.tanh(z3 + z4), scale=.1).log_prob(z5)

        x0 = torch.distributions.Normal(loc=z3, scale=.1).sample()
        log_lik += torch.distributions.Normal(loc=z3, scale=.1).log_prob(x0)

        x1 = torch.distributions.Normal(loc=z5, scale=.1).sample()
        log_lik += torch.distributions.Normal(loc=z5, scale=.1).log_prob(x1)

        z = torch.cat((z0, z1, z2, z3, z4, z5, x0, x1), 1)
        #print(log_lik.mean() + torch.log(z.std(0)).sum())
        #exit()
        return z
    else:
        return inf_train_gen("8gaussians", rng, batch_size)

def getA(toy):
    if toy == "8-MIX":
        A = torch.zeros(16, 16)
        for i in range(8):
            A[i * 2, 2*i+1] = 1
    elif toy == "7-MIX":
        A = torch.zeros(14, 14)
        for i in range(7):
            A[i * 2, 2 * i + 1] = 1

    elif toy == "3-MIX_DEP":
        A = torch.zeros(6, 6)
        for i in range(2):
            A[i * 2, 2 * i + 1] = 1
        A[4, 0] = 1
        A[4, 2] = 1
        A[4, 1] = 1
        A[4, 3] = 1
        A[5, 0] = 1
        A[5, 2] = 1
        A[5, 1] = 1
        A[5, 3] = 1
    elif toy == "woodStructural":
        A = torch.zeros(8, 8)
        A[2, 0] = 1
        A[2, 1] = 1
        A[3, 0] = 1
        A[3, 1] = 1
        A[5, 3] = 1
        A[5, 4] = 1
        A[6, 3] = 1
        A[7, 5] = 1
    elif toy == "7-pyramid":
        A = torch.zeros(7, 7)
        A[0, 1] = 1
        A[2, 3] = 1
        A[4, 0] = 1
        A[4, 1] = 1
        A[5, 2] = 1
        A[5, 3] = 1
        A[6, 4] = 1
        A[6, 5] = 1
    elif toy == '1Normal':
        A = torch.zeros(1, 1)
    else:
        return None
    return A


# -*- encoding: utf-8 -*-
##############################################################################
#
#    Copyright (C) 2014 Instituto de Desarrollo Urbano (<http://www.idu.gov.co>). All Rights Reserved
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published by
#    the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
##############################################################################

from osv import fields, osv
import gitlab
from openerp import SUPERUSER_ID
from datetime import datetime
from datetime import timedelta

def get_connection_gitlab(param_model, cr):
    token = param_model.get_param(cr, SUPERUSER_ID, 'gitlab.token', default=False)
    host = param_model.get_param(cr, SUPERUSER_ID, 'gitlab.host', default=False)
    c = gitlab.Gitlab(host, token)
    return c

class res_users(osv.osv):
    _inherit = "res.users"
    _name = "res.users"

    _columns = {
        'gitlab_id': fields.integer('GitLab ID'),
    }

class gitlab_issue(osv.osv):
    _inherit = [
        'mail.thread',
    ]

    def _get_url(self, cr, uid, ids, field, args, context=None):
        res = {}
        records = self.browse(cr, uid, ids, context=context)
        for record in records:
            res[record.id] = "{0}/issues/{1}".format(record.project_id.url, record.gitlab_idd)
        return res

    def _get_color(self, cr, uid, ids, field, args, context=None):
        res = {}
        records = self.browse(cr, uid, ids, context=context)
        for record in records:
            color = 0
            if record.state == 'closed':
                color = 1
            elif record.state == 'reopened':
                color = 2
            res[record.id] = color
        return res

    _name = "gitlab.issue"
    _order = "sequence, name, id"
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1, track_visibility='onchange',),
        'sequence': fields.integer('Sequence', help="Gives the sequence order when displaying a list."),
        'due_date': fields.datetime('Due Date', track_visibility='onchange',),
        'start_date': fields.datetime('Start Date', track_visibility='onchange',),
        'finish_date': fields.datetime('Finish Date', track_visibility='onchange',),
        'state': fields.selection(
            [('opened','Open'), ('closed','Closed'), ('reopened','Re-Opened')],
            'Estado',
            required=True,
            track_visibility='onchange',
        ),
        'gitlab_id': fields.integer('GitLab ID', track_visibility='onchange',),
        'gitlab_idd': fields.integer('GitLab idd', track_visibility='onchange',),
        'stage_id': fields.many2one('gitlab.stage', 'Stage', track_visibility='onchange',),
        'milestone_id': fields.many2one('gitlab.milestone', 'Milestone', track_visibility='onchange',),
        'project_id': fields.many2one('gitlab.project', 'Project', track_visibility='onchange',),
        'label_ids': fields.many2many('gitlab.label', 'gitlab_issue_label', 'issue_id', 'label_ids', 'Labels'),
        'user_id': fields.many2one('res.users', 'User', track_visibility='onchange',),
        'task_id': fields.many2one('project.task', 'Task', track_visibility='onchange',),
        'url': fields.function(
            _get_url,
            type="char",
            string='GitLab URL',
            store=False,
        ),
        'kanban_color': fields.function(
            _get_color,
            type="integer",
            string='Kanban Color',
            store=True,
        ),
    }

    def kanban_columns(self, cr, uid, ids, domain, read_group_order=None, access_rights_uid=None, context=None):
        stage_model = self.pool.get('gitlab.stage')
        column_ids = stage_model.search(cr, uid, [], context=context)
        columns = stage_model.name_get(cr, uid, column_ids, context=context)
        return columns, {}

    _group_by_full = {
        'stage_id': kanban_columns,
    }

    def set_due_date(self, cr, uid, ids, *args):
        """Set due date tomorrow
        """
        tomorrow = datetime.now() + timedelta(days=1)
        return self.write(cr, uid, ids, {'due_date': tomorrow.strftime("%Y-%m-%d %H:%M:%S")})

    def write(self, cr, uid, ids, vals, context=None):
        if len(vals) == 1 and (vals.get('stage_id') or vals.get('user_id')):
            gitlab_conn = get_connection_gitlab(self.pool.get('ir.config_parameter'), cr)
            if vals.get('stage_id'):
                stage = self.pool.get('gitlab.stage').browse(cr, uid, vals.get('stage_id'), context=context)
                status = stage.set_issue_status
                for issue in self.browse(cr, uid, ids, context=context):
                    labels = 'col:' + stage.name
                    for label in issue.label_ids:
                        labels = labels + ',' + label.name
                    res = gitlab_conn.editissue(issue.project_id.gitlab_id, issue.gitlab_id, labels = labels, state_event = status)
                    if res:
                        vals['state'] = res['state']
                        if status == 'close' and not issue.finish_date:
                            vals['finish_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                        if status == 'reopen' and not issue.start_date:
                            vals['start_date'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                        if status == 'reopen':
                            vals['finish_date'] = False

            if vals.get('user_id'):
                user = self.pool.get('res.users').browse(cr, uid, vals.get('user_id'), context=context)
                for issue in self.browse(cr, uid, ids, context=context):
                    res = gitlab_conn.editissue(issue.project_id.gitlab_id, issue.gitlab_id, assignee_id = user.gitlab_id)
        return super(gitlab_issue, self).write(cr, uid, ids, vals, context)

class gitlab_stage(osv.osv):
    _name = "gitlab.stage"
    _order = 'sequence ASC, name ASC'
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1),
        'sequence': fields.integer('Sequence', help="Gives the sequence order when displaying a list."),
        'issue_ids': fields.one2many('gitlab.issue', 'stage_id', 'Issues'),
        'set_issue_status': fields.selection(
            [('close','Close'), ('reopen','Re-Open')],
            'Issue set status to',
        ),
    }

class gitlab_project(osv.osv):
    _name = "gitlab.project"
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1),
        'namespace': fields.char('Namespace', size=255, required=True, select=1),
        'url': fields.char('url', size=255),
        'gitlab_id': fields.integer('GitLab ID'),
        'issue_ids': fields.one2many(
            'gitlab.issue',
            'project_id',
            'Issues',
       ),
    }

class gitlab_milestone(osv.osv):
    _name = "gitlab.milestone"
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1),
        'gitlab_id': fields.integer('GitLab ID'),
        'gitlab_idd': fields.integer('GitLab idd'),
        'issue_ids': fields.one2many(
            'gitlab.issue',
            'milestone_id',
            'Issues',
       ),
    }

class gitlab_label(osv.osv):
    _name = "gitlab.label"
    _columns = {
        'name': fields.char('Name', size=255, required=True, select=1),
        'issue_ids': fields.many2many('gitlab.issue', 'gitlab_issue_label', 'label_ids','issue_id','Issues'),
    }

"""
@startuml
Title gitlab integración openerp
class gitlab.issue
class gitlab.milestone
class gitlab.project

class project.task
class project.project

class res.user

gitlab.issue "*" -up- res.user
gitlab.issue "*" -- project.task
gitlab.milestone -- "*" gitlab.issue
gitlab.project -- "*" gitlab.issue
gitlab.issue "*" -- "*" gitlab.label

@enduml

"""


import functools
from typing import Callable

def check_permission(user: str) -> Callable:
    def check(func) -> Callable:
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            try:
                if user in user_permissions:
                    result = func(*args, **kwargs)
                    return result
                else:
                    raise PermissionError
            except PermissionError:
                print('{}: У пользователя недостаточно прав, чтобы выполнить функцию {}'.format(
                    PermissionError.__name__,
                    func.__name__))

        return wrapper

    return check

user_permissions = ['admin']

@check_permission('admin')
def delete_site():
    print('Удаляем сайт')

@check_permission('user_1')
def add_comment():
    print('Добавляем комментарий')

delete_site()
add_comment()


# -*- coding: utf-8 -*-
''' Module for running ZFS command
'''

# Import Python libs
import logging

# Some std libraries that are made
# use of.
import re
import sys

# Import Salt libs
import bonneville.utils
import bonneville.utils.decorators as decorators
import bonneville.modules.cmdmod as salt_cmd

log = logging.getLogger(__name__)

# Function alias to set mapping. Filled
# in later on.
__func_alias__ = {}

@decorators.memoize
def _check_zfs():
    '''
    Looks to see if zfs is present on the system.
    '''
    # Get the path to the zfs binary.
    return bonneville.utils.which('zfs')

def _available_commands():
    '''
    List available commands based on 'zfs help'. Returns a dict.
    '''
    zfs_path = _check_zfs()
    if not zfs_path:
        return False

    _return = {}
    # Note that we append '|| :' as a unix hack to force return code to be 0.
    res = salt_cmd.run_all('{0} help || :'.format(zfs_path))

    # This bit is dependent on specific output from `zfs help` - any major changes
    # in how this works upstream will require a change.
    for line in res['stderr'].splitlines():
        if re.match('	[a-zA-Z]', line):
            cmds = line.split(' ')[0].split('|')
            doc = ' '.join(line.split(' ')[1:])
            for cmd in [cmd.strip() for cmd in cmds]:
                if cmd not in _return:
                    _return[cmd] = doc
    return _return

def _exit_status(retcode):
    '''
    Translate exit status of zfs
    '''
    ret = {0: 'Successful completion.',
           1: 'An error occurred.',
           2: 'Usage error.'
          }[retcode]
    return ret

def __virtual__():
    '''
    Makes sure that ZFS is available.
    '''
    if _check_zfs():
        return 'zfs'
    return False

def _add_doc(func, doc, prefix='\n\n    '):
    if not func.__doc__:
        func.__doc__ = ''
    func.__doc__ += '{0}{1}'.format(prefix, doc)

def _make_function(cmd_name, doc):
    '''
    Returns a function based on the command name.
    '''
    def _cmd(*args):
        # Define a return value.
        ret = {}

        # Run the command.
        res = salt_cmd.run_all(
                '{0} {1} {2}'.format(
                    _check_zfs(),
                    cmd_name,
                    ' '.join(args)
                    )
                )

        # Make a note of the error in the return object if retcode
        # not 0.
        if res['retcode'] != 0:
            ret['Error'] = _exit_status(res['retcode'])

        # Set the output to be splitlines for now.
        ret = res['stdout'].splitlines()

        return ret

    _add_doc(_cmd, 'This function is dynamically generated.', '\n    ')
    _add_doc(_cmd, doc)
    _add_doc(_cmd, '\n    CLI Example:\n\n')
    _add_doc(_cmd, '\n        salt \'*\' zfs.{0} <args>'.format(cmd_name))

    # At this point return the function we've just defined.
    return _cmd

# Run through all the available commands
if _check_zfs():
    available_cmds = _available_commands()
    for available_cmd in available_cmds:

        # Set the output from _make_function to be 'available_cmd_'.
        # ie 'list' becomes 'list_' in local module.
        setattr(
                sys.modules[__name__],
                '{0}_'.format(available_cmd),
                _make_function(available_cmd, available_cmds[available_cmd])
                )

        # Update the function alias so that salt finds the functions properly.
        __func_alias__['{0}_'.format(available_cmd)] = available_cmd


from django.views.generic import View, ListView, TemplateView, DetailView
from django.shortcuts import render
from django.http import JsonResponse, HttpResponse, QueryDict
from django.urls import reverse

from .models import students
from .form import StuModelForm

class AddStu(TemplateView):
    template_name = 'stuadd_jq.html'

class ListStu(ListView):
    template_name = 'stulist_jq.html'
    model = students
    context_object_name = "all_students"
    '''通过上面3行，就可以将数据从数据库中全部拿到，然后渲染到stulist_jq.html中'''

    def post(self, request):
        print(request.POST)
        StuFormdata = StuModelForm(request.POST)
        if StuFormdata.is_valid():
            StuFormdata.save() #将通过表单验证的数据保存到数据库中
            res = {'code':0, 'msg':'用户添加成功', 'next_url':reverse('students:jlist')}
        else:
            #print(StuFormdata.errors.as_json())
            print(StuFormdata.errors.as_text())
            res = {'code':1, 'errmsg':StuFormdata.errors.as_text(), 'next_url':reverse('students:jadd')}
        #return render(request, 'stulist.html')
        return JsonResponse(res)

    '''删除用户'''
    def delete(self, request):
        data = QueryDict(request.body).dict() #通过前端ajax将ID传递到后端
        print(data)
        pk = data.get('id')
        try:
            user = self.model.objects.filter(pk=pk)
            user.delete()
            res = {'code':0, 'result':'删除用户成功'}
        except:
            res = {'code':1, 'errmsg':'删除用户失败'}
        return JsonResponse(res, safe=True)

class DetailStu(DetailView):
    template_name = 'stuupdate_jq.html'
    model = students
    context_object_name = "stu"

    def get_object(self, queryset=None):
        obj = super(DetailStu, self).get_object()
        return obj

    def post(self, request, **kwargs):
        print(request.POST)
        print(kwargs) #例子 {'pk': '3'}
        user = students.objects.get(pk=kwargs['pk'])
        StuFormdata = StuModelForm(request.POST, instance=user)  #modelform对象instance是什么作用？
        if StuFormdata.is_valid():
            try:
                print(StuFormdata)
                StuFormdata.save()#将数据保存到数据库
                res = {'code':0, 'result': '用户更新成功', 'next_url':reverse('students:jlist')}
            except:
                res = {'code':1, 'errmsg': '用户更新失败', 'next_url':reverse('students:jlist')}
        else:
            print(StuFormdata.errors)
            res = {'code': 1, 'errmsg': StuFormdata.errors, 'next_url': reverse('students:jlist')}
        return render(request,'jump_jq.html',res)

# class AddStu(View):
#     def get(self, request):
#         return render(request, 'stuadd_jq.html')
#
#     def post(self, request):
#         print(request.POST)
#         return render(request, 'stuadd_jq.html')


from pytube import YouTube

link = input("Paste a youtube video link here: ")

path = input(" Please insert a path to save your video: ")

yt = YouTube(link)

result = {

    "Title": yt.title,
    "Views": yt.views,
    "Length": yt.length,
    "Rating": yt.rating

}

print(result)

ys = yt.streams.get_highest_resolution()

print ("Downloading your file... Please wait!")

ys.download(path)

print("Your file is ready to use! Thanks for using the app! Follow me on my GitHub.")

# A basic YouTube Video Downloader with python!
# My GitHub: https://github.com/Madeline-Lea


from tkinter import Tk, FALSE, Button
import tkinter
from tkinter import *
from tkinter.messagebox import *
from Table_py import *
from PIL import Image, ImageTk #pip install pillow
from time import sleep

# Arseny: too many global variables
# Arseny: you will got problem on debug
# Arseny: pack your variables in one class or object for state
form1 = tkinter.Tk()
form1.title('form1')
form1.resizable(width=FALSE, height=FALSE)
form1.geometry('1300x662+100+100')
image = Image.open("image//suit.jpg")
suit = ImageTk.PhotoImage(image)

# Arseny: buf1, buf2, buf3 what is this?
# Arseny: the names of variables don't mean anything
# Arseny: I can't reach their purpose looking on the name. It's wrong
buf1 = StringVar()
buf2 = StringVar()
buf3 = StringVar()
buf1.set('YOUR NAME')
user_name=server_port = tkinter.Entry(form1,textvariable=buf1)
user_name.place(relx=0.20, rely=0.05, relwidth=0.07, relheight=0.03)
buf2.set('IP')
user_ip=server_port = tkinter.Entry(form1,textvariable=buf2)
user_ip.place(relx=0.30, rely=0.05, relwidth=0.06, relheight=0.03)
buf3.set('PORT')
server_port = tkinter.Entry(form1,textvariable=buf3)
server_port.place(relx=0.40, rely=0.05, relwidth=0.04, relheight=0.03)
user_connect = Button(text='connect', command=connect_click)
user_connect.place(relx=0.45, rely=0.05, relwidth=0.04, relheight=0.03)

def add_card_on_center(master, delta):
        bank = Button(text='bank of casino')
        bank.place(relx=0.3, rely=0.25, relwidth=0.09, relheight=0.03)
        card_server = Button(text='card')
        card_server.place(relx=delta+0.02, rely=0.31, relwidth=0.04, relheight=0.13)
        cards_center.append(card_server)
        return cards_center

def place_for_user(user, x, y):
    #suit = ImageTk.PhotoImage('lena.ppm')
    bank = Button(text='bank of '+user)
    bank.place(relx=x, rely=y, relwidth=0.09, relheight=0.03)

    card1 = Button(text='card1', image=suit)
    card1.place(relx=x, rely=y+0.04, relwidth=0.04, relheight=0.13)

    card2 = Button(text='card2', image=suit)
    card2.place(relx=x+0.05, rely=y+0.04, relwidth=0.04, relheight=0.13)

    bet_bank = Button(text='bet', state='disable')
    bet_bank.place(relx=x+0.11, rely=y+0.02, relwidth=0.04, relheight=0.03)

    raise_bet = Button(text='raise bet', command=raise_bet_click, state='disable')
    raise_bet.place(relx=x+0.11, rely=y+0.06, relwidth=0.04, relheight=0.03)

    pass_button = Button(text='pass', command=pass_button_click, state='disable')
    pass_button.place(relx=x+0.11, rely=y+0.1, relwidth=0.04, relheight=0.03)

    check = Button(text='check', command=check_click, state='disable')
    check.place(relx=x+0.11, rely=y+0.14, relwidth=0.04, relheight=0.03)

    user = Button(text='user '+user)
    user.place(relx=x, rely=y+0.19, relwidth=0.09, relheight=0.03)

    # Arseny: why do you use `dict` for scructs in your code?
    # Arseny: you should use object for this
    return {'bank': bank, 'raise_bet': raise_bet, 'bet': bet_bank, 'card1': card1,
            'card2': card2, 'check': check, 'user': user}

# Arseny: you have section global variables on the top of code
# Arseny: and then new section there. Thats hard to read and debug
cards_center = []
croupier = {'bank': 0, 'cards': cards_center}
player_places = []
size = 0.25
delta_center = 0
cards_center = add_card_on_center(form1, size+delta_center)

position = 3
shift = 0

x_default = 0.02
y_default = 0.75
user_name = {'1': 'player 1', '2': 'user 2', '3': 'player 3', '4': 'player 4', '5': 'player 5', '6': 'player 6', '7':
                    'player 7' }

player_places.append(place_for_user(user_name.get('1'), x_default, y_default-0.6))
player_places.append(place_for_user(user_name.get('2'), x_default, y_default-0.3))
player_places.append(place_for_user(user_name.get('3'), x_default, y_default))
player_places.append(place_for_user(user_name.get('4'), x_default+0.3, y_default))
player_places.append(place_for_user(user_name.get('5'), x_default+0.6, y_default))
player_places.append(place_for_user(user_name.get('6'), x_default+0.6, y_default-0.3))
player_places.append(place_for_user(user_name.get('7'), x_default+0.6, y_default-0.6))

#example for add some cards for the second player
# this wil be dict, where number of range of card will be map to face's card

# Arseny: bad names again. Why not `suite_bking`? Why `suit3`?
image = Image.open("image//king_black.jpg")
suit3 = ImageTk.PhotoImage(image)
image = Image.open("image//five_red.jpg")
suit4 = ImageTk.PhotoImage(image)

player_places[1].get('card1').config(image=suit3)
player_places[1].get('card2').config(image=suit4)
card = (croupier.get('cards'))[0]
card.config(image=suit3)

form1.mainloop()

#case with add new card in dynamic
#this code will be work only with new thred
# will be tinking
sleep(0.01)
cards_center = add_card_on_center(form1, size+0.02)
card = (croupier.get('cards'))[1]
card.config(image=suit3)
print('ok')
form1.mainloop()


from blinker import Namespace
from signals import executionSignals as execution

class Execution():
    def __init__(self):
        self.signals = Namespace()

        self.start = self.signals.signal("start")
        self.start.connect(execution.start)

        self.stop = self.signals.signal("stop")
        self.stop.connect(execution.stop)

        self.pause = self.signals.signal("pause")
        self.pause.connect(execution.pause)

        self.shutdown = self.signals.signal("shutdown")
        self.shutdown.connect(execution.shutdown)

    #Actions
    def post(self, action, **kwargs):
        if action == "start":
            self.start.send(self)
        elif action == "stop":
            self.stop.send(self)
        return {}


import numpy as np
import csv
import urllib3
import os

def maybe_download(filename, source_url, work_directory):
    """Download the filename from the source url into the working directory if it does not exist."""
    if not os.path.exists(work_directory):
        os.makedirs(work_directory)
    filepath = os.path.join(work_directory, filename)
    if not os.path.exists(filepath):
        urllib3.disable_warnings()
        with urllib3.PoolManager() as http:
            r = http.request('GET', source_url)
            with open(filepath, 'wb') as fout:
                fout.write(r.read())

        print('{} succesfully downloaded'.format(filename))
    return filepath

def write_file(file_path, text_path, num_tokens):
    total_num_tokens = 0
    print(f'Writing to {file_path}...')
    with open(file_path, 'w', encoding='utf-8') as f_out:
        writer = csv.writer(f_out)
        with open(text_path, 'r') as f:
            for i, text in enumerate(f):

                writer.writerow([text])
                # f_out.write(text)

                # calculate approximate length based on tokens
                total_num_tokens += len(text.split())
                if total_num_tokens > num_tokens:
                    break
                if i % 10000 == 0:
                    print('Processed {:,} documents. Total # tokens: {:,}.'.format(i, total_num_tokens))
    print('{}. # documents: {:,}. # tokens: {:,}.'.format(file_path, i, total_num_tokens))

if __name__=='__main__':
    write_file('assets/val.csv','assets/wikitext-103/valid.txt',1000000)
    write_file('assets/train.csv', 'assets/wikitext-103/train.txt', 1000000)


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import flags
import absl.logging as _logging  # pylint: disable=unused-import

import tensorflow as tf
try:
  import google3.experimental.users.zihangd.lm.data_utils as data_utils
  import google3.experimental.users.zihangd.lm.input_function as input_function
  import google3.experimental.users.zihangd.lm.tokenization as tokenization
except ImportError as e:
  import data_utils
  import input_function
  import tokenization

flags.DEFINE_string("record_info_dir", default="./proc_data/example",
                    help="Path to local directory containing *.tfrecord.")
flags.DEFINE_string("tokenizer_type", "sent_piece",
                    help="Type of the tokenizer.")
flags.DEFINE_string("tokenizer_paths", "",
                    help="Comma separated string.")
flags.DEFINE_integer("num_core_per_host", default=16,
                     help="number of cores per host.")
flags.DEFINE_integer("bsz_per_host", default=32, help="batch size per host.")
flags.DEFINE_integer("seq_len", default=512,
                     help="tgt len; 0 for not using it")
flags.DEFINE_bool("lower_case", default=False,
                  help="use lower_case inputs or not.")
flags.DEFINE_string("split", default="train",
                    help="Data split.")
flags.DEFINE_integer("num_passes", default=None,
                     help="Num of passes to use.")

FLAGS = flags.FLAGS

def main(unused_argv):
  del unused_argv  # Unused

  tokenizer = tokenization.get_tokenizer(
      tokenizer_type=FLAGS.tokenizer_type,
      paths=FLAGS.tokenizer_paths.split(","),
      do_lower_case=FLAGS.lower_case)
  data_utils.setup_special_ids(tokenizer)

  input_fn, record_info_dict = input_function.get_input_fn(
      tfrecord_dir=FLAGS.record_info_dir,
      split=FLAGS.split,
      bsz_per_host=FLAGS.bsz_per_host,
      seq_len=FLAGS.seq_len,
      num_hosts=1,
      num_core_per_host=FLAGS.num_core_per_host,
      lower_case=FLAGS.lower_case,
      num_passes=FLAGS.num_passes,
  )
  tf.logging.info(record_info_dict)

  bsz_per_core = FLAGS.bsz_per_host // FLAGS.num_core_per_host
  params = {
      "batch_size": bsz_per_core
  }

  dataset = input_fn(params)
  example = dataset.make_one_shot_iterator().get_next()

  with tf.Session() as sess:
    for i in range(10):
      example_np = sess.run(example)
      if i == 0:
        for key in example_np.keys():
          print(key, example_np[key].shape, len(example_np[key].shape))
          print(example_np[key][0].tolist())
          print(tokenizer.convert_ids_to_text(example_np[key][0].tolist()))
        print("=" * 80)

if __name__ == "__main__":
  tf.app.run()