from datetime import datetime

import pytz as pytz

from bin.rss.RSSFeedGen import RSSFeedGen
import requests
from bs4 import BeautifulSoup

def UnionMangaRssHelper(url):
    response = requests.get(url)
    soup = BeautifulSoup(response.text, 'html.parser')
    list_cap_tags = soup.select('div > div > a[href*="/leitor"]')
    list_cap_date_tags = soup.select('div > div > span[style*="font-size: 10px; color: #999;"]')
    name = soup.find('h2')
    rss = RSSFeedGen()
    rss.set_header(str(name.next), url)
    timezone = pytz.timezone("UTC")

    for i in range(list_cap_tags.__len__()-1, -1, -1):
        cap = list_cap_tags[i]
        cap_date = list_cap_date_tags[i]
        with_timezone = timezone.localize(datetime.strptime(str(cap_date.next), '(%d/%m/%Y)'))
        rss.add_entry(str(cap.next), cap['href'], with_timezone)
    return rss.get_rss()

if __name__ == '__main__':
    UnionMangaRssHelper("https://unionmangas.top/perfil-manga/shingeki-no-kyojin")


from copy import deepcopy, copy
from random import shuffle, choice
from string import ascii_uppercase
from timeit import default_timer as timer
from math import inf, floor
from random import random
from collections import Counter

#BOARD_BG_COLOR_1 = 40
BOARD_BG_COLOR_1 = 43
BOARD_BG_COLOR_2 = 0
WHITE_TEXT_COLOR = 0
BLACK_TEXT_COLOR = 31
COMPARISON_BG_COLOR = 46
DEFAULT_COLOR_CODE = '\x1b[0m'

BOARD_BG_FORMATS = [BOARD_BG_COLOR_1 if (i+int(i/8)) % 2 else BOARD_BG_COLOR_2 for i in range(64)]

# Black pieces have values below; white pieces have negatives of those values
EMPTY = None
ROOK = 1
KNIGHT = 2
BISHOP = 3
QUEEN = 4
KING = 5
BLACK_PAWN_MOVED = 6
BLACK_PAWN_UNMOVED = 7
BLACK = 8
WHITE = -BLACK
WHITE_PAWN_MOVED = -BLACK_PAWN_MOVED
WHITE_PAWN_UNMOVED = -BLACK_PAWN_UNMOVED
PAWNS = [BLACK_PAWN_MOVED, BLACK_PAWN_UNMOVED, WHITE_PAWN_MOVED, WHITE_PAWN_UNMOVED]

PIECE_STRINGS = {ROOK: 'R', KNIGHT: 'N', BISHOP: 'B', QUEEN: 'Q', KING: 'K', BLACK_PAWN_MOVED: 'P',
                 BLACK_PAWN_UNMOVED: 'P'}
FORMATTED_PIECE_STRINGS = {}
for key in list(PIECE_STRINGS.keys()):
    PIECE_STRINGS[-key] = PIECE_STRINGS[key]
    FORMATTED_PIECE_STRINGS[key] = '\x1b[1;' + str(BLACK_TEXT_COLOR) + ';{background}m' + PIECE_STRINGS[key]
    FORMATTED_PIECE_STRINGS[-key] = '\x1b[1;' + str(WHITE_TEXT_COLOR) + ';{background}m' + PIECE_STRINGS[-key]
PIECE_STRINGS[EMPTY] = ' '
FORMATTED_PIECE_STRINGS[EMPTY] = '\x1b[1;0;{background}m' + ' '

PIECE_VALUES = {BLACK_PAWN_MOVED: 1, BLACK_PAWN_UNMOVED: 1, WHITE_PAWN_MOVED: -1, WHITE_PAWN_UNMOVED: -1,
                KNIGHT: 3, -KNIGHT: -3, BISHOP: 3, -BISHOP: -3, ROOK: 5, -ROOK: -5, QUEEN: 9, -QUEEN: -9,
                KING: 200, -KING: -200}

state_counter = 0

def color_code(text, background):
    return '\x1b[1;{text};{background}m'.format(text=text, background=background)

def linearize_xy_coord(xy):
    return xy[1] * 8 + xy[0]

def calculate_move_table():
    def spaces_up(xy):
        return [(xy[0], y) for y in range(xy[1] + 1, 8) if y < 8]

    def spaces_down(xy):
        return [(xy[0], y) for y in range(xy[1] - 1, -1, -1) if y >= 0]

    def spaces_right(xy):
        return [(x, xy[1]) for x in range(xy[0] + 1, 8) if x < 8]

    def spaces_left(xy):
        return [(x, xy[1]) for x in range(xy[0] - 1, -1, -1) if x >= 0]

    def spaces_up_right(xy):
        if xy[0] == 7 or xy[1] == 7:
            return []
        return zip(range(xy[0] + 1, 8), range(xy[1] + 1, 8))

    def spaces_down_right(xy):
        if xy[0] == 7 or xy[1] == 0:
            return []
        return zip(range(xy[0] + 1, 8), range(xy[1] - 1, -1, -1))

    def spaces_up_left(xy):
        if xy[0] == 0 or xy[1] == 7:
            return []
        return zip(range(xy[0] - 1, -1, -1), range(xy[1] + 1, 8))

    def spaces_down_left(xy):
        if xy[0] == 0 or xy[1] == 0:
            return []
        return zip(range(xy[0] - 1, -1, -1), range(xy[1] - 1, -1, -1))

    def diagonal_move_set(xy):
        return [spaces_down_left(xy), spaces_down_right(xy), spaces_up_left(xy),
                spaces_up_right(xy)]

    def straight_move_set(xy):
        return [spaces_up(xy), spaces_down(xy), spaces_left(xy),
                spaces_right(xy)]

    def is_space_on_board(xy):
        return xy[0] >= 0 and xy[0] < 8 and xy[1] >= 0 and xy[1] < 8

    def move_set_from_deltas(xy, deltas):
        move_set = []
        for subset in deltas:
            move_set.append([(xy[0] + d[0], xy[1] + d[1]) for d in subset])
            move_set[-1] = [move_space for move_space in move_set[-1] if is_space_on_board(move_space)]
        return move_set

    def linearize_move_set(move_set):
        return [[linearize_xy_coord(xy) for xy in subset] for subset in move_set]

    def piece_move_set(piece, xy):
        if piece == BLACK_PAWN_UNMOVED:
            deltas = [[(0, -1), (0, -2)], [(-1, -1)], [(1, -1)]]
            move_set = move_set_from_deltas(xy, deltas)

        elif piece == BLACK_PAWN_MOVED:
            deltas = [[(0, -1)], [(-1, -1)], [(1, -1)]]
            move_set = move_set_from_deltas(xy, deltas)

        elif piece == WHITE_PAWN_UNMOVED:
            deltas = [[(0, 1), (0, 2)], [(-1, 1)], [(1, 1)]]
            move_set = move_set_from_deltas(xy, deltas)

        elif piece == WHITE_PAWN_MOVED:
            deltas = [[(0, 1)], [(-1, 1)], [(1, 1)]]
            move_set = move_set_from_deltas(xy, deltas)

        elif piece == ROOK:
            move_set = straight_move_set(xy)

        elif piece == KNIGHT:
            move_set = move_set_from_deltas(xy, [[(1, 2)], [(2, 1)], [(2, -1)], [(1, -2)], [(-1, -2)], [(-2, -1)],
                                                        [(-2, 1)], [(-1, 2)]])

        elif piece == BISHOP:
            move_set = diagonal_move_set(xy)

        elif piece == QUEEN:
            move_set = straight_move_set(xy) + diagonal_move_set(xy)

        elif piece == KING:
            move_set = move_set_from_deltas(xy, [[(0, 1)], [(1, 1)], [(1, 0)], [(1, -1)], [(0, -1)], [(-1, -1)],
                                                        [(-1, 0)], [(-1, 1)]])

        return move_set

    all_pieces = [WHITE_PAWN_UNMOVED, WHITE_PAWN_MOVED, BLACK_PAWN_UNMOVED, BLACK_PAWN_MOVED, KNIGHT, BISHOP, QUEEN,
                  KING, ROOK]

    move_table = {}
    for piece in all_pieces:
        move_table[piece] = {}
        for xy in [(x, y) for x in range(8) for y in range(8)]:
            move_table[piece][linearize_xy_coord(xy)] = linearize_move_set(piece_move_set(piece, xy))

    return move_table

move_table = calculate_move_table()

end_row_piece_order = (ROOK, KNIGHT, BISHOP, KING, QUEEN, BISHOP, KNIGHT, ROOK)
starting_state = [-piece for piece in end_row_piece_order] + \
                 [WHITE_PAWN_UNMOVED]*8 + \
                 [EMPTY]*32 + \
                 [BLACK_PAWN_UNMOVED]*8 + \
                 list(end_row_piece_order) + \
                 [BLACK]     # Active player

starting_state = tuple(starting_state)
full_piece_list = [piece for piece in starting_state[:-1] if piece is not None]
full_piece_counter = Counter(full_piece_list)

MATERIAL_INDEX = 0
PAWN_INDEX = 1
MOBILITY_INDEX = 2
EVALUATION_INDEX = 3
SEARCH_VALUE_INDEX = 4
SEARCH_DEPTH_INDEX = 5
TABLE_COL_NUM = 6
state_table = {}

def random_state():
    global starting_state
    board = list(starting_state)[:-1]
    shuffle(board)
    state = board+[WHITE if random()<0.5 else BLACK]

    return tuple(state)

def are_same_player(piece_a, piece_b):
    return (piece_a < 0) == (piece_b < 0)

def are_opponents(piece_a, piece_b):
    return not are_same_player(piece_a, piece_b)

def child_from_move(state, from_index, to_index, add_to_table):
    global state_counter, state_table

    if state[to_index] is not None:
        if (are_same_player(state[from_index], state[to_index]) or
                (state[from_index] in PAWNS and abs(from_index - to_index) % 8 == 0)):
            # Trying to move into a spot occupied by player's own piece, or
            # Pawn trying to capture straight ahead
            return None
    elif state[from_index] in PAWNS and abs(from_index - to_index) % 8 != 0:
        # Pawn is trying to move diagonally without a capture
        return None

    child = list(state)
    state_counter += 1

    if child[from_index] == WHITE_PAWN_UNMOVED:
        child[to_index] = WHITE_PAWN_MOVED
    elif child[from_index] == BLACK_PAWN_UNMOVED:
        child[to_index] = BLACK_PAWN_MOVED
    else:
        child[to_index] = child[from_index]

    child[from_index] = None
    child[-1] = -child[-1]

    child = tuple(child)
    if add_to_table:
        if child not in state_table.keys():
            state_table[child] = [None for _ in range(TABLE_COL_NUM)]

            child_material_value = state_table[state][MATERIAL_INDEX]
            if state[to_index] is not EMPTY:
                child_material_value -= PIECE_VALUES[state[to_index]]

            state_table[child][MATERIAL_INDEX] = child_material_value

    return child

def children_from_piece(state, piece_index, add_to_table):
    move_set = move_table[abs(state[piece_index]) if state[piece_index] not in PAWNS else state[piece_index]][
        piece_index]
    children = []

    for move_direction in move_set:
        for move_to_index in move_direction:

            child = child_from_move(state, piece_index, move_to_index, add_to_table)
            if child is not None:
                children.append(child)
            if state[move_to_index] is not None:
                break
    return children

def get_children(state, add_to_table=True):

    children = []
    for square_index in range(64):
        if state[square_index] is not None and are_same_player(state[square_index], state[-1]):
            children += children_from_piece(state, square_index, add_to_table)

    return children

def get_captures(state, stringify=False):
    current_pieces = []
    for piece in state[:-1]:
        if piece is not None:
            if piece == BLACK_PAWN_MOVED:
                current_pieces.append(BLACK_PAWN_UNMOVED)
            elif piece == WHITE_PAWN_MOVED:
                current_pieces.append(WHITE_PAWN_UNMOVED)
            else:
                current_pieces.append(piece)
    current_pieces = Counter(current_pieces)
    captured_pieces = full_piece_counter - current_pieces

    captures = {WHITE: [], BLACK: []}
    for piece in captured_pieces.elements():
        captures[WHITE if piece < 0 else BLACK].append(piece if not stringify else FORMATTED_PIECE_STRINGS[
            piece].format(background=0))

    return captures

def get_board_string(state, bg_formats=BOARD_BG_FORMATS):

    captures = get_captures(state, stringify=True)
    board_string = color_code(BLACK_TEXT_COLOR, 0) + '•' if state[-1] == BLACK else ' '
    board_string += ' '.join(captures[WHITE]) + '\n'

    for y in range(7, -1, -1):
        board_string += DEFAULT_COLOR_CODE + str(y + 1) + ' '
        for x in range(0, 8):
            linear_board_index = linearize_xy_coord((x, y))
            board_string += FORMATTED_PIECE_STRINGS[state[linear_board_index]].\
                format(background=bg_formats[linear_board_index]) + ' '
        board_string += DEFAULT_COLOR_CODE + '\n'

    board_string += '  ' + ' '.join([ascii_uppercase[i] for i in range(0, 8)]) + '\n'

    board_string += color_code(WHITE_TEXT_COLOR, 0) + '•' if state[-1] == WHITE else ' '
    board_string += ' '.join(captures[BLACK]) + DEFAULT_COLOR_CODE

    return board_string

def print_board(state):
    print(get_board_string(state))

def print_board_compare(board_a, board_b):
    """ Print board_a, highlighting any differences from board_b """

    board_bg_formats = [COMPARISON_BG_COLOR if board_a[i] != board_b[i] else BOARD_BG_FORMATS[i]
                        for i in range(64)]

    print(get_board_string(board_a, bg_formats=board_bg_formats))

def material_value(state):

    value = 0
    for i in state[:-1]:
        if i is not None:
            value += PIECE_VALUES[i]

    return value

def pawn_score(state):

    def doubled_pawns(pawns_in_file):
        return len([_ for _ in pawns_in_file if _ > 1])

    def isolated_pawns(pawns_in_file):
        num_isolated_pawns = 0
        for file, file_pawns in enumerate(pawns_in_file):
            if not ((file-1 <= 0 or pawns_in_file[file-1] > 0) and (file+1 >= 8 or pawns_in_file[file+1] > 0)):
                num_isolated_pawns += 1
        return num_isolated_pawns

    white_pawns_in_file = [0 for _ in range(8)]
    black_pawns_in_file = [0 for _ in range(8)]
    white_blocked_pawns = 0
    black_blocked_pawns = 0

    for piece_index, piece in enumerate(state):

        if piece == BLACK_PAWN_MOVED or piece == BLACK_PAWN_UNMOVED:
            black_pawns_in_file[piece_index % 8] += 1

            square_below = piece_index - 8
            if square_below >= 0 and state[square_below] is not EMPTY:
                black_blocked_pawns += 1

        elif piece == WHITE_PAWN_MOVED or piece == WHITE_PAWN_UNMOVED:
            white_pawns_in_file[piece_index % 8] += 1

            square_above = piece_index + 8
            if square_above < 64 and state[square_above] is not EMPTY:
                white_blocked_pawns += 1

    score = 0.5 * (white_blocked_pawns - black_blocked_pawns +
           doubled_pawns(white_pawns_in_file) - doubled_pawns(black_pawns_in_file) +
           isolated_pawns(white_pawns_in_file) - isolated_pawns(black_pawns_in_file))

    return score

def mobility_score(state):
    if state[-1] == BLACK:
        black_moving = state
        white_moving = copy(list(state))
        white_moving[-1] = WHITE
        white_moving = tuple(white_moving)

    else:
        white_moving = state
        black_moving = copy(list(state))
        black_moving[-1] = BLACK
        black_moving = tuple(black_moving)
    return 0.1*(len(get_children(black_moving, add_to_table=False)) - len(get_children(white_moving,
                                                                                add_to_table=False)))

def evaluate(state):
    global state_table

    table_entry = state_table[state]

    if table_entry[EVALUATION_INDEX] is not None:
        return table_entry[EVALUATION_INDEX]

    if table_entry[MOBILITY_INDEX] is None:
        table_entry[MOBILITY_INDEX] = mobility_score(state)

    if table_entry[PAWN_INDEX] is None:
        table_entry[PAWN_INDEX] = pawn_score(state)

    table_entry[EVALUATION_INDEX] = table_entry[MATERIAL_INDEX] + table_entry[MOBILITY_INDEX] + table_entry[PAWN_INDEX]

    return table_entry[EVALUATION_INDEX]

def ordering_function(state, state_table):
    table_entry = state_table[state]

    if table_entry[PAWN_INDEX] is None:
        table_entry[PAWN_INDEX] = pawn_score(state)

    score = table_entry[PAWN_INDEX] + table_entry[MATERIAL_INDEX]
    return score if state[-1] == BLACK else -score

def is_in_check(state, children):
    # Checks if the player whose turn is in state has her opponent in check
    opponents_king = KING if state[-1] == WHITE else -KING
    for child in children:
        if opponents_king not in child:
            return True
    return False

def alphabeta(state, value_function, ordering_function=ordering_function, forward_prune_schedule=None, \
                                                                                depth_limit=inf, \
                                                                                time_limit=inf):

    def alphabeta_recursive(state, state_table, alpha, beta, depth_left, value_function,
                            ordering_function=ordering_function,
                            forward_prune_schedule=None, start_time=0,
                                      time_limit=inf):

        if time_limit is not inf and timer() - start_time >= time_limit:
            return None, None
        if depth_left == 0 or KING not in state or -KING not in state:
            return state, value_function(state)

        children = get_children(state)

        if not children:
            return state, value_function(state)

        children.sort(key=lambda x: ordering_function(x, state_table))

        if forward_prune_schedule and depth_left <= len(forward_prune_schedule):
            fraction_to_prune = forward_prune_schedule[0]
            number_to_prune = floor(fraction_to_prune*len(children))
            if number_to_prune != 0:
                children = children[:-number_to_prune]

        # Black wants positive values; white wants negative values
        best_child_value = inf if state[-1] == WHITE else -inf
        best_child = children[0]
        depth = depth_left-1

        for child in children:
            if state_table[child][SEARCH_VALUE_INDEX] is not None and state_table[child][SEARCH_DEPTH_INDEX] >= depth:
                child_value = state_table[child][SEARCH_VALUE_INDEX]
            else:
                _, child_value = alphabeta_recursive(child, state_table, alpha, beta, depth,
                                                               value_function,
                                                     forward_prune_schedule=forward_prune_schedule[1:],
                                                                      start_time=start_time,
                                                                      time_limit=time_limit)

                if child_value is None:  # Time limit reached
                    return None, None

                state_table[child][SEARCH_VALUE_INDEX] = child_value
                state_table[child][SEARCH_DEPTH_INDEX] = depth

            if state[-1] == WHITE:

                if child_value < best_child_value:
                    best_child = child
                    best_child_value = child_value

                if child_value <= alpha:
                    break
                elif child_value < beta:
                    beta = child_value

            elif state[-1] == BLACK:
                if child_value > best_child_value:
                    best_child = child
                    best_child_value = child_value

                if child_value >= beta:
                    break
                elif child_value > alpha:
                    alpha = child_value

        return best_child, best_child_value

    global state_table, child_getting_time, pawn_score_time, state_counter
    child_getting_time = 0
    pawn_score_time = 0
    state_counter = 0

    state_table = {state: [None for _ in range(TABLE_COL_NUM)]}
    state_table[state][MATERIAL_INDEX] = material_value(state)

    if depth_limit < inf:

        for depth in range(1, depth_limit+1):

            best_child, best_child_value = alphabeta_recursive(state, state_table, -inf, inf, depth, value_function,
                        forward_prune_schedule=forward_prune_schedule[:depth] if forward_prune_schedule else
                        None)

    elif time_limit < inf:
        start_time = timer()
        depth = 1
        child = state
        child_value = None

        while child is not None:
            best_child = child
            best_child_value = child_value
            child, child_value = alphabeta_recursive(state, state_table, -inf, inf, depth, value_function,
                                forward_prune_schedule=forward_prune_schedule if forward_prune_schedule else None,
                                start_time=start_time, time_limit=time_limit)
            depth += 1

        print('Depth limit reached: ' + str(depth-2))

    if KING not in best_child or -KING not in best_child:
        return None
    else:
        return best_child

def minimax(state, depth_left, value_function, start_time=0, time_limit=inf):

    def recursive_depth_limited_minimax(state, depth_left, value_function, start_time=0, time_limit=inf):

        if time_limit is not inf and timer()-start_time >= time_limit:
            return None, None
        if depth_left == 0 or KING not in state or -KING not in state:
            return state, value_function(state)

        children = get_children(state)

        if not children:
            return state, value_function(state)

        # Black wants positive values; white wants negative values
        best_child_value = inf if state[-1] == WHITE else -inf
        best_child = children[0]

        for child in children:
            _, child_value = recursive_depth_limited_minimax(child, depth_left-1, value_function, start_time=start_time,
                                                             time_limit=time_limit)

            if child_value is None:     # Time limit reached
                return None, None

            if (state[-1] == WHITE and child_value < best_child_value) or \
               (state[-1] == BLACK and child_value > best_child_value):
                best_child_value = child_value
                best_child = child

        return best_child, best_child_value

    child, child_value = recursive_depth_limited_minimax(state, depth_left, value_function, start_time=0,
                                                         time_limit=inf)

    if child_value == (inf if state[-1] == WHITE else -inf):
        return None
    else:
        return child

def random_move_generator(state):
    children = get_children(state)

    # Make sure the king is not in check; if it is, move intelligently; otherwise, move randomly
    players_king = KING if state[-1] == BLACK else -KING

    shuffle(children)

    while children:

        child = children.pop()

        # Make sure King is not in check in the selected child
        grandchildren = get_children(child)
        not_in_check = True
        for grandchild in grandchildren:
            if players_king not in grandchild:
                not_in_check = False
                break

        if not_in_check:
            return child

    return None

def human_move_generator(state):

    def parse_input_xy(text):
        text = text.upper()
        try:
            xy = (ascii_uppercase.index(text[0]), int(text[1])-1)
        except (IndexError, ValueError):
            return None
        if xy[0] in range(0,8) and xy[1] in range(0,8):
            return xy[1]*8+xy[0]
            #return xy
        return None

    while True:
        move_from = input('Move from: ')
        move_from = parse_input_xy(move_from)
        if move_from and state[move_from] is not None \
                    and are_same_player(state[move_from], state[-1]):
            children = children_from_piece(state, move_from, False)
            if children:
                break

    while True:
        move_to = input('to: ')
        move_to = parse_input_xy(move_to)
        if move_to:
            moved_piece = state[move_from]
            if moved_piece == BLACK_PAWN_UNMOVED:
                moved_piece = BLACK_PAWN_MOVED
            elif moved_piece == WHITE_PAWN_UNMOVED:
                moved_piece = WHITE_PAWN_MOVED
            for child in children:
                if child[move_to] == moved_piece:
                    return child

def play_game(black_move_generator, white_move_generator, show_game_states=False):

    move_generators = {BLACK: black_move_generator, WHITE: white_move_generator}

    state = starting_state
    states = [state]
    if show_game_states:
        print(get_board_string(state))

    while True:
        move_time = timer()
        state = move_generators[state[-1]](state)
        print('Time for move: {:.1f}'.format(timer() - move_time))

        lookups = 0
        if state is None:
            break

        if show_game_states:
            print_board_compare(state, states[-1])

        states.append(state)

    if show_game_states:
        winner_str = 'Black ' if states[-1][-1] == WHITE else 'White'
        print(winner_str + ' won!')

    return states

def computer_move_generator(state):
    print('Computer is moving...')
    return alphabeta(state, evaluate, depth_limit=5, forward_prune_schedule=[0, 0, 0, 0.25, 0.25])

play_game(computer_move_generator, human_move_generator, show_game_states=True)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat 19 Oct 2019 02:34:47 AM EDT

@author: 77ph
"""
### версия использовавшаяся в тестах netem

import requests
import json

auth_token="127859963:525cec3227216c14addebfc4241793c6ef4e"
url="https://api.contest.com/voip" + auth_token + "/getConnection?call={call}"
querystring=""
r=requests.get(url, params=querystring)
acc = r.json()
print(json.dumps(acc, indent=2))

if acc['ok'] == True:
        cfg = acc["result"]["config"]
        encryption_key = acc["result"]["encryption_key"]
        id = acc["result"]["endpoints"][0]["id"]
        ipv4 = acc["result"]["endpoints"][0]["ip"]
        port = acc["result"]["endpoints"][0]["port"]
        peer_tags_caller = acc["result"]["endpoints"][0]["peer_tags"]["caller"]
        peer_tags_callee = acc["result"]["endpoints"][0]["peer_tags"]["callee"]

        print("----")
        print(cfg)
        print("----")
        print(encryption_key)
        print("----")
        print(id)
        print("----")
        print(ipv4)
        print("----")
        print(port)
        print("----")
        print(peer_tags_caller)
        print("----")
        print(peer_tags_callee)
        print("----")
        hs = open("config.json", 'w')
        hs.write(json.dumps(cfg))
        hs.close()

        #./tgvoipcall 127.0.0.1:555 ssssssssss -k encryption_key_hex -i /path/to/sound_A.ogg -o /path/to/sound_output_B.ogg -c config.json
        ss = open("start_A.sh","w")
        run = "../cpp/tgvoipcall " + ipv4 + ":" + port + " " + peer_tags_caller + " -k " + encryption_key + " -i ../python/sample17.ogg -o ../python/sound_output_B.raw -c ../python/config.json" + " -l " + id + " -r caller"
        ss.write(run)
        ss.close()
        ss = open("start_B.sh","w")
        run = "../cpp/tgvoipcall " + ipv4 + ":" + port + " " + peer_tags_callee + " -k " + encryption_key + " -i ../python/sample17.ogg  -o ../python/sound_output_A.raw -c ../python/config.json" + " -l " + id + " -r callee"
        ss.write(run)
        ss.close()


# This script queries MAST for WFC3 IR data and downloads the data from
# the AWS public dataset rather than from MAST servers.

# Working with http://astroquery.readthedocs.io/en/latest/mast/mast.html
from astroquery.mast import Observations
import boto3
import os
import datetime

t_start = datetime.datetime.now()

# Read in the AWS credentials
import configparser
config = configparser.ConfigParser()
config.read(os.path.expanduser('~/.aws/credentials'))
os.environ["AWS_ACCESS_KEY_ID"] = config.get('default','aws_access_key_id')
os.environ["AWS_SECRET_ACCESS_KEY"] = config.get('default','aws_secret_access_key')

# Create a directory for downloaded files
if not os.path.exists('DATA'):
  os.makedirs('DATA')

# This downloads all the F160W DRZ images from CANDELS in the GOODS-South field
print('\nQuerying the MAST archive.\n')
obsTable = Observations.query_criteria(obs_collection='HST',
                                       filters='F160W',
                                       instrument_name='WFC3/IR',
                                       proposal_id=['12062','12061','12062'])

# Grab the list of available data products for these observations
products = Observations.get_product_list(obsTable)

# Select only drizzled (DRZ) files
filtered = Observations.filter_products(products,
                                        mrp_only=False,
                                        productSubGroupDescription='DRZ')

# Enable 'S3 mode' for module which will return S3-like URLs for FITs files
# e.g. s3://stpubdata/hst/public/icde/icde43l0q/icde43l0q_drz.fits
Observations.enable_s3_hst_dataset()

# Grab the S3 URLs for each of the observations
s3_urls = Observations.get_hst_s3_uris(filtered)

print('Query returned {} entries.\n'.format(len(filtered)))

print('Downloading data from S3.\n')

s3 = boto3.resource('s3')

# Create an authenticated S3 session. Note, download within US-East is free
# e.g. to a node on EC2.
s3_client = boto3.client('s3')

bucket = s3.Bucket('stpubdata')

# For brevity, only download the first three images
for url in s3_urls[:3]:
  # Extract the S3 key from the S3 URL
  fits_s3_key = url.replace("s3://stpubdata/", "")
  root = url.split('/')[-1]
  if not os.path.exists('DATA/{}'.format(root)):
    bucket.download_file(fits_s3_key, 'DATA/{}'.format(root), ExtraArgs={"RequestPayer": "requester"})
    print('{} ---> DATA/{}'.format(fits_s3_key, root))
  else:
    print('DATA/{} already exists.'.format(root))
print('\n')

t_end = datetime.datetime.now()
duration = t_end - t_start

print('Done.\nDuration: {:6.2f} seconds.\n'.format(duration.total_seconds()))


from setuptools import setup, find_packages

version = '1.0.0'

setup(
    name='ODP-Identity',
    version=version,
    description='The SAEON Open Data Platform Identity Service',
    url='https://github.com/SAEONData/ODP-Identity',
    author='Mark Jacobson',
    author_email='mark@saeon.ac.za',
    license='MIT',
    packages=find_packages(),
    include_package_data=True,
    python_requires='~=3.8',
    install_requires=[
        'flask',
        'flask-login',
        'flask-wtf',
        'flask-mail',
        'itsdangerous',
        'python-dotenv',
        'wtforms',
        'argon2_cffi',
    ],
    extras_require={
        'test': ['pytest', 'pytest-cov']
    },
    dependency_links=[
        'git+https://github.com/SAEONData/ODP-AccountsLib.git#egg=ODP_AccountsLib',
        'git+https://github.com/SAEONData/Hydra-Admin-Client.git#egg=Hydra_Admin_Client',
        'git+https://github.com/SAEONData/Hydra-OAuth2-Blueprint.git#egg=Hydra_OAuth2_Blueprint',
    ],
)


#!/usr/bin/env python3
import datetime
import os
import queue
import re
import sys
import threading

import requests

from api import PixivApi
from model import PixivIllustModel

_THREADING_NUMBER = 5
_queue_size = 0
_finished_download = 0
_CREATE_FOLDER_LOCK = threading.Lock()
_PROGRESS_LOCK = threading.Lock()

def get_default_save_path():
    current_path = os.path.dirname(os.path.abspath(sys.argv[0]))
    return os.path.join(current_path, 'illustrations')

def get_file_path(illust, save_path='.'):
    """chose a file path by user id and name, if the current path has a folder start with the user id,
    use the old folder instead

    Return:
        file_path: a string represent complete folder path.
    """
    user_id = illust.user_id
    user_name = illust.user_name

    cur_dirs = list(filter(os.path.isfile, os.listdir(save_path)))
    cur_user_ids = [cur_dir.split()[0] for cur_dir in cur_dirs]
    if user_id not in cur_user_ids:
        dir_name = re.sub(r'[<>:"/\\|\?\*]', ' ', user_id + ' ' + user_name)
    else:
        dir_name = list(filter(lambda x: x.split()[0] == user_id, cur_dirs))[0]

    file_path = os.path.join(save_path, dir_name)

    return file_path

def print_progress():
    with _PROGRESS_LOCK:
        global _finished_download
        _finished_download += 1

        number_of_sharp = round(_finished_download / _queue_size * 60)
        number_of_space = 60 - number_of_sharp
        sys.stdout.write('\r' + str(_finished_download) + '/' + str(
            _queue_size) + '[' + '#' * number_of_sharp + ' ' * number_of_space + ']')

def download_threading(download_queue, save_path='.', add_rank=False):
    headers = {'Referer': 'http://www.pixiv.net/'}
    while not download_queue.empty():
        illust = download_queue.get()
        try:
            for url in illust.image_urls:
                file_name = url.split('/')[-1]
                if add_rank:
                    file_name = illust.rank + ' - ' + file_name
                file_path = os.path.join(save_path, file_name)
                if not os.path.exists(file_path):
                    with _CREATE_FOLDER_LOCK:
                        if not os.path.exists(os.path.dirname(file_path)):
                            os.makedirs(os.path.dirname(file_path))
                    r = requests.get(url, headers=headers, stream=True)
                    if r.status_code == requests.codes.ok:
                        temp_chunk = r.content
                        with open(file_path, 'wb') as f:
                            f.write(temp_chunk)
            print_progress()
        except Exception as e:
            print(e)
            download_queue.put(illust)
        finally:
            download_queue.task_done()

def start_and_wait_download_threadings(download_queue, save_path='.', add_rank=False):
    """start download threadings and wait till complete"""
    th = []
    for _ in range(_THREADING_NUMBER):
        t = threading.Thread(target=download_threading, args=(download_queue, save_path, add_rank))
        t.start()
        th.append(t)

    for t in th:
        t.join()

    print('\nFininsh')

def download_illusts(data_list, save_path='.', add_user_folder=False, add_rank=False):
    illusts = PixivIllustModel.from_data(data_list)

    if len(illusts) > 0:
        print('Start download, total illusts', len(illusts))

        if add_user_folder:
            save_path = get_file_path(illusts[0], save_path)

        download_queue = queue.Queue()
        for illust in illusts:
            download_queue.put(illust)

        global _queue_size, _finished_download
        _queue_size = len(illusts)
        _finished_download = 0

        start_and_wait_download_threadings(download_queue, save_path, add_rank)

    else:
        print('There is no new illust need to download')

def download_by_user_id(user):
    save_path = get_default_save_path()
    user_id = input('Input the user id:')
    data_list = user.get_user_illusts(user_id)
    download_illusts(data_list, save_path, add_user_folder=True)

def download_by_ranking(user):
    today = str(datetime.date.today())
    save_path = os.path.join(get_default_save_path(), today + ' ranking')
    data_list = user.get_ranking_illusts(per_page=100, mode='daily')
    download_illusts(data_list, save_path, add_rank=True)

def download_by_history_ranking(user):
    date = input('Input the date:(eg:2015-07-10)')
    save_path = os.path.join(get_default_save_path(), date + ' ranking')
    data_list = user.get_ranking_illusts(date=date, per_page=100, mode='daily')
    download_illusts(data_list, save_path, add_rank=True)

def main():
    print('Pixiv Downloader 2.2'.center(80, '#'))
    user = PixivApi()
    options = {
        '1': download_by_user_id,
        '2': download_by_ranking,
        '3': download_by_history_ranking
    }

    while True:
        choose = input('Which do you want to download:\n1 From user id\n2 From ranking\n3 From history ranking\n')

        if choose in [str(i) for i in range(1, 4)]:
            options[choose](user)
        else:
            print('Wrong input!')

if __name__ == '__main__':
    sys.exit(main())


# -*- coding:utf8 -*-

import time

from assertpy import assert_that as asserts

from common import sqlbase, page
from common.base import log, Base
from common.interface import createCustomer, createApartmentContract, addHouseContractAndFitment
from contract.houseContract.page import houseContractPage

@log
def test_1044():
    """修改委托合同租金"""

    # describe：在委托合同详情中，修改业绩核算周期对应时间段的委托合同租金策略中的租金，业绩中对应发生变化
    # data：1、业绩审核状态为待审核；2、委托合同状态为待审核或者已初审；3、提前记录原业绩中的核算收进价和差价业绩；
    # result：1、新产生的业绩中核算收进价与原记录不同；2、新的差价业绩与原来的值不同；

    fileName = 'apartmentAchievement_1044'

    with Base() as base:
        # 创建委托合同和出租合同
        houseSql = sqlbase.serach(
            "select house_id,residential_id,building_id,house_code from house where deleted=0 and city_code=330100 order by rand() limit 1")  # 获取随机开发房源
        houseInfo = {'houseID': houseSql[0], 'residentialID': houseSql[1], 'buildingID': houseSql[2],'houseCode': houseSql[3]}
        dateInfo = sqlbase.serach("select date(sysdate()),date_add(date(sysdate()),INTERVAL 1 day),date_add(date(sysdate()),interval 2 year),date_add(date(sysdate()),interval 27 month),"
                                  "date_add(date(sysdate()),INTERVAL 1 month),date_add(date(sysdate()),INTERVAL 6 month) from dual")  # 日期元素
        apartmentId = addHouseContractAndFitment(apartment_type='MANAGE', entrust_type='SHARE', sign_date=dateInfo[0],
                                                 owner_sign_date=dateInfo[0], entrust_start_date=dateInfo[0],
                                                 entrust_end_date=dateInfo[2], delay_date=dateInfo[3],
                                                 free_start_date=dateInfo[0], free_end_date=dateInfo[4],
                                                 first_pay_date=dateInfo[0], second_pay_date=dateInfo[4],
                                                 rent=3000, parking=100, year_service_fee=500, payment_cycle='MONTH',
                                                 fitment_start_date=dateInfo[0], fitment_end_date=dateInfo[4], rooms=3,
                                                 fitmentCost=88888,houseInfo=houseInfo)
        rentPriceInfo = sqlbase.serach("select rent_price,date(sysdate()) from apartment where apartment_id='%s'" % apartmentId)
        rentPrice = float(rentPriceInfo[0])
        customer = createCustomer()
        apartmentContractInfo = createApartmentContract(apartement_id=apartmentId, customerInfo=customer,
                                                        rent_price=rentPrice, sign_date=dateInfo[0],
                                                        rent_start_date=dateInfo[1], rent_end_date=dateInfo[5],  # 承租6个月
                                                        deposit=rentPrice, payment_cycle='MONTH')
        apartmentContractNum = apartmentContractInfo['contractNum']
        achievementsql = "select substring_index(aca.house_code,'-',1),aca.is_active,aca.audit_status,aca.contract_audit_status,aca.profits_fee,aca.rent_cost from apartment_contract_achievement aca " \
                          "inner join apartment a on a.apartment_code=aca.house_code and a.apartment_id='%s' where aca.contract_num='%s' and aca.deleted=0" % (
                          apartmentId, apartmentContractNum)
        base.diffAssert(lambda test: asserts(sqlbase.waitData(achievementsql, 1)).is_true(), 1057,
                        u'%s:合同 %s 对应业绩生成异常' % (fileName, apartmentContractNum))
        profits_feeOld = sqlbase.serach(achievementsql)[4]  # 差价业绩
        rent_cost_old = sqlbase.serach(achievementsql)[5]  # 核算收进价
        houseCode = sqlbase.serach(achievementsql)[0]
        # 修改租金策略并且复审
        base.open(page.entrustContractPage, houseContractPage.contractSearchMould['tr_contract'])
        # base.input_text(houseContractPage.contractSearchMould['contract_num_loc'], info[2])
        base.input_text(houseContractPage.contractSearchMould['residential_name_loc'], houseCode)
        base.click(houseContractPage.contractSearchMould['search_button_loc'])  # 搜索
        base.staleness_of(houseContractPage.contractSearchMould['tr_contract'])  # 等待列表刷新
        base.dblclick(houseContractPage.contractSearchMould['tr_contract'],
                      checkLoc=houseContractPage.addHouseContractMould['contract_num_loc'])  # 双击第一条数据
        base.click(houseContractPage.addHouseContractMould['contract_strategy1.1_loc'])
        base.input_text(houseContractPage.addHouseContractMould['contract_strategy1_loc'], 4000)  # 第一年租金策略
        base.click(houseContractPage.addHouseContractMould['page1_save_button'])  # 保存
        for i in range(3):
            try:
                base.check_submit()
                break
            except:
                message = base.script(
                    "var a=$('.panel.window.messager-window>div:nth-child(2)>div:nth-child(2)').text();return a", True)
                messagehope = u'用户数据已经被其他用户更新'
                if messagehope in message:
                    base.click(houseContractPage.addHouseContractMould['message_close_loc'])  # 关闭提示
                    base.click(houseContractPage.addHouseContractMould['rent_save_button'])  # 保存
                    base.check_submit()
        # 委托合同复审
        base.click(houseContractPage.addHouseContractMould['tab_info_loc'], index=1)  # 租金页面
        base.click(houseContractPage.addHouseContractMould['rent_detail_selectAll'])  # 全选
        base.click(houseContractPage.addHouseContractMould['rent_audit_loc'])  # 审核
        base.click(houseContractPage.addHouseContractMould['audit_pass_loc'])  # 通过
        base.click(houseContractPage.addHouseContractMould['rent_audit_confirm'])  # 确认
        # 初审
        base.click(houseContractPage.addHouseContractMould['tab_info_loc'], index=3)  # 最后一页
        time.sleep(2)
        base.script('$("button[status=\'PASS\']")[2].click()')  # 初审
        base.click(houseContractPage.addHouseContractMould['contract_audit_confirm'])  # 确认
        for i in range(3):
            try:
                base.staleness_of(houseContractPage.contractSearchMould['tr_contract'])  # 等待数据刷新
                break
            except:
                message = base.script(
                    "var a=$('.panel.window.messager-window>div:nth-child(2)>div:nth-child(2)').text();return a", True)
                messagehope = u'用户数据已经被其他用户更新'
                if messagehope in message:
                    base.click(houseContractPage.addHouseContractMould['message_close_loc'])  # 关闭提示
                    base.click(houseContractPage.addHouseContractMould['contract_audit_confirm'])  # 确认
                    base.staleness_of(houseContractPage.contractSearchMould['tr_contract'])  # 等待数据刷新
        # 复审
        base.dblclick(houseContractPage.contractSearchMould['tr_contract'],
                      checkLoc=houseContractPage.addHouseContractMould['contract_num_loc'])  # 双击第一条数据
        base.click(houseContractPage.addHouseContractMould['tab_info_loc'], index=3)  # 最后一页
        time.sleep(2)
        base.script('$("button[status=\'APPROVED\']")[1].click()')  # 复审
        base.click(houseContractPage.addHouseContractMould['rentdif_cofirm_loc'])  # 租金策略不同提示确定
        base.click(houseContractPage.addHouseContractMould['contract_audit_confirm'])  # 确认
        for i in range(3):
            try:
                base.check_submit()
                break
            except:
                message = base.script(
                    "var a=$('.panel.window.messager-window>div:nth-child(2)>div:nth-child(2)').text();return a", True)
                messagehope = u'用户数据已经被其他用户更新'
                if messagehope in message:
                    base.click(houseContractPage.addHouseContractMould['message_close_loc'])  # 关闭提示
                    base.click(houseContractPage.addHouseContractMould['contract_audit_confirm'])  # 确认
                    base.check_submit()
        #获取最新业绩信息
        time.sleep(10)
        profits_feeNew = sqlbase.serach(achievementsql)[4]
        rent_cost_new = sqlbase.serach(achievementsql)[5]
        base.diffAssert(lambda test:asserts(rent_cost_new).is_not_equal_to(rent_cost_old),1044,
                        u'%s:出租合同 %s 对应委托成本修改后业绩中核算收进价异常，修改前 %s 修改后 %s' % (fileName, apartmentContractNum, rent_cost_old, rent_cost_new))
        base.diffAssert(lambda test: asserts(profits_feeNew).is_not_equal_to(profits_feeOld),1044,
                        u'%s:出租合同 %s 对应委托成本修改后业绩中差价业绩异常，修改前 %s 修改后 %s' % (fileName, apartmentContractNum,profits_feeOld, profits_feeNew))

test_1044()


from flask import Flask
from flask import flash, redirect, render_template, request, url_for
from flask_sqlalchemy import SQLAlchemy
from flask_login.utils import current_user, login_user, logout_user

from config import app, db
from tables import Shuttles_A, Shuttles_B, Booking, Users
from functions import add_form, get_index_data, is_admin, debug_print, \
  is_safe_url, get_redirect_target, login_required

@app.route('/')
def index():
    shuttlesA, shuttlesB = get_index_data(db)
    return render_template('index.html', shuttlesA=shuttlesA, shuttlesB=shuttlesB)

@app.route('/login', methods = ['GET', 'POST'])
def login():
    if current_user.is_authenticated:
        return redirect(url_for('admin.index'))
    if request.method == 'GET':
        return render_template('login.html')
    user, verify = is_admin(app, db, request.form)
    if not verify:
        flash('Echec de l\'authentification.', 'error')
        return redirect(url_for('login'))
    login_user(user)
    flash('Vous êtes connecté en tant qu\'administrateur.', 'success')
    return redirect(get_redirect_target() or url_for('admin.index'))

@app.route('/logout')
@login_required
def logout():
    logout_user()
    flash('Déconnexion effectuée avec succès.', 'success')
    return redirect(url_for('index'))

@app.route('/add', methods = ['POST'])
def new():
    #fields = ['surname', 'firstname', 'presale']
    fields = Booking.__table__.columns.keys()[1:-2]
    if False in [ request.form[field] != '' for field in fields ]:
        flash('Il manque des informations pour réserver les navettes !', 'error')
        return redirect(url_for('index'))

    debug_print(app, request.form)
    add_form(app, db, request.form)

    shuttlesA, shuttlesB = Shuttles_A.query.all(), Shuttles_B.query.all()
    return redirect(url_for('index'))

@app.errorhandler(403)
def forbidden(error):
    flash('Error {}'.format(error), 'error')
    return redirect(url_for('index'))

@app.errorhandler(404)
def not_found(error):
    flash('Error {}'.format(error), 'error')
    return redirect(url_for('index'))

@app.errorhandler(405)
def method_not_allowed(error):
    flash('Error {}'.format(error), 'error')
    return redirect(url_for('index'))

@app.errorhandler(500)
def internal_server_error(error):
    flash('Error {}'.format(error), 'error')
    return redirect(url_for('index'))

if __name__=='__main__':
    app.run(host='0.0.0.0', debug=True)


import os
import sys
import numpy as np
import pandas as pd
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from torch.utils.data import Dataset
from sklearn.model_selection import train_test_split
import torch
TEST_SET_DIR = '/scratch/sr365/ML_MM_Benchmark/testsets'

def get_data_into_loaders(data_x, data_y, batch_size, DataSetClass, rand_seed=0, test_ratio=0.3):
    """
    Helper function that takes structured data_x and data_y into dataloaders
    :param data_x: the structured x data
    :param data_y: the structured y data
    :param rand_seed: the random seed
    :param test_ratio: The testing ratio
    :return: train_loader, test_loader: The pytorch data loader file
    """
    if test_ratio == 1:         # Test case
        print('This is testing mode!!! Test ratio = 1')
        test_data = DataSetClass(data_x, data_y)
        test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size)
        return None, test_loader

    x_train, x_test, y_train, y_test = train_test_split(data_x, data_y, test_size=test_ratio,
                                                        random_state=rand_seed)
    print('total number of training sample is {}, the dimension of the feature is {}'.format(len(x_train), len(x_train[0])))
    print('total number of test sample is {}'.format(len(y_test)))

    # Construct the dataset using a outside class
    train_data = DataSetClass(x_train, y_train)
    test_data = DataSetClass(x_test, y_test)

    # Construct train_loader and test_loader
    train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size)
    test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size)

    return train_loader, test_loader

def normalize_np(x, x_max_list=None, x_min_list=None):
    """
    Normalize the x into [-1, 1] range in each dimension [:, i]
    :param x: np array to be normalized
    :return: normalized np array
    """
    if x_max_list is not None:
        if x_min_list is None or len(x[0]) != len(x_max_list) or len(x_max_list) != len(x_min_list):
            print("In normalize_np, your dimension does not match with provided x_max, try again")
            quit()

    new_x_max_list = []
    new_x_min_list = []
    for i in range(len(x[0])):
        if x_max_list is None:
            x_max = np.max(x[:, i])
            x_min = np.min(x[:, i])
        else:
            x_max = x_max_list[i]
            x_min = x_min_list[i]
        x_range = (x_max - x_min ) /2.
        x_avg = (x_max + x_min) / 2.
        x[:, i] = (x[:, i] - x_avg) / x_range
        print("In normalize_np, row ", str(i), " your max is:", np.max(x[:, i]))
        print("In normalize_np, row ", str(i), " your min is:", np.min(x[:, i]))
        if x_max_list is None:
            assert np.max(x[:, i]) - 1 < 0.0001, 'your normalization is wrong'
            assert np.min(x[:, i]) + 1 < 0.0001, 'your normalization is wrong'
            new_x_max_list.append(x_max)
            new_x_min_list.append(x_min)
    return x, np.array(new_x_max_list), np.array(new_x_min_list)

def read_data_color_filter(flags, eval_data_all=False):
    """
    Data reader function for the gaussian mixture data set
    :param flags: Input flags
    :return: train_loader and test_loader in pytorch data set format (normalized)
    """

    # Read the data
    data_dir = os.path.join(flags.data_dir, 'Color')
    data_x = pd.read_csv(os.path.join(data_dir, 'data_x.csv'), header=None).astype('float32').values
    data_y = pd.read_csv(os.path.join(data_dir, 'data_y.csv'), header=None).astype('float32').values

    # This is to for the last test dataset
    if eval_data_all:
        data_dir = os.path.join(TEST_SET_DIR, 'Color')
        data_x = pd.read_csv(os.path.join(data_dir, 'test_x.csv'), header=None).astype('float32').values
        data_y = pd.read_csv(os.path.join(data_dir, 'test_y.csv'), header=None).astype('float32').values
        return get_data_into_loaders(data_x, data_y, flags.batch_size,  SimulatedDataSet_regress, rand_seed=flags.rand_seed,test_ratio=1)

    print("shape of data_x", np.shape(data_x))
    print("shape of data_y", np.shape(data_y))

    return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed, test_ratio=flags.test_ratio)

def read_data_Yang(flags, eval_data_all=False):
    """
    Data reader function for the gaussian mixture data set
    :param flags: Input flags
    :return: train_loader and test_loader in pytorch data set format (normalized)
    """

    # Read the data
    data_dir = os.path.join(flags.data_dir, 'Yang')
    data_x = pd.read_csv(os.path.join(data_dir, 'data_x.csv'), header=None).astype('float32').values
    data_y = pd.read_csv(os.path.join(data_dir, 'data_y.csv'), header=None).astype('float32').values
    # Normalize the dataset
    data_x, x_max, x_min = normalize_np(data_x)

    # This is to for the last test dataset
    if eval_data_all:
        data_dir = os.path.join(TEST_SET_DIR, 'Yang')
        data_x = pd.read_csv(os.path.join(data_dir, 'test_x.csv'), header=None).astype('float32').values
        data_y = pd.read_csv(os.path.join(data_dir, 'test_y.csv'), header=None).astype('float32').values
        data_x, _, _, = normalize_np(data_x, x_max, x_min)
        print('This is Yang dataset with data_x shape of', np.shape(data_x))
        return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed, test_ratio=1)

    print("shape of data_x", np.shape(data_x))
    print("shape of data_y", np.shape(data_y))

    return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed, test_ratio=flags.test_ratio)

def read_data_peurifoy(flags, eval_data_all=False):
    """
    Data reader function for the gaussian mixture data set
    :param flags: Input flags
    :return: train_loader and test_loader in pytorch data set format (normalized)
    """

    # Read the data
    data_dir = os.path.join(flags.data_dir, 'Peurifoy')
    data_x = pd.read_csv(os.path.join(data_dir, 'data_x.csv'), header=None).astype('float32').values
    data_y = pd.read_csv(os.path.join(data_dir, 'data_y.csv'), header=None).astype('float32').values
    # This is to for the last test dataset
    if eval_data_all:
        data_dir = os.path.join(TEST_SET_DIR, 'Peurifoy')
        data_x = pd.read_csv(os.path.join(data_dir, 'test_x.csv'), header=None).astype('float32').values
        data_y = pd.read_csv(os.path.join(data_dir, 'test_y.csv'), header=None).astype('float32').values
        data_x = (data_x - 50) / 20.
        print('This is Perifoy dataset with data_x shape of', np.shape(data_x))
        return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed,test_ratio=1)

    # The geometric boundary of peurifoy dataset is [30, 70], normalizing manually
    data_x = (data_x - 50) / 20.

    print("shape of data_x", np.shape(data_x))
    print("shape of data_y", np.shape(data_y))

    return get_data_into_loaders(data_x, data_y, flags.batch_size, SimulatedDataSet_regress, rand_seed=flags.rand_seed, test_ratio=flags.test_ratio)

def read_data(flags, eval_data_all=False):
    """
    The data reader allocator function
    The input is categorized into couple of different possibilities
    0. meta_material
    1. gaussian_mixture
    2. sine_wave
    3. naval_propulsion
    4. robotic_arm
    5. ballistics
    :param flags: The input flag of the input data set
    :param eval_data_all: The switch to turn on if you want to put all data in evaluation data
    :return:
    """
    print("In read_data, flags.data_set =", flags.data_set)
    if 'Yang' in flags.data_set or 'ADM' in flags.data_set:
        train_loader, test_loader = read_data_Yang(flags,eval_data_all=eval_data_all)
    elif 'Peurifoy' in  flags.data_set :
        train_loader, test_loader = read_data_peurifoy(flags,eval_data_all=eval_data_all)
    elif 'olor' in flags.data_set:
        train_loader, test_loader =read_data_color_filter(flags,eval_data_all=eval_data_all)
    else:
        sys.exit("Your flags.data_set entry is not correct, check again!")
    return train_loader, test_loader

class MetaMaterialDataSet(Dataset):
    """ The Meta Material Dataset Class """
    def __init__(self, ftr, lbl, bool_train):
        """
        Instantiate the Dataset Object
        :param ftr: the features which is always the Geometry !!
        :param lbl: the labels, which is always the Spectra !!
        :param bool_train:
        """
        self.ftr = ftr
        self.lbl = lbl
        self.bool_train = bool_train
        self.len = len(ftr)

    def __len__(self):
        return self.len

    def __getitem__(self, ind):
        return self.ftr[ind, :], self.lbl[ind, :]

class SimulatedDataSet_class_1d_to_1d(Dataset):
    """ The simulated Dataset Class for classification purposes"""
    def __init__(self, x, y):
        self.x = x
        self.y = y
        self.len = len(x)

    def __len__(self):
        return self.len

    def __getitem__(self, ind):
        return self.x[ind], self.y[ind]

class SimulatedDataSet_class(Dataset):
    """ The simulated Dataset Class for classification purposes"""
    def __init__(self, x, y):
        self.x = x
        self.y = y
        self.len = len(x)

    def __len__(self):
        return self.len

    def __getitem__(self, ind):
        return self.x[ind, :], self.y[ind]

class SimulatedDataSet_regress(Dataset):
    """ The simulated Dataset Class for regression purposes"""
    def __init__(self, x, y):
        self.x = x
        self.y = y
        self.len = len(x)

    def __len__(self):
        return self.len

    def __getitem__(self, ind):
        return self.x[ind, :], self.y[ind, :]


#!/usr/bin/env python
import argparse
import bugsy
import config
import datetime
import github
import json
import logging
import mozreview
import os
import psycopg2
import sys
import time
import traceback

sys.path.insert(0, os.path.normpath(os.path.join(os.path.normpath(
                os.path.abspath(os.path.dirname(__file__))), '..', '..',
                'pylib', 'mozautomation')))
from mozautomation.commitparser import parse_bugs
from mozlog.structured import commandline

import transplant
import treestatus

GITHUB_COMMENT_LIMIT = 10  # max updates to post to github / iteration
MOZREVIEW_COMMENT_LIMIT = 10  # max updates to post to reviewboard / iteration

def handle_pending_mozreview_pullrequests(logger, dbconn):
    gh = github.connect()
    if not gh:
        return

    bzurl = config.get('bugzilla')['url']

    cursor = dbconn.cursor()

    query = """
        select id,ghuser,repo,pullrequest,destination,bzuserid,bzcookie,bugid,
               pingback_url
        from MozreviewPullRequest
        where landed is null
    """
    cursor.execute(query)

    landed_revisions = []
    mozreview_updates = []
    for row in cursor.fetchall():
        (transplant_id, ghuser, repo, pullrequest, destination, bzuserid,
         bzcookie, bugid, pingback_url) = row

        logger.info('attempting to import pullrequest: %s' % transplant_id)

        # see if we can extract the bug from the commit message
        if bugid is None:
            title, body = github.retrieve_issue(gh, ghuser, repo, pullrequest)
            bugs = parse_bugs(title)
            if bugs:
                bugid = bugs[0]
                logger.info('using bug %s from issue title' % bugid)
                landed_revisions.append([bugid, None, None, transplant_id])

        # still no luck, attempt to autofile a bug on the user's behalf
        if bugid is None:
            logger.info('attempting to autofile bug for: %s' % transplant_id)

            b = bugsy.Bugsy(userid=bzuserid, cookie=bzcookie,
                            bugzilla_url=bzurl)
            if not b:
                logger.info('could not connect to bugzilla instance at %s for '
                            'pullrequest id %s' % (bzurl, transplant_id))
                error = 'could not connect to bugzilla. bad credentials?'
            else:
                bug = bugsy.Bug()

                # Summary is required, the rest have defaults or are optional
                bug.summary = title

                if config.testing():
                    bug.product = 'TestProduct'
                    bug.component = 'TestComponent'
                else:
                    # TODO: determine a better product & component than the
                    # defaults provided by Bugsy
                    pass

                pr_url = github.url_for_pullrequest(ghuser,repo, pullrequest)
                bug.add_comment('%s\n\nImported from: %s' % (body, pr_url))

                try:
                    b.put(bug)
                    bugid = bug.id
                    logger.info('created bug: %s ' % bugid)
                    landed_revisions.append([bugid, None, None, transplant_id])
                except bugsy.BugsyException as e:
                    logger.info('transplant failed: could not create new bug: %s '
                                % e.msg)
                    landed_revisions.append([None, False, e.msg, transplant_id])

                    # set up data to be posted back to mozreview
                    data = {
                        'request_id': transplant_id,
                        'bugid': None,
                        'landed': False,
                        'error_msg': 'could not create new bug: ' + e.msg,
                        'result': ''
                    }

                    mozreview_updates.append([transplant_id, pingback_url,
                                              json.dumps(data)])

        landed, result = transplant.transplant_to_mozreview(gh, destination,
                                                            ghuser, repo,
                                                            pullrequest,
                                                            bzuserid, bzcookie,
                                                            bugid)

        if landed:
            logger.info(('transplanted from'
                         ' https://github.com/%s/%s/pull/%s'
                         ' to destination: %s new revision: %s') %
                        (ghuser, repo, pullrequest, destination, result))
        else:
            logger.info(('transplant failed'
                         ' https://github.com/%s/%s/pull/%s'
                         ' destination: %s error: %s') %
                        (ghuser, repo, pullrequest, destination, result))

        landed_revisions.append([bugid, landed, result, transplant_id])

        # set up data to be posted back to mozreview
        data = {
            'request_id': transplant_id,
            'bugid': bugid,
            'landed': landed,
            'error_msg': '',
            'result': ''
        }

        if landed:
            data['result'] = result
        else:
            data['error_msg'] = result

        mozreview_updates.append([transplant_id, pingback_url, json.dumps(data)])

    if landed_revisions:
        query = """
            update MozreviewPullRequest set bugid=%s,landed=%s,result=%s
            where id=%s
        """
        cursor.executemany(query, landed_revisions)
        dbconn.commit()

    if mozreview_updates:
        query = """
            insert into MozreviewUpdate(request_id,pingback_url,data)
            values(%s,%s,%s)
        """
        cursor.executemany(query, mozreview_updates)
        dbconn.commit()

def handle_pending_transplants(logger, dbconn):
    cursor = dbconn.cursor()

    query = """
        select id,tree,rev,destination,trysyntax,push_bookmark,pingback_url
        from Transplant
        where landed is null
    """
    cursor.execute(query)

    landed_revisions = []
    mozreview_updates = []
    for row in cursor.fetchall():
        (transplant_id, tree, rev, destination, trysyntax, push_bookmark,
            pingback_url) = row

        if not treestatus.tree_is_open(destination):
            continue

        landed, result = transplant.transplant(tree, destination, rev,
                                               trysyntax, push_bookmark)
        if landed:
            logger.info(('transplanted from tree: %s rev: %s'
                         ' to destination: %s new revision: %s') %
                        (tree, rev, destination, result))
        else:
            if 'is CLOSED!' in result:
                logger.info('transplant failed: tree: %s is closed - '
                             ' retrying later.' % tree)

                # continuing here will skip updating the autoland request
                # so we will attempt to land it again later.
                continue
            else:
                logger.info('transplant failed: tree: %s rev: %s '
                            'destination: %s error: %s' %
                            (tree, rev, destination, result))

        # set up data to be posted back to mozreview
        data = {
            'request_id': transplant_id,
            'tree': tree,
            'rev': rev,
            'destination': destination,
            'trysyntax': trysyntax,
            'landed': landed,
            'error_msg': '',
            'result': ''
        }

        if landed:
            data['result'] = result
        else:
            data['error_msg'] = result

        mozreview_updates.append([transplant_id, pingback_url, json.dumps(data)])

        landed_revisions.append([landed, result, transplant_id])

    if landed_revisions:
        query = """
            update Transplant set landed=%s,result=%s
            where id=%s
        """
        cursor.executemany(query, landed_revisions)
        dbconn.commit()

    if mozreview_updates:
        query = """
            insert into MozreviewUpdate(request_id,pingback_url,data)
            values(%s,%s,%s)
        """
        cursor.executemany(query, mozreview_updates)
        dbconn.commit()

def handle_pending_github_updates(logger, dbconn):
    """Attempt to post updates to github"""

    gh = github.connect()
    if not gh:
        return

    cursor = dbconn.cursor()
    query = """
        select id,ghuser,repo,pullrequest,destination,landed,result,pingback_url
        from MozreviewPullRequest
        where landed is not NULL and pullrequest_updated is NULL
        limit %(limit)s
    """
    cursor.execute(query, {'limit': GITHUB_COMMENT_LIMIT})

    updated = []
    for row in cursor.fetchall():
        ghuser = row[1]
        repo = row[2]
        pullrequest = row[3]
        landed = row[5]
        result = row[6]
        pingback_url = row[7]

        if landed:
            message = 'Your new review request is available at: %s' % result
        else:
            message = 'Something went wrong importing to mozreview: %s' % result

        logger.info('attempting to add comment to pullrequest for request: %s' % row[0])

        worked = github.add_issue_comment(gh, ghuser, repo, pullrequest,
                                          message)
        if worked:
            updated.append([row[0]])
        else:
            logger.info('commenting on pull request failed')

    if updated:
        query = """
            update MozreviewPullRequest set pullrequest_updated=TRUE
            where id=%s
        """
        cursor.executemany(query, updated)
        dbconn.commit()

def handle_pending_mozreview_updates(logger, dbconn):
    """Attempt to post updates to mozreview"""

    cursor = dbconn.cursor()
    query = """
        select request_id,pingback_url,data
        from MozreviewUpdate
        limit %(limit)s
    """
    cursor.execute(query, {'limit': MOZREVIEW_COMMENT_LIMIT})

    mozreview_auth = mozreview.read_credentials()

    updated = []
    for row in cursor.fetchall():
        request_id, pingback_url, data = row
        logger.info('trying to post mozreview update to: %s for request: %s' %
                    (pingback_url, request_id))

        status_code, text = mozreview.update_review(mozreview_auth,
                                                    pingback_url, data)
        if status_code == 200:
            updated.append([request_id])
        else:
            logger.info('failed: %s - %s' % (status_code, text))

    if updated:
        query = """
            delete from MozreviewUpdate
            where request_id=%s
        """
        cursor.executemany(query, updated)
        dbconn.commit()

def get_dbconn(dsn):
    dbconn = None
    while not dbconn:
        try:
            dbconn = psycopg2.connect(dsn)
        except psycopg2.OperationalError:
            time.sleep(0.1)
    return dbconn

def main():
    parser = argparse.ArgumentParser()

    dsn = config.get('database')

    parser.add_argument('--dsn', default=dsn,
                        help="Postgresql DSN connection string")
    commandline.add_logging_group(parser)
    args = parser.parse_args()

    logging.basicConfig()
    logger = commandline.setup_logging('autoland', vars(args), {})
    logger.info('starting autoland')

    dbconn = get_dbconn(args.dsn)
    last_error_msg = None
    while True:
        try:
            handle_pending_mozreview_pullrequests(logger, dbconn)
            handle_pending_transplants(logger, dbconn)
            handle_pending_github_updates(logger, dbconn)
            handle_pending_mozreview_updates(logger, dbconn)
            time.sleep(0.25)
        except KeyboardInterrupt:
            break
        except psycopg2.InterfaceError:
            dbconn = get_dbconn(args.dsn)
        except:
            # If things go really badly, we might see the same exception
            # thousands of times in a row. There's not really any point in
            # logging it more than once.
            error_msg = traceback.format_exc()
            if error_msg != last_error_msg:
                logger.error(error_msg)
                last_error_msg = error_msg

if __name__ == "__main__":
    main()


# from config import Config
from keras import backend as K
from keras.layers import Input
from keras.models import Model
from app.models.frcnn_detector.model import *
from app.models.frcnn_detector.utils import *
import time
import os
import pickle
import math

os.environ['KMP_DUPLICATE_LIB_OK'] = 'True'

class Config:

    def __init__(self):
        # Print the process or not
        self.verbose = True

        # Name of base network
        self.network = 'vgg'

        # Setting for data augmentation
        self.use_horizontal_flips = False
        self.use_vertical_flips = False
        self.rot_90 = False

        # Anchor box scales
        # Note that if im_size is smaller, anchor_box_scales should be scaled
        # Original anchor_box_scales in the paper is [128, 256, 512]
        self.anchor_box_scales = [128, 256, 512]

        # Anchor box ratios
        self.anchor_box_ratios = [[1, 1],
                                  [1. / math.sqrt(2), 2. / math.sqrt(2)],
                                  [2. / math.sqrt(2), 1. / math.sqrt(2)]]

        # Size to resize the smallest side of the image
        # Original setting in paper is 600. Set to 300 in here to save training
        # time
        self.im_size = 600

        # image channel-wise mean to subtract
        self.img_channel_mean = [103.939, 116.779, 123.68]
        self.img_scaling_factor = 1.0

        # number of ROIs at once
        self.num_rois = 10

        # stride at the RPN (this depends on the network configuration)
        self.rpn_stride = 16

        self.balanced_classes = False

        # scaling the stdev
        self.std_scaling = 4.0
        self.classifier_regr_std = [8.0, 8.0, 4.0, 4.0]

        # overlaps for RPN
        self.rpn_min_overlap = 0.3
        self.rpn_max_overlap = 0.7

        # overlaps for classifier ROIs
        self.classifier_min_overlap = 0.1
        self.classifier_max_overlap = 0.5

        # placeholder for the class mapping, automatically generated by the
        # parser
        self.class_mapping = {0: 'shoe', 1: 'bg'}

        self.model_path = None

class detector_model:
    '''
    detector model class
    '''

    def __init__(self):
        self.C = Config()
        pass

    def load_model(self, detector_weights):
        '''
    	Loads FRCNN model from detector_path
    	'''

        # Read in Config for Testing
        # with open(detector_config, 'rb') as f_in:
        #    config = pickle.load(f_in)

        # Turn off any data augmentation at test time
        self.C.model_path = detector_weights
        self.C.use_horizontal_flips = False
        self.C.use_vertical_flips = False
        self.C.rot_90 = False
        class_mapping = self.C.class_mapping
        class_mapping = {v: k for k, v in class_mapping.items()}

        # Gather Model Inputs
        num_features = 512
        input_shape_img = (None, None, 3)
        input_shape_features = (None, None, num_features)
        img_input = Input(shape=input_shape_img)
        roi_input = Input(shape=(self.C.num_rois, 4))
        feature_map_input = Input(shape=input_shape_features)

        # Define Model Layers
        shared_layers = nn_base(img_input, trainable=True)
        num_anchors = len(self.C.anchor_box_scales) * len(
            self.C.anchor_box_ratios)
        rpn_layers = rpn_layer(shared_layers, num_anchors)
        classifier = classifier_layer(feature_map_input, roi_input,
                                      self.C.num_rois,
                                      nb_classes=len(self.C.class_mapping))

        # Build Models
        model_rpn = Model(img_input, rpn_layers)
        model_classifier_only = Model([feature_map_input, roi_input],
                                      classifier)
        model_classifier = Model([feature_map_input, roi_input], classifier)

        print('Loading weights from {}'.format(self.C.model_path))
        model_rpn.load_weights(self.C.model_path, by_name=True)
        model_classifier.load_weights(self.C.model_path, by_name=True)

        model_rpn.compile(optimizer='sgd', loss='mse')
        model_classifier.compile(optimizer='sgd', loss='mse')

        self.model_rpn = model_rpn
        self.model_classifier = model_classifier

    def predict(self, img, save_path):

        # Predict
        bbox_threshold = 0.95

        final_predictions = []

        T = {}
        P = {}

        # Create copy of original image
        img_o = img
        cv2.imwrite("app/models/frcnn_detector/output/test.jpg", img_o)
        img = img_o.copy()
        X, ratio = format_img(img, self.C)

        _, fx, fy = format_img_map(img, self.C)

        # Change X (img) shape from (1, channel, height, width) to (1, height,
        # width, channel)
        X = np.transpose(X, (0, 2, 3, 1))

        # get the feature maps and output from the RPN
        [Y1, Y2, F] = self.model_rpn.predict(X)

        R = rpn_to_roi(Y1, Y2, self.C, overlap_thresh=0.8)

        # convert from (x1,y1,x2,y2) to (x,y,w,h)
        R[:, 2] -= R[:, 0]
        R[:, 3] -= R[:, 1]

        # apply the spatial pyramid pooling to the proposed regions
        bboxes = {}
        probs = {}

        for jk in range(R.shape[0] // self.C.num_rois + 1):
            ROIs = np.expand_dims(
                R[self.C.num_rois * jk:self.C.num_rois * (jk + 1), :], axis=0)
            if ROIs.shape[1] == 0:
                break

            if jk == R.shape[0] // self.C.num_rois:
                # pad R
                curr_shape = ROIs.shape
                target_shape = (curr_shape[0], self.C.num_rois, curr_shape[2])
                ROIs_padded = np.zeros(target_shape).astype(ROIs.dtype)
                ROIs_padded[:, :curr_shape[1], :] = ROIs
                ROIs_padded[0, curr_shape[1]:, :] = ROIs[0, 0, :]
                ROIs = ROIs_padded

            [P_cls, P_regr] = self.model_classifier.predict([F, ROIs])

            # Calculate all classes' bboxes coordinates on resized image
            # (300, 400) Drop 'bg' classes bboxes
            for ii in range(P_cls.shape[1]):

                # If class name is 'bg', continue
                if np.max(P_cls[0, ii, :]) < bbox_threshold or np.argmax(
                        P_cls[0, ii, :]) == (P_cls.shape[2] - 1):
                    continue

                # Get class name
                cls_name = self.C.class_mapping[np.argmax(P_cls[0, ii, :])]

                if cls_name not in bboxes:
                    bboxes[cls_name] = []
                    probs[cls_name] = []

                (x, y, w, h) = ROIs[0, ii, :]

                cls_num = np.argmax(P_cls[0, ii, :])
                try:
                    (tx, ty, tw, th) = P_regr[0, ii,
                                       4 * cls_num:4 * (cls_num + 1)]
                    tx /= self.C.classifier_regr_std[0]
                    ty /= self.C.classifier_regr_std[1]
                    tw /= self.C.classifier_regr_std[2]
                    th /= self.C.classifier_regr_std[3]
                    x, y, w, h = roi_helpers.apply_regr(x, y, w, h, tx, ty, tw,
                                                        th)
                except:
                    pass
                bboxes[cls_name].append(
                    [16 * x, 16 * y, 16 * (x + w), 16 * (y + h)])
                probs[cls_name].append(np.max(P_cls[0, ii, :]))

        all_dets = []
        real_dets = []

        # if no predictions
        if bboxes == {}:
            real_dets.append(
                {'x1': 0, 'y1': 0, 'x2': img.shape[0], 'y2': img.shape[1]})

        for key in bboxes:
            bbox = np.array(bboxes[key])

            # Apply non-max-suppression on final bboxes to get the output
            # bounding boxe
            new_boxes, new_probs = non_max_suppression_fast(
                bbox, np.array(probs[key]), overlap_thresh=0.2)
            for jk in range(new_boxes.shape[0]):
                (x1, y1, x2, y2) = new_boxes[jk, :]
                det = {'x1': x1, 'x2': x2, 'y1': y1, 'y2': y2,
                       'class': key, 'prob': new_probs[jk]}
                all_dets.append(det)

                # Calculate real coordinates on original image
                (real_x1, real_y1, real_x2, real_y2) = get_real_coordinates(
                    ratio, x1, y1, x2, y2)
                real_det = {'x1': real_x1, 'x2': real_x2, 'y1': real_y1,
                            'y2': real_y2, 'prob': new_probs[jk]}
                real_dets.append(real_det)

        final_predictions.append([img_o, real_dets])
        print(len(final_predictions[0][1]))

        # Crop images and save
        img_paths = save_crop(final_predictions, save_path)

        return img_paths


from django.contrib import admin
from main.models import *
from django.utils.safestring import mark_safe

class CategoryAdmin(admin.ModelAdmin):
    list_display = ('id', 'title', 'is_published', 'views', 'photo', 'get_photo')
    list_display_links = ('id', 'title')
    search_fields = ('id', 'title')
    list_editable = ('is_published',)

    fields = ('title', 'is_published', 'views', 'slug', 'photo', 'get_photo')
    readonly_fields = ('views', 'get_photo')

    def get_photo(self, obj):
        if obj.photo:
            res = mark_safe(f'')
        else:
            res = "нет картинки"
        return res

    get_photo.short_description = 'Миниатюра'

class ImageGalleryInline(admin.TabularInline):
    model = ImageGallery
    extra = 1
    fields = ('additional_picture', 'created_at', 'get_photo')
    readonly_fields = ('get_photo', 'created_at')

    def get_photo(self, obj):
        if obj.additional_picture:
            res = mark_safe(f'')
        else:
            res = "нет картинки"
        return res

    get_photo.short_description = 'Доп. картинка'

class ProductAdmin(admin.ModelAdmin):
    inlines = [ImageGalleryInline, ]
    list_display = ('get_photo', 'title', 'article', 'is_published', 'category', 'price_unit', 'price_pack', 'views')
    list_display_links = ('title', 'article')
    search_fields = ('title', 'article', 'category')
    list_editable = ('price_unit', 'price_pack', 'is_published')
    fields = ('get_photo', 'main_image', 'title', 'article', 'category', 'is_published', 'description', 'price_unit',
              'price_pack', 'views', 'created_at', 'update_at')
    readonly_fields = ('get_photo', 'created_at', 'update_at', 'views')

    def get_photo(self, obj):
        if obj.main_image:
            res = mark_safe(f'')
        else:
            res = "нет картинки"
        return res

    get_photo.short_description = 'Фото'

admin.site.register(Category, CategoryAdmin)
admin.site.register(Product, ProductAdmin)

admin.site.site_title = "Админ панель"
admin.site.site_header = "Админ панель"


import numpy as np
import matplotlib.pyplot as plt
from models.model import Model

class HyperbolicModel(Model):

    def train(self):
        x_1 = 1 / self.x_train
        x_1_2 = x_1 * x_1
        x_1_y = x_1 * self.y_train
        x_1_avg = np.average(x_1)
        x_1_2_avg = np.average(x_1_2)
        x_1_y_avg = np.average(x_1_y)
        eq_left_side = np.array([[1, x_1_avg], [x_1_avg, x_1_2_avg]])
        eq_right_side = np.array([self.y_avg, x_1_y_avg])
        params = np.linalg.solve(eq_left_side, eq_right_side)
        self.a = params[0]
        self.b = params[1]

    def test(self):
        y_t = self.a + self.b * (1 / self.x_test)
        e = y_t - self.y_test
        e_pos_avg = np.average(e[e > 0])
        e_neg_avg = np.average(e[e < 0])
        y_t_e_pos = y_t + e_pos_avg
        y_t_e_neg = y_t + e_neg_avg

        plt.plot(self.x_test, y_t, 'b')
        plt.plot(self.x_test, y_t_e_pos, 'r--')
        plt.plot(self.x_test, y_t_e_neg, 'r--')
        r = np.corrcoef(self.y_test, y_t)[0, 1]
        self.r_2 = r * r
        print("Hyperbolic Model:\nCorrelation coefficient: ", r, "\nDetermination coefficient: ", self.r_2)

    def calculate_mse(self, x, y):
        y_t = self.a + self.b * (1 / x)
        self.mse = sum((y - y_t)**2)
        self.predictions = zip(x, y, y_t)


import numpy
import torch
import common.torch
import common.summary
import common.numpy
import attacks
from .adversarial_training import *

class ConfidenceCalibratedAdversarialTraining(AdversarialTraining):
    """
    Confidence-calibrated adversarial training.
    """

    def __init__(self, model, trainset, testset, optimizer, scheduler, attack, objective, loss, transition, fraction=0.5, augmentation=None, writer=common.summary.SummaryWriter(), cuda=False):
        """
        Constructor.

        :param model: model
        :type model: torch.nn.Module
        :param trainset: training set
        :type trainset: torch.utils.data.DataLoader
        :param testset: test set
        :type testset: torch.utils.data.DataLoader
        :param optimizer: optimizer
        :type optimizer: torch.optim.Optimizer
        :param scheduler: scheduler
        :type scheduler: torch.optim.LRScheduler
        :param attack: attack
        :type attack: attacks.Attack
        :param objective: objective
        :type objective: attacks.Objective
        :param loss: loss
        :type loss: callable
        :param loss: transition
        :type loss: callable
        :param fraction: fraction of adversarial examples per batch
        :type fraction: float
        :param augmentation: augmentation
        :type augmentation: imgaug.augmenters.Sequential
        :param writer: summary writer
        :type writer: torch.utils.tensorboard.SummaryWriter or TensorboardX equivalent
        :param cuda: run on CUDA device
        :type cuda: bool
        """

        assert fraction < 1

        super(ConfidenceCalibratedAdversarialTraining, self).__init__(model, trainset, testset, optimizer, scheduler, attack, objective, fraction, augmentation, writer, cuda)

        self.loss = loss
        """ (callable) Loss. """

        self.transition = transition
        """ (callable) Transition. """

        self.N_class = None
        """ (int) Number of classes. """

        if getattr(self.model, 'N_class', None) is not None:
            self.N_class = self.model.N_class

        self.writer.add_text('config/loss', self.loss.__name__)
        self.writer.add_text('config/transition', self.transition.__name__)

    def train(self, epoch):
        """
        Training step.

        :param epoch: epoch
        :type epoch: int
        """

        for b, (inputs, targets) in enumerate(self.trainset):
            if self.augmentation is not None:
                inputs = self.augmentation.augment_images(inputs.numpy())

            inputs = common.torch.as_variable(inputs, self.cuda)
            inputs = inputs.permute(0, 3, 1, 2)
            targets = common.torch.as_variable(targets, self.cuda)

            if self.N_class is None:
                _ = self.model.forward(inputs)
                self.N_class = _.size(1)

            distributions = common.torch.one_hot(targets, self.N_class)

            split = int(self.fraction * inputs.size()[0])
            # update fraction for correct loss computation
            fraction = split / float(inputs.size(0))

            clean_inputs = inputs[:split]
            adversarial_inputs = inputs[split:]
            clean_targets = targets[:split]
            adversarial_targets = targets[split:]
            clean_distributions = distributions[:split]
            adversarial_distributions = distributions[split:]

            self.model.eval()
            self.objective.set(adversarial_targets)
            adversarial_perturbations, adversarial_objectives = self.attack.run(self.model, adversarial_inputs, self.objective)
            adversarial_perturbations = common.torch.as_variable(adversarial_perturbations, self.cuda)
            adversarial_inputs = adversarial_inputs + adversarial_perturbations

            gamma, adversarial_norms = self.transition(adversarial_perturbations)
            gamma = common.torch.expand_as(gamma, adversarial_distributions)

            adversarial_distributions = adversarial_distributions*(1 - gamma)
            adversarial_distributions += gamma*torch.ones_like(adversarial_distributions)/self.N_class

            inputs = torch.cat((clean_inputs, adversarial_inputs), dim=0)

            self.model.train()
            self.optimizer.zero_grad()
            logits = self.model(inputs)
            clean_logits = logits[:split]
            adversarial_logits = logits[split:]

            adversarial_loss = self.loss(adversarial_logits, adversarial_distributions)
            adversarial_error = common.torch.classification_error(adversarial_logits, adversarial_targets)

            clean_loss = self.loss(clean_logits, clean_distributions)
            clean_error = common.torch.classification_error(clean_logits, clean_targets)
            loss = (1 - fraction) * clean_loss + fraction * adversarial_loss

            loss.backward()
            self.optimizer.step()
            self.scheduler.step()

            global_step = epoch * len(self.trainset) + b
            self.writer.add_scalar('train/lr', self.scheduler.get_lr()[0], global_step=global_step)

            self.writer.add_scalar('train/loss', clean_loss.item(), global_step=global_step)
            self.writer.add_scalar('train/error', clean_error.item(), global_step=global_step)
            self.writer.add_scalar('train/confidence', torch.mean(torch.max(torch.nn.functional.softmax(clean_logits, dim=1), dim=1)[0]).item(), global_step=global_step)

            self.writer.add_histogram('train/logits', torch.max(clean_logits, dim=1)[0], global_step=global_step)
            self.writer.add_histogram('train/confidences', torch.max(torch.nn.functional.softmax(clean_logits, dim=1), dim=1)[0], global_step=global_step)

            success = torch.clamp(torch.abs(adversarial_targets - torch.max(torch.nn.functional.softmax(adversarial_logits, dim=1), dim=1)[1]), max=1)
            self.writer.add_scalar('train/adversarial_loss', adversarial_loss.item(), global_step=global_step)
            self.writer.add_scalar('train/adversarial_error', adversarial_error.item(), global_step=global_step)
            self.writer.add_scalar('train/adversarial_confidence', torch.mean(torch.max(torch.nn.functional.softmax(adversarial_logits, dim=1), dim=1)[0]).item(), global_step=global_step)
            self.writer.add_scalar('train/adversarial_success', torch.mean(success.float()).item(), global_step=global_step)

            self.writer.add_histogram('train/adversarial_logits', torch.max(adversarial_logits, dim=1)[0], global_step=global_step)
            self.writer.add_histogram('train/adversarial_confidences', torch.max(torch.nn.functional.softmax(adversarial_logits, dim=1), dim=1)[0], global_step=global_step)

            self.writer.add_histogram('train/adversarial_objectives', adversarial_objectives, global_step=global_step)
            self.writer.add_histogram('train/adversarial_norms', adversarial_norms, global_step=global_step)

            if self.summary_gradients:
                for name, parameter in self.model.named_parameters():
                    self.writer.add_histogram('train_weights/%s' % name, parameter.view(-1), global_step=global_step)
                    self.writer.add_histogram('train_gradients/%s' % name, parameter.grad.view(-1), global_step=global_step)

            self.writer.add_images('train/images', inputs[:min(16, split)], global_step=global_step)
            self.writer.add_images('train/adversarial_images', inputs[split:split + 16], global_step=global_step)
            self.progress(epoch, b, len(self.trainset))

    def test(self, epoch):
        """
        Test on adversarial examples.

        :param epoch: epoch
        :type epoch: int
        """

        self.model.eval()

        # reason to repeat this here: use correct loss for statistics
        losses = None
        errors = None
        logits = None
        confidences = None

        for b, (inputs, targets) in enumerate(self.testset):
            inputs = common.torch.as_variable(inputs, self.cuda)
            inputs = inputs.permute(0, 3, 1, 2)
            targets = common.torch.as_variable(targets, self.cuda)

            if self.N_class is None:
                _ = self.model.forward(inputs)
                self.N_class = _.size(1)

            distributions = common.torch.as_variable(common.torch.one_hot(targets, self.N_class))

            outputs = self.model(inputs)
            losses = common.numpy.concatenate(losses, self.loss(outputs, distributions, reduction='none').detach().cpu().numpy())
            errors = common.numpy.concatenate(errors, common.torch.classification_error(outputs, targets, reduction='none').detach().cpu().numpy())
            logits = common.numpy.concatenate(logits, torch.max(outputs, dim=1)[0].detach().cpu().numpy())
            confidences = common.numpy.concatenate(confidences, torch.max(torch.nn.functional.softmax(outputs, dim=1), dim=1)[0].detach().cpu().numpy())
            self.progress(epoch, b, len(self.testset))

        global_step = epoch  # epoch * len(self.trainset) + len(self.trainset) - 1
        self.writer.add_scalar('test/loss', numpy.mean(losses), global_step=global_step)
        self.writer.add_scalar('test/error', numpy.mean(errors), global_step=global_step)
        self.writer.add_scalar('test/logit', numpy.mean(logits), global_step=global_step)
        self.writer.add_scalar('test/confidence', numpy.mean(confidences), global_step=global_step)

        self.writer.add_histogram('test/losses', losses, global_step=global_step)
        self.writer.add_histogram('test/errors', errors, global_step=global_step)
        self.writer.add_histogram('test/logits', logits, global_step=global_step)
        self.writer.add_histogram('test/confidences', confidences, global_step=global_step)

        self.model.eval()

        losses = None
        errors = None
        logits = None
        confidences = None
        successes = None
        norms = None
        objectives = None

        for b, (inputs, targets) in enumerate(self.testset):
            if b >= self.max_batches:
                break

            inputs = common.torch.as_variable(inputs, self.cuda)
            inputs = inputs.permute(0, 3, 1, 2)
            targets = common.torch.as_variable(targets, self.cuda)
            distributions = common.torch.as_variable(common.torch.one_hot(targets, self.N_class))

            self.objective.set(targets)
            adversarial_perturbations, adversarial_objectives = self.attack.run(self.model, inputs, self.objective)
            objectives = common.numpy.concatenate(objectives, adversarial_objectives)

            adversarial_perturbations = common.torch.as_variable(adversarial_perturbations, self.cuda)
            inputs = inputs + adversarial_perturbations

            gamma, adversarial_norms = self.transition(adversarial_perturbations)
            gamma = common.torch.expand_as(gamma, distributions)
            distributions = distributions * (1 - gamma) + gamma * torch.ones_like(distributions) / self.N_class

            outputs = self.model(inputs)
            losses = common.numpy.concatenate(losses, self.loss(outputs, distributions, reduction='none').detach().cpu().numpy())
            errors = common.numpy.concatenate(errors, common.torch.classification_error(outputs, targets, reduction='none').detach().cpu().numpy())
            logits = common.numpy.concatenate(logits, torch.max(outputs, dim=1)[0].detach().cpu().numpy())
            confidences = common.numpy.concatenate(confidences, torch.max(torch.nn.functional.softmax(outputs, dim=1), dim=1)[0].detach().cpu().numpy())
            successes = common.numpy.concatenate(successes, torch.clamp(torch.abs(targets - torch.max(torch.nn.functional.softmax(outputs, dim=1), dim=1)[1]), max=1).detach().cpu().numpy())
            norms = common.numpy.concatenate(norms, adversarial_norms.detach().cpu().numpy())
            self.progress(epoch, b, self.max_batches)

        global_step = epoch + 1# * len(self.trainset) + len(self.trainset) - 1
        self.writer.add_scalar('test/adversarial_loss', numpy.mean(losses), global_step=global_step)
        self.writer.add_scalar('test/adversarial_error', numpy.mean(errors), global_step=global_step)
        self.writer.add_scalar('test/adversarial_logit', numpy.mean(logits), global_step=global_step)
        self.writer.add_scalar('test/adversarial_confidence', numpy.mean(confidences), global_step=global_step)
        self.writer.add_scalar('test/adversarial_norm', numpy.mean(norms), global_step=global_step)
        self.writer.add_scalar('test/adversarial_objective', numpy.mean(objectives), global_step=global_step)
        self.writer.add_scalar('test/adversarial_success', numpy.mean(successes), global_step=global_step)

        self.writer.add_histogram('test/adversarial_losses', losses, global_step=global_step)
        self.writer.add_histogram('test/adversarial_errors', errors, global_step=global_step)
        self.writer.add_histogram('test/adversarial_logits', logits, global_step=global_step)
        self.writer.add_histogram('test/adversarial_confidences', confidences, global_step=global_step)
        self.writer.add_histogram('test/adversarial_norms', norms, global_step=global_step)
        self.writer.add_histogram('test/adversarial_objectives', objectives, global_step=global_step)