# ------------------------------------------------------------------------------
#
# ------------------------------------------------------------------------------

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os

import torch
import torch.nn.parallel
# import torch.backends.cudnn as cudnn
import torch.optim
import torch.utils.data
import torch.utils.data.distributed
# import torchvision.transforms as transforms

import _init_paths
from config import cfg
from config import update_config
# from core.loss import JointsMSELoss
# from core.function import validate
from core import inference
# from utils.utils import create_logger

# import dataset
import models

import cv2
import mmcv
from mmdet.apis import inference_detector
# from mmdet.datasets import to_tensor

import time
import json
import math
# import glob
# import numpy as np
# from PIL import Image

import magic

# "keypoints": [
#             "nose","left_eye","right_eye","left_ear","right_ear",
#             "left_shoulder","right_shoulder","left_elbow","right_elbow",
#             "left_wrist","right_wrist","left_hip","right_hip",
#             "left_knee","right_knee","left_ankle","right_ankle"
#         ],

def remove_trailing_slash(str):
    return str[:-1] if str[-1] is '/' else str

def read_bboxes(jspath, input_path, expand=1.0):
    '''
    :param jspath:
    :param input_path:
    :param expand:
    :return:
    '''
    with open(jspath, 'r') as f:
        json_data = json.load(f)

    image_size = json_data['image_size']
    detections_dict = json_data['results']
    bboxes = list()
    img_fnames = list()
    final_bbox_coords = list()
    vid_frames = list()

    if os.path.isfile(input_path):
        mime = magic.from_file(input_path, mime=True)
        if 'video' in mime:
            # extract the video into frames
            vid_frames = list(mmcv.VideoReader(input_path))

            # auto-generate file names: these files will never exist in disk
            # img_fnames = ['{}.jpg'.format(n * step + 1) for n in range(len(vid_frames))]
        else:
            raise Exception('Input file type not supported.')

    for idx, img_fname in enumerate(detections_dict.keys()):
        if os.path.isdir(input_path):
            full_img = mmcv.imread('{}/{}'.format(input_path, img_fname))
        elif vid_frames:
            full_img = vid_frames[idx]
        else:
            raise Exception('No valid input provided')

        for r in detections_dict[img_fname]:
            lt, rb = r['bbox']['lt'], r['bbox']['rb']  # left top, right bottom corners

            lt = [int(dim*(1-expand/100)) for dim in lt]  # apply expansion
            rb = [int(dim*(1+expand/100)) for dim in rb]

            final_bbox_coords.append((lt, rb))

            bbox = full_img[lt[1]:rb[1], lt[0]:rb[0]]  # crop
            bboxes.append(bbox)
            img_fnames.append(img_fname)
            # mmcv.imshow(bbox)  # debug

    return dict(img_fnames=img_fnames, image_size=image_size,
                bboxes=bboxes, final_bbox_coords=final_bbox_coords)

def calc_avg_aspect_ratio(bboxes):
    '''
    :param bboxes:
    :return: the averaged height/width ratio
    '''
    return sum(bbox.shape[0] for bbox in bboxes) / sum(bbox.shape[1] for bbox in bboxes)

def project_point_along_direction(ref_point, point_2, x_cut=0.0, y_cut=0.0, p_value=0.0):
    '''

    :param ref_point:
    :param point_2:
    :param x_cut:
    :param y_cut:
    :param p_value:
    :return:
    '''
    r_x, r_y = ref_point[0], ref_point[1]
    p_x, p_y = point_2[0], point_2[1]

    d_i = p_x - r_x
    d_j = p_y - r_y

    if (r_x == p_x):
        theta = math.pi / 2  # arctan limit x->inf
    else:
        theta = abs(math.atan(d_j / d_i))

    sign_x = -1 if d_i < 0 else 1
    sign_y = -1 if d_j < 0 else 1

    if theta == math.pi/2 or theta == 0:  # singularities, do nothing
        return int(p_x), int(p_y)

    if x_cut:
        new_x = x_cut
        new_magnitude = (new_x - r_x) / (sign_x * math.cos(theta))
        new_y = r_y + sign_y * new_magnitude * math.sin(theta)
    elif y_cut:
        new_y = y_cut
        new_magnitude = (new_y - r_y) / (sign_y * math.sin(theta))
        new_x = r_x + sign_x * new_magnitude * math.cos(theta)
    elif p_value:
        segment_magnitude = math.sqrt((r_x - p_x) ** 2 + (r_y - p_y) ** 2)
        new_magnitude = segment_magnitude * (1 + p_value)
        new_x = r_x + sign_x * new_magnitude * math.cos(theta)
        new_y = r_y + sign_y * new_magnitude * math.sin(theta)
    else:
        raise Exception('Either x_cut or y_cut or p_value must be provided')

    return int(new_x), int(new_y)

def get_valeo_pd_ann(coords_list, final_bbox_coords):
    '''
    :param coords_list:
    :param final_bbox_coords:
    :return:
    '''
    bbox_topleft = final_bbox_coords[0]
    bbox_rightbottom = final_bbox_coords[1]

    left_ankle = coords_list[15]
    left_ankle_to_bbox = abs(left_ankle[1] - bbox_rightbottom[1])
    right_ankle = coords_list[16]
    right_ankle_to_bbox = abs(right_ankle[1] - bbox_rightbottom[1])

    closest_dist = min(left_ankle_to_bbox, right_ankle_to_bbox)
    ears_middle_point = (int((coords_list[3][0] + coords_list[4][0]) / 2),
                         int((coords_list[3][1] + coords_list[4][1]) / 2))

    valeo_middle_point = (int((coords_list[11][0] + coords_list[12][0]) / 2),
                          int((coords_list[11][1] + coords_list[12][1]) / 2))

    valeo_head_top = project_point_along_direction(valeo_middle_point, ears_middle_point, y_cut=bbox_topleft[1])

    valeo_left_foot = (left_ankle[0], left_ankle[1] + closest_dist)
    valeo_right_foot = (right_ankle[0], right_ankle[1] + closest_dist)

    # Not good
    # valeo_left_foot = project_point_along_direction(valeo_middle_point, left_ankle, y_cut=bbox_rightbottom[1])
    # valeo_right_foot = project_point_along_direction(valeo_left_foot, right_ankle, x_cut=bbox_rightbottom[0])

    return dict(left_foot=valeo_left_foot, right_foot=valeo_right_foot,
                middle_point=valeo_middle_point, top_point=valeo_head_top, top_original=ears_middle_point)

def get_valeo_scores(scores):
    '''
    Simplistic score approximations

    :param scores:
    :return:
    '''
    scores = scores[0]
    return [str(scores[15][0])[:5],                      # left_ankle
            str(scores[16][0])[:5],                      # right_ankle
            str(0.5*(scores[11][0]+scores[12][0]))[:5],  # hip average
            str(0.5*(scores[3][0]+scores[4][0]))[:5]]    # ears average

def draw_valeo_keypoints(img, img_fname, valeo_ann, args):  ## debug
    '''
    :param img:
    :param img_fname:
    :param valeo_ann:
    :param args:
    :return:
    '''
    new_path = '{}/{}_valeo_annotation.jpg'.format(args.output_path, os.path.splitext(os.path.basename(img_fname))[0])

    if  os.path.exists(new_path):  # accumulate instances
        img = mmcv.imread(new_path)

    cv2.circle(img, valeo_ann[0], 4, (0, 0, 255), -1)
    cv2.circle(img, valeo_ann[1], 4, (0, 0, 255), -1)
    cv2.circle(img, valeo_ann[2], 4, (0, 0, 255), -1)
    cv2.circle(img, valeo_ann[3], 4, (0, 0, 255), -1)
    cv2.circle(img, valeo_ann[4], 4, (0, 255, 0), -1)

    cv2.line(img, valeo_ann[0], valeo_ann[1], (0, 0, 255), 2, 1)
    cv2.line(img, valeo_ann[1], valeo_ann[2], (0, 0, 255), 2, 1)
    cv2.line(img, valeo_ann[2], valeo_ann[3], (0, 0, 255), 2, 1)

    mmcv.imwrite(img, new_path)

def draw_all_keypoints(img, img_fname, coords_list, scores, final_bbox_coords, args):  ## debug
    '''
    :param img:
    :param img_fname:
    :param coords_list:
    :param scores:
    :param final_bbox_coords:
    :param args:
    :return:
    '''
    new_path = '{}/{}_poses.jpg'.format(args.output_path, os.path.splitext(os.path.basename(img_fname))[0])

    if  os.path.exists(new_path):  # accumulate instances
        img = mmcv.imread(new_path)

    for idx, coords in enumerate(coords_list):
        if idx in [1, 2, 3, 4]: continue

        cv2.circle(img, coords, 4, (0, 255, 255), -1)
        cv2.putText(img, str(idx)+': '+str(scores[0][idx][0])[:3], (coords[0]+5, coords[1]),
                    cv2.FONT_HERSHEY_PLAIN, 0.8, (0, 255, 0), 1, cv2.LINE_AA)

    # BBOX
    cv2.rectangle(img, tuple(final_bbox_coords[0]), tuple(final_bbox_coords[1]), (0, 0, 255), 2)

    ## Basic skeleton
    # line_color = (255, 255, 0)
    # line_thickness = 2
    # cv2.line(img, coords_list[0], coords_list[5], line_color, line_thickness)
    # cv2.line(img, coords_list[0], coords_list[6], line_color, line_thickness)

    # cv2.line(img, coords_list[5], coords_list[6], line_color, line_thickness)
    # cv2.line(img, coords_list[5], coords_list[7], line_color, line_thickness)
    # cv2.line(img, coords_list[5], coords_list[11], line_color, line_thickness)
    # cv2.line(img, coords_list[6], coords_list[12], line_color, line_thickness)
    # cv2.line(img, coords_list[6], coords_list[8], line_color, line_thickness)
    # cv2.line(img, coords_list[7], coords_list[9], line_color, line_thickness)
    # cv2.line(img, coords_list[8], coords_list[10], line_color, line_thickness)
    # cv2.line(img, coords_list[11], coords_list[12], line_color, line_thickness)
    # cv2.line(img, coords_list[11], coords_list[13], line_color, line_thickness)
    # cv2.line(img, coords_list[12], coords_list[14], line_color, line_thickness)
    # cv2.line(img, coords_list[13], coords_list[15], line_color, line_thickness)
    # cv2.line(img, coords_list[14], coords_list[16], line_color, line_thickness)

    mmcv.imwrite(img, new_path)

    # cv2.namedWindow('image', cv2.WINDOW_NORMAL)
    # cv2.imshow('image', img)
    # cv2.waitKey(0)
    # cv2.destroyAllWindows()

def process_heatmaps(outputs, bboxes_data, args):
    '''
    :param outputs:
    :param bboxes_data:
    :param input_path:
    :return:
    '''

    vid_frames = list()

    # Handle video input
    if os.path.isfile(args.input_path):
        mime = magic.from_file(args.input_path, mime=True)
        if 'video' in mime:
            # extract the video into frames
            vid_frames = list(mmcv.VideoReader(args.input_path))
            # auto-generate file names: these files will never exist in disk
            # img_fnames = ['{}.jpg'.format(n * step + 1) for n in range(len(vid_frames))]
        else:
            raise Exception('Input file type not supported.')

    result_dict = dict()

    for idx, output in enumerate(list(outputs)):
        if isinstance(output, tuple):
            heatmaps, _ = output
        else:
            heatmaps, _ = output, None

        heatmaps = heatmaps.clone().cpu().numpy()
        heatmap_shape = (heatmaps.shape[2], heatmaps.shape[3])

        keypoints, scores = inference.get_max_preds(heatmaps)

        bboxes = bboxes_data['bboxes']
        final_bbox_coords = bboxes_data['final_bbox_coords']
        bbox_img = mmcv.imread(bboxes[idx])
        bbox_shape = bbox_img.shape[:2]

        basefname = bboxes_data['img_fnames'][idx]
        img_fname = '{}/{}'.format(args.input_path, basefname)

        if os.path.isdir(args.input_path):
            full_img = mmcv.imread(img_fname)
        elif vid_frames:
            frame_idx = int(basefname.split('.')[0]) - 1
            print(frame_idx)
            full_img = vid_frames[frame_idx]
        else:
            raise Exception('No valid input provided')

        lefttop_point = bboxes_data['final_bbox_coords'][idx][0]
        coords_list = list()

        for _, kp in enumerate(keypoints[0]):
            bbox_coords = (int(kp[0] * bbox_shape[0]/heatmap_shape[0]), int(kp[1] * bbox_shape[1]/heatmap_shape[1]))
            full_img_coords = (bbox_coords[0] + lefttop_point[0], bbox_coords[1] + lefttop_point[1])
            coords_list.append(full_img_coords)

        if args.valeo:
            valeo_ann = get_valeo_pd_ann(coords_list, final_bbox_coords[idx])
            coords_list = list(valeo_ann.values())
            sc = get_valeo_scores(scores)
        else:
            sc = [str(s[0])[:5] for s in scores[0]]

        if basefname not in list(result_dict):
            result_dict[basefname] = dict(keypoints=coords_list, scores=sc)
        else:
            result_dict[basefname]['keypoints'] += coords_list
            result_dict[basefname]['scores'] += sc

        if args.save_images:
            if args.valeo:
                draw_valeo_keypoints(full_img, img_fname, coords_list, args)
            else:
                draw_all_keypoints(full_img, img_fname, coords_list, scores, final_bbox_coords[idx], args)

    # Save JSON file
    with open('{}/{}_keypoints.json'.format(args.output_path, time.strftime("%Y%m%d%H%M%S")), 'w') as out_file:
        json.dump(result_dict, out_file)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("-i", "--input_path", help="Path to directory containing images OR path to a single image",
                        required=True)
    parser.add_argument("-js", "--input_json", help="Path to JSON containing detections",
                        required=True)
    parser.add_argument("-o", "--output_path", help="Desired output dir",
                        required=True)
    parser.add_argument("-s", "--save_images", action='store_true')
    parser.add_argument('-v', '--valeo', help='Valeo PD annotation', action='store_true')
    args = parser.parse_args()

    args.input_path = remove_trailing_slash(args.input_path)
    args.output_path = remove_trailing_slash(args.output_path)

    # Static params
    config_file = '../experiments/coco/hrnet/w48_384x288_adam_lr1e-3.yaml'
    cfg_mmcv = mmcv.Config.fromfile('../lib/config/mmcv_config.py')
    checkpoint = '../lib/models/pytorch/pose_coco/pose_hrnet_w48_384x288.pth'

    # Trying to simplify bullshit cfg parsing without modifying get_pose_net() method
    cfg.cfg = config_file
    cfg.opts, cfg.modelDir, cfg.logDir, cfg.dataDir = [], '', '', ''
    update_config(cfg, cfg)
    ##

    bboxes_data = read_bboxes(args.input_json, args.input_path)
    bboxes = bboxes_data['bboxes']

    model_input_width = cfg.MODEL.IMAGE_SIZE[1]
    height_scale = calc_avg_aspect_ratio(bboxes)
    cfg_mmcv.data.test.img_scale = (model_input_width, int(height_scale*model_input_width))

    model = models.pose_hrnet.get_pose_net(cfg, is_train=False)
    model.load_state_dict(torch.load(checkpoint), strict=False)

    if len(bboxes) > 1:
        outputs = inference_detector(model, bboxes, cfg_mmcv)
    elif len(bboxes) == 1:
        outputs = [inference_detector(model, bboxes, cfg_mmcv)]
    else:
        raise Exception('There are no bboxes!')

    process_heatmaps(outputs, bboxes_data, args)

if __name__ == '__main__':
    main()


"""
Interface to interact with fireplace simulator.
Includes two classes: Board and SelfPlay
Board interacts with fireplace and returns state, possible actions, etc.
SelfPlay initiliazes self play games
"""
from fireplace.game import Game
from fireplace.player import Player
from fireplace.utils import random_draft
from fireplace import cards
from fireplace.exceptions import GameOver, InvalidAction
from hearthstone.enums import CardClass, CardType
from .exceptions import UnhandledAction
import random
import numpy as np
import sys

"""
TO DO:
Test do_move function
Def winner function
Redo all of it rip
"""
#current player return function = game.current_player
class Board(object):
    def __init__(self, is_basic=True):
        self.state = np.zeros(263, dtype=np.int32)
        self.players = ['player1', 'player2']
        self.is_basic = is_basic
        # self.game = Game()

    def init_envi(self):
        """
        Initializes the environment. All basic initialization goes here.
        """
        cards.db.initialize()
        self.last_move = -1

    def init_game(self):
        """
        initializes a game between two players
        Returns:
            game: A game entity representing the start of the game after the mulligan phase
        """
        if self.is_basic: #create quick simple game
            p1 = 6 #priest
            p2 = 7 #rogue
            deck1 = random_draft(CardClass(p1))
            deck2 = random_draft(CardClass(p2))
            self.players[0] = Player("Player1", deck1, CardClass(p1).default_hero)
            self.players[1] = Player("Player2", deck2, CardClass(p2).default_hero)
            self.game = Game(players=self.players)
            self.game.start()

            #Skip mulligan
            for player in self.game.players:
                cards_to_mulligan = random.sample(player.choice.cards, 0)
                player.choice.choose(*cards_to_mulligan)

            return self.game

        else:
            p1 = random.randint(1, 9)
            p2 = random.randint(1, 9)
            #initialize players and randomly draft decks
            #pdb.set_trace()
            deck1 = random_draft(CardClass(p1))
            deck2 = random_draft(CardClass(p2))
            self.players[0] = Player("Player1", deck1, CardClass(p1).default_hero)
            self.players[1] = Player("Player2", deck2, CardClass(p2).default_hero)
            #begin the game
            self.game = Game(players=self.players)
            self.game.start()

            #Skip mulligan for now
            for player in self.game.players:
                cards_to_mulligan = random.sample(player.choice.cards, 0)
                player.choice.choose(*cards_to_mulligan)

            return self.game

    def get_actions(self, player):
        """
        generate a list of tuples representing all valid actions
        format:
            (actiontype, index, target)
        card.requires_target!!
        """
        actions = []

        #If the player is being given a choice, return only valid choices
        if player.choice:
            for card in player.choice.cards:
                actions.append(("choose", card, None))

        else:
            # add cards in hand
            for index, card in enumerate(player.hand):
                if card.is_playable():
                    # summonable minions (note some require a target on play)
                    if card.type == 4:
                        if card.requires_target():
                            for target in card.targets:
                                actions.append(("summon", index, target))
                        else:
                            actions.append(("summon", index, None, None))
                    # playable spells and weapons
                    elif card.requires_target():
                        for target in card.targets:
                            actions.append(("spell", index, target))
                    else:
                        actions.append(("spell", index, None))
            # add targets avalible to minions that can attack
            for position, minion in enumerate(player.field):
                if minion.can_attack():
                    for target in minion.attack_targets:
                        actions.append(("attack", position, target))
            # add hero power and targets if applicable
            if player.hero.power.is_usable():
                if player.hero.power.requires_target():
                    for target in player.hero.power.targets:
                        actions.append(("hero_power", None, target))
                else:
                    actions.append(("hero_power", None, None))
            # add hero attacking if applicable
            if player.hero.can_attack():
                for target in player.hero.attack_targets:
                    actions.append(("hero_attack", None, target))
            # add end turn
            actions.append(("end_turn", None, None))
            self.actions = actions
        return actions

    def get_actionsize(self):
        self.action_size = len(self.actions)
        return self.action_size

    def perform_action(self, a, player, game):
        """
        utilty to convert an action tuple
        into an action input
        Args:
            a, a tuple representing (action, index, target)
        """

        try:

            if a[0] == "summon":
                if a[2] is None:
                    player.hand[a[1]].play()
                else:
                    player.hand[a[1]].play(a[2])
            elif a[0] == "spell":
                if a[2] is None:
                    player.hand[a[1]].play()
                else:
                    player.hand[a[1]].play(a[2])
            elif a[0] == "attack":
                player.field[a[1]].attack(a[2])
            elif a[0] == "hero_power":
                if a[2] is None:
                    player.hero.power.use()
                else:
                    player.hero.power.use(a[2])
            elif a[0] == "hero_attack":
                player.hero.attack(a[2])
            elif a[0] == "end_turn":
                game.end_turn()
            elif a[0] == "choose":
                #print("Player choosing card %r, " % a[1])
                player.choice.choose(a[1])
            else:
                raise UnhandledAction
        except UnhandledAction:
            print("Attempted to take an inappropriate action!\n")
            print(a)
        except GameOver:
            raise

    def get_state(self):
        """
        Args:
            game, the current game object
            player, the player from whose perspective to analyze the state
        return:
            a numpy array features extracted from the
            supplied game.
        """
        s = np.zeros(263, dtype=np.int32)
        c = np.zeros(263, dtype=np.int32)
        for player in self.players:
            p1 = player
            p2 = player.opponent

            #0-9 player1 class, we subtract 1 here because the classes are from 1 to 10
            s[p1.hero.card_class-1] = 1
            #10-19 player2 class
            s[10 + p2.hero.card_class-1] = 1
            i = 20
            # 20-21: current health of current player, then opponent
            s[i] = p1.hero.health
            s[i + 1] = p2.hero.health

            # 22: hero power usable y/n
            s[i + 2] = p1.hero.power.is_usable()*1
            # 23-24: # of mana crystals for you opponent
            s[i + 3] = p1.max_mana
            s[i + 4] = p2.max_mana
            # 25: # of crystals still avalible
            s[i + 5] = p1.mana
            #26-31: weapon equipped y/n, pow., dur. for you, then opponent
            s[i + 6] = 0 if p1.weapon is None else 1
            s[i + 7] = 0 if p1.weapon is None else p1.weapon.damage
            s[i + 8] = 0 if p1.weapon is None else p1.weapon.durability

            s[i + 9] = 0 if p2.weapon is None else 1
            s[i + 10] = 0 if p2.weapon is None else p2.weapon.damage
            s[i + 11] = 0 if p2.weapon is None else p2.weapon.durability

            # 32: number of cards in opponents hand
            s[i + 12] = len(p2.hand)
            #in play minions

            i = 33
            #33-102, your monsters on the field
            p1_minions = len(p1.field)
            for j in range(0, 7):
                if j < p1_minions:
                    # filled y/n, pow, tough, current health, can attack
                    s[i] = 1
                    s[i + 1] = p1.field[j].atk
                    s[i + 2] = p1.field[j].max_health
                    s[i + 3] = p1.field[j].health
                    s[i + 4] = p1.field[j].can_attack()*1
                    # deathrattle, div shield, taunt, stealth y/n
                    s[i + 5] = p1.field[j].has_deathrattle*1
                    s[i + 6] = p1.field[j].divine_shield*1
                    s[i + 7] = p1.field[j].taunt*1
                    s[i + 8] = p1.field[j].stealthed*1
                    s[i + 9] = p1.field[j].silenced*1
                i += 10

            #103-172, enemy monsters on the field
            p2_minions = len(p2.field)
            for j in range(0, 7):
                if j < p2_minions:
                    # filled y/n, pow, tough, current health, can attack
                    s[i] = 1
                    s[i + 1] = p2.field[j].atk
                    s[i + 2] = p2.field[j].max_health
                    s[i + 3] = p2.field[j].health
                    s[i + 4] = p2.field[j].can_attack()*1
                    # deathrattle, div shield, taunt, stealth y/n
                    s[i + 5] = p2.field[j].has_deathrattle*1
                    s[i + 6] = p2.field[j].divine_shield*1
                    s[i + 7] = p2.field[j].taunt*1
                    s[i + 8] = p2.field[j].stealthed*1
                    s[i + 9] = p2.field[j].silenced*1
                i += 10

            #in hand

            #173-262, your cards in hand
            p1_hand = len(p1.hand)
            for j in range(0, 10):
                if j < p1_hand:
                    #card y/n
                    s[i] = 1
                    # minion y/n, attk, hp, battlecry, div shield, deathrattle, taunt
                    s[i + 1] = 1 if p1.hand[j].type == 4 else 0
                    s[i + 2] = p1.hand[j].atk if s[i + 1] == 1 else 0
                    s[i + 2] = p1.hand[j].health if s[i + 1] == 1 else 0
                    s[i + 3] = p1.hand[j].divine_shield*1 if s[i + 1] == 1 else 0
                    s[i + 4] = p1.hand[j].has_deathrattle*1 if s[i + 1] == 1 else 0
                    s[i + 5] = p1.hand[j].taunt*1 if s[i + 1] == 1 else 0
                    # weapon y/n, spell y/n, cost
                    s[i + 6] = 1 if p1.hand[j].type == 7 else 0
                    s[i + 7] = 1 if p1.hand[j].type == 5 else 0
                    s[i + 8] = p1.hand[j].cost
                i += 9
            if player == self.players[0]:
                c = s
                continue
            else:
                self.state = np.append(c, s)
                return self.state
        #     self.state = s
        # return self.state

    def get_state_data(self):
        state = []
        self.get_actions()
        pass

    def check_winner(self):
        for player in self.players:
            if player.playstate == PlayState.WON:
                return True, player
            if player.playstate == PlayState.TIED:
                return True, -1
        return False, -1

class SelfPlay(object):
    def __init__(self, board):
        self.board = board

    def start_play(self, ai_player, mcts_benchmark):
        self.board.init_envi()
        game = self.board.init_game()
        p1, p2 = self.board.players
        while True:
            current_player = game.current_player
            turn_player = players[current_player]
            move = get_actions(turn_player)
            self.board.perform_action(move)
            end, winner = self.board.check_winner()
            if end:
                return winner

    def start_self_play(self, mcts_player, temp=1e-3):
        """ start a self-play game using a MCTS player, reuse the search tree,
        and store the self-play data: (state, mcts_probs, z) for training
        """
        self.board.init_envi()
        game = self.board.init_game()

        p1, p2 = self.board.players
        states, mcts_probs, current_players = [], [], []
        while True:
            move, move_probs = mcts_player.get_action(self.board,
                temp=temp, return_prob=1)

            #store data for training
            states.append(self.board.get_state())
            mcts_probs.append(move_probs)
            current_players.append(game.current_player)

            #perform move
            self.board.perform_action(move, game.current_player, game)
            #evaluate game end
            end, winner = self.board.check_winner()
            if end:
                # winner from the perspective of the current player of each state
                winners_z = np.zeros(len(current_players))
                if winner != -1:
                    winners_z[np.array(current_players) == winner] = 1.0
                    winners_z[np.array(current_players) != winner] = -1.0
                mcts_player.reset_player()
                return winner, zip(states, mcts_probs, winners_z)

"""
    if __name__ == "__main__":
        initialize()
        game = setup_game()
        try:
            while True:
                if game.current_player.choice:
                    actions = get_actions(game.current_player)
                index = random.randint(0, len(actions)-1)
                perform_action(actions[index], game.current_player, game)
        except GameOver:
            print('Game ended successfully')
        except InvalidAction as err:
            print('Invalid Action: ', err)
        except:
            print('Unexcepted error!!', sys.exc_info()[0])
"""


# coding=utf-8
import time

import ccxt
import numpy as np
import cbo_order_market
import cbo_order_stop
import line

# ロット #単位はドル
LOT = 5000
# 本番用(mex1)
bitmex = ccxt.bitmex({
    'apiKey': '自分のbitmex api キー',
    'secret': '自分のbitmex api シークレット',
    'enableRateLimit': True
})
bitmex.urls['api'] = bitmex.urls['www']

# 過去18時間のhigh_lowを取得
def high_low18(last,line_num):
    timest = bitmex.nonce()
    timest = timest - 1000 * 60 * 1140
    candles = bitmex.fetch_ohlcv('BTC/USD', timeframe='1h', since=timest)
    high = np.array([])
    low = np.array([])
    for i in candles:
        high = np.append(high, i[2])
        low = np.append(low, i[3])
    high_max = np.max(high[0:-1])
    low_min = np.min(low[0:-1])
    body = "過去 ： 高値 => " + str(high_max) + " 安値 => " + str(low_min)
    print(body)
    if line_num % 600 == 0:
        positions = bitmex.private_get_position()
        entry_price = positions[0]['avgEntryPrice']
        line.execute("[MEX_DTN生存確認]" + "現在価格：" + str(last) + " " + body + "保有vol：" + str(entry_price))
    if last > high_max:
        return "上昇ドテン",high_max,low_min
    elif last < low_min:
        return "下降ドテン",high_max,low_min
    else:
        return "",high_max,low_min

# 現状のポジションを確認
def check_position():
    positions = bitmex.private_get_position()
    position_volume = positions[0]['currentQty']
    if position_volume == 0:
        side = "NO POSITION"
    else:
        if position_volume > 0:
            side = "Buy"
            position_volume = positions[0]['currentQty']
        elif positions[0]['currentQty'] < 0:
            side = "Sell"
            position_volume = positions[0]['currentQty']
    print("現在の保有ポジション = " + str(side) + ":" + str(position_volume))
    return side

# ドテンタイミングの監視をスタート
line_num = 0
now_position = check_position()
status = ""
cur_high_max = 0
cur_low_min = 0
while True:
    try:
        now_position = check_position()
        last = (bitmex.fetch_order_book('BTC/USD')['bids'][0][0] + bitmex.fetch_order_book('BTC/USD')['bids'][0][0]) / 2
        print("MEX1現在価格：" + str(last))
        done = high_low18(last,line_num)
        line_num += 1

        if status != done[0]:
            print(done[0])
            if done[0] == "上昇ドテン" and (now_position == "Sell" or now_position == "NO POSITION"):
                order = cbo_order_market.execute('b', done[1], done[2],bitmex,LOT)
            elif done[0] == "下降ドテン" and (now_position == "Buy" or now_position == "NO POSITION"):
                order = cbo_order_market.execute('s', done[1], done[2],bitmex,LOT)
            if done[0] != "":
                status = done[0]
        orders = bitmex.fetch_open_orders()
        if (cur_low_min != 0 and cur_low_min != done[2] and now_position == 'Buy') or (cur_high_max != 0 and cur_high_max != done[1] and now_position == 'Sell') or (len(orders) == 0 and now_position != "NO POSITION"):
            print('stop価格の更新があったので、現在のorderをキャンセルして再注文')
            cbo_order_stop.update_only_stop(now_position, done[1], done[2],bitmex,LOT)

        cur_high_max = done[1]
        cur_low_min = done[2]
        time.sleep(10)
    except Exception as e:
        print("エラー:" + str(e))
        pass


import os
import sys
import ctypes
import platform
import os
import numpy as np
from random import gauss
import win32com.client as com
def get_acceleration(lead_veh_acceleration,lead_veh_id,lead_veh_rel_velocity,lead_veh_type,timestep,
                     veh_id,veh_setspeed,veh_type,veh_used_distance_headway,veh_used_rel_vel,veh_velocity,
                     veh_distance_headway,prev_veh_executed_acceleration,
                                       prev_veh_cc_setpoint,prev_veh_cruisecontrol_acceleration):
    if platform.system() == 'Windows':
        print('Running on win')
        filepath = "L:\\UserData\\Kingsley\\PythonEnsembleTestBed"

        file_dll = os.path.join(filepath, 'OperationalDLL.dll')
        #file_dll = './OperationalDLL.dll'
    elif platform.system() == 'Darwin':
        print('Running on mac')
        file_dll = 'OperationalDLL.dylib'
    else:
        print('System not supported')
        sys.exit()

    # Load operational DLL
    lib = None
    try:
        lib = ctypes.cdll.LoadLibrary(file_dll)
    except:
        print('Error: DLL file could not be found')
        quit()

    # Set input values: Write value's for current vehicle, in current timestep
    curr_lead_veh_acceleration = ctypes.c_double(lead_veh_acceleration) #2.0
    curr_lead_veh_id = ctypes.c_long(lead_veh_id) #40
    curr_lead_veh_rel_velocity = ctypes.c_double(lead_veh_rel_velocity ) #-1.0
    curr_lead_veh_type = ctypes.c_long(lead_veh_type) #10
    curr_timestep = ctypes.c_double(timestep) #55.0
    curr_ts_length = ctypes.c_double(0.1)
    curr_veh_id = ctypes.c_long(veh_id) #10
    curr_veh_setspeed = ctypes.c_double(veh_setspeed) #88/3.6
    curr_veh_type = ctypes.c_long(veh_type) #10
    curr_veh_controller_in_use = ctypes.c_long(1) # from tactical layer 1=ACC,2=CACC
    curr_veh_ACC_h = ctypes.c_double(1.6)
    curr_veh_CACC_h = ctypes.c_double(0.6)
    curr_veh_used_distance_headway = ctypes.c_double(veh_used_distance_headway)#20.0
    curr_veh_used_rel_vel = ctypes.c_double(veh_used_rel_vel) #-1.0
    curr_veh_velocity = ctypes.c_double(veh_velocity) #85/3.6
    curr_veh_autonomous_operational_warning = ctypes.c_long(10)
    curr_veh_platooning_max_acceleration = ctypes.c_double(2.0)

    prev_veh_cc_setpoint = ctypes.c_double(prev_veh_cc_setpoint)
    prev_veh_cruisecontrol_acceleration = ctypes.c_double(prev_veh_cruisecontrol_acceleration)
    prev_veh_distance_headway = ctypes.c_double(veh_distance_headway) #20.0
    prev_veh_executed_acceleration = ctypes.c_double(prev_veh_executed_acceleration) #-2.0

    # Define variables for return values: These are placeholders, no action required
    veh_autonomous_operational_acceleration = ctypes.c_double(1)
    veh_autonomous_operational_mixingmode = ctypes.c_long(1)
    veh_autonomous_operational_warning = ctypes.c_double(1)
    veh_cc_setpoint = ctypes.c_double(1)
    veh_cruisecontrol_acceleration = ctypes.c_double(1)
    success = ctypes.c_int(0)

    print("Now call the OL itself...")

    # Call operational controller
    lib.operational_controller(
        curr_lead_veh_acceleration,
        curr_lead_veh_id,
        curr_lead_veh_rel_velocity,
        curr_lead_veh_type,
        curr_timestep,
        curr_ts_length,
        curr_veh_id,
        curr_veh_setspeed,
        curr_veh_type,
        curr_veh_controller_in_use,
        curr_veh_ACC_h,
        curr_veh_CACC_h,
        curr_veh_used_distance_headway,
        curr_veh_used_rel_vel,
        curr_veh_velocity,
        curr_veh_autonomous_operational_warning,
        curr_veh_platooning_max_acceleration,
        prev_veh_cc_setpoint,
        prev_veh_cruisecontrol_acceleration,
        prev_veh_distance_headway,
        prev_veh_executed_acceleration,
        ctypes.byref(veh_autonomous_operational_acceleration),
        ctypes.byref(veh_autonomous_operational_mixingmode),
        ctypes.byref(veh_autonomous_operational_warning),
        ctypes.byref(veh_cc_setpoint),
        ctypes.byref(veh_cruisecontrol_acceleration),
        ctypes.byref(success))

    # Print the return values
    if success.value > 0:
        veh_acceleration=veh_autonomous_operational_acceleration.value
        #print(veh_autonomous_operational_mixingmode.value)
        #print(veh_autonomous_operational_warning.value)
        veh_cc_set_point=veh_cc_setpoint.value
        veh_cruise_control_acceleration=veh_cruisecontrol_acceleration.value
    else:
        veh_acceleration=-999
        veh_cc_setpoint=-999
        veh_cruise_control_acceleration=-999
        print('An error occurred while calling DLL')
    return veh_acceleration,veh_cc_setpoint,veh_cruise_control_acceleration

Vissim = com.gencache.EnsureDispatch("Vissim.Vissim")
GlosaNetworkPath='D:\\Projects\\ENSEMBLE\\Vissim_networks\\Pipeline'#'L:\\UserData\\Kingsley\\Ensemble'
    #'L:\\UserData\\Kingsley\\SafeDriving'
    #'L:\\UserData\\Kingsley\\Ensemble'#'C:\\Users\\Public\\Documents\\GLOSA\\GlosaTrafficLight'
Filename= os.path.join(GlosaNetworkPath, 'Pipeline.inpx')
#os.path.join(GlosaNetworkPath, 'KnooppuntZonzeelBackup.inpx') #os.path.join(GlosaNetworkPath, 'GlosaTestNetwork2.inpx')
#os.path.join(GlosaNetworkPath, 'TestNetwork.inpx')
flag_read_additionally  = False # you can read network(elements) additionally, in this case set "flag_read_additionally" to true
Vissim.LoadNet(Filename, flag_read_additionally)
## Load a Layout:
Filename = os.path.join(GlosaNetworkPath, 'Pipeline.layx')
#os.path.join(GlosaNetworkPath, 'KnooppuntZonzeelBackup.layx')
    #os.path.join(GlosaNetworkPath, 'TestNetwork.layx')
    #os.path.join(GlosaNetworkPath, 'KnooppuntZonzeelBackup.layx')#os.path.join(GlosaNetworkPath, 'GlosaTestNetwork2.layx')
Vissim.LoadLayout(Filename)

End_of_simulation = 6000 # simulation second [s]
Simulation_Resolution = 10 # simulation second [s]
Number_Runs=4
Simulation_Period=300
Vissim.Simulation.SetAttValue('SimRes', Simulation_Resolution)
Vissim.Simulation.SetAttValue('NumRuns', Number_Runs)
Vissim.Simulation.SetAttValue('SimPeriod', Simulation_Period)
#UDA6
#Vissim.Net.UserDefinedAttributes.AddUserDefinedDataAttribute(6,'Vehicle','vehAcceleration','vehAcceleration',2,0)
#Vissim.Net.UserDefinedAttributes.ItemByKey(6).SetAttValue('DefValue',-1)
#UDA6
Vissim.Net.UserDefinedAttributes.AddUserDefinedDataAttribute(3,'Vehicle','COM_cruise_control_Ac','COM_cruise_control_Ac',2,0)
Vissim.Net.UserDefinedAttributes.ItemByKey(3).SetAttValue('DefValue',-1)

#UDA
Vissim.Net.UserDefinedAttributes.AddUserDefinedDataAttribute(4,'Vehicle','COM_cc_setpoint','COM_cc_setpoint',2,0)
Vissim.Net.UserDefinedAttributes.ItemByKey(4).SetAttValue('DefValue',-1)
def get_leader_info(Vehicle):
    lead_veh_id = Vehicle.AttValue('LeadTargNo')
    lead_veh_type = Vehicle.AttValue('LeadTargType')
    if lead_veh_type == 'VEHICLE' and lead_veh_id != None:
        try:
            front_vehicle = Vissim.Net.Vehicles.ItemByKey(lead_veh_id)
        except:
            front_vehicle = -1
    else:
        front_vehicle=-1
    return front_vehicle
#prev_veh_cc_setpoint = np.zeros(number_of_vehicles)
for i in range(6000):
    for Vehicle in Vissim.Net.Vehicles.FilteredBy("[VEHTYPE\\NO]=210"):
        lead_veh=get_leader_info(Vehicle)
        if lead_veh!=-1 and (Vehicle.AttValue('Lane')==lead_veh.AttValue('Lane')):
            #simulation info
            timestep=Vehicle.AttValue('SimSec')
            #Ego Info
            print((Vehicle.AttValue('Lane')))
            veh_id=Vehicle.AttValue('No')
            veh_setspeed=Vehicle.AttValue('DesSpeed')/3.6
            veh_type=Vehicle.AttValue('VehType\\No')
            veh_used_distance_headway=Vehicle.AttValue('FollowDistNet')
            veh_used_rel_vel=(Vehicle.AttValue('Speed')-lead_veh.AttValue('Speed'))/3.6
            veh_velocity=Vehicle.AttValue('Speed')/3.6
            veh_distance_headway=Vehicle.AttValue('FollowDistNet')
            prev_veh_executed_acceleration = Vehicle.AttValue('COM_Ac')
            prev_veh_cc_setpoint = Vehicle.AttValue('COM_cc_setpoint')
            prev_veh_cruisecontrol_acceleration=Vehicle.AttValue('COM_cruise_control_Ac')
            #veh_executed_acceleration=a_prev[veh_id]
            #veh_cc_setpoint=prev_veh_cc_setpoint[veh_id]
            # Leader Info
            lead_veh_acceleration=lead_veh.AttValue('Acceleration')
            lead_veh_id=lead_veh.AttValue('No')
            lead_veh_rel_velocity=(Vehicle.AttValue('Speed')-lead_veh.AttValue('Speed'))/3.6
            lead_veh_type=lead_veh.AttValue('VehType\\No')

            curr_veh_executed_acceleration,curr_veh_cc_set_point,curr_veh_cruisecontrol_acceleration=get_acceleration(lead_veh_acceleration,lead_veh_id,lead_veh_rel_velocity,lead_veh_type,timestep,
                     veh_id,veh_setspeed,veh_type,veh_used_distance_headway,veh_used_rel_vel,veh_velocity,
                     veh_distance_headway,prev_veh_executed_acceleration,
                                       prev_veh_cc_setpoint,prev_veh_cruisecontrol_acceleration)
            # a=call vehicle_model(curr_veh_executed_acceleration)
            write_file('file_variables.txt')
            Vehicle.SetAttValue('COM_Ac',curr_veh_executed_acceleration)
            #Vehicle.SetAttValue('COM_Ac', a)
            Vehicle.SetAttValue('COM_At',3)
            Vehicle.SetAttValue('COM_cc_setpoint', curr_veh_cc_set_point)
            Vehicle.SetAttValue('COM_cruise_control_Ac', curr_veh_cruisecontrol_acceleration)

        else:
            continue
    Vissim.Simulation.RunSingleStep()


import numpy as np
import os,json
import pathlib
import argparse

from pygama.pargen.aoe_cal import cal_aoe

argparser = argparse.ArgumentParser()
argparser.add_argument("files", help="files", nargs='*',type=str)
argparser.add_argument("--ecal_file", help="ecal_file",type=str, required=True)
argparser.add_argument("--configs", help="configs", type=str, required=True)
argparser.add_argument("--plot_file", help="plot_file",type=str)
argparser.add_argument("--hit_pars", help="hit_pars",type=str)
args = argparser.parse_args()

with open(args.files[0]) as f:
    files = f.read().splitlines()
    files = sorted(files)

with open(args.ecal_file, 'r') as o:
    cal_dict = json.load(o)

energy_param = 'cuspEmax'
cal_energy_param = 'cuspEmax_ctc'

#out_dict = cal_aoe(files, cal_dict, energy_param, cal_energy_param, dt_corr=False, cut_parameters=cut_parameters, plot_savepath=args.plot_file)

out_dict = {"ecal_pars":[1,1],"aoe_pars":[1,1]}

if args.hit_pars is not None:
    pathlib.Path(os.path.dirname(args.hit_pars)).mkdir(parents=True, exist_ok=True)
    with open(args.hit_pars, 'w') as w:
        json.dump(out_dict,w, indent=4)
pathlib.Path(args.plot_file).touch()


import argparse
import datetime
import numpy as np
import pandas as pd
import pysam

def convert_table_to_vcf(genotypes_filename, calls_filename, reference_filename, vcf_filename):
    # Load genotypes in long format.
    genotypes = pd.read_table(genotypes_filename)

    # Get all SVs.
    vcf = pysam.VariantFile(calls_filename)
    calls = []
    for record in vcf:
        if isinstance(record.info["SVLEN"], tuple):
            sv_length = record.info["SVLEN"][0]
        else:
            sv_length = record.info["SVLEN"]

        calls.append([
            record.chrom,
            record.start,
            record.info["END"],
            record.info["SVTYPE"],
            sv_length,
            record.info["SEQ"],
            record.info["CONTIG"],
            record.info["CONTIG_START"],
            record.info["CONTIG_END"],
            record.info["CONTIG_SUPPORT"],
            record.info["CONTIG_DEPTH"],
            record.info["REPEAT_TYPE"]
        ])

    calls = pd.DataFrame(calls, columns=("chr", "start", "end", "sv_call", "event_size", "sv_sequence", "contig", "contig_start", "contig_end", "support", "depth", "repeat_type"))
    calls["call_id"] = calls.apply(lambda row: "-".join(map(str, (row.chr, row.start, row.end, row.sv_call))), axis=1)
    calls["quality"] = calls.apply(lambda row: int(min(100, round(-10 * np.log10(1 - (row.support / float(row.depth))) * np.log(row.depth), 0))), axis=1)

    # Get the reference base at the position of the variant start.
    reference = pysam.FastaFile(reference_filename)
    calls["reference"] = calls.apply(lambda row: reference.fetch(row.chr, row.start, row.start + 1).upper(), axis=1)

    # Update start position to be 1-based.
    calls["start"] = calls["start"] + 1

    # Assign an id to each call.
    genotypes["call_id"] = genotypes.apply(lambda row: "-".join(map(str, (row.chr, row.start, row.end, row.sv_call))), axis=1)
    sample_specific_fields = ["sample", "genotype", "genotype_quality", "genotype_likelihoods", "concordant", "discordant"]
    genotypes = genotypes[sample_specific_fields + ["call_id"]]

    # Build a genotype field for each call/sample combination.
    format = "GT:GQ:PL:DPR:DPA"
    genotypes["genotype_data"] = genotypes.apply(lambda row: ":".join(map(str, [row.genotype, row.genotype_quality, row.genotype_likelihoods, int(round(row.discordant, 0)), int(round(row.concordant, 0))])), axis=1)

    # Pivot genotypes to wide format to match VCF style.
    wide_genotypes = genotypes.pivot("call_id", "sample", "genotype_data")
    wide_genotypes = wide_genotypes.sort_index(axis=0)
    wide_genotypes.insert(0, "FORMAT", format)

    # Build an INFO field for each call.
    calls["info"] = calls.apply(lambda row: ";".join(["=".join(map(str, item)) for item in (("END", row.end), ("SVTYPE", row.sv_call), ("SVLEN", row.event_size), ("CONTIG", row.contig), ("CONTIG_START", row.contig_start), ("CONTIG_END", row.contig_end), ("REPEAT_TYPE", row.repeat_type), ("DPA", row.support), ("DP", row.depth), ("SEQ", row.sv_sequence))]), axis=1)

    calls["sv_sequence"] = calls.apply(lambda row: "<DEL>" if row["sv_call"] == "deletion" else "<INS>", axis=1)
    simple_calls = calls[["chr", "start", "call_id", "reference", "sv_sequence", "quality", "info"]].rename_axis({"chr": "#CHROM", "start": "POS", "reference": "REF", "call_id": "ID", "quality": "QUAL", "info": "INFO", "sv_sequence": "ALT"}, axis=1)

    # Annotate distinct calls with wide genotypes.
    calls_with_genotypes = simple_calls.merge(wide_genotypes, how="inner", left_on="ID", right_index=True)

    # Add reference, alternate allele, quality, and filter fields.
    calls_with_genotypes.insert(6, "FILTER", "PASS")

    # Save genotypes as tab-delimited file.
    with open(vcf_filename, "w") as vcf:
        vcf.write("##fileformat=VCFv4.2\n")
        vcf.write("##fileDate=%s\n" % datetime.date.strftime(datetime.date.today(), "%Y%m%d"))
        vcf.write("##source=SMRT_SV_genotyper\n")
        vcf.write('##INFO=<ID=DP,Number=1,Type=Integer,Description="Total depth of local assemblies">' + "\n")
        vcf.write('##INFO=<ID=DPA,Number=1,Type=Integer,Description="Depth of local assemblies supporting variant">' + "\n")
        vcf.write('##INFO=<ID=SVTYPE,Number=1,Type=String,Description="Type of structural variant">' + "\n")
        vcf.write('##INFO=<ID=SVLEN,Number=1,Type=Integer,Description="Difference in length between REF and ALT alleles">' + "\n")
        vcf.write('##INFO=<ID=END,Number=1,Type=Integer,Description="End coordinate of this variant">' + "\n")
        vcf.write('##INFO=<ID=CONTIG,Number=1,Type=String,Description="Name of alternate assembly contig">' + "\n")
        vcf.write('##INFO=<ID=CONTIG_START,Number=1,Type=Integer,Description="Start coordinate of this variant in the alternate assembly contig">' + "\n")
        vcf.write('##INFO=<ID=CONTIG_END,Number=1,Type=Integer,Description="End coordinate of this variant in the alternate assembly contig">' + "\n")
        vcf.write('##INFO=<ID=REPEAT_TYPE,Number=1,Type=String,Description="Repeat classification of variant content">' + "\n")
        vcf.write('##INFO=<ID=SEQ,Number=1,Type=String,Description="Sequence associated with variant">' + "\n")
        vcf.write('##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">' + "\n")
        vcf.write('##FORMAT=<ID=GQ,Number=1,Type=Integer,Description="Genotype quality">' + "\n")
        vcf.write('##FORMAT=<ID=PL,Number=G,Type=String,Description="Genotype likelihood">' + "\n")
        vcf.write('##FORMAT=<ID=DPR,Number=1,Type=String,Description="Read depth supporting reference allele">' + "\n")
        vcf.write('##FORMAT=<ID=DPA,Number=1,Type=String,Description="Read depth supporting alternate allele">' + "\n")
        calls_with_genotypes.to_csv(vcf, sep="\t", index=False)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("genotypes", help="tab-delimited file of sample, call coordinates in query sequence, call type, and genotypes in standard format (e.g., 0/0)")
    parser.add_argument("calls", help="VCF of original SV calls with repeat type from SMRT SV")
    parser.add_argument("reference", help="FASTA file for reference used with SMRT SV")
    parser.add_argument("output", help="genotypes in VCF format")
    args = parser.parse_args()

    convert_table_to_vcf(args.genotypes, args.calls, args.reference, args.output)


import numpy as np
import pandas as pd
import os.path as op
from glob import glob

DU2EN = {
    'Verrassing': 'surprise',
    'Verdrietig': 'sadness',
    'Bang': 'fear',
    'Walging': 'disgust',
    'Blij': 'happiness',
    'Boos': 'anger',
    'Geen van allen': 'none'
}

tsvs = sorted(glob('data/ratings_complete/*expressive_ratings.tsv'))
dfs_evsn = []
dfs_emo = []
for tsv in tsvs:
    sub = op.basename(tsv).split("_")[0]
    print(f"Processing {sub} ...")
    df = pd.read_csv(tsv, sep='\t', index_col=0).query("rating_type == 'emotion'")
    y = df.loc[:, ['rating']]
    y['rating'] = [DU2EN[s] for s in y['rating']]
    uniq_idx = y.index.unique()
    emos = sorted(y['rating'].unique())
    counts = pd.DataFrame(np.zeros((uniq_idx.size, 7)), columns=emos, index=uniq_idx)

    for i, idx in enumerate(uniq_idx):
        this = y.loc[idx, 'rating']
        if isinstance(this, str):
            this = [this]

        for r in this:
            counts.loc[idx, r] = counts.loc[idx, r] + 1

    # Pre-allocate best prediction array
    labels = np.zeros_like(counts, dtype=float)

    for ii in range(counts.shape[0]):
        # Determine most frequent label across reps
        opt_class = np.where(counts.iloc[ii, :] == counts.iloc[ii, :].max())[0]
        rnd_class = np.random.choice(opt_class, size=1)
        labels[ii, rnd_class] = 1

    # Repeat best possible prediction R times
    labels = pd.DataFrame(labels, index=uniq_idx, columns=emos)
    emo_vs_none = (1 - labels[['none']]).rename({'none': 'emo'}, axis=1).sort_index()
    emo_vs_none['sub'] = sub
    dfs_evsn.append(emo_vs_none)

    emo = labels.drop('none', axis=1)
    emo['sub'] = sub
    dfs_emo.append(emo)
    # TODO: add data_split

df_evsn = pd.concat(dfs_evsn, axis=0)
df_evsn.to_csv('data/featurespace-emoVSnone.tsv', sep='\t')

df_emo = pd.concat(dfs_emo, axis=0)
df_emo.to_csv('data/featurespace-categoricalemotion.tsv', sep='\t')


from __future__ import print_function
from __future__ import division

import argparse
import random
import torch
import torch.nn.functional as F
import torch.nn as nn
import torch.backends.cudnn as cudnn
import torch.optim as optim
import torch.utils.data
from torch.autograd import Variable
import numpy as np
from warpctc_pytorch import CTCLoss
import os
import utils
import dataset
import time
from torch.optim import lr_scheduler
from warmup_scheduler import GradualWarmupScheduler

import models.crnn as crnn

parser = argparse.ArgumentParser()
parser.add_argument('--trainRoot', required=True, help='path to dataset')
parser.add_argument('--valRoot', required=False, help='path to dataset')
parser.add_argument('--workers', type=int, help='number of data loading workers', default=2)
parser.add_argument('--batchSize', type=int, default=64, help='input batch size')
parser.add_argument('--imgH', type=int, default=32, help='the height of the input image to network')
parser.add_argument('--imgW', type=int, default=100, help='the width of the input image to network')
parser.add_argument('--nh', type=int, default=256, help='size of the lstm hidden state')
parser.add_argument('--nepoch', type=int, default=25, help='number of epochs to train for')
# TODO(meijieru): epoch -> iter
# parser.add_argument('--cuda', action='store_true', help='enables cuda')
parser.add_argument('--ngpu', type=int, default=1, help='number of GPUs to use')
parser.add_argument('--pretrained', default='', help="path to pretrained model (to continue training)")
parser.add_argument('--alphabet',  help = "path to alphabet", default='0123456789abcdefghijklmnopqrstuvwxyz')
parser.add_argument('--expr_dir', default='expr', help='Where to store samples and models')
parser.add_argument('--displayInterval', type=int, default=500, help='Interval to be displayed')
parser.add_argument('--n_test_disp', type=int, default=10, help='Number of samples to display when test')
parser.add_argument('--valInterval', type=int, default=500, help='Interval to be displayed')
parser.add_argument('--saveInterval', type=int, default=500, help='Interval to be displayed')
parser.add_argument('--lr', type=float, default=0.01, help='learning rate for Critic, not used by adadealta')
parser.add_argument('--beta1', type=float, default=0.5, help='beta1 for adam. default=0.5')
parser.add_argument('--adam', action='store_true', help='Whether to use adam (default is rmsprop)')
parser.add_argument('--adadelta', action='store_true', help='Whether to use adadelta (default is rmsprop)')
parser.add_argument('--keep_ratio', action='store_true', help='whether to keep ratio for image resize')
parser.add_argument('--manualSeed', type=int, default=1234, help='reproduce experiemnt')
parser.add_argument('--random_sample', action='store_true',default=True, help='whether to sample the dataset with random sampler')
opt = parser.parse_args()
print(opt)

debug = True
train_num, val_num = 192023, 20000

# log config
if not os.path.exists("./logs/"):
    os.mkdir("./logs/")
log_file = "./logs/"+str(opt.expr_dir)
log = utils.Logger()
if not os.path.exists(log_file):
    os.mkdir(log_file)
log.open(log_file+"/log_train.txt", mode="a")

# gpu devices
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

if not os.path.exists(opt.expr_dir):
    os.makedirs(opt.expr_dir)

# set seed
random.seed(opt.manualSeed)
np.random.seed(opt.manualSeed)
torch.manual_seed(opt.manualSeed)
torch.cuda.manual_seed_all(opt.manualSeed)

cudnn.benchmark = True

# if torch.cuda.is_available() and not opt.cuda:
#     print("WARNING: You have a CUDA device, so you should probably run with --cuda")

train_ = dataset.lmdbDataset(root=opt.trainRoot)
train_dataset, test_dataset = torch.utils.data.random_split(train_, [train_num, val_num])

assert train_dataset
if not opt.random_sample:
    sampler = dataset.randomSequentialSampler(train_dataset, opt.batchSize)
else:
    sampler = None
train_loader = torch.utils.data.DataLoader(
    train_dataset, batch_size=opt.batchSize,drop_last=True,
    shuffle=True, sampler=sampler,
    num_workers=int(opt.workers),
    collate_fn=dataset.alignCollate(imgH=opt.imgH, imgW=opt.imgW, keep_ratio=opt.keep_ratio))
# test_dataset = dataset.lmdbDataset(
#     root=opt.valRoot, transform=dataset.resizeNormalize((100, 32)))
test_loader = torch.utils.data.DataLoader(
    test_dataset, shuffle=False, batch_size=opt.batchSize, num_workers=int(opt.workers),drop_last=True,
    collate_fn=dataset.alignCollate(imgH=opt.imgH, imgW=opt.imgW, keep_ratio=True))

alphabet_str = utils.generate_alphabet(opt.alphabet)
nclass = len(opt.alphabet) + 1
nc = 1

converter = utils.strLabelConverter(alphabet_str)
criterion = CTCLoss()
# criterion = nn.CTCLoss()

# custom weights initialization called on crnn
def weights_init(m):
    classname = m.__class__.__name__
    if classname.find('Conv') != -1:
        m.weight.data.normal_(0.0, 0.02)
    elif classname.find('BatchNorm') != -1:
        m.weight.data.normal_(1.0, 0.02)
        m.bias.data.fill_(0)

crnn = crnn.CRNN(opt.imgH, nc, nclass, opt.nh)
crnn.apply(weights_init)
if opt.pretrained != '':
    print('loading pretrained model from %s' % opt.pretrained)
    crnn.load_state_dict(torch.load(opt.pretrained))
print(crnn)

image = torch.FloatTensor(opt.batchSize, 3, opt.imgH, opt.imgW)
text = torch.IntTensor(opt.batchSize * 5)
length = torch.IntTensor(opt.batchSize)

if torch.cuda.device_count() > 1:
    crnn = nn.DataParallel(crnn)
crnn.to(device)
image = image.to(device)
criterion = criterion.to(device)

image = image.to(device)
# text = text.to(device)
# length = length.to(device)

image = Variable(image)
text = Variable(text)
length = Variable(length)

# loss averager
loss_avg = utils.averager()
epoch_loss_avg = utils.averager()

# setup optimizer
if opt.adam:
    optimizer = optim.Adam(crnn.parameters(), lr=opt.lr,
                           betas=(opt.beta1, 0.999))
elif opt.adadelta:
    optimizer = optim.Adadelta(crnn.parameters())
else:
    optimizer = optim.RMSprop(crnn.parameters(), lr=opt.lr)

def val(net, test_loader, criterion, max_iter=100):
    print('Start val')

    for p in crnn.parameters():
        p.requires_grad = False

    net.eval()

    val_iter = iter(test_loader)

    # i = 0
    n_correct = 0
    loss_avg = utils.averager()

    # max_iter = min(max_iter, len(test_loader))
    # for i in range(max_iter):
    for data in test_loader:
        # data = val_iter.next()
        # i += 1
        cpu_images, cpu_texts = data
        batch_size = cpu_images.size(0)
        utils.loadData(image, cpu_images)
        t, l = converter.encode(cpu_texts)
        utils.loadData(text, t)
        utils.loadData(length, l)

        preds = crnn(image)
        preds_size = Variable(torch.IntTensor([preds.size(0)] * batch_size))
        cost = criterion(preds, text, preds_size, length) / batch_size
        loss_avg.add(cost)

        _, preds = preds.max(2)
        # preds = preds.squeeze(2)
        preds = preds.transpose(1, 0).contiguous().view(-1)
        sim_preds = converter.decode(preds.data, preds_size.data, raw=False)
        for pred, target in zip(sim_preds, cpu_texts):
            if pred == target.lower():
                n_correct += 1

    # raw_preds = converter.decode(preds.data, preds_size.data, raw=True)[:opt.n_test_disp]
    # for raw_pred, pred, gt in zip(raw_preds, sim_preds, cpu_texts):
    #     print('%-20s => %-20s, gt: %-20s' % (raw_pred, pred, gt))

    accuracy = n_correct / float(val_num)
    val_output_str = 'Test loss: %f, correct/total:%d / %d accuray: %f ' % (loss_avg.val(), n_correct,val_num,accuracy)
    # print(val_output_str)
    log.write(val_output_str)
    log.write("\n")

    return [loss_avg.val(), accuracy]

def trainBatch(net, criterion, optimizer):
    data = train_iter.next()
    cpu_images, cpu_texts = data
    batch_size = cpu_images.size(0)
    utils.loadData(image, cpu_images)
    t, l = converter.encode(cpu_texts)
    utils.loadData(text, t)
    utils.loadData(length, l)

    preds = net(image)

    preds_size = Variable(torch.IntTensor([preds.size(0)] * batch_size))
    cost = criterion(preds, text, preds_size, length) / batch_size
    net.zero_grad()
    cost.backward()
    optimizer.step()
    return cost

# scheduler = lr_scheduler.ReduceLROnPlateau(optimizer, mode='min', patience=4, factor=0.1)
scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=[40,80], gamma=0.1)
# scheduler = GradualWarmupScheduler(optimizer, multiplier=0.001, total_epoch=5, after_scheduler=scheduler_ms)
total_time_start = time.time()
iter_time_start = time.time()
epoch_time_start = time.time()

temp_val_acc = 0
best_model = crnn.state_dict()

for epoch in range(opt.nepoch):
    scheduler.step()

    train_iter = iter(train_loader)
    i = 0
    while i < len(train_loader):
        for p in crnn.parameters():
            p.requires_grad = True
        crnn.train()

        cost = trainBatch(crnn, criterion, optimizer)
        loss_avg.add(cost)
        epoch_loss_avg.add(cost)
        i += 1

        if i % opt.displayInterval == 0:
            iter_time_end = time.time()
            output_log = 'Epoch: [%d/%d]; iter: [%d/%d]; Loss: %f; time: %.2f s; lr: %s ' % (epoch, opt.nepoch, i, len(train_loader), loss_avg.val(), iter_time_end-iter_time_start,  utils.get_learning_rate(optimizer))
            # print(output_log)
            log.write(output_log)
            log.write("\n")
            iter_time_start = time.time()
            loss_avg.reset()

        if i % opt.valInterval == 0:
            val_metric = val(crnn, test_dataset, criterion)
            if val_metric[1] > temp_val_acc:
                temp_val_acc = val_metric[1]
                best_model = crnn.state_dict()
                torch.save(
                    crnn.state_dict(), '{0}/CRNN_ep{1}_it{2}_acc{3}.pth'.format(opt.expr_dir, epoch, i, temp_val_acc))

        # # do checkpointing
        # if i % opt.saveInterval == 0:
        #     torch.save(
        #         crnn.state_dict(), '{0}/CRNN_ep{1}_it{2}.pth'.format(opt.expr_dir, epoch, i))

    # val_metric = val(crnn, test_loader, criterion)
    # if val_metric[1] > temp_val_acc:
    #     temp_val_acc = val_metric[1]
    #     best_model = crnn.state_dict()
    # scheduler acc
    # scheduler.step(val_metric[1])

    torch.save(
        crnn.state_dict(), '{0}/CRNN_ep{1}_acc{2}.pth'.format(opt.expr_dir, epoch, temp_val_acc))

    epoch_time_end = time.time()
    epoch_output_log =  'Epoch: [%d/%d]; Total Loss: %f; time: %.2f s; \n' \
                        % (epoch, opt.nepoch,  loss_avg.val(), epoch_time_end-epoch_time_start)
    # print(epoch_output_log)
    log.write(epoch_output_log)
    log.write("\n")

    # 每个epoch的loss都reset
    epoch_loss_avg.reset()

torch.save(best_model, '{0}/CRNN_best_model.pth'.format(opt.expr_dir))
print("All cost time: {:.2f}s".format(time.time()-total_time_start))


#!/usr/bin/env python
import yaml

# ----------------------------------------------------------------------------
# The 2017 DAVIS Challenge on Video Object Segmentation
#-----------------------------------------------------------------------------
# Copyright (c) 2017 Federico Perazzi
# Licensed under the BSD License [see LICENSE for details]
# Written by Federico Perazzi (federico@disneyresearch.com)
# Adapted from FAST-RCNN (Ross Girshick)
# ----------------------------------------------------------------------------

""" Configuration file."""

import os
import os.path as osp

import sys
from easydict import EasyDict as edict
import json
from enum import Enum

class phase(Enum):
    VAL      = 'all_val'
    VAL_SUBSET = 'val_subset'
    TESTSEEN = 'test-seen'
    TESTUNSEEN = 'test-unseen'

__C = edict()

# Public access to configuration settings
cfg = __C

# Number of CPU cores used to parallelize evaluation.
__C.N_JOBS = 32

# Paths to dataset folders
__C.PATH = edict()

__C.IM_SIZE = (448, 256)
__C.PHASE = phase.VAL

# Multiobject evaluation (Set to False only when evaluating DAVIS 2016)
__C.MULTIOBJECT = True

# Root folder of project
__C.PATH.ROOT = osp.abspath(osp.join(osp.dirname(__file__), '../../../..'))

# Data folder
__C.PATH.DATA = osp.abspath('/raid/ljyang/data/LSVOS')

# Path to input images
__C.PATH.SEQUENCES = __C.PATH.DATA

# Path to annotations
__C.PATH.ANNOTATIONS = __C.PATH.DATA

# Color palette
__C.PATH.PALETTE = osp.abspath(osp.join(__C.PATH.ROOT, 'data/palette.txt'))

# Paths to files
__C.FILES = edict()

__C.FILES.DB_INFO = osp.join(__C.PATH.DATA,'{}_seqs.json')
# Measures and Statistics
__C.EVAL = edict()

# Metrics: J: region similarity, F: contour accuracy, T: temporal stability
__C.EVAL.METRICS = ['J','F']

# Statistics computed for each of the metrics listed above
__C.EVAL.STATISTICS= ['mean','recall','decay']

def db_read_sequences(db_phase=None):
  """ Read list of sequences. """
  print db_phase.value
  info_path = __C.FILES.DB_INFO.format(db_phase.value)
  db_info = json.load(open(info_path))

  sequences = db_info

  return sequences

import numpy as np
__C.palette = np.loadtxt(__C.PATH.PALETTE,dtype=np.uint8).reshape(-1,3)


"""
Code to convert an eimage to individual sensor segments, applying
electronics readout effects.

 * Retrieve pixel geometry for each segment
 * Copy imaging segment pixels from eimage
 * Add dark current
 * Add defects (bright defects, dark defects, traps)
 * Apply CTE
 * Apply gain
 * Apply crosstalk
 * Add read noise and bias offset
 * Write FITS file for each amplifier
"""
import os
import warnings
from collections import namedtuple, OrderedDict
import tempfile
import sqlite3
import numpy as np
import scipy
from astropy.io import fits
import astropy.time
import galsim
import lsst.geom as lsst_geom
import lsst.afw.image as afwImage
with warnings.catch_warnings():
    warnings.simplefilter('ignore')
    from lsst.sims.catUtils.utils import ObservationMetaDataGenerator
    from lsst.sims.utils import \
        getRotSkyPos, ObservationMetaData, altAzPaFromRaDec
    from .sims_GalSimInterface import LsstObservatory
from .camera_info import CameraInfo, getHourAngle
from .imSim import get_logger, get_config, airmass, get_version_keywords
from .cosmic_rays import CosmicRays

__all__ = ['ImageSource', 'set_noao_keywords', 'cte_matrix']

class ImageSource(object):
    '''
    Class to create single segment images based on the pixel geometry
    described by a Camera object from lsst.afw.cameraGeom.

    Attributes
    ----------
    eimage: astropy.io.fits.HDUList
        The input eimage data.  This is used as a container for both
        the pixel data and image metadata.
    eimage_data: np.array
        The data attribute of the eimage PrimaryHDU.
    exptime: float
        The exposure time of the image in seconds.
    sensor_id: str
        The raft and sensor identifier, e.g., 'R22_S11'.
    amp_images: OrderedDict
        Dictionary of amplifier images.
    camera_info: CameraInfo object
        Object containing the readout properties of the sensors in the
        focal plane, provided by lsst.obs.lsst.imsim.ImsimMapper().camera.
    '''
    def __init__(self, image_array, exptime, sensor_id, visit=42, logger=None):
        """
        Class constructor.

        Parameters
        ----------
        image_array: np.array
            A numpy array containing the pixel data for an eimage.
        exptime: float
            The exposure time of the image in seconds.
        sensor_id: str
            The raft and sensor identifier, e.g., 'R22_S11'.
        visit: int [42]
            Visit number to be used with the sensor_id for generating
            the random seed for the dark current and read noise.
        logger: logging.Logger [None]
            logging.Logger object to use. If None, then a logger with level
            INFO will be used.
        """
        if logger is None:
            self.logger = get_logger('INFO')
        else:
            self.logger = logger

        self.config = get_config()

        self.eimage = fits.HDUList()
        self.eimage.append(fits.PrimaryHDU(image_array))
        self.eimage_data = self.eimage[0].data.transpose()

        self.exptime = exptime
        self.sensor_id = sensor_id
        self.visit = visit

        self._rng = None
        self.seed = None

        self.camera_info = CameraInfo()

        self._make_amp_images()

        self.ratel = 0
        self.dectel = 0
        self.rotangle = 0

    def __del__(self):
        self.eimage.close()

    @staticmethod
    def create_from_galsim_image(gs_image, logger=None):
        """
        Create an ImageSource object from a galsim Image object
        produced by a GalSimInterpreter.
        """
        # Extract the exptime and sensor_id from the
        # sims_GalSimInterface.GalSim_afw_TanSimWCS object.
        wcs = gs_image.wcs
        exptime = wcs.photParams.exptime*wcs.photParams.nexp
        sensor_id = 'R{}{}_S{}{}'\
                    .format(*[x for x in wcs.detectorName if x.isdigit()])

        raw_image = ImageSource(gs_image.array, exptime, sensor_id,
                                visit=wcs.fitsHeader.getScalar('OBSID'))
        raw_image.eimage_data = gs_image.array

        # Add the keywords from wcs.fitsHeader to the raw_image.eimage
        # attribute so that they are propagated to the raw file.
        for name in wcs.fitsHeader.names():
            raw_image.eimage[0].header[name] = wcs.fitsHeader.getScalar(name)
        raw_image._read_pointing_info(None)
        return raw_image

    @staticmethod
    def create_from_eimage(eimage_file, sensor_id=None, opsim_db=None,
                           logger=None):
        """
        Create an ImageSource object from a PhoSim eimage file.

        Parameters
        ----------
        eimage_file: str
           Filename of the eimage FITS file from which the amplifier
           images will be extracted.
        sensor_id: str [None]
            The raft and sensor identifier, e.g., 'R22_S11'.  If None,
            then extract the CHIPID keyword in the primarey HDU.
        opsim_db: str [None]
            OpSim db file to use to find pointing information for each
            visit.  This is needed for older imSim eimage files that
            do not have the RATEL, DECTEL, and ROTANGLE keywords
            set in the FITS header.   If None and if an eimage file without
            those keywords is provided, then a RuntimeError will be raised.
        logger: logging.Logger [None]
            logging.Logger object to use. If None, then a logger with level
            INFO will be used.

        Returns
        -------
        ImageSource object
            An ImageSource object with the pixel data and metadata from
            the eimage file.
        """
        eimage = fits.open(eimage_file)
        exptime = eimage[0].header['EXPTIME']
        if sensor_id is None:
            sensor_id = eimage[0].header['CHIPID']

        image_source = ImageSource(eimage[0].data, exptime, sensor_id,
                                   visit=eimage[0].header['OBSID'],
                                   logger=logger)
        image_source.eimage = eimage
        image_source.eimage_data = eimage[0].data
        image_source._read_pointing_info(opsim_db)
        return image_source

    def _read_pointing_info(self, opsim_db):
        try:
            self.ratel = self.eimage[0].header['RATEL']
            self.dectel = self.eimage[0].header['DECTEL']
            self.rotangle = self.eimage[0].header['ROTANGLE']
            return
        except KeyError:
            if opsim_db is None:
                raise RuntimeError("eimage file does not have pointing info. "
                                   "Need an opsim db file.")
        # Read from the opsim db.
        # We need an ObservationMetaData object to use the getRotSkyPos
        # function.
        obs_gen = ObservationMetaDataGenerator(database=opsim_db,
                                               driver="sqlite")
        obs_md = obs_gen.getObservationMetaData(obsHistID=self.visit,
                                                boundType='circle',
                                                boundLength=0)[0]
        # Extract pointing info from opsim db for desired visit.
        conn = sqlite3.connect(opsim_db)
        query = """select descDitheredRA, descDitheredDec,
        descDitheredRotTelPos from summary where
        obshistid={}""".format(self.visit)
        curs = conn.execute(query)
        ra, dec, rottelpos = [np.degrees(x) for x in curs][0]
        conn.close()
        self.ratel, self.dectel = ra, dec
        obs_md.pointingRA = ra
        obs_md.pointingDec = dec
        self.rotangle = getRotSkyPos(ra, dec, obs_md, rottelpos)

    def get_amp_image(self, amp_info, imageFactory=afwImage.ImageI):
        """
        Return an amplifier afwImage.Image object with electronics
        readout effects applied.  This method is only provided so that
        the pixel data and geometry can be displayed using
        lsst.afw.cameraGeom.utils.showAmp.

        Parameters
        ----------
        amp_info : lsst.afw.cameraGeom.amplifier.amplifier.Amplifier
            Data structure used by cameraGeom to contain the amplifier
            information such as pixel geometry, gain, noise, etc..
        imageFactory : lsst.afw.image.Image[DFIU], optional
            Image factory to be used for creating the return value.

        Returns
        -------
        lsst.afw.Image[DFIU]
            The image object containing the pixel data.
        """
        amp_name = self.amp_name(amp_info)
        float_image = self.amp_images[amp_name]
        if imageFactory == afwImage.ImageF:
            return float_image
        # Return image as the type given by imageFactory.
        output_image = imageFactory(amp_info.getRawBBox())
        output_image.getArray()[:] = float_image.getArray()
        return output_image

    def amp_name(self, amp_info):
        """
        The ampifier name derived from a
        lsst.afw.cameraGeom.amplifier.amplifier.Amplifier.

        Parameters
        ----------
        amp_info: lsst.afw.cameraGeom.amplifier.amplifier.Amplifier.

        Returns
        -------
        str
             The amplifier name, e.g., "R22_S22_C00".
        """
        return '_'.join((self.sensor_id, amp_info.getName()))

    def _make_amp_images(self):
        """
        Make the amplifier images for all the amps in the sensor.
        """
        self.amp_images = OrderedDict()
        amp_names = self.camera_info.get_amp_names(self.sensor_id)
        for amp_name in amp_names:
            self._make_amp_image(amp_name)
        self._apply_crosstalk()
        if self.config['electronics_readout']\
           ['disable_readnoise_bias_darkcurrent']:
            return
        for amp_name in amp_names:
            self._add_read_noise_and_bias(amp_name)

    def _make_amp_image(self, amp_name):
        """
        Create the segment image for the amplier geometry specified in amp.

        Parameters
        ----------
        amp_name : str
            The amplifier name, e.g., "R22_S11_C00".
        """
        amp_info = self.camera_info.get_amp_info(amp_name)
        bbox = self.camera_info.mosaic_section(amp_info)
        full_segment = afwImage.ImageF(amp_info.getRawBBox())

        # Get the imaging segment (i.e., excluding prescan and
        # overscan regions), and fill with data from the eimage.
        imaging_segment \
            = full_segment.Factory(full_segment, amp_info.getRawDataBBox())
        data = self.eimage_data[bbox.getMinY():bbox.getMaxY()+1,
                                bbox.getMinX():bbox.getMaxX()+1].copy()

        # Apply flips in x and y relative to assembled eimage in order
        # to have the pixels in readout order.
        if amp_info.getRawFlipX():
            data = data[:, ::-1]
        if amp_info.getRawFlipY():
            data = data[::-1, :]
        imaging_segment.getArray()[:] = data

        # Add dark current.
        if (self.exptime > 0 and not self.config['electronics_readout']
            ['disable_readnoise_bias_darkcurrent']):
            dark_current = self.config['electronics_readout']['dark_current']
            imaging_arr = imaging_segment.getArray()
            rng = galsim.PoissonDeviate(seed=self.rng,
                                        mean=dark_current*self.exptime)
            dc_data = np.zeros(np.prod(imaging_arr.shape))
            rng.generate(dc_data)
            imaging_arr += dc_data.reshape(imaging_arr.shape)

        # Add defects.

        # Apply CTE.
        full_arr = full_segment.getArray()
        pcti = self.config['electronics_readout']['pcti']
        pcte_matrix = cte_matrix(full_arr.shape[0], pcti)
        for col in range(0, full_arr.shape[1]):
            full_arr[:, col] = np.dot(pcte_matrix, full_arr[:, col])

        scti = self.config['electronics_readout']['scti']
        scte_matrix = cte_matrix(full_arr.shape[1], scti)
        for row in range(0, full_arr.shape[0]):
            full_arr[row, :] = np.dot(scte_matrix, full_arr[row, :])

        # Convert to ADU.
        full_arr /= amp_info.getGain()

        self.amp_images[amp_name] = full_segment

    def _add_read_noise_and_bias(self, amp_name):
        """
        Add read noise and bias.  This should be done as the final
        step before returning the processed image.

        Parameters
        ----------
        amp_name : str
            The amplifier name, e.g., "R22_S11_C00".
        """
        amp_info = self.camera_info.get_amp_info(amp_name)
        full_arr = self.amp_images[amp_name].getArray()
        rng = galsim.GaussianDeviate(seed=self.rng,
                                     sigma=amp_info.getReadNoise())
        rn_data = np.zeros(np.prod(full_arr.shape))
        rng.generate(rn_data)
        full_arr += rn_data.reshape(full_arr.shape)
        full_arr += self.config['electronics_readout']['bias_level']

    @property
    def rng(self):
        """
        galsim.BaseDeviate seeded with an integer derived from a hash
        of the visit and sensor id.  This is used as the underlying
        random number generator for both the read noise and dark
        current calculations.
        """
        if self._rng is None:
            self.seed = CosmicRays.generate_seed(self.visit, self.sensor_id)
            self._rng = galsim.BaseDeviate(self.seed)
        return self._rng

    def _apply_crosstalk(self):
        """
        Apply intra-CCD crosstalk using the cross-talk matrix
        from obs_lsst.  This should be run only once and
        only after ._make_amp_image has been run for each amplifier.
        """
        if not self.camera_info.det_catalog[self.sensor_id].hasCrosstalk():
            return
        xtalk = self.camera_info.det_catalog[self.sensor_id].getCrosstalk()
        amp_names = self.camera_info.get_amp_names(self.sensor_id)
        imarrs = np.array([self.amp_images[amp_name].getArray()
                           for amp_name in amp_names])
        for amp_index, amp_name, xtalk_row in zip(range(len(amp_names)),
                                                  amp_names, xtalk):
            self.amp_images[amp_name].getArray()[:, :] \
                = (imarrs[amp_index] +
                   sum([x*y for x, y in zip(imarrs, xtalk_row)]))

    def get_amplifier_hdu(self, amp_name, compress=True):
        """
        Get an astropy.io.fits.HDU for the specified amplifier.

        Parameters
        ----------
        amp_name: str
            The amplifier name, e.g., "R22_S11_C00".
        compress: bool [True]
            Use RICE_1 compression.

        Returns
        -------
        astropy.io.fits.ImageHDU
            Image HDU with the pixel data and header keywords
            appropriate for the requested sensor segment.
        """
        data = self.amp_images[amp_name].getArray().astype(np.int32)
        if compress:
            hdu = fits.CompImageHDU(data=data, compression_type='RICE_1')
        else:
            hdu = fits.ImageHDU(data=data)
        hdr = hdu.header
        amp_info = self.camera_info.get_amp_info(amp_name)
        # Copy keywords from eimage primary header.
        with warnings.catch_warnings():
            warnings.simplefilter("ignore")
            for key in self.eimage[0].header.keys():
                if key in ('BITPIX', 'NAXIS'):
                    continue
                try:
                    hdr[key] = self.eimage[0].header[key]
                except ValueError:
                    # eimages produced by phosim contain non-ASCII or
                    # non-printable characters resulting in a ValueError.
                    self.logger.warn("ValueError raised while attempting to "
                                     "read %s from eimage header", key)

        # Transpose the WCS matrix elements to account for the use of the
        # Camera Coordinate System in the eimage.  These changes
        # neglect any implied changes in the SIP coefficients.
        channels = '10 11 12 13 14 15 16 17 07 06 05 04 03 02 01 00'.split()
        amp_nums = dict(kv_pair for kv_pair in zip(channels, range(16)))
        amp_num = amp_nums[amp_name[-2:]]
        # These keywords seem to give approximately correct per amp
        # WCS's when viewed with ds9.
        x_pos = (list(range(1, 9)) + list(range(8, 0, -1)))[amp_num]
        hdr['CRPIX1'], hdr['CRPIX2'] \
            = (hdr['CRPIX2'] - amp_info.getRawDataBBox().getWidth()*(8 - x_pos),
               hdr['CRPIX1'])
        if amp_num < 8:
            hdr['CD1_1'], hdr['CD1_2'] = -hdr['CD1_2'], hdr['CD1_1']
            hdr['CD2_1'], hdr['CD2_2'] = -hdr['CD2_2'], hdr['CD2_1']
        else:
            hdr['CD1_1'], hdr['CD1_2'] = -hdr['CD1_2'], -hdr['CD1_1']
            hdr['CD2_1'], hdr['CD2_2'] = -hdr['CD2_2'], -hdr['CD2_1']

        # Set NOAO geometry keywords.
        hdr['DATASEC'] = self._noao_section_keyword(amp_info.getRawDataBBox())
        hdr['DETSEC'] = \
            self._noao_section_keyword(self.camera_info.mosaic_section(amp_info),
                                       flipx=amp_info.getRawFlipX(),
                                       flipy=amp_info.getRawFlipY())
        hdr['GAIN'] = amp_info.getGain()

        return hdu

    def write_fits_file(self, outfile, overwrite=True, run_number=None,
                        lsst_num='LCA-11021_RTM-000', compress=True,
                        image_type='SKYEXP', added_keywords=None):
        """
        Write the processed eimage data as a multi-extension FITS file.

        Parameters
        ----------
        outfile: str
            Name of the output FITS file.
        overwrite: bool [True]
            Flag whether to overwrite an existing output file.
        run_number: int [None]
            Run number.  If None, then the visit number is used.
        compress: bool [True]
            Use RICE_1 compression for each image HDU.
        image_type: str ['SKYEXP']
            Image type to write to the `OBSTYPE` and `IMGTYPE` keywords.
        added_keywords: dict [None]
            Python dict of key/value pairs to add to the primary HDU.
            These will be set just before writing the FITS file, so
            they will override any existing keywords.
        """
        output = fits.HDUList(fits.PrimaryHDU())
        output[0].header = self.eimage[0].header
        # Since `.header` acts like an unordered dict when set like
        # this, we need to re-insert the WCSAXES keyword to precede
        # any other WCS keywords as stipulated in the FITS Standard,
        # version 4.0, section 8.2.
        wcsaxes = output[0].header['WCSAXES']
        del output[0].header['WCSAXES']
        output[0].header.insert(5, ('WCSAXES', wcsaxes, ''))
        if run_number is None:
            run_number = self.visit
        output[0].header['RUNNUM'] = str(run_number)
        output[0].header['DARKTIME'] = output[0].header['EXPTIME']
        output[0].header['TIMESYS'] = 'TAI'
        output[0].header['LSST_NUM'] = lsst_num
        output[0].header['TESTTYPE'] = 'IMSIM'
        output[0].header['IMGTYPE'] = image_type
        output[0].header['OBSTYPE'] = output[0].header['IMGTYPE']
        output[0].header['MONOWL'] = -1
        raft, ccd = output[0].header['CHIPID'].split('_')
        output[0].header['RAFTNAME'] = raft
        output[0].header['SENSNAME'] = ccd
        output[0].header['OUTFILE'] = os.path.basename(outfile)
        # Add boresight pointing angles and rotskypos (angle of sky
        # relative to Camera coordinates) from which obs_lsst can
        # infer the CCD-wide WCS.
        output[0].header['RATEL'] = self.ratel
        output[0].header['DECTEL'] = self.dectel
        output[0].header['ROTANGLE'] = self.rotangle

        # Add various keywords needed for jointcal, including  hour angle
        # and airmass values at the start and end of the observation.
        mjd_obs = output[0].header['MJD-OBS']
        mjd_end = astropy.time.Time(output[0].header['DATE-END'], format='isot',
                                    scale='tai').mjd
        observatory = LsstObservatory()
        output[0].header['HASTART'] \
            = getHourAngle(observatory, mjd_obs, self.ratel)
        output[0].header['HAEND'] \
            = getHourAngle(observatory, mjd_end, self.ratel)

        # Set the airmass from the start of the observation using the
        # opsim db value and compute the airmass from the altitude at
        # the end of the observation.
        output[0].header['AMSTART'] = output[0].header['AIRMASS']
        obs_md = ObservationMetaData(mjd=mjd_end, pointingRA=self.ratel,
                                     pointingDec=self.dectel,
                                     rotSkyPos=self.rotangle)
        alt_end, _, _ = altAzPaFromRaDec(self.ratel, self.dectel, obs_md)
        output[0].header['AMEND'] = airmass(alt_end)

        # Write the added_keywords.
        if added_keywords is not None:
            for key, value in added_keywords.items():
                output[0].header[key] = value

        # Write the seed used by the read noise and dark current.
        output[0].header['RN_SEED'] = self.seed
        # Use seg_ids to write the image extensions in the order
        # specified by LCA-10140.
        seg_ids = '10 11 12 13 14 15 16 17 07 06 05 04 03 02 01 00'.split()
        for seg_id in seg_ids:
            amp_name = '_C'.join((self.sensor_id, seg_id))
            output.append(self.get_amplifier_hdu(amp_name, compress=compress))
            output[-1].header['EXTNAME'] = 'Segment%s' % seg_id

        # Set the imSim version and LSST Stack product versions and
        # tags in the primary HDU.
        output[0].header.update(get_version_keywords())
        self.fits_atomic_write(output, outfile, overwrite=overwrite)

    @staticmethod
    def fits_atomic_write(hdulist, outfile, overwrite=True):
        """
        Perform an atomic write of a FITS file using astropy.io.fits by
        writing to a temporary file then renaming to the final filename.

        Parameters
        ----------
        hdulist: astropy.io.HDUList
            HDU list object to write as a FITS file.
        outfile: str
            Destination filename.
        overwrite: bool [True]
            Flag to overwrite an existing output file.  If False and the
            file already exisits, this function will raise a RuntimeError.
        """
        with tempfile.NamedTemporaryFile(mode='wb', delete=False,
                                         dir='.') as tmp:
            hdulist.writeto(tmp)
            tmp.flush()
            os.fsync(tmp.fileno())
            os.chmod(tmp.name, 0o660)
        if not overwrite and os.path.isfile(outfile):
            raise RuntimeError(
                'File {} exists already. Cannot overwrite.'.format(outfile))
        os.rename(tmp.name, outfile)

    @staticmethod
    def _noao_section_keyword(bbox, flipx=False, flipy=False):
        """
        Convert bounding boxes into NOAO section keywords.

        Parameters
        ----------
        bbox : lsst.geom.Box2I
            Bounding box.
        flipx : bool
            Flag to indicate that data should be flipped in the x-direction.
        flipy : bool
            Flag to indicate that data should be flipped in the y-direction.
        """
        xmin, xmax = bbox.getMinX()+1, bbox.getMaxX()+1
        ymin, ymax = bbox.getMinY()+1, bbox.getMaxY()+1
        if flipx:
            xmin, xmax = xmax, xmin
        if flipy:
            ymin, ymax = ymax, ymin
        return '[%i:%i,%i:%i]' % (xmin, xmax, ymin, ymax)

PixelParameters = namedtuple('PixelParameters',
                             ('''dimv dimh ccdax ccday ccdpx ccdpy gap_inx
                                 gap_iny gap_outx gap_outy preh'''.split()))
pixel_parameters \
    = {'E2V': PixelParameters(2002, 512, 4004, 4096, 4197, 4200, 28,
                              25, 26.5, 25, 10),
       'ITL': PixelParameters(2000, 509, 4000, 4072, 4198, 4198, 27,
                              27, 26.0, 26, 3)}

# The image extension headers do not include the CCD vendor so we
# infer that from the values of the DETSIZE keyword.
vendors = {'[1:4096,1:4004]': 'E2V',
           '[1:4072,1:4000]': 'ITL'}

def set_noao_keywords(hdu, slot_name):
    """
    Update the image header for one of the readout segments.  Adds raft-level
    coordinates (one set in Camera coordinates and one set rotated so the CCD
    orientation has serial direction horizontal).  Adds amplifier and rotated
    CCD coordinates as well.  See LCA-13501.

    Parameters
    ----------
    hdu : fits.ImageHDU
        FITS image whose header is being updated
    slot_name : str
        Name of the slot within the raft

    Returns
    -------
    fits.ImageHDU : The modified hdu.
    """
    hdu.header['SLOT'] = slot_name

    # Infer the CCD vendor from the DETSIZE keyword.
    try:
        vendor = vendors[hdu.header['DETSIZE']]
    except KeyError:
        raise RuntimeError("DETSIZE not recognized.")

    # Pixel geometries differ between the two CCD vendors
    pixel_pars = pixel_parameters[vendor]

    # Get the segment 'coordinates' from the extension name, e.g., 'Segment00'.
    extname = hdu.header['EXTNAME']
    sx = int(extname[-2])
    sy = int(extname[-1])

    # For convenience of notation in LCA-13501 these are also defined
    # as 'serial' and 'parallel' indices, with Segment = Sp*10 + Ss.
    sp = sx
    ss = sy

    # Extract the x and y location indexes in the raft from the slot
    # name.  Also define the serial and parallel versions.
    cx = int(slot_name[-2])
    cy = int(slot_name[-1])
    cp = cx
    cs = cy

    # Define the WCS and Mosaic keywords.
    hdu.header['WCSNAMEA'] = 'AMPLIFIER'
    hdu.header['CTYPE1A'] = 'Seg_X   '
    hdu.header['CTYPE2A'] = 'Seg_Y   '

    hdu.header['WCSNAMEC'] = 'CCD     '
    hdu.header['CTYPE1C'] = 'CCD_X   '
    hdu.header['CTYPE2C'] = 'CCD_Y   '

    hdu.header['WCSNAMER'] = 'RAFT    '
    hdu.header['CTYPE1R'] = 'RAFT_X  '
    hdu.header['CTYPE2R'] = 'RAFT_Y  '

    hdu.header['WCSNAMEF'] = 'FOCAL_PLANE'

    hdu.header['WCSNAMEB'] = 'CCD_SERPAR'
    hdu.header['CTYPE1B'] = 'CCD_S   '
    hdu.header['CTYPE2B'] = 'CCD_P   '

    hdu.header['WCSNAMEQ'] = 'RAFT_SERPAR'
    hdu.header['CTYPE1Q'] = 'RAFT_S  '
    hdu.header['CTYPE2Q'] = 'RAFT_P  '

    # Keyword values that are common betweem E2V and ITL CCDs.
    hdu.header['PC1_1A'] = 0
    hdu.header['PC1_2A'] = 1 - 2*sx
    hdu.header['PC2_2A'] = 0
    hdu.header['CDELT1A'] = 1
    hdu.header['CDELT2A'] = 1
    hdu.header['CRPIX1A'] = 0
    hdu.header['CRPIX2A'] = 0
    hdu.header['CRVAL1A'] = sx*(pixel_pars.dimv + 1)

    hdu.header['PC1_1C'] = 0
    hdu.header['PC1_2C'] = 1 - 2*sx
    hdu.header['PC2_2C'] = 0
    hdu.header['CDELT1C'] = 1
    hdu.header['CDELT2C'] = 1
    hdu.header['CRPIX1C'] = 0
    hdu.header['CRPIX2C'] = 0
    hdu.header['CRVAL1C'] = sx*(2*pixel_pars.dimv + 1)

    hdu.header['PC1_1R'] = 0
    hdu.header['PC1_2R'] = 1 - 2*sx
    hdu.header['PC2_2R'] = 0
    hdu.header['CDELT1R'] = 1
    hdu.header['CDELT2R'] = 1
    hdu.header['CRPIX1R'] = 0
    hdu.header['CRPIX2R'] = 0
    hdu.header['CRVAL1R'] = (sx*(2*pixel_pars.dimv + 1) + pixel_pars.gap_outx
                             + (pixel_pars.ccdpx - pixel_pars.ccdax)/2.
                             + cx*(2*pixel_pars.dimv + pixel_pars.gap_inx
                                   + pixel_pars.ccdpx - pixel_pars.ccdax))

    hdu.header['PC1_1B'] = 0
    hdu.header['PC1_2B'] = 0
    hdu.header['PC2_2B'] = 1 - 2*sp
    hdu.header['CDELT1B'] = 1
    hdu.header['CDELT2B'] = 1
    hdu.header['CRPIX1B'] = 0
    hdu.header['CRPIX2B'] = 0

    hdu.header['PC1_1Q'] = 0
    hdu.header['PC1_2Q'] = 0
    hdu.header['PC2_2Q'] = 1 - 2*sp
    hdu.header['CDELT1Q'] = 1
    hdu.header['CDELT2Q'] = 1
    hdu.header['CRPIX1Q'] = 0
    hdu.header['CRPIX2Q'] = 0
    hdu.header['CRVAL2Q'] = (sp*(2*pixel_pars.dimv + 1) + pixel_pars.gap_outx
                             + (pixel_pars.ccdpx - pixel_pars.ccdax)/2.
                             + cp*(2*pixel_pars.dimv + pixel_pars.gap_inx
                                   + pixel_pars.ccdpx - pixel_pars.ccdax))

    if vendor == 'ITL':
        hdu.header['PC2_1A'] = -1
        hdu.header['CRVAL2A'] = pixel_pars.dimh + 1 - pixel_pars.preh

        hdu.header['PC2_1C'] = -1
        hdu.header['CRVAL2C'] \
            = pixel_pars.dimh + 1 + sy*pixel_pars.dimh - pixel_pars.preh

        hdu.header['PC2_1R'] = -1
        hdu.header['CRVAL2R'] = (pixel_pars.dimh + 1 + sy*pixel_pars.dimh
                                 + pixel_pars.gap_outy
                                 + (pixel_pars.ccdpy - pixel_pars.ccday)/2.
                                 + cy*(8*pixel_pars.dimh + pixel_pars.gap_iny
                                       + pixel_pars.ccdpy - pixel_pars.ccday)
                                 - pixel_pars.preh)

        hdu.header['PC1_1B'] = -1
        hdu.header['CRVAL1B'] = (ss + 1)*pixel_pars.dimh + 1 - pixel_pars.preh
        hdu.header['CRVAL2B'] = sp*(2*pixel_pars.dimv + 1)

        hdu.header['PC1_1Q'] = -1
        hdu.header['CRVAL1Q'] \
            = (pixel_pars.gap_outy + (pixel_pars.ccdpy - pixel_pars.ccday)/2.
               + cs*(8*pixel_pars.dimh + pixel_pars.gap_iny
                     + pixel_pars.ccdpy - pixel_pars.ccday)
               + (ss + 1)*pixel_pars.dimh + 1 - pixel_pars.preh)

        hdu.header['DTM1_1'] = -1
        hdu.header['DTV1'] \
            = (pixel_pars.dimh + 1) + sy*pixel_pars.dimh + pixel_pars.preh
        hdu.header['DTV2'] = (2*pixel_pars.dimv + 1)*(1 - sx)
    else:
        # vendor == 'E2V'
        hdu.header['PC2_1A'] = 1 - 2*sx
        hdu.header['CRVAL2A'] \
            = sx*(pixel_pars.dimh + 1) + (2*sx - 1)*pixel_pars.preh

        hdu.header['PC2_1C'] = 1 - 2*sx
        hdu.header['CRVAL2C'] = (sx*(pixel_pars.dimh+1) + sy*pixel_pars.dimh
                                 + (2*sx - 1)*pixel_pars.preh)

        hdu.header['PC2_1R'] = 1 - 2*sx
        hdu.header['CRVAL2R'] \
            = (sx*(pixel_pars.dimh + 1) + sy*pixel_pars.dimh
               + pixel_pars.gap_outy
               + (pixel_pars.ccdpy - pixel_pars.ccday)/2.
               + cy*(8*pixel_pars.dimh + pixel_pars.gap_iny + pixel_pars.ccdpy
                     - pixel_pars.ccday) + (2*sx - 1)*pixel_pars.preh)
        hdu.header['PC1_1B'] = 1 - 2*sp
        hdu.header['CRVAL1B'] = (sp*(pixel_pars.dimh + 1) + ss*pixel_pars.dimh
                                 + (2*sp - 1)*pixel_pars.preh)
        hdu.header['CRVAL2B'] = sp*(2*pixel_pars.dimv + 1)

        hdu.header['PC1_1Q'] = 1 - 2*sp
        hdu.header['CRVAL1Q'] \
            = (pixel_pars.gap_outy + (pixel_pars.ccdpy - pixel_pars.ccday)/2.
               + cs*(8*pixel_pars.dimh + pixel_pars.gap_iny
                     + pixel_pars.ccdpy - pixel_pars.ccday)
               + sp*(pixel_pars.dimh + 1) + ss*pixel_pars.dimh
               + (2*sp - 1)*pixel_pars.preh)

        hdu.header['DTM1_1'] = 1 - 2*sx
        hdu.header['DTV1'] = ((pixel_pars.dimh + 1 + 2*pixel_pars.preh)*sx
                              + sy*pixel_pars.dimh - pixel_pars.preh)
        hdu.header['DTV2'] = (2*pixel_pars.dimv + 1)*(1 - sx)

    return hdu

def cte_matrix(npix, cti, ntransfers=20, nexact=30):
    """
    Compute the CTE matrix so that the apparent charge q_i in the i-th
    pixel is given by

    q_i = Sum_j cte_matrix_ij q0_j

    where q0_j is the initial charge in j-th pixel.  The corresponding
    python code would be

    >>> cte = cte_matrix(npix, cti)
    >>> qout = numpy.dot(cte, qin)

    Parameters
    ----------
    npix : int
        Total number of pixels in either the serial or parallel
        directions.
    cti : float
        The charge transfer inefficiency.
    ntransfers : int, optional
        Maximum number of transfers to consider as contributing to
        a target pixel.
    nexact : int, optional
        Number of transfers to use exact the binomial distribution
        expression, otherwise use Poisson's approximation.

    Returns
    -------
    numpy.array
        The npix x npix numpy array containing the CTE matrix.

    Notes
    -----
    This implementation is based on
    Janesick, J. R., 2001, "Scientific Charge-Coupled Devices", Chapter 5,
    eqs. 5.2a,b.

    """
    ntransfers = min(npix, ntransfers)
    nexact = min(nexact, ntransfers)
    my_matrix = np.zeros((npix, npix), dtype=np.float)
    for i in range(1, npix):
        jvals = np.concatenate((np.arange(1, i+1), np.zeros(npix-i)))
        index = np.where(i - nexact < jvals)
        j = jvals[index]
        my_matrix[i-1, :][index] \
            = scipy.special.binom(i, j)*(1 - cti)**i*cti**(i - j)
        if nexact < ntransfers:
            index = np.where((i - nexact >= jvals) & (i - ntransfers < jvals))
            j = jvals[index]
            my_matrix[i-1, :][index] \
                = (j*cti)**(i-j)*np.exp(-j*cti)/scipy.special.factorial(i-j)
    return my_matrix

import numpy,import scipy,import astropy,from astropy