#!/usr/bin/env python
# coding: utf-8

#
#
# # Project:  (Medical Appointment No Shows)
# # Hessa Alqahtani
# ## Table of Contents
#

#
Introduction

#
Data Wrangling

#
Exploratory Data Analysis

#
Conclusions

#

#
# ## Introduction
# > This is an attempt to investigate Medical Appointment No Shows dataset of 110527 patient appointments in Brazil from late April to early June 2016. A number of characteristics about each patient are included in each row. Data shows that 30% patients miss thier appointments. The data sets is provided on Kaggle.
#  My goal is trying to answer the questions by investigating the dataset.\
# **Questions:**
# > - Who is more committed to attending the appointment, female or male ?
# > - Are no-show appointments associated with a certain gender?
# > - Do the SMS reminders decrease the number of absences?
# > - Are illnesses like Handicap affecting by patient's age?
#

# In[1]:

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
get_ipython().run_line_magic('matplotlib', 'inline')
import seaborn as sns

#
# ## Data Wrangling
#
#
#
# ### General Properties

# In[2]:

df = pd.read_csv('noshowappointments.csv',                        parse_dates=['ScheduledDay', 'AppointmentDay'])
df.head()

# > I used **head()** to display the first five rows of the data.

# In[3]:

df.info()

# In[4]:

#this function takes data frame (df) as a parameter and prints out the number of rows and columns.
def shape(df):
    print('Number of columns is',len(df.columns))
    print('Number of rows is',len(df.index))

# In[5]:

shape(df)

# In[6]:

df.info()

# > After i loaded my data, I found out that the type of "ScheduledDay and AppointmentDay" was object while it's supposed to be datetime. So, I used the argument "parse_dates" to change it's type.

# In[7]:

df.describe()

# > **describe()** function help me to look at basic statistics of the numerical features in my data.

# In[8]:

df.nunique()

# > I used **unique()** function to get the unique values in my data.

#
#
# ### Data Cleaning (the process of removing incorrect, corrupted, incorrectly formatted, duplicate, or missing values)

# In[9]:

df.duplicated().sum()

# In[10]:

df.isnull().sum()

# > As shown above, The data does not have any missing values nor duplicate.

# In[11]:

df.drop('PatientId',axis=1, inplace=True)

# In[12]:

df.drop('AppointmentID',axis=1, inplace=True)

# > Here, I removed some columns that were not necessary to make the data processing faster.

# In[13]:

df= df.rename(columns={'No-show': 'miss_their_appointments'})

# > I renamed “No-show” column to “miss_their_appointments” to help me avoid any confusion. First time i saw the data, I thought that the "No" values mean that they missed their appointments while it's the opposite.

# In[14]:

df=df.rename(columns={'Handcap':'Handicap'})

# > I renamed misspelled column name.

# In[15]:

df.head()

# > i called the head() again just to check the data after the clean process.

#
# ## Exploratory Data Analysis
#
#
#
# ### Research Question 1 (Is the gender going to affect  the appointments?)

# In[16]:

df.Gender.value_counts()

# In[17]:

df.Gender.unique()

# In[18]:

df.groupby('Gender').size().plot(kind='pie' ,autopct='%.2f', colors = ['pink', 'lightblue']);
plt.title("Gender");
plt.ylabel('pie chart');

# > As shown above, Female is the greater proportion, They takes way more care of their health in comparison to man.
# In total 71840 F against 38687 M.
# > 1. I checked the value counts and unique value in Gender column.
# > 2. I used pie chart for this data because it's single-variable **1D** explorations, I learned how to use autopct in this chart.

# In[19]:

df['miss_their_appointments'].value_counts()

# In[20]:

df.miss_their_appointments.unique()

# In[21]:

df_gender=df.groupby('Gender').miss_their_appointments.value_counts()

# In[22]:

df_gender.unstack().plot.bar(color=['pink','lightblue'])
plt.title("Is the Gender going to affect the appointments?");
plt.ylabel('patients');
plt.xlabel('Gender');

# > As shown above, +88K appeared in their appointments, 57k were female and 30k male. And 22k miss their appointments were 13k females and 7k males. So, After i calculated the ratio for female (57:13 =4.38) and male(30:7=4.28) as we can see it's almost the same ratio for both who came to their appointments and who didn't. As a result Gender doesn't affect the appointments.
# > 1. As noted previously, We need to check the value count and unique value for the columns needed.
# > 2.  This time I used "GroupBy" function. It allows me to split my data into separate groups to perform computations for better analysis.
# > 3. I used bar chart because i want to compare more than one column and it helps to show the comparison values clearly.

# ### Research Question 2  (Dose the SMS help the patients to avoid missing their appointments? )

# In[23]:

df.SMS_received.value_counts()

# In[24]:

df_sms=df.groupby('SMS_received').miss_their_appointments.value_counts()

# In[25]:

df_sms.unstack().plot.bar(color=['pink','lightblue'])
plt.title("Is the SMS going to affect the appointments?")
plt.ylabel('patients');
plt.xlabel('messages sent to the patient');

# > in this bar chart i took two columns (SMS_received and miss_their_appointments) to see if the SMS will help patients to attend their appointments. As shown in the bar, +60k came and around 11k miss their appointments without the sms received. And from total people received messages (35482) around 27k came to their appointment after the message! and only 10k missed their appointments which show that sms messages really help the patient to remember his appointments.

# ### Research Question 3 (Relation between Age and handicap )

# In[26]:

df.Handicap.value_counts()

# In[27]:

df.Handicap.unique()

# In[28]:

df.plot.scatter(x="Handicap", y="Age", color=['lightblue']);

# >**Add at least one scatter plot or correlation matrix to show relationships between variables**
# i decided to add the two(Age,Handicap) in scatter to show the relationships between variables.
# As shown above there are 108286 patients doesn't suffer from any handicap. and then it drops to 2042 patients with level 1 handicap(serious difficulty walking or climbing stairs).
# I took "age" varible  because it's the only one that have a better distribution between the amount of data.

#
# ## Conclusions
# > **Results**:\
# 1- Female is the greater proportion in this dataset with 65% compaere to 35% M.\
# 2- The gender doesn't affect the appointments.\
# 3- There is a higher percentage of patients who recevied SMS and didn't show up when compared to patients who recevied SMS and did show up. Which means SMS reminders help the patients to come to their appointments.
#
# **Limitations**:
# >- The data set explored in this analysis was just over a 6 month period.
# >- Patients with repeat "miss_their_appointments" would be a good cohort to remove from the analysis to understand environmental or health factors influencing their appointments.
# >- We don't have a lot of details to draw conclusions.
# > -We don't have a good distribution between the amount of data.
# ## Submitting your Project
#
#


import numpy as np
import tensorflow as tf
import pandas as pd
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.preprocessing import StandardScaler, OneHotEncoder
import gc

from Auto_Tabular.utils.data_utils import ohe2cat
from Auto_Tabular.utils.log_utils import log, timeit
from Auto_Tabular.utils.data_utils import fill_na
from Auto_Tabular.utils.sample import AutoSample
from Auto_Tabular import CONSTANT

class TabularDataSpace:
    def __init__(self, metadata, eda_info, main_df, y, lgb_info):
        self.metadata = metadata

        self.lgb_info = lgb_info

        self.data = main_df
        self.all_idxs = main_df.index
        self.all_train_idxs = main_df.index[main_df.index >= 0]
        self.test_idxs = main_df.index[main_df.index < 0]

        self.y = y

        self.cat_cols = eda_info['cat_cols']
        self.num_cols = eda_info['num_cols']

        self.col2type = {}

        self.post_drop_set = set()

        self.init_col2type()

        self.splits = {}

        self.train_valid_split_idxs()

        self.update = False

    @timeit
    def train_valid_split_idxs(self, ratio=0.2):
        sss = StratifiedShuffleSplit(n_splits=5, test_size=ratio, random_state=0)
        idxs = np.arange(len(self.y))
        i = 0
        for train, val in sss.split(idxs, ohe2cat(self.y)):
            self.splits[i] = (train, val)
            i += 1

        self.train_idxs, self.val_idxs = self.splits[0]

        self.m = len(self.train_idxs)
        self.auto_sample = AutoSample(self.y[self.train_idxs])

    @timeit
    def get_dataloader(self, train_loop_num, round_num, run_num, use_all_data, model_type):

        self.train_idxs, self.val_idxs = self.splits[round_num-1]
        print('round {}'.format(round_num))

        data_loader = {}
        do_sample_col = False

        if use_all_data:
            log('all data')
            sample_idxs = self.all_train_idxs
        else:
            log('all train')
            sample_idxs = self.train_idxs

        log('sample ratio {}'.format(len(sample_idxs)/len(self.train_idxs)))

        cat_cols = self.get_categories(self.data)

        if model_type == 'nn_keras':
            feats = self.nn_process(self.data, cat_cols)
            feats = pd.DataFrame(feats, index=self.all_idxs)
            data_loader['X'], data_loader['y'] = feats, self.y
            data_loader['shape'] = feats.shape[1]

        elif model_type == 'emb_nn':
            feats = pd.DataFrame(self.data, index=self.all_idxs)
            self.label_encode(feats, cat_cols)
            data_loader['X'], data_loader['y'] = feats, self.y
            data_loader['shape'] = feats.shape[1]
        elif model_type == 'tree':
            feats = self.data
            if do_sample_col:
                data_loader['X'] = feats
            else:
                data_loader['X'] = feats
            data_loader['y'] = self.y
            data_loader['cat_cols'] = cat_cols
        elif model_type == 'lr':
            feats = self.nn_process(self.data, cat_cols)
            feats = pd.DataFrame(feats, index=self.all_idxs)
            data_loader['X'], data_loader['y'] = feats, self.y
            data_loader['cat_cols'] = cat_cols

        data_loader['all_train_idxs'], data_loader['train_idxs'], \
        data_loader['val_idxs'], data_loader['test_idxs'], \
        data_loader['splits'], data_loader['cat_cols']\
            = self.all_train_idxs, sample_idxs, self.val_idxs, self.test_idxs, self.splits, cat_cols

        return data_loader

    def to_tfdataset(self, feats, y=None, mode=None):
        if mode == 'train':
            return tf.data.Dataset.from_tensor_slices((feats, y))
        elif mode == 'val':
            return tf.data.Dataset.from_tensor_slices(feats), y
        elif mode == 'test':
            return tf.data.Dataset.from_tensor_slices(feats)

    def drop_post_drop_column(self, df):
        if len(self.post_drop_set) != 0:
            drop_cols = list(self.post_drop_set)
            df.drop(drop_cols, axis=1, inplace=True)
            gc.collect()
            log('post drop cols:{}'.format(drop_cols))

    def get_categories(self, df):
        categories = []
        col_set = set(df.columns)
        for col in self.cat_cols:
            if col in col_set:
                if df[col].nunique() <= 10:
                    categories.append(col)
        log('get categories: {}'.format(categories))
        return categories

    def update_data(self, df, col2type):
        self.data = df
        self.update_col2type(col2type)

    def update_col2type(self, col2type):
        self.col2type.update(col2type)
        self.type_reset()

    def type_reset(self):

        cat_cols = []
        num_cols = []

        for cname, ctype in self.col2type.items():
            if ctype == CONSTANT.CATEGORY_TYPE:
                cat_cols.append(cname)
            elif ctype == CONSTANT.NUMERICAL_TYPE:
                num_cols.append(cname)

        self.cat_cols = sorted(cat_cols)
        self.num_cols = sorted(num_cols)

    def init_col2type(self):
        for col in self.cat_cols:
            self.col2type[col] = CONSTANT.CATEGORY_TYPE
        for col in self.num_cols:
            self.col2type[col] = CONSTANT.NUMERICAL_TYPE

    def num_fit_transform(self, df, num_cols):
        if not num_cols:
            return []
        scaler = StandardScaler()
        df = df[list(num_cols)]
        arr = df.values
        norm_arr = scaler.fit_transform(arr)
        norm_arr[np.isnan(norm_arr)] = 0
        return norm_arr

    def cat_fit_transform(self, df, cat_cols):
        if len(cat_cols) == 0:
            return []
        res = []
        for cat in cat_cols:
            enc = OneHotEncoder(handle_unknown='ignore')
            arr = enc.fit_transform(df[cat].values.reshape(-1, 1)).toarray()
            res.append(arr)
        cat_feats = np.concatenate(res, axis=1)
        return cat_feats

    def nn_process(self, X, cat_cols):
        num_cols = [col for col in X.columns if col not in cat_cols]

        log('nn_process num col: {}'.format(num_cols))
        log('nn_process cat col:{}'.format(cat_cols))

        X = fill_na(X)

        cat_feats = self.cat_fit_transform(X, cat_cols)
        num_feats = self.num_fit_transform(X, num_cols)

        if len(cat_feats) > 0 and len(num_feats) > 0:
            feats = np.concatenate([cat_feats, num_feats], axis=1)
        elif len(cat_feats) > 0:
            feats = cat_feats
        elif len(num_feats) > 0:
            feats = num_feats
        return feats

    def label_encode(self, X, cat_cols):
        if cat_cols == []:
            return

        for col in cat_cols:
            X[col] = pd.Categorical(X[col]).codes


import numpy as np
import torch

from detectron2.modeling.proposal_generator.proposal_utils import add_ground_truth_to_proposals
from detectron2.modeling.roi_heads.fast_rcnn import FastRCNNOutputLayers
from detectron2.modeling.roi_heads.roi_heads import StandardROIHeads, ROI_HEADS_REGISTRY
from detectron2.structures import Boxes, pairwise_iou
from detectron2.utils.events import get_event_storage
from detectron2.utils.registry import Registry

from .fast_rcnn import RepLossFastRCNNOutputs
from .matcher import Top2Matcher

@ROI_HEADS_REGISTRY.register()
class RepLossROIHeads(StandardROIHeads):

    def __init__(self, cfg, input_shape):
        super().__init__(cfg, input_shape)

        self.rep_gt_factor = cfg.MODEL.ROI_HEADS.REPULSION_LOSS.REP_GT_FACTOR
        self.rep_box_factor = cfg.MODEL.ROI_HEADS.REPULSION_LOSS.REP_BOX_FACTOR
        self.rep_gt_sigma = cfg.MODEL.ROI_HEADS.REPULSION_LOSS.REP_GT_SIGMA
        self.rep_box_sigma = cfg.MODEL.ROI_HEADS.REPULSION_LOSS.REP_BOX_SIGMA
        self.d2_normalize = cfg.MODEL.ROI_HEADS.REPULSION_LOSS.D2_NORMALIZE

        self.proposal_matcher = Top2Matcher(
            cfg.MODEL.ROI_HEADS.IOU_THRESHOLDS,
            cfg.MODEL.ROI_HEADS.IOU_LABELS,
            allow_low_quality_matches=False
        )

    def forward(self, images, features, proposals, targets=None):
        """
        See :class:`StandardROIHeads.forward`.

        Only included here because there are new calls to
            self.label_and_sample_proposals, and
            self._forward_boxes
        """
        del images
        if self.training:
            proposals = self.label_and_sample_proposals(proposals, targets)
        del targets

        features_list = [features[f] for f in self.in_features]

        if self.training:
            losses = self._forward_box(features_list, proposals)
            losses.update(self._forward_mask(features_list, proposals))
            losses.update(self._forward_keypoint(features_list, proposals))
            return proposals, losses
        else:
            pred_instances = self._forward_box(features_list, proposals)
            pred_instances = self.forward_with_given_boxes(features, pred_instances)
            return pred_instances, {}

    def _forward_box(self, features, proposals):
        """
        See :class:`StandardROIHeads._forward_box`.

        Only change is that RepLossFastRCNNOutputs is called.

        Args:
            features (list[Tensor]): #level input features for box prediction
            proposals (list[Instances]): the per-image object proposals with
                their matching ground truth.
                Each has fields "proposal_boxes", and "objectness_logits",
                "gt_classes", "gt_boxes", "gp_rep_boxes", "gt_box_inds"

        Returns:
            In training, a dict of losses.
            In inference, a listf of `Instances`, the predicted instances.
        """
        box_features = self.box_pooler(features, [x.proposal_boxes for x in proposals])
        box_features = self.box_head(box_features)
        pred_class_logits, pred_proposal_deltas = self.box_predictor(box_features)
        del box_features

        outputs = RepLossFastRCNNOutputs(
            self.box2box_transform,
            pred_class_logits,
            pred_proposal_deltas,
            proposals,
            self.smooth_l1_beta,
            self.rep_gt_factor,
            self.rep_box_factor,
            self.rep_gt_sigma,
            self.rep_box_sigma,
            self.d2_normalize
        )
        if self.training:
            return outputs.losses()
        else:
            pred_instances, _ = outputs.inference(
                self.test_score_thresh, self.test_nms_thresh, self.test_detections_per_img
            )
            return pred_instances

    @torch.no_grad()
    def label_and_sample_proposals(self, proposals, targets):
        """
        Prepare some proposals to be used to train the ROI heads.
        It performs box matching between `proposals` and `targets`, and assigns
        training labels to the proposals.
        It returns ``self.batch_size_per_image`` random samples from proposals and groundtruth
        boxes, with a fraction of positives that is no larger than
        ``self.positive_sample_fraction``.

        Args:
            See :meth:`ROIHeads.forward`

        Returns:
            list[Instances]:
                length `N` list of `Instances`s containing the proposals
                sampled for training. Each `Instances` has the following fields:

                - proposal_boxes: the proposal boxes
                - gt_boxes: the ground-truth box that the proposal is assigned to
                  (this is only meaningful if the proposal has a label > 0; if label = 0
                  then the ground-truth box is random)
                - gt_box_inds - index in targets of each box in gt_boxes
                - gt_rep_boxes - repulsion target for each proposal

                Other fields such as "gt_classes", "gt_masks", that's included in `targets`.
        """
        gt_boxes = [x.gt_boxes for x in targets]
        # Augment proposals with ground-truth boxes.
        # In the case of learned proposals (e.g., RPN), when training starts
        # the proposals will be low quality due to random initialization.
        # It's possible that none of these initial
        # proposals have high enough overlap with the gt objects to be used
        # as positive examples for the second stage components (box head,
        # cls head, mask head). Adding the gt boxes to the set of proposals
        # ensures that the second stage components will have some positive
        # examples from the start of training. For RPN, this augmentation improves
        # convergence and empirically improves box AP on COCO by about 0.5
        # points (under one tested configuration).
        if self.proposal_append_gt:
            proposals = add_ground_truth_to_proposals(gt_boxes, proposals)

        proposals_with_gt = []

        num_fg_samples = []
        num_bg_samples = []
        for proposals_per_image, targets_per_image in zip(proposals, targets):
            has_gt = len(targets_per_image) > 0
            has_repel_target = len(targets_per_image) > 1

            match_quality_matrix = pairwise_iou(
                targets_per_image.gt_boxes, proposals_per_image.proposal_boxes
            )
            matched_idxs, matched_labels, repel_matched_idxs = self.proposal_matcher(match_quality_matrix)
            sampled_idxs, gt_classes = self._sample_proposals(
                matched_idxs, matched_labels, targets_per_image.gt_classes
            )

            # Set target attributes of the sampled proposals:
            proposals_per_image = proposals_per_image[sampled_idxs]
            proposals_per_image.gt_classes = gt_classes
            sampled_targets = matched_idxs[sampled_idxs]

            # also store the index of the target box - useful for creating subsets later
            proposals_per_image.set("gt_box_inds", sampled_targets)

            # We index all the attributes of targets that start with "gt_"
            # and have not been added to proposals yet (="gt_classes").
            if has_gt:
                # NOTE: here the indexing waste some compute, because heads
                # like masks, keypoints, etc, will filter the proposals again,
                # (by foreground/background, or number of keypoints in the image, etc)
                # so we essentially index the data twice.
                for (trg_name, trg_value) in targets_per_image.get_fields().items():
                    if trg_name.startswith("gt_") and not proposals_per_image.has(trg_name):
                        proposals_per_image.set(trg_name, trg_value[sampled_targets])
            else:
                gt_boxes = Boxes(
                    targets_per_image.gt_boxes.tensor.new_zeros((len(sampled_idxs), 4))
                )
                proposals_per_image.gt_boxes = gt_boxes

            # add in the repulsion loss info - store the box we are repelling away from
            # if no repulsion target, use a box { 0, 0, 0, 0 } like for attraction target
            if has_repel_target:
                sampled_repels = repel_matched_idxs[sampled_idxs]
                proposals_per_image.set("gt_rep_boxes", targets_per_image.gt_boxes[sampled_repels])
            else:
                gt_rep_boxes = Boxes(
                    targets_per_image.gt_boxes.tensor.new_zeros((len(sampled_idxs), 4))
                )
                proposals_per_image.set("gt_rep_boxes", gt_rep_boxes)

            num_bg_samples.append((gt_classes == self.num_classes).sum().item())
            num_fg_samples.append(gt_classes.numel() - num_bg_samples[-1])
            proposals_with_gt.append(proposals_per_image)

        # Log the number of fg/bg samples that are selected for training ROI heads
        storage = get_event_storage()
        storage.put_scalar("roi_head/num_fg_samples", np.mean(num_fg_samples))
        storage.put_scalar("roi_head/num_bg_samples", np.mean(num_bg_samples))

        return proposals_with_gt


# Multiscale sampling (MSS) with VASP and LAMMPS
# Xiaohong Zhang
# Getman Research Group
# Apr 14, 2019
import os, sys
import pandas as pd
import numpy as np
from collections import OrderedDict
import math
import fnmatch
from vasp import VASP

class LmpsToVasp(VASP):
	""" convert LAMMPS file to VASP file """
	def __init__(self, curr_dir):
		poscar_file = os.path.join(curr_dir, '2-lammps/0-add-sol/POSCAR_0.POSCAR') # solvated poscar file
		mss_input = os.path.join(curr_dir, 'input/master_input.txt')
		lmps_input = os.path.join(curr_dir, '2-lammps/prod/input.prod')
		for f in os.listdir(os.path.join(curr_dir, '2-lammps/prod')):
			if fnmatch.fnmatch(f, 'dump.*.lammpstrj'):
				lmps_dump = os.path.join(curr_dir, '2-lammps/prod/', f)

		super().__init__(poscar_file, mss_input)
		self.equil_run = 2000000  # remove first 2 ns
		self.shift_z = 14.0 # add more vacuum space in z direction for vasp simulation
		self.frames = [] # store e.g. 10 frames into list, include both title and coords
		self.cell_vec = [[0]*3 for i in range(3)]  ## from dump file generated POSCAR with unshifted cell vector
		self.elem_type_list = [] #all element type list for a frame (same for all frames)
		# self.indices is the atom index used for calc HB, this indices cannot be obtained from vac POSCAR or
		# after Jeremy's solvent added, b/c re-ordered the coordinates from POSCAR to LAMMPS
		self.indices = {'surface':[],'adsorbate':[],'solvent':[],
						'Oads':[],'Hads':[],'Osol':[],'Hsol':[]} #need ads list for rigid ads

		## cannot move below statement to above
		self.readLmps(lmps_input, lmps_dump)
		self.setElemInfo() # must call after read lammps

	def readLmps(self, lmps_input, lmps_dump):
		""" Extract 10 frames from LAMMPS dump trajectory of production run """
		with open(lmps_input) as f:
			for line in f:
				if "name string" in line:
					self.name = line.strip().split()[3]
				if "dumpFreq equal" in line:
					self.dump_freq = int(line.strip().split()[3])
				if "thermFreq equal" in line:
					self.therm_freq = int(line.strip().split()[3])
				if "runStep equal" in line:
					self.final_step = int(line.strip().split()[3])

		frame_interval = (self.final_step - self.equil_run) / self.frame_num   # 300000
		with open(lmps_dump, 'r') as f:
			dump_vec = []
			for i, line in enumerate(f):
				if i == 3:
					self.total_atom = int(line.strip().split()[0]) # 211
					## self.total_atom+9 is lineno for each frame
					self.frame_line = self.total_atom + 9 # 220
					## skip the equilibrium time
					start_line = self.frame_line*(self.equil_run+frame_interval*0.5)/self.dump_freq
					# print(start_line)
				elif 5 <= i  <= 7 :
					dump_vec.append([float(j) for j in line.strip().split()])
					if len(dump_vec) == 3:
						self.setCellVec(dump_vec)
				elif i > 7 and start_line <= i < start_line+self.frame_line:
					self.frames.append(line.strip().split())
					### read to the last line of this frame
					if i == start_line+self.frame_line - 1:
						start_line += self.frame_line*(frame_interval/self.dump_freq)
						# print(start_line)

	def setCellVec(self, dump_vec):
		""" convert lammps dump raw cell format to poscar cell format """
				xlo_bound = dump_vec[0][0]
		xhi_bound = dump_vec[0][1]
		ylo_bound = dump_vec[1][0]
		yhi_bound = dump_vec[1][1]
		zlo_bound = dump_vec[2][0]
		zhi_bound = dump_vec[2][1]
		xy = dump_vec[0][2] if len(dump_vec[0]) > 2 else 0.0
		xz = dump_vec[1][2] if len(dump_vec[1]) > 2 else 0.0
		yz = dump_vec[2][2] if len(dump_vec[2]) > 2 else 0.0

		xlo = xlo_bound - min(0.0,xy,xz,xy+xz)
		xhi = xhi_bound - max(0.0,xy,xz,xy+xz)
		ylo = ylo_bound - min(0.0,yz)
		yhi = yhi_bound - max(0.0,yz)
		zlo = zlo_bound
		zhi = zhi_bound
		self.cell_vec = [[xhi-xlo,0,0],
						[xy,yhi-ylo,0],
						[xz,yz,zhi-zlo]]

	def setElemInfo(self):

		"""only read the first of the 10 selected frames.
		get the information of each atom element and number.
		Also get a list of each element
		"""
		frame = self.frames[0:self.frame_line]
		for i in range(9, len(frame)):
			_type = int(frame[i][2])
			elem = self.atom.loc[_type-1, 'elem'].split('_')[0] # inheritant self.atom from VASP(ReadInput)
			self.elem_type_list.append(elem)  ## total of 211 elem type

		coords = self.reorderCoords(frame)
		self.setIndices(coords) # use coords rather than frame

	def reorderCoords(self, frame):
		""" process one frame at a time, get all info of GROUPED elements
		must call self.getElemInfo(frame) at first in genVASP() function
		to define self.elem_type_list.
		Group coords according to their element type. Previous dump file O,H,H,
		Now all O are together, all H are together.
		frame is a list of list that is extracted from dump, including header
		"""

		# reordered according to Pt C O H
		coords = pd.DataFrame(columns=['id','mol','type','xs','ys','zs','opt']) #optimization
		#### loop over Pt, C, O,H in order to group the same elements together
		for elem in self.elem['elem']: # loop over elem type
			for i in range(9, len(frame)):
				## all frames have same order, thus instead of get elem type for each
				## frame, here use the general elem_type_list
				if self.elem_type_list[i-9] == elem:
					# set the default selective dynamics as F
					coords.loc[len(coords)] = [frame[i][0],frame[i][1],frame[i][2],
											   frame[i][3],frame[i][4],frame[i][5],'F   F   F']
		return coords

	def setIndices(self, coords):
		"""get all group information"""
		# self.indices = {'Osol':[],'Hsol':[],'Oads':[],'Hads':[],'adsorbate':[]}
		# only need to be called once
		surface = [int(i) for i in self.group[self.group['name']=='surface']['type'].item().split()] # 1
		adsorbate = [int(i) for i in self.group[self.group['name']=='adsorbate']['type'].item().split()] # [2, 3, 4, 5]
		solvent = [int(i) for i in self.group[self.group['name']=='solvent']['type'].item().split()] # [6, 7, 8, 9, 10, 11]
		Oads = [int(i) for i in self.group[self.group['name']=='Oads']['type'].item().split()] # 3
		Hads = [int(i) for i in self.group[self.group['name']=='Hads']['type'].item().split()] # 5
		Osol = [int(i) for i in self.group[self.group['name']=='Osol']['type'].item().split()] # 6 9
		Hsol = [int(i) for i in self.group[self.group['name']=='Hsol']['type'].item().split()] # 7 11

		# print (surface)
		# print (adsorbate)
		# print (solvent)
		# print (Oads)
		# print (Hads)
		# print (Osol)
		# print (Hsol)

		for i in range(len(coords)):
			_type = int(coords.loc[i,'type'])
			if _type in surface:
				self.indices['surface'].append(i)
			if _type in adsorbate:
				self.indices['adsorbate'].append(i)
			if _type in solvent:
				self.indices['solvent'].append(i)
			if _type in Oads:
				self.indices['Oads'].append(i)
			if _type in Hads:
				self.indices['Hads'].append(i)
			if _type in Osol:
				self.indices['Osol'].append(i) # coords start from 0
			if _type in Hsol:
				self.indices['Hsol'].append(i)
		# print ('surface  ', self.indices['surface'])
		# print ('\nadsorbate  ', self.indices['adsorbate'])
		# print ('\nsolvent  ', self.indices['solvent'])
		# print ('\nOads  ', self.indices['Oads'])
		# print ('\nHads  ', self.indices['Hads'])
		# print ('\nOsol  ', self.indices['Osol'])
		# print ('\nHsol  ', self.indices['Hsol'])

	def calcHB(self, coords):

		""" calculate the HB between adsorbates and solvent molecules
			process one frame's coords at one time
		"""
		relax_sol = {'sol_acc':[], 'sol_don':[]} # index starts from 0

		for i in self.indices['Oads']: # all index in coords is based on start from 0
			p_Oads = [float(coords.loc[i,'xs']), float(coords.loc[i,'ys']), float(coords.loc[i,'zs'])]
			# work for different adsorbate coverage
			mol_ads_type = coords.loc[i,'mol']
			mol_ads = coords.loc[coords['mol'] == mol_ads_type].index.tolist() # index for a whole molecule
			# print(mol_ads) #int,[27, 28, 29, 45, 46, 47, 90, 91, 92, 93, 94, 95, 96]

			for j in self.indices['Osol']:
				p_Osol = [float(coords.loc[j,'xs']), float(coords.loc[j,'ys']), float(coords.loc[j,'zs'])]
				d_oo = self.getDist(p_Oads, p_Osol)
				# print("%d\t%d: %f" % (i,j,dist))
				if d_oo > 0.0 and d_oo <= 3.5:
					mol_sol_type = coords.loc[j,'mol']
					mol_sol = coords.loc[coords['mol'] == mol_sol_type].index.tolist() # index for a whole molecule,start 0

					######################################################
					##### Calc HB-ed H2O mol when solvent as acceptor (ads as donor) #####
					for k in mol_ads:
						if k in self.indices['Hads']: # 95，96
							p_Hads = [float(coords.loc[k,'xs']), float(coords.loc[k,'ys']), float(coords.loc[k,'zs'])]
							d_hyd = self.getDist(p_Oads, p_Hads)  # doner H and its bonded O/N
							if d_hyd <= 1.2:
								d_oh = self.getDist(p_Osol, p_Hads) # dist of O...H, i.e., HB
								if d_oh <= 2.5:
									a_ooh = self.getAngle(d_oo, d_hyd, d_oh)
									a_oho = self.getAngle(d_hyd, d_oh, d_oo)
									if a_ooh <= 30 and a_oho >= 120:
										relax_sol['sol_acc'].append(mol_sol)

					######################################################
					##### Calc HB-ed H2O mol when solvent as donor #####
					for k in mol_sol: # e.g., H2O has two H
						if k in self.indices['Hsol']: # 95，96
							p_Hsol = [float(coords.loc[k,'xs']), float(coords.loc[k,'ys']), float(coords.loc[k,'zs'])]
							d_hyd = self.getDist(p_Osol, p_Hsol)  # hydroxyl group, i.e., doner H and its bonded O/N
							if d_hyd <= 1.2:
								d_oh = self.getDist(p_Oads, p_Hsol) # dist of O...H, i.e., HB
								if d_oh <= 2.5:
									a_ooh = self.getAngle(d_oo, d_hyd, d_oh)
									a_oho = self.getAngle(d_hyd, d_oh, d_oo)
									if a_ooh <= 30 and a_oho >= 120:
										relax_sol['sol_don'].append(mol_sol)

		return relax_sol

	def fracToCartesian(self, p):
		xa = self.cell_vec[0][0]
		xb = self.cell_vec[0][1]
		xc = self.cell_vec[0][2]
		ya = self.cell_vec[1][0]
		yb = self.cell_vec[1][1]
		yc = self.cell_vec[1][2]
		za = self.cell_vec[2][0]
		zb = self.cell_vec[2][1]
		zc = self.cell_vec[2][2]
		m = 1.0 # self.multiplier

		###must return array in order to use lambda in getDist()
		cp = m * np.array( [p[0]*xa + p[1]*ya + p[2]*za,
		                    p[0]*xb + p[1]*yb + p[2]*zb,
		                    p[0]*xc + p[1]*yc + p[2]*zc] )
		return cp

	def getDist(self, p0, p1):
		""" test all images """
		x = p1[0]
		y = p1[1]
		z = p1[2]
		images = np.array([[x,y,z],     [x+1,y,z],      [x-1,y,z],
		                   [x,y+1,z],   [x+1,y+1,z],    [x-1,y+1,z],
		                   [x,y-1,z],   [x+1,y-1,z],    [x-1,y-1,z]])

		return min(np.sqrt(sum(map(lambda x:x*x, self.fracToCartesian(p0-p)))) for p in images)

	def getAngle(self, d1, d2, d3): # Note: square of the distances
		return math.degrees(math.acos((d1**2 + d2**2 - d3**2)/(2.0*d1*d2)))

	def genVASP(self, frame_idx, model='full', jobtype='geomopt', queue='workq', version='cpu'):
		""" if generate 10 VASP folders with index "file_id"
			model choices: full, sol, ads
			jobtype choices: singpt, geomopt, cineb, dimer
			queue choices: workq, curium
			version choices: cpu, gpu, neb
		"""
		# frame: list of list. including the frame header. frame is the raw data
		if model not in ['full', 'surface', 'ads', 'sol']:
			sys.exit('\nERROR: wrong poscar model\n')

		# print(type(frame_idx))
		frame = self.frames[self.frame_line*frame_idx:self.frame_line*(frame_idx+1)]
		jobname = self.ads_name + '_' + str(frame_idx) + '_' + model

		if model == 'full':
			path = os.path.join('./3-vasp-eint/', str(frame_idx))
			coords = self.reorderCoords(frame)
			relax_sol = self.calcHB(coords) # return a dict
			contcar_file = ''
		else:
			coords = pd.DataFrame()
			relax_sol = {}
			contcar_file = os.path.join('./3-vasp-eint/', str(frame_idx), 'CONTCAR')
			if model == 'surface':
				path = os.path.join('./3-vasp-eint/', str(frame_idx)+'p')
			elif model == 'ads':
				path = os.path.join('./3-vasp-eint/', str(frame_idx)+'a')
			elif model == 'sol':
				path = os.path.join('./3-vasp-eint/', str(frame_idx)+'s')

		self.createFolder(path)
		self.writePOSCAR(path, coords, relax_sol, self.cell_vec, self.shift_z,
						 self.indices, contcar_file, model, jobtype)
		self.writeINCAR(path, model, jobtype)
		self.writeKPOINTS(path)
		self.writePOTCAR(path, model)
		self.writeSubVASP(path, jobname, queue, version)	#subVasp no need model b/c jobname includes model

def main():

	curr_dir = sys.argv[1]
	vasp = LmpsToVasp(curr_dir)

	"""command line model choices: full, surface, ads, sol"""
	model = sys.argv[2]  # CHANGE TO ARGPARSE LATER

	if model == 'full':
		for i in range(vasp.frame_num):
			vasp.genVASP(frame_idx=i, model='full', jobtype='geomopt', queue='workq', version='cpu')

	else:
		idx = int(sys.argv[3])
		vasp.genVASP(frame_idx=idx, model=model, jobtype='singpt', queue='workq', version='cpu')

if __name__ == '__main__':
	main()


import numpy as np
import torch as t
from torch import nn
from torch.nn import functional as F
from torchvision.models import vgg16
from torchvision.ops import RoIPool

from utils.anchors import Anchors
from utils.helper import init_params, totensor
from utils.config import opt

def get_rcnn_vgg16():
    model = vgg16(not opt.load_path)

    features = list(model.features)[:30]
    for layer in features[:10]:
        for p in layer.parameters():
            p.requires_grad = False

    classifier = list(model.classifier)
    del classifier[6]
    classifier = nn.Sequential(*classifier)

    return nn.Sequential(*features), classifier

class VGG16RoIHead(nn.Module):

    """
    FasterRCNN VGG-16 Head.
    This outputs class-wise localizations and classification based on feature
    maps in the given RoIs.

    Arguments:
        n_class         :- The number of classes including the background.
        spatial_scale   :- Scale of the roi is resized.
        classifier      :- Two layer Linear ported from vgg16
        roi_size        :- ROI Pool Size
    """

    def __init__(self, n_class, spatial_scale, classifier, roi_size = 7):
        super(VGG16RoIHead, self).__init__()

        self.classifier = classifier
        self.roi_pool = RoIPool( (roi_size, roi_size), spatial_scale)
        self.pred_locs = nn.Linear(4096, n_class * 4)
        self.pred_scores = nn.Linear(4096, n_class)

        init_params(self.pred_locs, 0, 0.001)
        init_params(self.pred_scores, 0, 0.01)

    def forward(self, features, rois, roi_indices):

        """
        Arguments:
            features        :- Image features
            rois            :- A bounding box array containing coordinates of proposal boxes.
            roi_indices     :- An array of images indices for the bounding boxes.
        Returns:-
            roi_pred_locs   :- The predicted locations from the VGG head for each class
            roi_pred_scores :- The predicted scores from the VGG head for each class
        """

        roi_indices = totensor(roi_indices).float()
        rois = totensor(rois).float()
        indices_and_rois = t.cat([roi_indices[:, None], rois], dim=1)
        xy_indices_and_rois = indices_and_rois[:, [0, 2, 1, 4, 3]]
        indices_and_rois =  xy_indices_and_rois.contiguous()

        roi_pool = self.roi_pool(features, indices_and_rois)
        roi_pool = roi_pool.view(roi_pool.size(0), -1)

        classifier_output = self.classifier(roi_pool)

        roi_pred_locs = self.pred_locs(classifier_output)
        roi_pred_scores = self.pred_scores(classifier_output)

        return roi_pred_locs, roi_pred_scores


#!/usr/bin/env python
# -*- coding: utf-8 -*-

" The code for the player, including MainPlayer, MainExploiter, LeagueExploiter"

# modified from AlphaStar pseudo-code

import numpy as np

from alphastarmini.core.ma.pfsp import pfsp

from alphastarmini.core.rl.alphastar_agent import AlphaStarAgent

__author__ = "Ruo-Ze Liu"

debug = False

class Player(object):
    @property
    def learner(self):
        return self._learner

    def set_learner(self, learner):
        self._learner = learner

    @property
    def actors(self):
        return self._actors

    def add_actor(self, actor):
        self._actors.append(actor)

    def get_match(self):
        pass

    def ready_to_checkpoint(self):
        return False

    def _create_checkpoint(self):
        # AlphaStar： return Historical(self, self.payoff)
        return Historical(self.agent, self._payoff)

    @property
    def payoff(self):
        return self._payoff

    @property
    def race(self):
        return self._race

    def checkpoint(self):
        raise NotImplementedError

    def setup(self, obs_spec, action_spec):
        self.agent.setup(obs_spec, action_spec)

    def reset(self):
        self.agent.reset()

class Historical(Player):

    def __init__(self, agent, payoff):
        # AlphaStar： self._agent = Agent(agent.race, agent.get_weights())
        self.agent = AlphaStarAgent(name="Historical", race=agent.race, initial_weights=agent.get_weights())
        self._payoff = payoff
        self._race = agent.race
        self._parent = agent
        self.name = "Historical"
        self._actors = []

    @property
    def parent(self):
        return self._parent

    def get_match(self):
        raise ValueError("Historical players should not request matches")

    def ready_to_checkpoint(self):
        return False

class MainPlayer(Player):

    def __init__(self, race, agent, payoff):
        self.agent = AlphaStarAgent(name="MainPlayer", race=race, initial_weights=agent.get_weights())
        # actually the _payoff maintains all the players and their fight results
        # maybe this should be the league, making it more reasonable
        self._payoff = payoff
        self._race = agent.race
        self._checkpoint_step = 0
        self.name = "MainPlayer"
        self._actors = []

    def _pfsp_branch(self):
        historical = [
            player for player in self._payoff.players
            if isinstance(player, Historical)
        ]
        win_rates = self._payoff[self, historical]
        return np.random.choice(
            historical, p=pfsp(win_rates, weighting="squared")), True

    def _selfplay_branch(self, opponent):
        # Play self-play match
        if self._payoff[self, opponent] > 0.3:
            return opponent, False

        # If opponent is too strong, look for a checkpoint
        # as curriculum
        historical = [
            player for player in self._payoff.players
            if isinstance(player, Historical) and player.parent == opponent
        ]
        win_rates = self._payoff[self, historical]
        return np.random.choice(
            historical, p=pfsp(win_rates, weighting="variance")), True

    def _verification_branch(self, opponent):
        # Check exploitation
        exploiters = set([
            player for player in self._payoff.players
            if isinstance(player, MainExploiter)
        ])
        # Q: What is the player.parent?
        # A: This is only the property of Historical
        exp_historical = [
            player for player in self._payoff.players
            if isinstance(player, Historical) and player.parent in exploiters
        ]
        win_rates = self._payoff[self, exp_historical]
        if len(win_rates) and win_rates.min() < 0.3:
            return np.random.choice(
                exp_historical, p=pfsp(win_rates, weighting="squared")), True

        # Check forgetting
        historical = [
            player for player in self._payoff.players
            if isinstance(player, Historical) and player.parent == opponent
        ]
        win_rates = self._payoff[self, historical]

        def remove_monotonic_suffix(win_rates, players):
            if not win_rates:
                return win_rates, players

            for i in range(len(win_rates) - 1, 0, -1):
                if win_rates[i - 1] < win_rates[i]:
                    return win_rates[:i + 1], players[:i + 1]

            return np.array([]), []

        win_rates, historical = remove_monotonic_suffix(win_rates, historical)
        if len(win_rates) and win_rates.min() < 0.7:
            return np.random.choice(
                historical, p=pfsp(win_rates, weighting="squared")), True

        return None

    def get_match(self):
        coin_toss = np.random.random()

        # Make sure you can beat the League
        if coin_toss < 0.5:
            return self._pfsp_branch()

        main_agents = [
            player for player in self._payoff.players
            if isinstance(player, MainPlayer)
        ]
        opponent = np.random.choice(main_agents)

        # Verify if there are some rare players we omitted
        if coin_toss < 0.5 + 0.15:
            request = self._verification_branch(opponent)
            if request is not None:
                return request

        return self._selfplay_branch(opponent)

    def ready_to_checkpoint(self):
        steps_passed = self.agent.get_steps() - self._checkpoint_step
        if steps_passed < 2e9:
            return False

        historical = [
            player for player in self._payoff.players
            if isinstance(player, Historical)
        ]
        win_rates = self._payoff[self, historical]
        return win_rates.min() > 0.7 or steps_passed > 4e9

    def checkpoint(self):
        self._checkpoint_step = self.agent.get_steps()
        return self._create_checkpoint()

class MainExploiter(Player):

    def __init__(self, race, agent, payoff):
        self.agent = AlphaStarAgent(name="MainExploiter", race=race, initial_weights=agent.get_weights())
        self._initial_weights = agent.get_weights()
        self._payoff = payoff
        self._race = agent.race
        self._checkpoint_step = 0
        self.name = "MainExploiter"
        self._actors = []

    def get_match(self):
        main_agents = [
            player for player in self._payoff.players
            if isinstance(player, MainPlayer)
        ]
        opponent = np.random.choice(main_agents)

        if self._payoff[self, opponent] > 0.1:
            return opponent, True

        historical = [
            player for player in self._payoff.players
            if isinstance(player, Historical) and player.parent == opponent
        ]
        win_rates = self._payoff[self, historical]

        return np.random.choice(
            historical, p=pfsp(win_rates, weighting="variance")), True

    def checkpoint(self):
        self.agent.set_weights(self._initial_weights)
        self._checkpoint_step = self.agent.get_steps()
        return self._create_checkpoint()

    def ready_to_checkpoint(self):
        steps_passed = self.agent.get_steps() - self._checkpoint_step
        if steps_passed < 2e9:
            return False

        main_agents = [
            player for player in self._payoff.players
            if isinstance(player, MainPlayer)
        ]
        win_rates = self._payoff[self, main_agents]
        return win_rates.min() > 0.7 or steps_passed > 4e9

class LeagueExploiter(Player):

    def __init__(self, race, agent, payoff):
        self.agent = AlphaStarAgent(name="LeagueExploiter", race=race, initial_weights=agent.get_weights())
        self._initial_weights = agent.get_weights()
        self._payoff = payoff
        self._race = agent.race
        self._checkpoint_step = 0
        self.name = "LeagueExploiter"
        self._actors = []

    def get_match(self):
        historical = [
            player for player in self._payoff.players
            if isinstance(player, Historical)
        ]
        win_rates = self._payoff[self, historical]
        return np.random.choice(

            historical, p=pfsp(win_rates, weighting="linear_capped")), True

    def checkpoint(self):
        if np.random.random() < 0.25:
            self.agent.set_weights(self._initial_weights)
        self._checkpoint_step = self.agent.get_steps()
        return self._create_checkpoint()

    def ready_to_checkpoint(self):
        steps_passed = self._agent.get_steps() - self._checkpoint_step
        if steps_passed < 2e9:
            return False
        historical = [
            player for player in self._payoff.players
            if isinstance(player, Historical)
        ]
        win_rates = self._payoff[self, historical]
        return win_rates.min() > 0.7 or steps_passed > 4e9


import random
import os
import shutil
import numpy as np

# download shoes from http://www.makehumancommunity.org/clothes/dudoc_domsjeans1.html
# download hair from http://www.makehumancommunity.org/clothes/mhair02.html
# download top from http://www.makehumancommunity.org/clothes/ballet_costume.html
# download bottom from http://www.makehumancommunity.org/clothes/m_trousers_01.html
# put these files into folder named 'models'
# generate_list: index, cloth texture, model folder name, model file name

cloth_pic_Folder = 'cloth_pic/'
person_Folder = 'create_person/'
person_save_Folder = person_Folder + 'deepfashion_pair_models/'
basic_personID = 0

def readCSV2List(readCSV2List_filePath):
    try:
        readCSV2List_file = open(readCSV2List_filePath, 'r', encoding="gbk")
        readCSV2List_context = readCSV2List_file.read()
        readCSV2List_result = readCSV2List_context.split("\n")
        readCSV2List_result_length = len(readCSV2List_result)
        for readCSV2List_i in range(readCSV2List_result_length):
            readCSV2List_result[readCSV2List_i] = readCSV2List_result[readCSV2List_i].split(",")
        return readCSV2List_result
    except Exception:
        print("Load data has some problem")
    finally:
        readCSV2List_file.close();

def generateHuman(cloth_list, person_id, sex):
    haveAcc = 0
    # load acc
    hair = open('modeleTxt/hair.txt', 'r').readlines()
    shoe = open('modeleTxt/shoe.txt', 'r').readlines()
    pifu = open('modeleTxt/skin.txt', 'r').readlines()

    if not os.path.exists(person_save_Folder):
        os.makedirs(person_save_Folder)

    if sex > 0:
        Gender1 = 1000000
    else:
        Gender1 = 0
    #     setting
    Gender = '%.6f' % (Gender1 / 1000000)
    Muscle = '%.6f' % (random.randint(0, 1000000) / 1000000)
    African_1 = random.randint(0, 1000000)
    African = '%.6f' % (African_1 / 1000000)
    Asian_1 = random.randint(0, 1000000 - African_1)
    Asian = '%.6f' % (Asian_1 / 1000000)
    Caucasian = '%.6f' % ((1000000 - Asian_1 - African_1) / 1000000)
    if Gender1 > 1000000 / 2:
        m_height = random.gauss(170, 5.7) / 200
        while m_height > 1:
            m_height = random.gauss(170, 5.7) / 200
        Height = '%.6f' % (m_height)
    else:
        m_height = random.gauss(160, 5.2) / 200
        while m_height > 1:
            m_height = random.gauss(160, 5.2) / 200
        Height = '%.6f' % (m_height)
    BreastSize = '%.6f' % (random.randint(0, 70) / 100)
    Age = '%.6f' % (random.randint(20, 90) / 100)
    BreastFirmness = '%.6f' % (random.randint(30, 100) / 100)
    Weight = '%.6f' % (random.randint(0, 1000000) / 1000000)

    file_name = 'B' + str(person_id)
    # creating person file
    f = open(person_save_Folder + file_name + ".mhm", 'a')
    f.write('# Written by MakeHuman 1.1.1\n')
    f.write('version v1.1.1\n')
    f.write('tags ' + file_name + '\n')
    f.write('camera 0.0 0.0 0.0 0.0 0.0 1.0\n')
    f.write('modifier macrodetails-universal/Muscle ' + Muscle + '\n')
    f.write('modifier macrodetails/African ' + African + '\n')
    f.write('modifier macrodetails-proportions/BodyProportions 0.500000\n')
    f.write('modifier macrodetails/Gender ' + Gender + '\n')
    f.write('modifier macrodetails-height/Height ' + Height + '\n')
    f.write('modifier breast/BreastSize ' + BreastSize + '\n')
    f.write('modifier macrodetails/Age ' + Age + '\n')
    f.write('modifier breast/BreastFirmness ' + BreastFirmness + '\n')
    f.write('modifier macrodetails/Asian ' + Asian + '\n')
    f.write('modifier macrodetails/Caucasian ' + Caucasian + '\n')
    f.write('modifier macrodetails-universal/Weight ' + Weight + '\n')
    f.write('skeleton cmu_mb.mhskel\n')
    f.write('eyes HighPolyEyes 2c12f43b-1303-432c-b7ce-d78346baf2e6\n')

    # adding clothes
    if Gender1 > 1000000 / 2:
        f.write(hair[random.randint(0, len(hair) - 1)])
    else:
        f.write(hair[random.randint(0, len(hair) - 1)])
    f.write(shoe[random.randint(0, len(shoe) - 1)])
    for i in range(0, len(cloth_list)):
        f.write(cloth_list[i]+'\n')
    f.write('clothesHideFaces True\n')
    f.write(pifu[random.randint(0, len(pifu) - 1)])
    f.write('material Braid01 eead6f99-d6c6-4f6b-b6c2-210459d7a62e braid01.mhmat\n')
    f.write('material HighPolyEyes 2c12f43b-1303-432c-b7ce-d78346baf2e6 eyes/materials/brown.mhmat\n')
    f.write('subdivide False\n')

def createClothes(modelFolder,modelName, uuid, saveName, saveFolder, imgName):
    def create_mhclo(modelFolder,modelName, uuid, saveName, saveFolder):
        file = 'models/' + modelFolder + '/' + modelName + '.mhclo'
        savePath = saveFolder + '/' + modelFolder + '/'
        if not os.path.exists(savePath):
            os.makedirs(savePath)
        with open(file, "r", encoding="utf-8") as f1, open(savePath + '' + saveName + ".mhclo", "a",
                                                           encoding="utf-8") as f2:
            for line in f1:
                if 'uuid' in line:
                    line = 'uuid ' + uuid + '\n'
                if 'name' in line:
                    line = 'name ' + str(saveName) + '\n'
                if 'material' in line:
                    line = 'material ' + str(saveName) + '.mhmat' + '\n'
                f2.write(line)
        for filename in os.listdir('models/' + modelFolder + '/'):
            if 'mhclo' not in filename and 'mhmat' not in filename:
                shutil.copy('models/' + modelFolder + '/' + filename, savePath)

    def create_mhmat(modelFolder,modelName, uuid, saveName, saveFolder, imgName):
        file = 'models/' + modelFolder + '/' + modelName + '.mhmat'
        savePath = saveFolder + '/' + modelFolder + '/'
        if not os.path.exists(savePath):
            os.makedirs(savePath)
        with open(file, "r", encoding="utf-8") as f1, open(savePath + saveName + ".mhmat", "a",
                                                           encoding="utf-8") as f2:
            for line in f1:
                if 'name' in line:
                    line = 'name ' + str(saveName) + '\n'
                if 'diffuseTexture' in line:
                    line = 'diffuseTexture ' + imgName + '\n'
                f2.write(line)
        shutil.move(cloth_pic_Folder+imgName, savePath + imgName)

    create_mhclo(modelFolder,modelName, uuid, saveName, saveFolder)
    create_mhmat(modelFolder,modelName, uuid, saveName, saveFolder, imgName)

def create_pair_person():
    clothlist = readCSV2List('generate_list.csv')
    personID = basic_personID
    sex = 0
    for i in range(0, len(clothlist), 2):
        try:
            print(i)
            sex = (sex+1)%2
            pair_list = []
            for j in range (i,i+2):
                uuid = '20210919-0000-0000-0000-' + str(j).zfill(12)
                save_name = 'create_'+str(j).zfill(7)
                save_folder = person_Folder+str(personID)
                if not os.path.exists(save_folder):
                    os.makedirs(save_folder)
                img_name = clothlist[j][1]
                pair_list.append('clothes ' + save_name + ' ' + uuid)
                createClothes(clothlist[j][2],clothlist[j][3], uuid, save_name, save_folder, img_name)
            generateHuman(pair_list, personID, sex)
            personID += 1
        except:
            print(i, personID)
create_pair_person()


# coding: utf-8

import tensorflow as tf
import keras
from keras import applications
import os
from keras.models import load_model,model_from_json
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import inline
import cv2
import dlib
from sklearn.svm import SVC
from sklearn.preprocessing import LabelEncoder
import imutils
from imutils.face_utils import FaceAligner
import time

from telegrambot import*
from facenet import*

alarm_time=0
# Распознавание нескольких лиц в реальном времени с использованием веб-камеры
cam=cv2.VideoCapture(0)
while True:
    ret,img=cam.read()
    temp1=cv2.cvtColor(img,cv2.COLOR_BGR2GRAY)
    faces=detector(img,1)
    x,y,w,h,F=[],[],[],[],[]
    Fimages=[]
    flag=False
    for i,f in enumerate(faces):
        flag=True
        a,b=f.left(),f.top()
        x.append(a)
        y.append(b)
        w.append(f.right()-a)
        h.append(f.bottom()-b)
        F.append(f)

    if flag==True:
        for i in range(len(x)):
            temp=face_aligner.align(temp1,temp1,F[i])
            temp=cv2.resize(temp,(160,160))
            cv2.imwrite(os.path.join(r'temp',r'temp.jpg'),temp)
            temp=cv2.imread(os.path.join(r'temp',r'temp.jpg'))
            os.remove(os.path.join(r'temp',r'temp.jpg'))
            Fimages.append(temp)

        Fimages=Standardize(np.array(Fimages))
        facenetpred=L2_Norm(model.predict(Fimages))
        pred=[]
        dist=[]
        for i in range(len(facenetpred)):
            t=clf.predict(np.reshape(facenetpred[i],(1,len(facenetpred[i]))))
            #print(t)
            tt=clf.predict_proba(np.reshape(facenetpred[i],(1,len(facenetpred[i]))))
            # FMax and SMax первая и вторая высокая возможность прогнозирования
            FMax=np.max(tt)
            #print(FMax)
            SMax=np.max(np.delete(tt,tt.argmax()))
            # Рассчет значений незнакомого лица
            if Euclied_dist(labels_emb[t[0]],facenetpred[i])<0.6 and FMax-SMax>0.3:
                pred.append(le.inverse_transform(t)[0])
            else:
                pred.append('Unknown')
            dist.append(str(FMax))

        c=0
        n='Unknown'
        for i in pred:
            if n in pred[c]:
                cv2.putText(img,i,(x[c]-10,y[c]-10),cv2.FONT_HERSHEY_PLAIN,2,(125,176,209),2)
                cv2.rectangle(img,(x[c],y[c]),(x[c]+w[c],y[c]+h[c]),(255,0,255),2)
                time2=time.time()
                if (time2-alarm_time)>10:
                    #print('Обнаружено неустановленное лицо')
                    alarm_time=time.time()
                    #send_screen(img)
            else:
                cv2.putText(img,i,(x[c]-10,y[c]-40),cv2.FONT_HERSHEY_PLAIN,2,(125,176,209),2)
                cv2.putText(img,'dist: ' + "{0:.2f}".format(float(dist[c])),(x[c]-10,y[c]-10),cv2.FONT_HERSHEY_PLAIN,2,(79,79,196),2)
                cv2.rectangle(img,(x[c],y[c]),(x[c]+w[c],y[c]+h[c]),(255,0,255),2)

            c+=1

    cv2.imshow('Video',img)
    if cv2.waitKey(10) == 27:
        break

cv2.destroyAllWindows()
cam.release()


# Licensed to Modin Development Team under one or more contributor license agreements.
# See the NOTICE file distributed with this work for additional information regarding
# copyright ownership.  The Modin Development Team licenses this file to you under the
# Apache License, Version 2.0 (the "License"); you may not use this file except in
# compliance with the License.  You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under
# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
# ANY KIND, either express or implied. See the License for the specific language
# governing permissions and limitations under the License.

import time
import logging
from typing import Dict, Optional
from multiprocessing import cpu_count

import xgboost as xgb
import ray
from ray.services import get_node_ip_address
import numpy as np
import pandas

from modin.distributed.dataframe.pandas import unwrap_partitions
from .utils import RabitContext, RabitContextManager

LOGGER = logging.getLogger("[modin.xgboost]")

@ray.remote
class ModinXGBoostActor:
    def __init__(self, ip, nthread=cpu_count()):
        self._evals = []
        self._dpredict = []
        self._ip = ip
        self._nthreads = nthread

        LOGGER.info(f"Actor <{self._ip}>, nthread = {self._nthreads} was initialized.")

    def _get_dmatrix(self, X_y):
        s = time.time()
        X = X_y[: len(X_y) // 2]
        y = X_y[len(X_y) // 2 :]

        assert (
            len(X) == len(y) and len(X) > 0
        ), "X and y should have the equal length more than 0"

        X = pandas.concat(X, axis=0)
        y = pandas.concat(y, axis=0)
        LOGGER.info(f"Concat time: {time.time() - s} s")

        return xgb.DMatrix(X, y)

    def set_train_data(self, *X_y, add_as_eval_method=None):
        self._dtrain = self._get_dmatrix(X_y)

        if add_as_eval_method is not None:
            self._evals.append((self._dtrain, add_as_eval_method))

    def set_predict_data(
        self,
        *X,
    ):
        for x in X:
            self._dpredict.append(xgb.DMatrix(x, None))

    def add_eval_data(self, *X_y, eval_method):
        self._evals.append((self._get_dmatrix(X_y), eval_method))

    def train(self, rabit_args, params, *args, **kwargs):
        local_params = params.copy()
        local_dtrain = self._dtrain
        local_evals = self._evals

        local_params["nthread"] = self._nthreads

        evals_result = dict()

        s = time.time()
        with RabitContext(self._ip, rabit_args):
            bst = xgb.train(
                local_params,
                local_dtrain,
                *args,
                evals=local_evals,
                evals_result=evals_result,
                **kwargs,
            )
            LOGGER.info(f"Local training time: {time.time() - s} s")
            return {"booster": bst, "history": evals_result}

    def predict(self, booster: xgb.Booster, *args, **kwargs):
        local_dpredict = self._dpredict
        booster.set_param({"nthread": self._nthreads})

        s = time.time()
        predictions = [booster.predict(X, *args, **kwargs) for X in local_dpredict]
        LOGGER.info(f"Local prediction time: {time.time() - s} s")
        return np.concatenate(predictions)

    def exit_actor(self):
        ray.actor.exit_actor()

def create_actors(num_cpus=1, nthread=cpu_count()):
    num_nodes = len(ray.nodes())

    # Create remote actors
    actors = {
        node_info.split("node:")[-1]: ModinXGBoostActor.options(
            num_cpus=num_cpus, resources={node_info: 1.0}
        ).remote(node_info.split("node:")[-1], nthread=nthread)
        for node_info in ray.cluster_resources()
        if "node" in node_info
    }

    assert num_nodes == len(
        actors
    ), f"Number of nodes {num_nodes} is not equal to number of actors {len(actors)}."

    return actors

def _split_data_across_actors(
    actors: Dict, set_func, X_parts, y_parts=None, evenly_data_distribution=True
):
    """
    Split row partitions of data between actors.

    Parameters
    ----------
    actors : dict
        Dictionary of used actors.
    set_func : callable
        The function for setting data in actor.
    X_parts : list
        Row partitions of X data.
    y_parts : list, default None
        Row partitions of y data.
    evenly_data_distribution : boolean, default True
        Whether make evenly distribution of partitions between nodes or not.
        In case `False` minimal datatransfer between nodes will be provided
        but the data may not be evenly distributed.
    """
    X_parts_by_actors = _assign_row_partitions_to_actors(
        actors, X_parts, evenly_data_distribution=evenly_data_distribution
    )

    if y_parts is not None:
        y_parts_by_actors = _assign_row_partitions_to_actors(
            actors,
            y_parts,
            X_parts_by_actors,
            evenly_data_distribution=evenly_data_distribution,
        )

    for ip, actor in actors.items():
        X_parts = X_parts_by_actors[ip][0]
        if y_parts is None:
            set_func(actor, *X_parts)
        else:
            y_parts = y_parts_by_actors[ip][0]
            set_func(actor, *(X_parts + y_parts))

def _assign_row_partitions_to_actors(
    actors: Dict, row_partitions, data_for_aligning=None, evenly_data_distribution=True
):
    """
    Assign row_partitions to actors.

    Parameters
    ----------
    actors : dict
        Dictionary of used actors.
    row_partitions : list
        Row partitions of data to assign.
    data_for_aligning : dict, default None
        Data according to the order of which should be
        distributed row_partitions. Used to align y with X.
    evenly_data_distribution : boolean, default True
        Whether make evenly distribution of partitions between nodes or not.
        In case `False` minimal datatransfer between nodes will be provided
        but the data may not be evenly distributed.

    Returns
    -------
    dict
        Dictionary of assigned to actors partitions
        as {ip: (partitions, order)}.
    """
    row_partitions_by_actors = {ip: ([], []) for ip in actors}
    if evenly_data_distribution:
        _assign_partitions_evenly(
            actors,
            row_partitions,
            False,
            row_partitions_by_actors,
        )
    else:
        if data_for_aligning is None:
            actors_ips = list(actors.keys())
            partitions_ips = [ray.get(row_part[0]) for row_part in row_partitions]
            unique_partitions_ips = set(partitions_ips)
            empty_actor_ips = []
            for ip in actors_ips:
                if ip not in unique_partitions_ips:
                    empty_actor_ips.append(ip)

            # In case portion of nodes without data is less than 10%,
            # no data redistribution between nodes will be performed.
            if len(empty_actor_ips) / len(actors_ips) < 0.1:
                import warnings

                for ip in empty_actor_ips:
                    actors[ip].exit_actor.remote()
                    actors.pop(ip)
                    row_partitions_by_actors.pop(ip)
                    warnings.warn(
                        f"Node {ip} isn't used as it doesn't contain any data."
                    )
                for i, row_part in enumerate(row_partitions):
                    row_partitions_by_actors[partitions_ips[i]][0].append(row_part[1])
                    row_partitions_by_actors[partitions_ips[i]][1].append(i)
            else:
                _assign_partitions_evenly(
                    actors,
                    row_partitions,
                    True,
                    row_partitions_by_actors,
                )
        else:
            for ip, (_, order_of_indexes) in data_for_aligning.items():
                row_partitions_by_actors[ip][1].extend(order_of_indexes)
                for row_idx in order_of_indexes:
                    row_partitions_by_actors[ip][0].append(row_partitions[row_idx][1])

    return row_partitions_by_actors

def _assign_partitions_evenly(
    actors: Dict,
    row_partitions,
    is_partitions_have_ip,
    row_partitions_by_actors: Dict,
):
    """
    Make evenly assigning of row_partitions to actors.

    Parameters
    ----------
    actors : dict
        Dictionary of used actors.
    row_partitions : list
        Row partitions of data to assign.
    is_partitions_have_ip : boolean
        Whether each value of row_partitions is (ip, partition).
    row_partitions_by_actors : dict
        Dictionary of assigned to actors partitions
        as {ip: (partitions, order)}. Output parameter.
    """
    num_actors = len(actors)
    row_parts_last_idx = (
        len(row_partitions) // num_actors
        if len(row_partitions) % num_actors == 0
        else len(row_partitions) // num_actors + 1
    )

    start_idx = 0
    for ip, actor in actors.items():
        if is_partitions_have_ip:
            last_idx = (
                (start_idx + row_parts_last_idx)
                if (start_idx + row_parts_last_idx < len(row_partitions))
                else len(row_partitions)
            )
            row_partitions_by_actors[ip][1].extend(list(range(start_idx, last_idx)))
            for idx in range(start_idx, last_idx):
                row_partitions_by_actors[ip][0].append(row_partitions[idx][1])
        else:
            idx_slice = (
                slice(start_idx, start_idx + row_parts_last_idx)
                if start_idx + row_parts_last_idx < len(row_partitions)
                else slice(start_idx, len(row_partitions))
            )
            row_partitions_by_actors[ip][0].extend(row_partitions[idx_slice])
        start_idx += row_parts_last_idx

def _train(
    dtrain,
    nthread,
    evenly_data_distribution,
    params: Dict,
    *args,
    evals=(),
    **kwargs,
):
    s = time.time()

    X, y = dtrain
    assert len(X) == len(y)

    X_row_parts = unwrap_partitions(X, axis=0, bind_ip=not evenly_data_distribution)
    y_row_parts = unwrap_partitions(y, axis=0, bind_ip=not evenly_data_distribution)
    assert len(X_row_parts) == len(y_row_parts), "Unaligned train data"

    # Create remote actors
    actors = create_actors(nthread=nthread)

    add_as_eval_method = None
    if evals:
        for (eval_data, method) in evals[:]:
            if eval_data is dtrain:
                add_as_eval_method = method
                evals.remove((eval_data, method))

        for ((eval_X, eval_y), eval_method) in evals:
            # Split data across workers
            _split_data_across_actors(
                actors,
                lambda actor, *X_y: actor.add_eval_data.remote(
                    *X_y, eval_method=eval_method
                ),
                unwrap_partitions(eval_X, axis=0, bind_ip=not evenly_data_distribution),
                unwrap_partitions(eval_y, axis=0, bind_ip=not evenly_data_distribution),
                evenly_data_distribution=evenly_data_distribution,
            )

    # Split data across workers
    _split_data_across_actors(
        actors,
        lambda actor, *X_y: actor.set_train_data.remote(
            *X_y, add_as_eval_method=add_as_eval_method
        ),
        X_row_parts,
        y_row_parts,
        evenly_data_distribution=evenly_data_distribution,
    )
    LOGGER.info(f"Data preparation time: {time.time() - s} s")

    s = time.time()
    with RabitContextManager(len(actors), get_node_ip_address()) as env:
        rabit_args = [("%s=%s" % item).encode() for item in env.items()]

        # Train
        fut = [
            actor.train.remote(rabit_args, params, *args, **kwargs)
            for _, actor in actors.items()
        ]

        # All results should be the same because of Rabit tracking. So we just
        # return the first one.
        result = ray.get(fut[0])
        LOGGER.info(f"Training time: {time.time() - s} s")
        return result

def _predict(
    booster,
    data,
    nthread: Optional[int] = cpu_count(),
    evenly_data_distribution: Optional[bool] = True,
    **kwargs,
):
    s = time.time()

    X, _ = data
    X_row_parts = unwrap_partitions(X, axis=0, bind_ip=not evenly_data_distribution)

    # Create remote actors
    actors = create_actors(nthread=nthread)

    # Split data across workers
    _split_data_across_actors(
        actors,
        lambda actor, *X: actor.set_predict_data.remote(*X),
        X_row_parts,
        evenly_data_distribution=evenly_data_distribution,
    )

    LOGGER.info(f"Data preparation time: {time.time() - s} s")
    s = time.time()

    # Predict
    predictions = [
        actor.predict.remote(booster, **kwargs) for _, actor in actors.items()
    ]
    result = ray.get(predictions)
    LOGGER.info(f"Prediction time: {time.time() - s} s")

    return np.concatenate(result)


"""
Use this script to create JSON-Line description files that can be used to
train deep-speech models through this library.
This works with data directories that are organized like LibriSpeech:
data_directory/group/speaker/[file_id1.wav, file_id2.wav, ...,
                              speaker.trans.txt]

Where speaker.trans.txt has in each line, file_id transcription
"""

from __future__ import absolute_import, division, print_function

import argparse
import json
import os
import wave
from glob import glob
import pickle
from tqdm import tqdm
import numpy as np

def main(data_directory, ref_directory, output_file):
    refs = []
    durations = []
    keys = []
    noisetypes = []
    snrs = []
    for condition in tqdm(os.listdir(data_directory)):
        condition_path = os.path.join(data_directory, condition)
        noise_type, snr = condition.split('_')
        snr = float(snr[:-2])
        for f in glob(os.path.join(condition_path, '*.wav')):
            file_id = os.path.basename(f)
            reference = os.path.join(ref_directory, file_id)

            audio = wave.open(f)
            duration = float(audio.getnframes()) / audio.getframerate()
            audio.close()

            keys.append(f)
            durations.append(duration)
            refs.append(reference)
            snrs.append(snr)
            noisetypes.append(noise_type)

    with open(output_file, 'w') as out_file:
        for i in range(len(keys)):
            line = json.dumps({'key': keys[i],
                'duration': durations[i],
                'ref': refs[i],
                'snr': snrs[i],
                'noisetype': noisetypes[i]})
            out_file.write(line + '\n')

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('data_directory', type=str,
                        help='Path to data directory')
    parser.add_argument('ref_directory', type=str,
                        help='Path to ref directory')
    parser.add_argument('output_file', type=str,
                        help='Path to output file')
    args = parser.parse_args()

    main(args.data_directory, args.ref_directory, args.output_file)


from collections import namedtuple, defaultdict
import cProfile
import logging
import os.path
import os
from multiprocessing import Process
import pickle
import sys
import math
import re

import numpy as np

from pbcore.io import GffWriter, Gff3Record

from kineticsTools.pipelineTools import consumer

DEFAULT_NCHUNKS = 256
# Labels for modified fraction:
FRAC = 'frac'
FRAClow = 'fracLow'
FRACup = 'fracUp'

log = logging.getLogger(__name__)

class ResultCollectorProcess(Process):

    """
    Gathers results and writes to a file.
    """

    def __init__(self, options, resultsQueue):
        Process.__init__(self)
        self.daemon = True
        self.options = options
        self._resultsQueue = resultsQueue

    def _run(self):
        log.info("Process %s (PID=%d) started running" % (self.name, self.pid))

        self.onStart()

        nextChunkId = 0
        chunkCache = {}

        sentinelsReceived = 0
        while sentinelsReceived < self.options.numWorkers:
            result = self._resultsQueue.get()
            self._resultsQueue.task_done()

            if result is None:
                sentinelsReceived += 1
            else:
                # Write out chunks in chunkId order.
                # Buffer received chunks until they can be written in order
                (chunkId, datum) = result
                chunkCache[chunkId] = datum

                # The rawData field is large and unused. Delete it to mitigate
                # risk of OOM problems
                for column in datum:
                    if 'rawData' in column:
                        del column['rawData']

                # Write out all the chunks that we can
                while nextChunkId in chunkCache:
                    nextChunk = chunkCache.pop(nextChunkId)
                    self.onResult(nextChunk)

                    nextChunkId += 1

        log.info("Result thread shutting down...")
        self.onFinish()

    def run(self):

        if self.options.doProfiling:
            cProfile.runctx("self._run()",
                            globals=globals(),
                            locals=locals(),
                            filename="profile-%s.out" % self.name)
        else:
            self._run()

    # ==================================
    # Overridable interface begins here.
    #
    def onStart(self):
        pass

    def onResult(self, result):
        pass

    def onFinish(self):
        pass

class KineticsWriter(ResultCollectorProcess):

    def __init__(self, options, resultQueue, refInfo, ipdModel):
        ResultCollectorProcess.__init__(self, options, resultQueue)

        self.refInfo = refInfo
        self.ipdModel = ipdModel

    @consumer
    def msCsvConsumer(self, filename):
        """
        Consume IPD summary rows and write them to csv
        """

        # Open the csv file
        f = self.openWriteHandle(filename)
        delim = ","

        cols = ["refName", "tpl", "strand", "base", "score", "tMean", "tErr",
                "modelPrediction", "ipdRatio", "coverage", "signal", "variance", "MSscore"]

        # Special cases for formatting columns of the csv
        handlers = dict()
        def threeF(x): return "%.3f" % x

        handlers["refName"] = lambda x: "\"%s\"" % x

        handlers["tpl"] = lambda x: str(x.item() + 1)
        handlers["score"] = lambda x: "%d" % x

        handlers["tMean"] = threeF
        handlers["modelPrediction"] = threeF
        handlers["caseMean"] = threeF
        handlers["controlMean"] = threeF
        handlers["ipdRatio"] = threeF
        handlers["pvalue"] = lambda x: "%.3e" % x

        handlers["controlStd"] = threeF
        handlers["controlStd"] = threeF
        handlers["tErr"] = threeF

        def fourF(x): return "%.4f" % x
        handlers["signal"] = fourF
        handlers["variance"] = fourF
        handlers["MSscore"] = lambda x: "%d" % x

        print(delim.join(cols), file=f)

        def fmt(rowData, colName):
            if colName not in rowData:
                return ""

            if colName in handlers:
                return handlers[colName](rowData[colName])
            else:
                return str(rowData[colName])

        try:
            while True:
                # Pull a list of record in from the producer
                itemList = (yield)

                for item in itemList:
                    if "signal" in item:
                        values = [fmt(item, col) for col in cols]
                        print(delim.join(values), file=f)

        except GeneratorExit:
            f.close()
            return
        except Exception as e:
            print(e)

    @consumer
    def csvConsumer(self, filename):
        """
        Consume IPD summary rows and write them to csv
        """

        # Open the csv file
        f = self.openWriteHandle(filename)
        delim = ","

        if self.options.control is None:

            # Columns for in-silico control
            if self.options.methylFraction:
                cols = ["refName", "tpl", "strand", "base", "score", "tMean", "tErr",
                        "modelPrediction", "ipdRatio", "coverage", FRAC, FRAClow, FRACup]
            else:
                if self.options.useLDA:
                    # FIXME: For testing LDA model, to look at LDA scores in
                    # csv output (run without --methylFraction or --control):
                    cols = ["refName", "tpl", "strand", "base", "score", "tMean",
                            "tErr", "modelPrediction", "ipdRatio", "coverage", "Ca5C"]
                else:
                    cols = ["refName", "tpl", "strand", "base", "score",
                            "tMean", "tErr", "modelPrediction", "ipdRatio", "coverage"]

        else:
            # Columns for case-control
            if self.options.methylFraction:
                cols = ["refName", "tpl", "strand", "base", "score", "pvalue", "caseMean", "controlMean", "caseStd",
                        "controlStd", "ipdRatio", "testStatistic", "coverage", "controlCoverage", "caseCoverage", FRAC, FRAClow, FRACup]
            else:
                cols = ["refName", "tpl", "strand", "base", "score", "pvalue", "caseMean", "controlMean", "caseStd",
                        "controlStd", "ipdRatio", "testStatistic", "coverage", "controlCoverage", "caseCoverage"]

        # Special cases for formatting columns of the csv
        handlers = dict()
        def threeF(x): return "%.3f" % x

        handlers["refName"] = lambda x: "\"%s\"" % x

        handlers["tpl"] = lambda x: str(x.item() + 1)
        handlers["score"] = lambda x: "%d" % x

        handlers["tMean"] = threeF
        handlers["modelPrediction"] = threeF
        handlers["caseMean"] = threeF
        handlers["controlMean"] = threeF
        handlers["ipdRatio"] = threeF
        handlers["pvalue"] = lambda x: "%.3e" % x

        handlers["controlStd"] = threeF
        handlers["controlStd"] = threeF
        handlers["tErr"] = threeF

        # FIXME: remove this line later:
        handlers["Ca5C"] = threeF

        handlers[FRAC] = threeF
        handlers[FRAClow] = threeF
        handlers[FRACup] = threeF

        print(delim.join(cols), file=f)

        def fmt(rowData, colName):
            if colName not in rowData:
                return ""

            if colName in handlers:
                return handlers[colName](rowData[colName])
            else:
                return str(rowData[colName])

        try:
            while True:
                # Pull a list of record in from the producer
                itemList = (yield)

                for item in itemList:
                    values = [fmt(item, col) for col in cols]
                    print(delim.join(values), file=f)

        except GeneratorExit:
            f.close()
            return
        except Exception as e:
            print(e)

    @consumer
    def bigWigConsumer(self, filename):
        import pyBigWig
        records = []
        records_by_pos = defaultdict(list)
        ranges = {}
        BaseInfo = namedtuple("BaseInfo", ("seqid", "pos", "sense", "ipd"))
        try:
            while True:
                chunk = (yield)
                if len(chunk) == 0:
                    continue
                # Fill out the ipd observations into the dataset
                for x in chunk:
                    pos = int(x['tpl']) + 1
                    seqid = x['refName']
                    ranges.setdefault(seqid, (sys.maxsize, 0))
                    ranges[seqid] = (min(ranges[seqid][0], pos),
                                     max(ranges[seqid][1], pos + 1))
                    rec = BaseInfo(
                        seqid=seqid,
                        pos=pos,
                        sense=int(x['strand']),
                        ipd=float(x['ipdRatio']))
                    records.append(rec)
                    records_by_pos[(rec.seqid, rec.pos)].append(rec)
        except GeneratorExit:
            records.sort(key=lambda x: x.pos)
            records.sort(key=lambda x: x.seqid)
            regions = [(s, ranges[s][1] - 1) for s in sorted(ranges.keys())]
            if len(regions) == 0:
                with open(filename, "wb") as _:
                    return
            bw = pyBigWig.open(filename, "w")
            bw.addHeader(regions)
            k = 0
            seqids = []
            starts = []
            ends = []
            ipd_enc = []
            # records are not necessarily consecutive or two per base!
            have_pos = set()

            def encode_ipds(plus, minus):
                def enc(x): return min(65535, int(round(100 * x)))
                return float(enc(minus) + 65536 * enc(plus))
            for rec in records:
                if (rec.seqid, rec.pos) in have_pos:
                    continue
                have_pos.add((rec.seqid, rec.pos))
                strand_records = records_by_pos[(rec.seqid, rec.pos)]
                if len(strand_records) == 2:
                    rec_minus = strand_records[k] if strand_records[k].sense else strand_records[k + 1]
                    rec_plus = strand_records[k +
                                              1] if strand_records[k].sense else strand_records[k]
                    assert rec_plus.pos == rec_minus.pos, (rec_plus, rec_minus)
                    seqids.append(rec_plus.seqid)
                    starts.append(rec_plus.pos - 1)
                    ends.append(rec_plus.pos)
                    ipd_enc.append(encode_ipds(rec_plus.ipd, rec_minus.ipd))
                else:
                    seqids.append(rec.seqid)
                    starts.append(rec.pos - 1)
                    ends.append(rec.pos)
                    if rec.sense == 0:
                        ipd_enc.append(encode_ipds(rec.ipd, 0))
                    else:
                        ipd_enc.append(encode_ipds(0, rec.ipd))
            log.info("Writing records for {n} bases".format(n=len(seqids)))
            bw.addEntries(seqids, starts, ends=ends, values=ipd_enc)
            bw.close()
            return

    def openWriteHandle(self, filename):
        if filename[-2:] == 'gz':
            import gzip
            fileobj = gzip.GzipFile(filename, mode="w", compresslevel=3)
        else:
            fileobj = open(filename, "w", 2 << 15)

        return fileobj

    @consumer
    def pickleConsumer(self, fileName):
        """
        Consume IPD summary rows and pickle to a 'None' terminated stream
        """

        f = open(fileName, "w")
        pickleStream = pickle.Pickler(f)

        try:
            while True:
                # Pickle a record
                n = (yield)
                pickleStream.dump(n)
                pickleStream.clear_memo()

        except GeneratorExit:
            # Write an end sentinel to the pickle stream
            pickleStream.dump(None)
            f.close()
            return

    def makeGffRecord(self, siteObs):
        """
        Convert the internal site observation object into a GFF entry
        """
        # Some useful attributes about the observation
        # - cognate base
        # - context snippet
        # - ipd ratio
        # - coverage
        snippet = self.snippetFunc(siteObs['tpl'], siteObs['strand'])
        attributes = [('coverage', siteObs['coverage']),
                      ('context', snippet),
                      ('IPDRatio', siteObs['ipdRatio'])]

        # Base of detected mod -- single position, closed,open
        # interval.
        # Note -- internally the tool uses 0-based reference
        # coordinates, however in gff the template indices are
        # 1-based.  Make that adjustment here.
        # On start vs. end: My reading of the gff spec
        # (http://www.sequenceontology.org/resources/gff3.html) says
        # to me that 1-base long feature (e.g. a modified base) should
        # have start + 1 == end, and 0-base long features
        # (e.g. insertions) should have start == end. This is not the
        # convention that Marco has apdopted in SMRTView, or the
        # convention that EviCons originally used.  We will adopt
        # their convention here, for now.
        start = siteObs['tpl'] + 1
        end = siteObs['tpl'] + 1

        if 'motif' in siteObs:
            attributes.append(('motif', "%s" % siteObs['motif']))

        if 'id' in siteObs:
            attributes.append(('id', "%s" % siteObs['id']))

        if self.options.methylFraction and FRAC in siteObs:
            attributes.append(('frac', "%.3f" % siteObs[FRAC]))
            attributes.append(('fracLow', "%.3f" % siteObs[FRAClow]))
            attributes.append(('fracUp', "%.3f" % siteObs[FRACup]))

        if 'modificationScore' in siteObs:
            # Report the QV from the modification identification module as a
            # special tag
            attributes.append(('identificationQv', "%d" %
                               int(round(siteObs['modificationScore']))))

        if 'modification' in siteObs:

            if siteObs['modification'] == '.':
                recordType = 'modified_base'

            elif siteObs['modification'] == 'nMd':
                recordType = '.'

            else:
                # if we have an identified mod, use it; otherwise use the old
                # generic term
                recordType = siteObs['modification']

        else:
            recordType = 'modified_base'

        refName = siteObs['refName']
        score = int(round(siteObs['score']))
        strand = '+' if siteObs['strand'] == 0 else '-'

        return Gff3Record(refName, start, end,
                          type=recordType,
                          score=score,
                          strand=strand,
                          source='kinModCall',
                          attributes=attributes)

    def makeM5CgffRecord(self, siteObs):

        start = siteObs['tpl'] + 1
        end = siteObs['tpl'] + 1

        attributes = [('coverage', siteObs['coverage']),
                      ('IPDRatio', siteObs['ipdRatio'])]

        recordType = 'CG'
        refName = siteObs['refId']
        score = "%.3f" % siteObs['Ca5C']
        strand = '+' if siteObs['strand'] == 0 else '-'

        return Gff3Record(refName, start, end,
                          type=recordType,
                          score=score,
                          strand=strand,
                          source='kinModCall', attributes=attributes)

    @consumer
    def gffConsumer(self, filename):
        """
        Consume IPD summary rows, filter them and write to GFF
        """

        #f = file(filename, 'w', 2<<15)
        f = self.openWriteHandle(filename)
        gff = GffWriter(f)

        # write headers describing the program that generated the data
        gff.writeHeader('##source ipdSummary v2.0')
        gff.writeHeader('##source-commandline %s' % self.options.cmdLine)

        # Write the reference renaming info into the gff headers ala evicons
        for entry in self.refInfo:
            gff.writeHeader("##sequence-region %s 1 %d"
                            % (entry.Name, entry.Length))

        minScore = -10 * math.log10(self.options.pvalue)
        snippetRef = -1
        try:
            while True:
                # Pull a record in from the
                siteObsList = (yield)

                for siteObs in siteObsList:
                    true_modif = False
                    # self.snippetFunc is a function that return a reference snippet given a template position and a strand
                    if snippetRef != siteObs['refId']:
                        self.snippetFunc = self.ipdModel.snippetFunc(
                            siteObs['refId'], 20, 20)
                        snippetRef = siteObs['refId']

                    # Two cases for gff entries:
                    # 1. 'Identified modification' - will have a 'modification' key
                    #     - use the modification name as the gff event type
                    #     - use 'modificationScore' for the gff score
                    # 2. Detected - no 'modification' key
                    #     - use 'modified_base' as the event type
                    #     - use the single site 'score' property as the gff score
                    #     - do not put this kind into the gff if it contains the a 'offTargetPeak' tag

                    gffline = self.makeGffRecord(siteObs)
                    attributes = str(gffline).split()[8].split(';')

                    real_modif = str(gffline).split()[2]
                    score_modif = str(gffline).split()[5]
                    siteObs['type_modif'] = real_modif
                    siteObs['score_modif'] = score_modif

                    for attribute in attributes:
                        key = attribute.split('=')[0]
                        value = attribute.split('=')[1]
                        siteObs[str(key)] = value

                    if siteObs['coverage'] > self.options.minCoverage:

                        # Case 1
                        if siteObs.has_key('modification') and siteObs['modification'] != '.':
                            gff.writeRecord(gffline)
                            true_modif = True
                        # Case 2
                        elif siteObs['score'] > minScore and not siteObs.has_key('offTargetPeak'):
                            gff.writeRecord(gffline)
                            true_modif = True
                        else:
                            true_modif = False
                    if true_modif:
                        siteObs['modified'] = True
                    else:
                        siteObs['modified'] = False
                        siteObs['type_modif'] = 'None'

                    # Will be printed in sys.stdout for every position (then the pt_methyl or whatsoever will chose it it wants to keep anything)

                    current_list = []

                    # exec("siteObs = "+str(siteObs))

                    for key in siteObs:
                        current_list.append('{}:'.format(str(key)))
                        if key != 'capped_values' and key != 'rawData':
                            for elt in str(siteObs[str(key)]):
                                if elt != '\n':
                                    current_list.append("{}".format(str(elt)))
                        else:
                            current_list.append('None')
                        current_list.append(';')
                    # remove the last ';'
                    current_list.pop(-1)
                    # Adding the \n
                    current_list.append('\n')
                    output = ''.join(current_list)
                    sys.stdout.write(output)
        except GeneratorExit:
            f.close()
            return

    @consumer
    def m5CgffConsumer(self, filename):
        f = self.openWriteHandle(filename)
        gff = GffWriter(f)

        # write headers describing the program that generated the data
        gff.writeHeader('##source ipdSummary v2.0')
        gff.writeHeader('##source-commandline %s' % self.options.cmdLine)

        # Write the reference renaming info into the gff headers ala evicons
        # for entry in self.refInfo:
        #     gff.writeHeader("##sequence-region %s 1 %d"
        #                     % (entry.Name, entry.Length))

        try:
            while True:
                # Pull in a single record?
                siteObsList = (yield)

                for siteObs in siteObsList:
                    if 'Ca5C' in siteObs and siteObs['strand'] == 0:
                        gff.writeRecord(self.makeM5CgffRecord(siteObs))

        except GeneratorExit:
            f.close()
            return

    def onStart(self):

        # Spec for what kinds of output files we can generate.
        # Entry format is (<option field name>, <extension>, <writer consumer
        # function>)
        fileSpec = [
            ('m5Cgff', 'm5C.gff', self.m5CgffConsumer),
            ('gff', 'gff', self.gffConsumer),
            ('csv', 'csv', self.csvConsumer),
            ('bigwig', 'bw', self.bigWigConsumer),
            ('ms_csv', 'ms.csv', self.msCsvConsumer),
            ('pickle', 'pickle', self.csvConsumer),
        ]

        sinkList = []

        # Go through the possible output file types and
        # determine if they should be output
        for (fileType, ext, func) in fileSpec:
            name = None

            # The 'outfile argument causes all outputs to be generated
            if self.options.outfile:
                if ext == "bw":
                    try:
                        import pyBigWig
                    except ImportError:
                        pass
                    else:
                        name = self.options.outfile + '.' + ext
                else:
                    name = self.options.outfile + '.' + ext

            # Individual outputs can specified - these filename override the
            # default
            if self.options.__getattribute__(fileType):
                name = self.options.__getattribute__(fileType)

            if name:
                sinkList.append(func(name))

        self.sinkList = sinkList

    def onResult(self, resultChunk):
        for sink in self.sinkList:
            sink.send(resultChunk)

    def onFinish(self):
        for sink in self.sinkList:
            sink.close()


import numpy as np
from PIL import ImageGrab
import pyautogui
import time

time.sleep(2)
pyautogui.PAUSE = 0
kecepatan = 0
pyautogui.press('space')

while True:
    img = np.array(ImageGrab.grab(bbox=(484, 207, 530 + kecepatan, 215)))
    if np.unique(img).size > 1:
        pyautogui.keyDown('space')
    if kecepatan < 170:
        kecepatan += 0.084