import sys
import cv2
import functools
from functools import reduce
import numpy as np
import tensorflow as tf
import tensorflow.contrib as tc
from baselines import logger
from baselines.common.mpi_adam import MpiAdam
import baselines.common.tf_util as U
from baselines.common.tf_util import save_variables, load_variables
from tensorflow.keras.applications.resnet50 import ResNet50
from tensorflow.keras.preprocessing import image
from tensorflow.keras.applications.resnet50 import preprocess_input, decode_predictions

class IMIT(object):

    def __init__(self, *, actor, memory, observation_shape, action_shape, batch_size=128,
                 action_range=(-1., 1.), actor_l2_reg=0., actor_lr=1e-4, clip_norm=None,
                 use_keras=False):
        """IMIT.

        For images, cast to a new variable which gets cast to the float.  Then we let *that*
        (obs0_f_imgs) be input to subsequent code. NOTE THE DIVISION BY 255 HERE!!

        Alas, things do change if we are using keras or not ... because then we
        just get a feature at the end.
        """
        self.use_keras    = use_keras
        self.actions      = tf.placeholder(tf.float32, shape=(None,) + action_shape, name='actions')
        self.flag_teacher = tf.placeholder(tf.float32, shape=(None, 1), name='flag_teacher')

        # Set up the pre-trained model.
        if self.use_keras:
            raise NotImplementedError()
            #self.pretrained_model = ResNet50(weights='imagenet', include_top=False)
            # The model predictions before last layer result in a (?,7,7,2048)-sized tensor.
            #self.obs0_f_imgs = tf.placeholder(tf.float32, shape=(None,7,7,2048), name='obs0_f_imgs')
        else:
            self.obs0 = tf.placeholder(tf.int32, shape=(None,) + observation_shape, name='obs0_imgs')
            self.obs0_f_imgs = tf.cast(self.obs0, tf.float32) / 255.0

        # Parameters.
        self.memory = memory
        self.obs_shape = observation_shape
        self.action_range = action_range
        self.actor = actor
        self.actor_lr = actor_lr
        self.actor_l2_reg = actor_l2_reg
        self.clip_norm = clip_norm
        self.batch_size = batch_size

        # Create actor network, with observations as input.
        self.actor_tf = actor(self.obs0_f_imgs)
        self.setup_actor_optimizer()

    def setup_actor_optimizer(self):
        """Make actor loss, grads, and optimizer.

        For this it's pretty simple: a behavior cloning loss, using actor_tf
        for the current actor's output given the state, and actions as
        placeholder for what was sampled from the buffer.

        Apply on all samples, no filtering for teachers as in DDPG, which means
        we simply divide by the batch size. Or just do tf.reduce_mean, but be
        sure to do row-wise first (to get error  per sample in the batch) and
        then take the mean (to get average in the BATCH)!

        We apply L2 reg if desired (following DeepMind's DDPGfD). Careful
        w/variable names if we switch network construction code!!
        """
        logger.info('\nSetting up actor optimizer, with behavior cloning')
        self._diff_m = self.actor_tf - self.actions
        self._diff_v = tf.reduce_mean(tf.square(self._diff_m), axis=1, keepdims=True)
        self.bc_loss = tf.reduce_mean(self._diff_v)
        self.actor_loss = self.bc_loss
        logger.info('  diff_matrix: {}'.format(self._diff_m))
        logger.info('  diff_vector: {}'.format(self._diff_v))

        if self.actor_l2_reg > 0.:
            actor_reg_vars = [var for var in self.actor.trainable_vars if
                    ((var.name.endswith('/w:0') or var.name.endswith('/kernel:0')) and 'output' not in var.name)]
            for var in actor_reg_vars:
                logger.info('  regularizing: {}'.format(var.name))
            logger.info('  applying l2 regularization with {}'.format(self.actor_l2_reg))
            self.actor_reg = tc.layers.apply_regularization(
                tc.layers.l2_regularizer(self.actor_l2_reg),
                weights_list=actor_reg_vars
            )
            self.actor_loss += self.actor_reg

        actor_shapes = [var.get_shape().as_list() for var in self.actor.trainable_vars]
        actor_nb_params = sum([reduce(lambda x, y: x * y, shape) for shape in actor_shapes])
        logger.info('  actor shapes: {}'.format(actor_shapes))
        logger.info('  actor params: {}\n'.format(actor_nb_params))
        self.actor_grads = U.flatgrad(self.actor_loss, self.actor.trainable_vars, clip_norm=self.clip_norm)
        self.actor_optimizer = MpiAdam(var_list=self.actor.trainable_vars, beta1=0.9, beta2=0.999, epsilon=1e-08)

    def store_transition(self, obs0, action, is_teacher=False):
        """Store transitions for IMIT (only need obs0, action).

        Daniel: collected via VecEnv, so iterate through batch size and append
        individual components. It's serial but shouldn't be a time bottleneck.
        """
        B = obs0.shape[0]
        for b in range(B):
            obs = obs0[b]
            if not self.use_keras:
                obs = cv2.resize(obs, (self.obs_shape[0], self.obs_shape[1]))
            self.memory.append(obs, action[b], is_teacher=is_teacher)

    def train(self, return_l2=False):
        """Sample a batch, run tensorflow, update actor parameters.
        """
        batch = self.memory.sample(batch_size=self.batch_size)

        # Daniel: I have more items than needed, for debugging purposes.
        ops = [self.actor_grads, self.actor_loss, self.actor_reg, self.flag_teacher,
                self._diff_v, self.actor_tf, self.actions]

        # Form the feed-dict.
        feed = {}
        if self.use_keras:
            raise NotImplemetedError()
            # TODO: does not work! Says graph has been finalized.
            #x = batch['obs0']
            #x = preprocess_input(x)
            #features = self.pretrained_model.predict(x)
            #feed[self.obs0_f_imgs] = features
        else:
            feed[self.obs0] = batch['obs0']
        feed[self.actions] = batch['actions']
        feed[self.flag_teacher] =  batch['flag_teacher']

        actor_grads, actor_loss, actor_l2_loss, flag_t, diff_v, act_tf, act_ph = \
                self.sess.run(ops, feed_dict=feed)
        assert act_tf.shape == act_ph.shape and len(act_tf.shape) == 2
        assert diff_v.shape == flag_t.shape and len(diff_v.shape) == 2
        #print('\nact_tf {}:\n{}'.format(act_tf.shape, act_tf.T))
        #print('\nact_ph {}:\n{}'.format(act_ph.shape, act_ph.T))
        #print('\ndiff_v {}:\n{}'.format(diff_v.shape, diff_v.T))
        #print('\nflag_t {}:\n{}'.format(flag_t.shape, flag_t.T))

        self.actor_optimizer.update(actor_grads, stepsize=self.actor_lr)
        if return_l2:
            return actor_loss, actor_l2_loss
        else:
            return actor_loss

    def step(self, obs, apply_noise=False):
        """Apply the policy, no noise added.

        Returns a 4-tuple, only for compatibility with other code. We just care
        about returning the action as the first argument.

        Leaving apply_noise for compatibility with `baselines/run.py`.
        """
        if not self.use_keras:
            # obs is from env, which is (B,224,224,3), but for calling our
            # model, we want it (B,x,x,3) where x is our chosen smaller dimension
            obs_new = []
            for b in range(obs.shape[0]):
                resized = cv2.resize(obs[b], (self.obs_shape[0], self.obs_shape[1]))
                obs_new.append(resized)
            obs_new = np.array(obs_new)
            assert obs_new.shape == (obs.shape[0], self.obs_shape[0], self.obs_shape[1], 4)
            obs = obs_new

        actor_tf = self.actor_tf
        feed_dict = {self.obs0: U.adjust_shape(self.obs0, [obs])}
        action = self.sess.run(actor_tf, feed_dict=feed_dict)
        action = np.clip(action, self.action_range[0], self.action_range[1])
        return action, None, None, None

    def initialize(self, sess):
        # Daniel: following PPO2 code outline, hoping to save/load models.
        self.save = functools.partial(save_variables, sess=sess)
        self.load = functools.partial(load_variables, sess=sess)
        # Daniel: back to normal.
        self.sess = sess
        self.sess.run(tf.global_variables_initializer())
        self.actor_optimizer.sync()


import numpy as np
import pandas as pd
import sklearn
import joblib

sample_text = "Latest industry research report on the Photography Studio Software Market report is an investigative study executed by experts on the basis of global market, which studies the aggressive structure of the global industry all over the globe. Constructed by the practice of proficient systematic tools such SWOT analysis, the global market report offers a complete judgment of global market. Download free sample report @ https://www.fiormarkets.com/report-detail/146313/request-sample The evaluation for CAGR (Compound Annual Growth Rate) is entirely provided by the Photography Studio Software Market report in terms of percentage for accurate period. This will assist users to make beyond question choice-based decisions on predicted chart. The report also wraps up leading and major players in the global market. Income (US$) and volume of the production are the two main units on which the global market size is calculated by the experts in this report. Intense analysis of key fragments of the market as well as the geological division all over the world is also carried out. Multiple properties of the global market such as growth drivers, limitations, and the upcoming aspects of every section have been communicated profoundly. On the basis of these characteristics, the Photography Studio Software Market report decides the standing future of the market globally. This report wraps each and every characteristics of the global market commencing from the fundamental information of the market and moving further to different vital criteria, on the basis of which, the global market is fragmented. Main application areas of the global market are also covered based on their performance. Browse Full Report With TOC @ https://www.fiormarkets.com/report/global-photography-studio-software-market-size-status-and-146313.html The global market report wraps a nearest analysis of current rules, policies, and regulations as well as global industrial chain. Apart from this, other factors such as chain of production, goods, key producers, supply and demand for these goods, and revenue as well as price structures for global market are also wrapped in this report. The report also enumerates the properties of demand and supply, manufacture capacity, the chronological presentation, and detail analysis of the global market all over the world. For more inquiry contact our sales team at sales@fiormarkets.com"

def predict(text):

    filename_model = '/var/www/app/nb_model.joblib.pkl'
    filename_tfidf = '/var/www/app/tfidf.joblib.pkl'
    model = joblib.load(filename_model)
    tfidf = joblib.load(filename_tfidf)
    df = pd.read_pickle('/var/www/app/cleaned_df.pkl')
    dfText = df['clean_text'].values

    result = tfidf.fit_transform(dfText)
    x_text = tfidf.transform([text])

    prediction = model.predict_proba(x_text)

    print(prediction)
    return prediction

if __name__ == "__main__":
    predict(sample_text)


import os
import cv2
import face_recognition
import numpy as np
from flask import Flask, flash, request, redirect, url_for, \
    send_from_directory, render_template
from werkzeug.utils import secure_filename
from face_swap import read_points, apply_affine_transform, rect_contains, \
    calculate_delaunay_triangles, warp_triangle

UPLOAD_FOLDER = 'uploads'
ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg'}

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

def face_swap(filename2):
    (major_ver, minor_ver, subminor_ver) = cv2.__version__.split('.')

    filename1 = 'putin.jpeg'

    img1 = cv2.imread(filename1)
    img2 = cv2.imread(filename2)
    img1_warped = np.copy(img2)

    points1 = read_points(face_recognition.face_landmarks(img1)[0])
    points2 = read_points(face_recognition.face_landmarks(img2)[0])

    # Find convex hull
    hull1 = []
    hull2 = []

    hull_index = cv2.convexHull(np.array(points2), returnPoints=False)

    for i in range(0, len(hull_index)):
        hull1.append(points1[int(hull_index[i])])
        hull2.append(points2[int(hull_index[i])])

    size_img2 = img2.shape
    rect = (0, 0, size_img2[1], size_img2[0])

    dt = calculate_delaunay_triangles(rect, hull2)

    if len(dt) == 0:
        quit()

    for i in range(0, len(dt)):
        t1 = []
        t2 = []

        for j in range(0, 3):
            t1.append(hull1[dt[i][j]])
            t2.append(hull2[dt[i][j]])

        warp_triangle(img1, img1_warped, t1, t2)

    hull8U = []
    for i in range(0, len(hull2)):
        hull8U.append((hull2[i][0], hull2[i][1]))

    mask = np.zeros(img2.shape, dtype=img2.dtype)

    cv2.fillConvexPoly(mask, np.int32(hull8U), (255, 255, 255))

    r = cv2.boundingRect(np.float32([hull2]))

    center = (r[0] + int(r[2] / 2), r[1] + int(r[3] / 2))

    output = cv2.seamlessClone(np.uint8(img1_warped), img2, mask, center, cv2.NORMAL_CLONE)

    return output

@app.route('/', methods=['GET', 'POST'])
def upload_file():
    if request.method == 'POST':
        if 'file' not in request.files:
            flash('No file part')
            return redirect(request.url)
        file = request.files['file']
        if file.filename == '':
            flash('No selected file')
            return redirect(request.url)
        if file and allowed_file(file.filename):
            filename = secure_filename(file.filename)
            path = os.path.join(app.config['UPLOAD_FOLDER'], filename)
            file.save(path)
            changed_filename = "changed_" + filename
            changed_path = os.path.join(app.config['UPLOAD_FOLDER'], changed_filename)
            changed_img = face_swap(path)
            cv2.imwrite(changed_path, changed_img)
            # file.save(changed_path)
            # img = cv2.imread(os.path.join(app.config['UPLOAD_FOLDER'], filename))
            # cv2.imwrite(os.path.join(app.config['UPLOAD_FOLDER'], ("changed_" + filename)), img)
            return redirect(url_for('uploaded_file', original=filename, changed=changed_filename))
    return render_template("base.html")

@app.route('/uploads/<filename>')
def send_file(filename):
    return send_from_directory(UPLOAD_FOLDER, filename)

@app.route('/show_images')
def uploaded_file():
    url = 'http://127.0.0.1:5000/uploads/'
    # it doesn't work with using explicit parameters, so here comes request
    img_original = url + request.args.get('original')
    img_changed = url + request.args.get('changed')
    return render_template('images.html', image1=img_original, image2=img_changed)

if __name__ == "__main__":
    app.run(debug=True)


from calorie import calories
from cnn_model import get_model
import os
import cv2
import numpy as np
import tensorflow as tf
from tensorflow import keras

IMG_SIZE = 400
LR = 1e-3
no_of_fruits=7

MODEL_NAME = 'Fruits_dectector-{}-{}.model'.format(LR, '5conv-basic')

model_save_at=os.path.join("model",MODEL_NAME)

model=get_model(IMG_SIZE,no_of_fruits,LR)

model.load(model_save_at)
labels=list(np.load('labels.npy'))

test_data='test_image.JPG'
img=cv2.imread(test_data)
img1=cv2.resize(img,(IMG_SIZE,IMG_SIZE))
model_out=model.predict([img1])
result=np.argmax(model_out)
name=labels[result]
cal=round(calories(result+1,img),2)

import matplotlib.pyplot as plt
plt.imshow(img)
plt.title('{}({}kcal)'.format(name,cal))
plt.axis('off')
plt.show()


import copy
import networkx as nx
from sfcsim.classes.node import *
from sfcsim.classes.vnf import *
from sfcsim.classes.network_matrix import *
import matplotlib.pyplot as plt
class network(nodes):
    '''
************************************************************************************************

    network类,继承nodes类，全局只应该有一个nodes实例,详情见network基础类设计文档
    network类除了继承nodes类之外还包含一个networkx.Graph实例，两者分别用于存储节点和链路信息，同时很多networkx
    原生的方法都可以使用，但修改节点链路等基础属性算法应该使用network类给出的下列方法
    属性值：
        包含nodes原有属性值
            number               node数量
            nodes                node类的所有实例，表示网络中存储的所有node实例
            __access_number      接入node数量
            __access_nodes       接入node类的所有实例
            __server_number      服务node数量
            __server_nodes       服务node类的所有实例
         G                       networkx.Graph实例，主要用于存储链路属性，节点属性为nodes类实例的引用
    属性方法：
        太多了，我不想写，包含nodes类原有方法，有些方法已经重载
        注意，对于添加节点链路等基础修改属性相关的功能应该使用network类方法，不用networkx原有方法(因为设计到节点属性)

************************************************************************************************
    '''
    def __init__(self,nodes1=[],network_matrix1=[]):
        nodes.__init__(self,nodes1)
        self.G=nx.Graph()
        self.generate()
        self.figure=''
        if(network_matrix1!=[]):
            data=self.update_edge(network_matrix1)
            self.change_edge(data[0],data[1],data[2])
    def generate(self):
        for node in self.nodes:  #如果nodes的节点不再G中，则加入节点
            if node not in self.G:
                self.G.add_node(node)
        n=[]
        for node in self.G.nodes:           #如果G中的节点不再nodes中，则删除G中节点，一切以nodes中为准
            if node not in self.nodes:
                n.append(node)
        self.G.remove_nodes_from(n)
    def set_network_matrix(network_matrix1):
        data=self.update_edge(network_matrix1)
        self.change_edge(data[0],data[1],data[2])
    def change_edge(self,edge_delete,edge_add,edge_att_change):
        self.delete_edges(edge_delete)
        self.add_edges(edge_add)
        self.set_edges_atts(edge_att_change)
        edges=edge_delete
        for edge in edge_add:
            edges.append([edge[0],edge[1]])
        for edge in edge_att_change:
            edges.append([edge0],[edge1])
        return edges

    def update_edge(self,network_matrix):
        edge_add=[] #[(node1,node2,atts)]
        edge_delete=[] #[[node1,node2],...]
        edge_att_change=[]# [[node1,node2,atts],...]
        lens=len(network_matrix.node_list)
        edges=self.G.edges
        for i in range(lens):
            for j in range(i+1,lens):
                if(network_matrix.edge[i][j]==0):  #下一状态不存在此edge
                    node0=self.get_node(network_matrix.node_list[i])
                    node1=self.get_node(network_matrix.node_list[j])
                    if (node0,node1) in edges:  #此状态中存在此edge，需要删除
                        edge_delete.append([node0,node1])

                else:                          #下一状态存在此edge
                    node0=self.get_node(network_matrix.node_list[i])
                    node1=self.get_node(network_matrix.node_list[j])
                    if (node0,node1) not in edges:  #此状态中不存在此edge，需要添加
                        atts={}
                        for att in network_matrix.edge_atts:
                            atts[att]=network_matrix.edge_atts[att][i][j]
                        edge_add.append([node0,node1,atts])
        return [edge_delete,edge_add,edge_att_change]
    def add_node(self,node):
        nodes.add_node(self,node)
        self.generate()
    def add_nodes(self,node_list):
        nodes.add_nodes(self,node_list)
        self.generate()
    def delete_node(self,node1):
        if type(node()) == type(node1):
            if(nodes.delete_node(self,node1.get_id())!=False):
                self.G.remove_node(node1)
        else:
            data=nodes.delete_node(self,node1)
            if(data!=False):
                self.G.remove_node(data)
    def delete_nodes(self,node_list):
        for node in node_list:
            self.delete_node(node)
    def add_edge(self,node1,node2,**link):
        if type(node()) != (type(node1)):
            node1=self.get_node(node1)
        if type(node()) != (type(node2)):
            node2=self.get_node(node2)
        if node1==False or node2==False:
            print('log: error!!!node1 or node2 not in node list, can\'t add edge to node doesn\'t exists')
        else:
            if 'bandwidth' in link:
                link['remain_bandwidth']=link['bandwidth']
            self.G.add_edge(node1,node2,**link)
    def add_edges_from(self,edges):#edges格式为[(node1,node2,atts)]
        i=0
        edges2=[]
        for edge in edges:
            if type(node()) != (type(edge[0])):
                edge[0]=self.get_node(edge[0])
            if type(node()) != (type(edge[1])):
                edge[1]=self.get_node(edge[1])
            if edge[0] ==False or edge[1] ==False:
                print('log: error!!!can\'t add edge between nodes doesn\'t exist')
            else:
                edges2.append(edge)
            if 'bandwidth' in edge[2]:
                edge[2]['remain_bandwidth']=edge[2]['bandwidth']
            i+=1
        self.G.add_edges_from(edges2)
    def add_edges(self,edges):#edges格式为[(node1,node2,atts)]
        self.add_edges_from(edges)
    def delete_edge(self,node1,node2):
        if type(node()) != (type(node1)):
            node1=self.get_node(node1)
        if type(node()) != (type(node2)):
            node2=self.get_node(node2)
        if node1==False or node2==False:
            print('log: error!!!node1 or node2 not in node list, can\'t delete edge to node doesn\'t exists')
            return False
        else:
            try:
                self.G.remove_edge(node1,node2)
            except:
                return False
            else:
                return True
    def delete_edges(self,edges):
        for edge in edges:
            self.delete_edge(edge[0],edge[1])
    def delete_edges_from(self,edges):
        for edge in edges:
            self.delete_edge(edge[0],edge[1])
    def set_edge_atts(self,node1,node2,atts):
        if type(node()) != (type(node1)):
            node1=self.get_node(node1)
        if type(node()) != (type(node2)):
            node2=self.get_node(node2)
        if node1==False or node2==False:
            print('log: error!!!node1 or node2 not in node list, can\'t add edge atts to node doesn\'t exists')
            return False
        else:
            if 'bandwidth' in atts:
                atts['remain_bandwidth']=atts['bandwidth']
            nx.set_edge_attributes(self.G,{(node1,node2):atts})
            return True
    def set_edges_atts(self,atts):
        atts2={}
        for key in atts:
            if type(node()) != (type(key[0])):
                a=self.get_node(key[0])
            else:
                a=key[0]
            if type(node()) != (type(key[1])):
                b=self.get_node(key[1])
            else:
                b=key[1]
            if a==False or b==False:
                print('log: error!!!node1 or node2 not in node list, can\'t add edge atts to node doesn\'t exists')
            else:
                atts2[(a,b)]=atts[key]
        nx.set_edge_attributes(self.G,atts2)
    def show_nodes(self):
        nodes.show(self)
    def show_edges(self):
        i=1
        print('*****     there are',len(self.G.edges),'edge in network     *****')
        print('    number  node1       node2       atts')
        for edge in self.G.edges.data():
            print('    %-6d  %-10s  %-10s  %-s' %(i,edge[0].get_id(),edge[1].get_id(),edge[2]))
            i+=1
    def show(self):
        print('**********************     print nodes and edges in network     *********************')
        self.show_nodes()
        self.show_edges()

    # def draw(self,pos=[],figsize=[36,20],node_size=10000,node_fone_size=8,link_fone_size=9,node_shape='H',path=''):     #画图
    #     plt.figure(figsize=figsize)
    #     if(path!=''):
    #         fig=plt.gcf()
    #     node_labels ={}# nx.get_node_attributes(G)
    #     edge_label ={}# nx.get_edge_attributes(G)
    #     node_colors=[]
    #     node_sizes=[]
    #     for node in self.G.nodes:
    #         strs='ID: '+ str(node.get_id())+'\n'
    #         if(node.is_access()==False):
    #             str1=('%-6s ' %('att'))
    #             str2=('%-6s ' %('all'))
    #             vnf_strs=[]
    #             for vnf in node.get_vnfs():
    #                 vnf_strs.append('%-6s ' %(str(vnf.get_name())))
    #             for key in node.get_atts():
    #                 if key != 'access':
    #                     str1=str1+('%-8s' %(str(key)))
    #                     stra=('%.3g/%.3g' %(node.get_remain_resource()[key],node.get_atts()[key]))
    #                     str2=str2+('%-8s' %(stra))
    #                     i=0
    #                     for vnf in node.get_vnfs():
    #                         stra=('%.3g/%.3g' %(vnf.get_remain_resource()[key],+vnf.get_atts()[key]))
    #                         vnf_strs[i]=vnf_strs[i]+('%-8s' %(stra))
    #                         i+=1
    #             node_colors.append('yellow')
    #             node_sizes.append(node_size)
    #             strs=strs+str1+'\n'+str2
    #             for vnf_str in vnf_strs:
    #                 strs=strs+'\n'+vnf_str
    #         else:
    #             node_colors.append('red')
    #             node_sizes.append(18000)
    #         node_labels[node] = strs.rstrip('\n')

    #     for edge in self.G.edges:
    #         strs=' '
    #         for key in self.G.edges[edge]:
    #             if key =='bandwidth':
    #                 stra=('%.3g/%.3g' %(self.G.edges[edge]['remain_bandwidth'],self.G.edges[edge][key]))
    #                 strs=strs+'BW'+':'+stra+'\n'
    #             elif key!='remain_bandwidth':
    #                 strs=strs+key+':'+str(self.G.edges[edge][key])+'\n'
    #         edge_label[edge] = strs.rstrip('\n')
    #     nx.draw(self.G,pos=pos,node_size=node_sizes,node_color=node_colors,width=2,edge_color='black',node_shape=node_shape)
    #     nx.draw_networkx_labels(self.G, pos=pos, labels=node_labels,font_size=node_fone_size)
    #     nx.draw_networkx_edge_labels(self.G, pos=pos,edge_labels=edge_label,font_size=link_fone_size)
    #     plt.show()
    #     plt.close()
    #     if(path!=''):
    #         fig.savefig(path)
    def draw(self,pos=[],figsize=[36,20],node_size=10000,node_fone_size=8,link_fone_size=9,node_shape='H',path=''):     #画图，根据资源占用比显示10级别色差

        node_labels ={}# nx.get_node_attributes(G)
        edge_label ={}# nx.get_edge_attributes(G)
        node_colors=[]
        edge_colors=[]
        node_sizes=[]
        color_list=[]
        color_list1=[(200,0,0),(255,0,0),(255,30,30),(255,60,60),(255,90,90),(255,120,120),(255,150,150),(255,180,180),(255,210,210),(255,240,240)]
        for data in color_list1:
            color_list.append((data[0]/255,data[1]/255,data[2]/255))
        for node in self.G.nodes:
            color=0
            strs='ID: '+ str(node.get_id())+'\n'
            if(node.is_access()==False):
                str1=('%-6s ' %('att'))
                str2=('%-6s ' %('all'))
                vnf_strs=[]
                for vnf in node.get_vnfs():
                    vnf_strs.append('%-6s ' %(str(vnf.get_name())))
                for key in node.get_atts():
                    if key != 'access':
                        str1=str1+('%-8s' %(str(key)))
                        stra=('%.3g/%.3g' %(node.get_remain_resource()[key],node.get_atts()[key]))
                        color+=node.get_remain_resource()[key]/node.get_atts()[key]
                        str2=str2+('%-8s' %(stra))
                        i=0
                        for vnf in node.get_vnfs():
                            stra=('%.3g/%.3g' %(vnf.get_remain_resource()[key],+vnf.get_atts()[key]))
                            vnf_strs[i]=vnf_strs[i]+('%-8s' %(stra))
                            i+=1
                color=int(round(10*color/(len(node.get_atts())-1))-1)
                if(color<0):
                    color=0
                node_colors.append(color_list[color])
                node_sizes.append(node_size)
                strs=strs+str1+'\n'+str2
                for vnf_str in vnf_strs:
                    strs=strs+'\n'+vnf_str
            else:
                node_colors.append('red')
                node_sizes.append(18000)
            node_labels[node] = strs.rstrip('\n')
        for edge in self.G.edges:
            strs=' '
            color=0
            for key in self.G.edges[edge]:
                if key =='bandwidth':
                    stra=('%.3g/%.3g' %(self.G.edges[edge]['remain_bandwidth'],self.G.edges[edge][key]))
                    color=int(round(10*self.G.edges[edge]['remain_bandwidth']/self.G.edges[edge][key])-1)
                    if(color<0):
                        color=0
                    strs=strs+'BW'+':'+stra+'\n'
                elif key!='remain_bandwidth':
                    strs=strs+key+':'+str(self.G.edges[edge][key])+'\n'
            edge_label[edge] = strs.rstrip('\n')
            edge_colors.append(color_list[color])
        plt.figure(figsize=figsize)
        plt.ioff()
        if(path!=''):
            fig=plt.gcf()
        nx.draw(self.G,pos=pos,node_size=node_sizes,node_color=node_colors,width=4,edge_color=edge_colors,node_shape=node_shape)
        nx.draw_networkx_labels(self.G, pos=pos, labels=node_labels,font_size=node_fone_size)
        nx.draw_networkx_edge_labels(self.G, pos=pos,edge_labels=edge_label,font_size=link_fone_size)
        plt.show()
        plt.close()
        if(path!=''):
            fig.savefig(path)
    def draw_dynamic(self,pos=[],figsize=[36,20],node_size=10000,node_fone_size=8,link_fone_size=9,node_shape='H',path=''):     #画图，根据资源占用比显示10级别色差
        node_labels ={}# nx.get_node_attributes(G)
        edge_label ={}# nx.get_edge_attributes(G)
        node_colors=[]
        edge_colors=[]
        node_sizes=[]
        color_list=[]
        color_list1=[(200,0,0),(255,0,0),(255,30,30),(255,60,60),(255,90,90),(255,120,120),(255,150,150),(255,180,180),(255,210,210),(255,240,240)]
        for data in color_list1:
            color_list.append((data[0]/255,data[1]/255,data[2]/255))
        for node in self.G.nodes:
            color=0
            strs='ID: '+ str(node.get_id())+'\n'
            if(node.is_access()==False):
                str1=('%-6s ' %('att'))
                str2=('%-6s ' %('all'))
                vnf_strs=[]
                for vnf in node.get_vnfs():
                    vnf_strs.append('%-6s ' %(str(vnf.get_name())))
                for key in node.get_atts():
                    if key != 'access':
                        str1=str1+('%-8s' %(str(key)))
                        stra=('%.3g/%.3g' %(node.get_remain_resource()[key],node.get_atts()[key]))
                        color+=node.get_remain_resource()[key]/node.get_atts()[key]
                        str2=str2+('%-8s' %(stra))
                        i=0
                        for vnf in node.get_vnfs():
                            stra=('%.3g/%.3g' %(vnf.get_remain_resource()[key],+vnf.get_atts()[key]))
                            vnf_strs[i]=vnf_strs[i]+('%-8s' %(stra))
                            i+=1
                color=int(round(10*color/(len(node.get_atts())-1))-1)
                if(color<0):
                    color=0
                node_colors.append(color_list[color])
                node_sizes.append(node_size)
                strs=strs+str1+'\n'+str2
                for vnf_str in vnf_strs:
                    strs=strs+'\n'+vnf_str
            else:
                node_colors.append('red')
                node_sizes.append(18000)
            node_labels[node] = strs.rstrip('\n')
        for edge in self.G.edges:
            strs=' '
            color=0
            for key in self.G.edges[edge]:
                if key =='bandwidth':
                    stra=('%.3g/%.3g' %(self.G.edges[edge]['remain_bandwidth'],self.G.edges[edge][key]))
                    color=int(round(10*self.G.edges[edge]['remain_bandwidth']/self.G.edges[edge][key])-1)
                    if(color<0):
                        color=0
                    strs=strs+'BW'+':'+stra+'\n'
                elif key!='remain_bandwidth':
                    strs=strs+key+':'+str(self.G.edges[edge][key])+'\n'
            edge_label[edge] = strs.rstrip('\n')
            edge_colors.append(color_list[color])
        if self.figure=='':
            self.figure=plt.figure(figsize=figsize)
            plt.ion() #开启interactive mode 成功的关键函数
        plt.clf() #清空画布上的所有内容
        if(path!=''):
            fig=plt.gcf()
        nx.draw(self.G,pos=pos,node_size=node_sizes,node_color=node_colors,width=4,edge_color=edge_colors,node_shape=node_shape)
        nx.draw_networkx_labels(self.G, pos=pos, labels=node_labels,font_size=node_fone_size)
        nx.draw_networkx_edge_labels(self.G, pos=pos,edge_labels=edge_label,font_size=link_fone_size)
        plt.show()
        if(path!=''):
            fig.savefig(path)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import cv2
import numpy as np
import torch
from torch import Tensor

class CLAHE:
    r"""Contrast Limited Adaptive Histogram Equalization. Uses CV2 as the the
    computational backend. Input should have either a single luminance channel or three RGB channels.
    Expected datatypes for ``input`` are ``torch.uint8``, ``torch.uint16``, or any floating point
    datatype with values in the range ``[0, 1]``.

    Args:

        ``*args``:
            Passed to :class:`cv2.createCLAHE`.

        ``*kwargs``:
            Passed to :class:`cv2.createCLAHE`.

    Shape:
        * ``inputs`` - :math:`(N, C, H, W)` where :math:`C \in \{1, 3\}`
        * Outputs - same as inputs

    """

    def __init__(self, *args, **kwargs):
        self.clahe = cv2.createCLAHE(*args, **kwargs)  # type: ignore
        self._args = args
        self._kwargs = kwargs

    def __repr__(self):
        s = "CLAHE("
        for arg in self._args:
            s += f"{arg}, "
        for key, value in self._kwargs.items():
            s += f"{key}={value}, "
        s = s[:-2] + ")"
        return s

    def __call__(self, inputs: Tensor) -> Tensor:
        if inputs.dtype != torch.uint8 and not inputs.is_floating_point():
            raise TypeError(f"CLAHE only supports torch.uint8 or float in range [0, 1], found {inputs.dtype}")
        if inputs.ndim != 4:
            raise ValueError(f"Expected inputs.ndim == 4, found {inputs.ndim}")

        batch_size, channels, height, width = inputs.shape
        result = torch.empty_like(inputs)
        inputs = inputs.cpu()

        # cv2 clahe doesn't support float, so if needed we will convert to uint16
        is_float = inputs.is_floating_point()

        for i in range(batch_size):
            batch_elem = inputs[i]

            # grayscale CLAHE
            if channels == 1:
                batch_elem = batch_elem.view(height, width)
                if is_float:
                    batch_elem = batch_elem.mul(2 ** 16).numpy().astype(np.uint16)
                assert batch_elem.dtype in [np.uint8, np.uint16]
                output = self.clahe.apply(batch_elem)
                if is_float:
                    output = output.astype(float)
                output = torch.from_numpy(output)

            # color CLAHE
            else:
                # RGB -> YUV and extract luminance for CLAHE
                batch_elem = batch_elem.permute(1, 2, 0)
                if is_float:
                    batch_elem = batch_elem.mul(2 ** 16).numpy().astype(np.uint16)
                batch_elem = cv2.cvtColor(batch_elem, cv2.COLOR_RGB2YUV)  # type: ignore
                luminance = batch_elem[0, ...]

                # apply CLAHE to luminance channel and update
                assert batch_elem.dtype in [np.uint8, np.uint16]
                luminance = self.clahe.apply(luminance)
                batch_elem[0, ...] = luminance

                # convert CLAHE YUV channel back to RGB
                output = cv2.cvtColor(batch_elem, cv2.COLOR_YUV2RGB)  # type: ignore
                if is_float:
                    output = output.astype(float)

                # restore numpy array to channels first tensor
                output = torch.from_numpy(output).permute(-1, 0, 1)

            # restore floats to range [0, 1]
            if is_float:
                output = output.type_as(result).div_(2 ** 16)

            result[i] = output.type_as(result).view(channels, height, width)

        return result


import random as rd

import cv2
import numpy as np

from .elements import add_single_element, add_elements
from . import mods

def vaporize(image_path):
    face_cascade = cv2.CascadeClassifier('cascade/haarcascade_frontalface_default.xml')
    eye_cascade = cv2.CascadeClassifier('cascade/haarcascade_eye.xml')

    # load main image from local file
    img = cv2.imread(image_path)
    # height, width, depth = img.shape

    # turn image gray for detecting face and eyes
    gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)

    # find all the faces in the image
    faces = face_cascade.detectMultiScale(gray, 1.3, 5)

    # go through each face
    for face in faces:
        y = face[1]
        x = face[0]
        w = face[2]
        h = face[3]
        roi_gray = gray[y:y + h, x:x + w]
        eyes = eye_cascade.detectMultiScale(roi_gray, 1.2, 6)
        for eye in eyes:
            eye[0] += face[0]
            eye[1] += face[1]

        # randomize which face modification will be performed
        eyes_present = len(eyes) >= 2
        mod_function, operates_on = mods.determine_face_mod(eyes_present)
        if operates_on == mods.EYES:
            modded_img = mod_function(img, eyes)
            if modded_img is not None:
                img = modded_img
        elif operates_on == mods.FACE:
            modded_img = mod_function(img, face)
            if modded_img is not None:
                img = modded_img

    # Add elements to image
    add_elements(img)
    # if there are no faces, just add more elements!
    if len(faces) < 1:
        add_elements(img)

    # randomize if high contrast is used
    choice = rd.randint(0, 1)
    if choice == 1:
        # edit alpha and beta to adjust contrast levels
        img = cv2.convertScaleAbs(img, alpha=1.2, beta=35)

    # randomize if high noise is used
    choice = rd.randint(0, 1)
    if choice == 1:
        row, col, ch = img.shape
        mean = 0
        # edit var to modify the amount of noise in the image
        var = 15
        sigma = var ** 1
        gauss = np.random.normal(mean, sigma, (row, col, ch))
        gauss = gauss.reshape(row, col, ch)
        noisy = (img + gauss)
        cv2.normalize(noisy, noisy, 0, 1, cv2.NORM_MINMAX)
        img = noisy

    return img


"""Set of functions used to generate reads derived from a set of sequences (i.e. full genome FASTA file).

* You can specify a mutation rate, a VCF file in order to add known SNPs, or a mutation matrix obtained from snpAD, :title-reference:`Kay Prüfer, snpAD: an ancient DNA genotype caller, Bioinformatics, Volume 34, Issue 24, 15 December 2018, Pages 4165–4171, https://doi.org/10.1093/bioinformatics/bty507`.
* Default read length distribution is derived from empirical data, but you can provide your own or length will be uniformly sampled.
* Work is processed in parallel in order to decrease computation time.

"""

import gzip
from Bio import SeqIO, SeqRecord
from Bio.Seq import Seq
import multiprocessing

# for creating partial function for feeding pool.map
from functools import partial
import numpy as np
import pandas as pd
import random
import sys
import pysam
import re
import argparse
import contextlib

def get_sequence_with_substitution(sequence):
    """Generate a sequence with mutations relative to the given mutation probability matrix

    Parameters
    ----------
    sequence : Bio.Seq.MutableSeq

    Returns
    -------
    Bio.Seq.Seq
    """
    read_length = len(sequence)
    # create a set of draws
    choices = np.random.random(read_length)
    # create a profile sequence with as many profile 30 relative to read_length
    positions = np.concatenate([np.arange(15), np.arange(29, 14, -1)])
    positions = np.insert(positions, 15, np.full(read_length - 30, 30))
    newSeq = list(sequence)
    # iterate over the sequence, choices and positions
    for idx, (nc, (choice, pos)) in enumerate(zip(sequence, zip(choices, positions))):
        # we draw a probability which allows us to mutate
        if choice > same_nuc[pos][nc]:
            t = tbl[nc].loc[pos]
            try:
                newSeq[idx] = t[choice < t].idxmin()
            except:  # rounding error when over choice 0.999999
                newSeq[idx] = t.idxmax()
    return Seq("".join(newSeq))

def get_chrom(refseq):
    """Get the chromosome from a RefSeq entry.

    Parameters
    ----------
    refseq : Bio.Seq.Seq
        A RefSeq entry

    Returns
    -------
    str
        The chromosome extracted from the sequence id
    """
    chrom = refseq.id  # in case of scaffold genome
    if "|" in chrom:
        chrom = chrom.split("|")[0]
    return chrom

# sort the list of requences according to chromosome

def sort_recs(recs, chromosome_list=[], split_char="|"):
    """Sort a list of Bio.SeqRecord according to the chromosome, position, length.

    Parameters
    ----------
    recs : list of Bio.SeqRecord
    chromosome_list : list of str
        A list specifying the order of the chromosomes (as python doesn't really natural sort lists)
    split_char : str
        The delimiter used for splitting the record.id

    Returns
    -------
    list of Bio.SeqRecord
        Records properly sorted
    """

    def sort_func(item):
        """we sort according to chromosomes
        our header is >16|kraken:taxid|9906|69694935_57 ...
        use split to access the first element
        last element is pos_length
        """
        chrom = item.id.split(split_char)[0]
        pos, l = item.id.split(split_char)[-1].split("_")
        if chromosome_list:
            return (chromosome_list.index(chrom), int(pos), int(l))
        else:
            return (int(pos), int(l))

    return sorted(recs, key=sort_func)

def simulate_deamination(sequence, nbases=3):
    """Add C -> T mutations to 5' and 3' ends

    Parameters
    ----------
    sequence : Bio.Seq.MutableSeq
    nbases : int
        Number of bases to mutate for both ends

    Returns
    -------
    Bio.Seq.Seq
        The deaminated sequence
    """
    while "C" in sequence[:nbases]:  # 5' C to T
        sequence[sequence.index("C")] = "T"
    while "C" in sequence[-nbases:]:  # 3' C to T
        sequence[-nbases + sequence[-nbases:].index("C")] = "T"
    return sequence.toseq()

def mutate_unif(sequence, unif):
    """Add mutations uniformly throughout the sequence

    Parameters
    ----------
    sequence: Bio.Seq.Seq
    unif: float
        The mutaion rate

    Returns
    -------
    Bio.Seq.Seq
        A new mutated sequence
    """
    same_nuc = set("ACGT")
    read_length = len(sequence)
    # we will mutate a base only if our sample is < unif
    (choices,) = np.where(np.random.random(read_length) < unif)
    newSeq = list(sequence)
    for idx in choices:
        # we call tuple in order to do random.choice (and remove the current nc from the possibilities)
        newSeq[idx] = random.choice(tuple(same_nuc.difference(sequence[idx])))
    return Seq("".join(newSeq))

# the idea is to generate a set of coordinates/size pairs and only take those substrings:
# sample 0:length_genome (N = #chunks desired)
# sample N sizes (N = chunks desired)
def chunk_fast(
    record,
    n_samples,
    vcf_in=None,
    chrom=None,
    specie="|",
    individual=0,
    unif=False,
    len_distrib=False,
    deaminate=0,
    minlength=35,
    maxlength=100,
    nthreads=4,
):
    """Split the sequence into chunks and add mutations (in parallel)

    The function will generate a list of positions to extract reads from as well as a list of read length (following the given distribution), it will then create a list of reads which will be mutated according to the mutation rule given (unif, VCF, or matrix)

    Parameters
    ----------
    record: Bio.SeqRecord
    n_samples: int
        How many reads should be sampled
    vcf_in: pysam.VariantFile, optional
        File providing mutations informations (VCF or Substitution matrix)
    chrom: int, optional
    specie: str, optional
    individual: int, optional
    unif: float, optional
        The mutaion rate, will be added on top of VCF/substitution Matrix
    len_distrib: str, optional
        If False, uniform distribution will be used
    deaminate: int, optional
    minlength: int, optional
    maxlength: int, optional
    nthreads: int, optional

    Returns
    -------
    list of Bio.Seq.Seq
    """
    try:
        positions = random.sample(range(0, len(record) - minlength), n_samples)
    except:
        print(
            "sample too small {}bp, sampling {} reads with replacements".format(
                len(record) - minlength, n_samples
            ),
            file=sys.stderr,
        )
        positions = [
            random.choice(range(0, len(record) - minlength)) for _ in range(n_samples)
        ]
    if len_distrib:  # we gave a file with distribution per length
        p = read_len_distrib(len_distrib, minlength, maxlength)
        length = np.random.choice(np.arange(minlength, maxlength + 1), n_samples, p=p)
    else:
        # length = random.choices(range(minlength, maxlength), k = n_samples) #only from python3.6
        # onwards
        length = np.random.choice(np.arange(minlength, maxlength + 1), n_samples)
    samples = []
    for pos, l in zip(positions, length):
        rec = record[pos : pos + l]
        while not set(rec).issubset("ACGT"):  # control for unknown bases
            # print ("Unknown base (N) in read at {} {},
            # resampling...".format(pos,l), file=sys.stderr) # debug purpose
            pos = random.choice(range(0, len(record) - minlength))
            if len_distrib:
                l = np.random.choice(np.arange(minlength, maxlength + 1), 1, p=p)[0]
            else:
                l = random.choice(range(minlength, maxlength + 1))
            rec = record[pos : pos + l]

        # create a new header which includes the specie read pos, read length
        samples += [
            SeqRecord.SeqRecord(
                rec.seq,
                id="{}{}{}_{}".format(rec.id, specie, pos, l),
                description=" ".join(rec.description.split(" ")[1:]),
            )
        ]
    if vcf_in:  # insert variation from VCF
        res = []
        if isinstance(vcf_in, pysam.VariantFile):  # we use a VCF
            for sample in samples:
                sequence = sample.seq.tomutable()
                l = len(sequence)
                for vcf_rec in vcf_in.fetch(chrom, start=pos, end=pos + l):
                    # Insert alt only if GT != 0/0, use the first individual by
                    # default
                    if any(vcf_rec.samples[0]["GT"]):
                        if len(vcf_rec.alts[0]) == 1:  # only SNPs
                            sequence[vcf_rec.start - pos] = vcf_rec.alts[0]
                if deaminate or unif:  # save the seqs for later deam/unif
                    res.append(sequence)
                else:
                    sample.seq = sequence.toseq()
                # print(record[vcf_rec.start-3:vcf_rec.start+4].seq,vcf_rec.ref,vcf_rec.alts,vcf_rec.start,
                # pos, l) # debug purpose
        else:  # we use a variation probability matrix
            # run on multithreaded code as it is the bottleneck (each base is possibly mutated)
            with multiprocessing.Pool(processes=nthreads) as pool:
                res = pool.map(
                    get_sequence_with_substitution, [s.seq.tomutable() for s in samples]
                )
        if len(res):  # run multi-threaded code
            with multiprocessing.Pool(processes=nthreads) as pool:
                if deaminate:
                    deam_func = partial(simulate_deamination, deaminate=deaminate)
                    res = pool.map(deam_func, res)
                if unif:
                    unif_func = partial(mutate_unif, unif=unif)
                    res = pool.map(unif_func, res)
            for s, seq in zip(samples, res):
                s.seq = seq
    return samples

def read_len_distrib(filename, minlen=35, maxlen=100):
    """Reads a tsv file with counts of read per length

    Parameters
    ----------
    filename : str
        tsv file containing the read length count
    minlen : int
        minimum length for the distribution
    maxlen : int
        maximum length for the distribution

    Returns
    -------
    list of float
        The computed proportion of each read length
    """
    from csv import reader

    with open(filename, "r", newline="") as csvfile:
        csvreader = reader(csvfile, delimiter="\t")
        first_row = csvreader.__next__()
        read_distrib = [0] * int(first_row[1]) + [int(first_row[0])]
        read_distrib.extend([int(row[0]) for row in csvreader])
    if len(read_distrib) < maxlen:
        print(
            "Read length distribution provides values only until",
            len(read_distrib),
            "; expected:",
            maxlen,
        )
    s = sum(read_distrib[minlen : maxlen + 1])
    return [r / s for r in read_distrib[minlen : maxlen + 1]]

def estimate_read_distribution(file_in, num_seq, n_chromosomes=None):
    """Reads a fasta file and computes the number of sequences to extract per chromosome (relative to chromosome length)

    Parameters
    ----------
    file_in : str
        FASTA file you want to estimate the distribution from.
    num_seq : int
        The total number of sequences you want to sample.
    n_chromosomes : int
        If you want to take only the first *n* chromosomes of your dataset.

    Returns
    -------
    list of int
        The number of sequences to extract per chromosome.
    """
    try:
        with pysam.FastaFile(file_in) as fa:
            print("The file contains", fa.nreferences, "sequences", file=sys.stderr)
            if num_seq:
                if n_chromosomes:
                    print("Working only with the first", n_chromosomes, file=sys.stderr)
                    full_size = sum(fa.lengths[:n_chromosomes])
                    reads_per_chrom = [
                        int(s / full_size * num_seq) + 1
                        for s in fa.lengths[:n_chromosomes]
                    ]
                else:
                    n_chromosomes = len(fa.lengths)
                    full_size = sum(fa.lengths)
                    reads_per_chrom = [
                        int(s / full_size * num_seq) + 1 for s in fa.lengths
                    ]
    except OSError:  # fasta is not indexed do a naive fallback
        print("Error, fasta file not indexed, doing naive sampling...", file=sys.stderr)
        if n_chromosomes:
            reads_per_chrom = [int(num_seq / n_chromosomes) + 1] * n_chromosomes
        else:
            print(
                "Naive sampling needs the option --chromosomes...Aborting...",
                file=sys.stderr,
            )
            sys.exit(1)
    # our samplig tends to slightly over sample, readjust by removing reads on random chromosomes
    extra_samples = sum(reads_per_chrom) - num_seq
    for idx in [random.randint(0, n_chromosomes - 1) for _ in range(extra_samples)]:
        # some scaffolds are so small that the don't even get a read to sample.
        while reads_per_chrom[idx] == 0:
            idx = random.randint(0, n_chromosomes - 1)
        reads_per_chrom[idx] -= 1
    return reads_per_chrom

# dummy function used for opening a file non-gzipped
@contextlib.contextmanager
def _ret_file(f):
    yield f

def read_mutation_matrix(file_in):
    """Create a table from a mutation matrix provided by SNPad

    Returns
    -------
    tbl : Pandas.table
        A table containing the mutation probability for base at each position of a read.
    sn : list of dict
        The list contains the highest probability for which we can have a mutation on a given position.
        Used to optimize the computation when we draw for a mutation.
    """
    t = pd.read_table(
        file_in, sep="\t", header=None, names=["profile", "from", "to", "prob"]
    )
    tbl = pd.pivot_table(
        t, values="prob", index=["profile", "to"], columns=["from"], aggfunc=np.sum
    )
    # contains, for each nc, the highest probability for which we can have
    # a mutation
    # e.g.
    # 'A': 0.00797800000000004, -> ~0.79% chances to have a mutation, any sampled number above this would result in keeping the nc
    # this allows us to optimize the algorithm by not using the lookup
    # table and directly keep the nc.
    sn = [
        {nc: prob for nc, prob in zip(tbl.columns, [tbl[x][pos][x] for x in tbl])}
        for pos in range(31)
    ]
    return tbl, sn

def _main():
    random.seed()
    # Process command line
    parser = argparse.ArgumentParser(description="Split a genome into chuncks")
    parser.add_argument(
        "--num_seq", default=1, type=int, help="Number of sequences to output"
    )
    parser.add_argument("--outfile", help="Fasta file where to extract sequence")
    parser.add_argument("--specie", help="Specie Taxa")
    parser.add_argument("file_in", metavar="genome.fa[.gz], [bgzip] and indexed")
    parser.add_argument(
        "--chromosomes", type=int, help="How many chromosomes are expected", default=0
    )
    parser.add_argument(
        "--minlen",
        type=int,
        help="Minimum read length (lengths are sampled uniformly)",
        default=35,
    )
    parser.add_argument(
        "--maxlen",
        type=int,
        help="Maximum read length (lengths are sampled uniformly)",
        default=100,
    )
    parser.add_argument(
        "--threads",
        type=int,
        default=multiprocessing.cpu_count(),
        help="Specify number of threads to use",
    )
    # TODO, control for option dependency properly
    parser.add_argument(
        "--deaminate",
        type=int,
        help="How many bases should be deaminated of each end",
        default=0,
    )
    parser.add_argument(
        "--unif", type=float, help="Add mutations uniformly distributed", default=0.0
    )
    parser.add_argument(
        "--individual",
        type=int,
        help="The individual used for variation in your VCF",
        default=0,
    )
    parser.add_argument(
        "--length",
        help="2 columns (#reads, length) TSV file containing read length distribution",
        default=False,
    )
    # either provide a VCF OR a substitution matrix, you probably don't want
    # to deaminate if you choose the latter
    group = parser.add_mutually_exclusive_group()
    group.add_argument(
        "--vcf", help="VCF containing the variation to be included in the samples"
    )
    group.add_argument(
        "--substitution_file",
        type=str,
        help="File containing the substitution probability matrix",
    )
    sorting = parser.add_mutually_exclusive_group()
    sorting.add_argument(
        "--sorted",
        action="store_true",
        help="Natural sort the reads by chromosome name, position and length",
    )
    sorting.add_argument(
        "--shuffled", action="store_true", help="Output shuffled reads"
    )
    args = parser.parse_args()

    print("Loading dataset...", end="", file=sys.stderr)

    if args.substitution_file:
        global tbl
        global same_nuc
        tbl, same_nuc = read_mutation_matrix(args.substitution_file)
        for nc in "ACGT":
            for pos in range(31):
                # do cumsum on sorted values, assuming that the highest proba is the nc itself: C->C, A->A, etc
                tbl.loc[pos][nc] = \
                    tbl.loc[pos][nc].sort_values(ascending=False).cumsum().sort_index()
    num_reads_to_sample = estimate_read_distribution(
        args.file_in, args.num_seq, args.chromosomes
    )
    specie = "|"
    if args.specie:
        specie += args.specie.replace(" ", "_") + "|"
    with gzip.open(args.file_in, "rt") if args.file_in.endswith("gz") else _ret_file(
        args.file_in
    ) as file_in, open(args.outfile, "w") if args.outfile else sys.stdout as file_out:
        all_chunks = []
        vcf_in = args.substitution_file  # will be none if we use a VCF file
        chromosomes = []
        if args.vcf:
            vcf_in = pysam.VariantFile(args.vcf)
            # subset the VCF to 1 individual only
            vcf_in.subset_samples([vcf_in.header.samples[args.individual]])
            # our chromosome can contain up to 3 characters (most likely 2 max:
            # e.g. chromosome 21,)
        p = re.compile(r"chromosome \w{1,3},", re.IGNORECASE)
        for num_record, record in enumerate(SeqIO.parse(file_in, "fasta")):
            # we want reads only to assigned chromosomes
            if args.chromosomes:
                if num_record > args.chromosomes - 1:  # num_record is 0-based
                    # assume the genome is sorted, with chromosomes first...
                    break
                print("Parsing chromosome", num_record + 1, end="\r", file=sys.stderr)
            if num_reads_to_sample[num_record] == 0:  # skip this sequence
                continue
            chrom = p.search(record.description)
            if not chrom:
                chrom = (
                    get_chrom(record),
                    get_chrom(record),
                )  # in case of scaffold genome
            else:  # save both the id and chromosome info from fasta description
                chrom = (get_chrom(record), chrom.group()[len("chromosome ") : -1])
            # we write chromosomes in the same order as the given input.
            if not args.shuffled and not args.sorted:
                all_chunks = sort_recs(
                    chunk_fast(
                        record,
                        num_reads_to_sample[num_record],
                        vcf_in,
                        chrom,
                        specie,
                        unif=args.unif,
                        deaminate=args.deaminate,
                        len_distrib=args.length,
                        minlength=args.minlen,
                        maxlength=args.maxlen,
                        nthreads=args.threads,
                    )
                )
                SeqIO.write(all_chunks, file_out, "fasta")
            else:
                chromosomes.append(chrom)
                all_chunks += chunk_fast(
                    record,
                    num_reads_to_sample[num_record],
                    vcf_in,
                    chrom,
                    specie,
                    unif=args.unif,
                    deaminate=args.deaminate,
                    len_distrib=args.length,
                    minlength=args.minlen,
                    maxlength=args.maxlen,
                    nthreads=args.threads,
                )
            if num_record % 100 == 99:  # show progress
                print(num_record + 1, "sequences parsed...", end="\r", file=sys.stderr)
        if args.vcf:
            vcf_in.close()
        print("\nDone\nWritting down records...", end="", file=sys.stderr)
        if args.shuffled:
            random.shuffle(all_chunks)
        if args.sorted:  # we will do a natural sort on the chromosomes
            # if we find a chromosome in the description part of the header
            # use it for sorting, otherwise sort with fasta id
            chromosomes.sort(
                key=lambda key: [
                    int(text) if text.isdigit() else text
                    for text in re.split(r"([0-9]+)", key[1])
                ]
            )
            all_chunks = sort_recs(
                all_chunks, chromosome_list=[c[0] for c in chromosomes]
            )
        if args.shuffled or args.sorted:
            SeqIO.write(all_chunks, file_out, "fasta")
    print("Done", file=sys.stderr)

if __name__ == "__main__":
    _main()


# Run inference on batches of data
# uses default configuration from detectron2
#
# Fatemeh Saleh <fatemehsadat.saleh@anu.edu.au>

import numpy as np
import cv2
import os
from detectron2 import model_zoo
from detectron2.modeling import build_model
from detectron2.config import get_cfg
import torch
import detectron2.data.transforms as T
from detectron2.checkpoint import  DetectionCheckpointer
import glob
import json
from skimage import measure
from pycocotools import mask
import time
import argparse

parser = argparse.ArgumentParser()
parser.add_argument('-s', help='name of sample such as, [0002_white_floor_05_02_2019_08_19_17_47]', required=True)
parser.add_argument('-f', help='name of furniture from [Kallax_Shelf_Drawer, Lack_Side_Table, Lack_Coffee_Table, Lack_TV_Bench]', required=True)
parser.add_argument('-root', default='/path/to/dataset/', required=True)
parser.add_argument('-batch', default=10, required=True, type=int)
parser.add_argument('-model', help='name of the model in cfg.OUTPUT_DIR such as [ResNeXt.pth] ', required=True)

args = parser.parse_args()

if __name__ == '__main__':

    test_dir = os.path.join(args.root, args.f, args.s, 'dev3', 'images')
    test_imgs = glob.glob(test_dir + '/*')
    test_imgs.sort()
    test_imgs_transform = []
    test_json = {'images': [], 'annotations': [], 'categories': []}

    cfg = get_cfg()
    cfg.merge_from_file(model_zoo.get_config_file("COCO-InstanceSegmentation/mask_rcnn_X_101_32x8d_FPN_3x.yaml"))
    cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.8  # set threshold for this model
    cfg.MODEL.WEIGHTS = os.path.join(cfg.OUTPUT_DIR, args.model)
    cfg.MODEL.ROI_HEADS.NUM_CLASSES = 7
    transform_gen = T.ResizeShortestEdge([cfg.INPUT.MIN_SIZE_TEST, cfg.INPUT.MIN_SIZE_TEST], cfg.INPUT.MAX_SIZE_TEST)

    model = build_model(cfg)
    DetectionCheckpointer(model).load(cfg.MODEL.WEIGHTS)
    model.train(False)

    for i in range(len(test_imgs)):
        original_image = cv2.imread(test_imgs[i])
        height, width = original_image.shape[:2]
        image = transform_gen.get_transform(original_image).apply_image(original_image)
        image = torch.as_tensor(image.astype("float32").transpose(2, 0, 1))
        input = {"image": image, "height": height, "width": width}
        test_imgs_transform.append(input)
        test_json['images'].append(
            {"id": i + 1, "file_name": args.f + '/' + args.s + '/dev3/images/' + str.split(test_imgs[i], '/')[-1],
             "height": height, "width": width})

    segment_id = 1
    start = time.time()
    with torch.no_grad():
        for b in range(0, len(test_imgs_transform), args.batch):
            if b + args.batch < len(test_imgs_transform):
                outputs = model(test_imgs_transform[b:b + args.batch])
                for j in range(args.batch):
                    for s in range(len(outputs[j]['instances'])):
                        bimask = outputs[j]['instances'][s].pred_masks.cpu().numpy().astype("uint8")[0]
                        segment = []
                        contours = measure.find_contours(bimask, 0.5)
                        for contour in contours:
                            contour = np.flip(contour, axis=1)
                            segmentation = contour.ravel().tolist()
                            segment.append(segmentation)
                        bimask = np.expand_dims(bimask, axis=-1)
                        Rs = mask.encode(np.asfortranarray(np.uint8(bimask)))
                        area = int(mask.area(Rs)[0]),
                        bbox_seg = mask.toBbox(Rs)[0].tolist()
                        test_json['annotations'].append(
                            {'image_id': int(b + j + 1), 'id': segment_id, 'segmentation': segment,
                             'bbox': outputs[j]['instances'][s].pred_boxes.tensor.cpu().numpy()[0].tolist()
                                , 'category_id': int(outputs[j]['instances'][s].pred_classes.cpu().numpy()[0]) + 1,
                             'score': float(outputs[j]['instances'][s].scores.cpu().numpy()[0]), 'area': int(area[0])})
                        segment_id += 1
            print("Inference Done for ", b, "Frames")
        if b < len(test_imgs_transform):
            outputs = model(test_imgs_transform[b:])
            for j in range(len(test_imgs_transform) - b):
                for s in range(len(outputs[j]['instances'])):
                    bimask = outputs[j]['instances'][s].pred_masks.cpu().numpy().astype("uint8")[0]
                    segment = []
                    contours = measure.find_contours(bimask, 0.5)
                    for contour in contours:
                        contour = np.flip(contour, axis=1)
                        segmentation = contour.ravel().tolist()
                        segment.append(segmentation)
                    bimask = np.expand_dims(bimask, axis=-1)
                    Rs = mask.encode(np.asfortranarray(np.uint8(bimask)))
                    area = int(mask.area(Rs)[0]),
                    bbox_seg = mask.toBbox(Rs)[0].tolist()
                    test_json['annotations'].append(
                        {'image_id': int(b + j + 1), 'id': segment_id, 'segmentation': segment,
                         'bbox': outputs[j]['instances'][s].pred_boxes.tensor.cpu().numpy()[0].tolist()
                            , 'category_id': int(outputs[j]['instances'][s].pred_classes.cpu().numpy()[0]) + 1,
                         'score': float(outputs[j]['instances'][s].scores.cpu().numpy()[0]), 'area': int(area[0])})
                    segment_id += 1
    print('Total inference time for ', len(test_imgs_transform), ' frames:', time.time()-start)
    test_json['categories'] =[{'id': 1, 'name': 'table_top'}, {'id': 2, 'name': 'leg'}, {'id': 3, 'name': 'shelf'},
                          {'id': 4, 'name': 'side_panel'}, {'id': 5, 'name': 'front_panel'}, {'id': 6, 'name': 'bottom_panel'},
                          {'id': 7, 'name': 'rear_panel'}]
    with open(os.path.join(cfg.OUTPUT_DIR, args.s + '.json'), 'w') as outfile:
        json.dump(test_json, outfile)
        print('Output json file successfully created!')


# Copyright (C) ATHENA AUTHORS; Ruixiong Zhang
# All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# pylint: disable=invalid-name
""" language model scorer used in speech recognition model decoding """
import tensorflow as tf
import numpy as np
import kenlm

class NGramScorer(object):
    """
    KenLM language model
    """

    def __init__(self, lm_path, sos, eos, num_syms, lm_weight=0.1):
        """
        Basic params will be initialized, the kenlm model will be created from
        the lm_path
        Args:
            lm_path: the saved lm model path
            sos: start symbol
            eos: end symbol
            num_syms: number of classes
            lm_weight: the lm weight
        """
        self.lang_model = kenlm.Model(lm_path)
        self.state_index = 0
        self.sos = sos
        self.eos = eos
        self.num_syms = num_syms
        self.lm_weight = lm_weight
        kenlm_state = kenlm.State()
        self.lang_model.BeginSentenceWrite(kenlm_state)
        self.cand_kenlm_states = np.array([[kenlm_state] * num_syms])

    def reset(self):
        """
        Call this function to reset the lm to predict on a new sequence
        """
        kenlm_state = kenlm.State()
        self.lang_model.BeginSentenceWrite(kenlm_state)
        self.cand_kenlm_states = np.array([[kenlm_state] * self.num_syms])

    def score(self, candidate_holder, new_scores):
        """
        Call this function to compute the NGram score of the next prediction
        based on historical predictions, the scoring function shares a common interface
        Args:
            candidate_holder:
        Returns:
            score: the NGram weighted score
            cand_states:
        """
        cand_seqs = candidate_holder.cand_seqs
        cand_parents = candidate_holder.cand_parents
        cand_syms = cand_seqs[:, -1]
        score = self.get_score(cand_parents, cand_syms, self.lang_model)
        score = self.lm_weight * score
        return score, candidate_holder.cand_states

    def get_score(self, cand_parents, cand_syms, lang_model):
        """
        the saved lm model will be called here
        Args:
            cand_parents: last selected top candidates
            cand_syms: last selected top char index
            lang_model: the language model
        Return:
            scores: the lm scores
        """
        scale = 1.0 / np.log10(np.e)  # convert log10 to ln

        num_cands = len(cand_syms)
        scores = np.zeros((num_cands, self.num_syms))
        new_states = np.zeros((num_cands, self.num_syms), dtype=object)
        chars = [str(x) for x in range(self.num_syms)]
        chars[self.sos] = "<s>"
        chars[self.eos] = "</s>"
        chars[0] = "<space>"

        for i in range(num_cands):
            parent = cand_parents[i]
            kenlm_state_list = self.cand_kenlm_states[parent]
            kenlm_state = kenlm_state_list[cand_syms[i]]
            for sym in range(self.num_syms):
                char = chars[sym]
                out_state = kenlm.State()
                score = scale * lang_model.BaseScore(kenlm_state, char, out_state)
                scores[i, sym] = score
                new_states[i, sym] = out_state
        self.cand_kenlm_states = new_states
        return scores

class RNNScorer(object):
    """
    RNN language model
    """

    def __init__(self, lm_model, lm_weight=0.1):
        """
        Basic params will be initialized
        Args:
            lm_model: the loaded rnn language model
        """
        self.lang_model = lm_model
        self.lm_weight = lm_weight

    def reset(self):
        """
        Call this function to reset the lm to predict on a new sequence
        """
        pass

    def score(self, candidate_holder, new_scores):
        """
        Call this function to compute the score of the next prediction
        based on historical predictions, the scoring function shares a common interface
        Args:
            candidate_holder:
        Returns:
            score: the weighted score
            cand_states:
        """
        cand_seqs = candidate_holder.cand_seqs
        score = self.lang_model.rnnlm(cand_seqs, training=False)
        score = score[:, -1, :]
        score = self.lm_weight * tf.math.log(tf.nn.softmax(score))
        return score, candidate_holder.cand_states