import luigi
import os
import nestedtext as nt
import shutil
import pandas as pd
import hashlib
import subprocess
import gzip
import tarfile
from misc import *
from getglobals import PARAMSET_GLOBALS

class INSTINCT_pipeline:

    #run() defined in specific pipeline

    def __init__(self,params,n,pipe_args,upstream,namespace,compdef,n_ref,dag):

        #print(n)

        self.params=params
        self.n=n
        self.n_ref=n_ref
        self.namespace = namespace

        if pipe_args!={}:

            pipe_args = pipelink_unpack(pipe_args) #contains a hidden 'if' to check for pipe_link

            if 'params_drop' in pipe_args:
                #print("******did params drop!*******")
                #import code
                #code.interact(local=dict(globals(), **locals()))
                #print(self.pipeID)
                #print(pipe_args['params_drop'])
                #print("******did params drop!*******")
                #print(pipe_args['params_drop'])
                self.params = param_smoosh(self.params,pipe_args['params_drop'])
                #print(self.params)

            #test if .pipe is referenced in pipeline#
            pipe_args,pipe_val,compdef,pipeID =pipetest_pargs_pval_compdef(pipe_args,namespace)

            #print(pipe_args)
            self.pipe_args=pipe_args[self.__class__.__name__] #drop to name of pipeline in pipe_args

        #print(self.pipe_args)
        self.upstream = upstream

        self.pipeID = pipeID

        #pass copies, so the originals aren't modified up the pipe
        self.params_copy = self.params.copy()
        self.pipe_args_copy = self.pipe_args.copy()

        #compdef will be specified as an attribute for defined pipelines (the not .pipe syntax version)

        self.compdef=compdef
        self.dag = dag

        #import code
        #code.interact(local=dict(globals(), **locals()))

    def run_component(self,component=None,final_process=False,upstream=[None]):

        #print(component)
        #print(self.n)
        #print(self.n_ref)

        if final_process==True:
            #print(self.namespace[self.pipe_args['process']])
                    #if cls.__name__ == "DLmodel_Test":
                        #import code
                        #code.interact(local=dict(globals(), **locals()))
            #
            component_name = self.pipe_args['process']
            n_use = pipeline_n_extract(self.n,self.n_ref,component_name)

            #print(n_use)
            #print(self.n_ref)
            #print(self.namespace[component_name])

            #I changed the params passing from self.params_copy to self.params... keep an eye on other ramifications of this change.
            process = self.namespace[component_name].invoke(self.params_copy,n=n_use,n_ref=self.n_ref,pipe_args=self.pipe_args_copy,compdef=None,\
                                                                            upstream=upstream,pipeID=self.pipeID,namespace = self.namespace,dag=self.dag)
            #print(self.pipe_args_copy)
            #print(self.params_copy["Job"]["DLmodel*"])
            #if self.pipe_args_copy['process'] == "DLmodel_Test":
            #    import code
            #    code.interact(local=dict(globals(), **locals()))
            return process
        elif component in self.pipe_args:

            #here, test if pipe or process
            #if component =="GetDETx":

            #assign temporary pipe args so any modifications will not be retained
            pipe_args = self.pipe_args_copy.copy()

            #test- similar to above.
            params = self.params_copy.copy()

            #import code
            #code.interact(local=dict(globals(), **locals()))

            #print(pipe_args[component])

            #component will never have both pipe and process in the same level. (?)
            if 'process' in pipe_args[component]:

                component_name = pipe_args[component]['process']
                n_use = pipeline_n_extract(self.n,self.n_ref,component_name)
                #print(n_use)
                #print(self.n_ref)
                #print(self.namespace[component_name])

                return self.namespace[component_name].invoke(params,n=n_use,n_ref=self.n_ref,pipe_args=pipe_args[component],upstream=upstream,\
                                                                       pipeID=self.pipeID,namespace = self.namespace,dag=self.dag,compdef=None)
            elif 'pipe' in pipe_args[component]:

                #test turn this off.
                #pipe_args,pipe_val,compdef,pipeID =pipetest_pargs_pval_compdef(self.pipe_args[component],self.namespace)

                #test possible bugfix
                pipe_args,pipe_val,compdef,pipeID =pipetest_pargs_pval_compdef(pipe_args[component],self.namespace)

                #although this didn't fix the current bug, it is probably more correct so leave it.
                #now, test saving a copy of params in the style of pipe_args above to avoid this persisting between components

                    #here  is where I'd test for the loop conditional

                if 'loop_on' in pipe_args: #and self.n==False: #not sure if this does anything..?

                    #check to see if there is an novr

                    #print(component)

                    #if component== "GetPE1_S1":
                    #    import code
                    #    code.interact(local=dict(globals(), **locals()))

                    #print(pipe_args['loop_on'])

                    #import code
                    #code.interact(local=dict(globals(), **locals()))
                    if 'reset_loop' in pipe_args:
                        if pipe_args['reset_loop'] == 'y':
                            do_reset = True
                    else:
                        do_reset = False

                    #print(component)
                    #print(upstream)
                    #print(self.n_ref)
                    #print(self.n)

                    #if len(self.n)==2 and len(self.n_ref)==1:
                    #    import code
                    #    code.interact(local=dict(globals(), **locals()))
                    #print(self.n)
                    #print(self.n_ref)

                    return CombineExtLoop.invoke(params=params,component=pipe_val,pipe_args=pipe_args,pipeID=pipeID,\
                                                 namespace = self.namespace,compdef=compdef,process_peek = pipe_args['loop_on'],dag=self.dag,\
                                                 n=self.n, n_ref = self.n_ref, reset_loop = do_reset)
                else:

                    #reworking looping. Need to see what vars are available here to plug into pipeline_n_extract
                    #import code
                    #code.interact(local=dict(globals(), **locals()))

                    #assert 1==2
                    #print("THIS HAPPENED")

                    return pipe_val.invoke(params,n=self.n,n_ref=self.n_ref,pipe_args=pipe_args,upstream=upstream,\
                                           pipeID=pipeID,compdef=compdef,namespace = self.namespace,dag=self.dag)
        else:

            #import code
            #code.interact(local=locals())
            if component in self.namespace:
                #this means that there is a hardcoded process or pipeline as the component.
                #When hardcoding compdef, use a string to identify process or pipeline in namespace.
                #once you hardcode a component, you cannot go back to pipeargs (everything upstream also must be hardcoded
                self.namespace[component].invoke(self.params_copy,n=self.n,n_ref=self.n_ref,pipe_args={},upstream=upstream,\
                                               pipeID=pipeID,compdef=None,namespace = self.namespace,dag=self.dag)
            else:
                #this means that the component is not assigned. So, we return None. This can be done intentionally or unintentionally, be careful
                return None

            #what this also means, is that components cannot share namespace with pipelines/processes!

    def loop_component(self,component=None,final_process=False,upstream=[None],process_peek=None):

        #unsure if this is being used, but if so want to see if I can
        import code
        code.interact(local=dict(globals(), **locals()))
        if 'reset_loop' in pipe_args:
            if pipe_args['reset_loop'] == 'y':
                do_reset = True
        else:
            do_reset = False

        return CombineExtLoop.invoke(params=self.params_copy,component=component,pipe_args={},pipeID=self.pipeID,\
                                             namespace = self.namespace,compdef=None,process_peek = process_peek,dag=self.dag,\
                                     n=self.n,n_ref=self.n_ref,reset_loop = do_reset)

    @classmethod
    def invoke(cls,params,pipe_args={},upstream=[None],n='default',pipeID=None,namespace = None,compdef = None,dag=[],n_ref=None):

        #drop to pipeline name param level if present in keys.
        if cls.__name__ in params:
            params = param_smoosh(params,cls.__name__)

        #print(params["DLmodel*"])

        return cls(params,n,pipe_args,upstream,namespace,compdef,n_ref,dag=dag).run()

class INSTINCT_task(luigi.Task):

    ports = luigi.Parameter(significant=False) #these are the upstream ports , in order.

    ports_hash = luigi.ListParameter() #these are included to be hashed.

    #internal variable definitions
    hash_length = int(os.environ.get('HASHLEN'))

    hash_file = False #for processes which need to do this, specify this as true

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self._Task__hash = int(str(self._Task__hash)[1:(1+self.hash_length)])

    def requires(self):
        if self.ports[0]!=None:
            for L in range(len(self.ports)):
                yield self.ports[L]

    def outfilegen(self):

        return self.outpath() + "/" + self.outfile

    def outpath(self):

        if self.ports[0]==None:
            #default to CacheRoot
            path = PARAMSET_GLOBALS['cache_root'] +'/'+ str(self._Task__hash)

        else:
            path = self.ports[0].outpath() +'/'+ str(self._Task__hash)

        if not os.path.exists(path): #might be redundant, make sure not slowing anything down
            os.makedirs(path)

        return path

    #convenience
    def getports(upstream):
        #print(upstream)
        #print(len(upstream))
        if upstream[0]!=None:
            ports = [upstream[L] for L in range(len(upstream))]
            ports_hash = [upstream[L]._Task__hash for L in range(len(upstream))]
        else:
            ports = [None]
            ports_hash = [None]
        return [ports,ports_hash]

class INSTINCT_process(INSTINCT_task):

    parameters = luigi.DictParameter(significant=False) #this is referenced within task
    param_string = luigi.Parameter() #this is only hashed

    param_string2 = luigi.Parameter(significant=False) #this is passed to run_cmd

    rerun_key = luigi.IntParameter() #all this does is change the hash to a unique value

    arguments = luigi.DictParameter(significant=False)
    descriptors = luigi.DictParameter(significant=False)

    processID = luigi.Parameter(significant=False)

    pipeID= luigi.Parameter(significant=False) #reference the pipeline that called the process

    def output(self):
        return luigi.LocalTarget(self.outfilegen())
    def run_cmd(self):
        #1st iteration of this, might modify

        methodjoin = '-'

        if self.descriptors['language']=='R':
            executable1 = 'Rscript '
            executable2 = '.R'
        elif self.descriptors['language']=='Python':
            executable1 = 'python '
            executable2 = '.py'
        elif self.descriptors['language']=='MATLAB':
            executable1 = ''
            executable2 = '.mat'
        elif self.descriptors['language']=='batch':
            executable1 = ''
            executable2 = '.bat'

        if self.descriptors['runtype']=='bin':
            executable2 = '.exe'
            methodjoin = '' #matlab does not allow for '-' in name
        #print(self.cmd_args)

        command0 = executable1 + PARAMSET_GLOBALS['project_root'] + 'lib/user/methods/' + self.processID +\
                       '/' + self.parameters['methodID'] + '/' + self.parameters['methodID'] + methodjoin + self.parameters['methodvers'] + executable2

        wrapper,wrapper_value = keyassess('wrapper',self.descriptors)
        if wrapper_value!='True':
            #import code
            #code.interact(local=locals())
            command2 = os.environ["INS_ARG_SEP"].join(self.cmd_args)

            command = command0 + ' '+ command2

        else:
            command1 = executable1 + PARAMSET_GLOBALS['project_root'] + 'lib/user/methods/' + self.processID + '/'+\
            self.processID  +"Wrapper" + executable2

            command2 = PARAMSET_GLOBALS['project_root'] + os.environ["INS_ARG_SEP"]+ os.environ["INS_ARG_SEP"].join(self.cmd_args)

            command = command1 + ' '+ command2

        if 'venv' in self.descriptors:
            print("******************************\nActivating virtual environment " + self.descriptors['venv_name'] + " with " + self.descriptors['venv'] + "\n******************************")
            if(self.descriptors['venv']=='Conda'):
                command_venv = self.descriptors['venv'] + ' activate ' + self.descriptors['venv_name'] + ' & '
                command = command_venv + command #append venv call to start of command. This assumes conda venv is set up to work on command line.

                pass_sub,pass_sub_val = keyassess('pass_sub_to_wrapper',self.descriptors)
                if pass_sub_val=='True':

                    command = command + os.environ["INS_ARG_SEP"]+ command0

                #import code
                #code.interact(local=dict(globals(), **locals()))
            else:
                print("VENV NOT YET CONFIGURED TO WORK ON BASE VENV")

        print("******************************\nRunning " + self.descriptors['language'] + " method " + self.parameters['methodID'] + methodjoin + self.parameters['methodvers'] +\
              " for process " + self.processID + "\n******************************")

        #temp:
        #print(command)

        print("******************************\nCommand params (can copy and paste): " + command2 +"\n******************************")

        #import code
        #code.interact(local=locals())
        subprocess.run(command,shell=True)

    #pipe_args is not refered to here since it is a placeholder to allow for dynamic calling between pipelines/processes
    @classmethod
    def invoke(cls,params,upstream=[None],n='default',pipe_args={},pipeID=None,namespace=None,dag=None,compdef=None,n_ref=None):

        #print(cls)
        #print(n)

        keytest,value = keyassess('drop',pipe_args)

        if keytest and value!=False:

            params = param_smoosh(params,value)

        paramssave = params

        processID = cls.__name__ #overwrite this if have to drop further to find params
        #print(cls.__name__)

        #if process name present, drop to process
        if cls.__name__ in params:
            params = params[cls.__name__] #always drop to processes available in params: control overrides in param file
        #if not present, try to use character matching.
        else:
            for f in range(len(list(params.keys()))):
                string_candidate = list(params.keys())[f]
                if '*' in string_candidate:

                    string_candidate_test = string_candidate[0:string_candidate.find('*')]

                    if string_candidate_test in cls.__name__:
                        params = params[string_candidate] #if candidate contains string and wildcard
                        processID = string_candidate_test
                        break
        #print(params)

        if 'parameters' in params:

            #if cls.__name__!="FormatGT":

            parameters = params['parameters'].copy()

            if len(parameters)>0:

                #if n==1:
                #import code
                #code.interact(local=locals())

                #extract methodID and method vers
                if params['descriptors']['runtype']!= "no_method":
                    methodparams = { k:v for k,v in parameters.items() if 'methodID' in k}
                    methodparams.update({ k:v for k,v in parameters.items() if 'methodvers' in k})

                    del parameters['methodID']
                    del parameters['methodvers']

                prm_tlist= sorted(parameters.items())

                prm_vals = [prm_tlist[x][1] for x in range(len(prm_tlist))]

                #import code
                #code.interact(local=dict(globals(), **locals()))

                if n != 'default':
                    #print('didit')
                    #print(n)
                    prm_vals_sort = [sorted(prm_vals[x])[n] if isinstance(prm_vals[x],list) else prm_vals[x] for x in range(len(prm_vals))]

                else:
                    #or resort the list so that it hashes in a standard way.
                    prm_vals_sort = [','.join(sorted(prm_vals[x])) if isinstance(prm_vals[x],list) else prm_vals[x] for x in range(len(prm_tlist))]

                #here, check for global parameters and inject in case of match
                for i in range(len(prm_vals_sort)):
                    while prm_vals_sort[i].startswith("[") and prm_vals_sort[i].endswith("]"): #attempt a bugfix

                        prm_vals_sort[i] = PARAMSET_GLOBALS['parameters'][prm_vals_sort[i][1:len(prm_vals_sort[i])-1]]
                    #also, check for leading and trailing whitespace
                    #prm_vals_sort[i] = prm_vals_sort[i].strip()

                #remove /n from strings if present
                for i in range(len(prm_vals_sort)):
                    if "\n" in prm_vals_sort[i]:
                        prm_vals_sort[i] = prm_vals_sort[i].replace('\n','')
                        #import code
                        #code.interact(local=dict(globals(), **locals()))

                param_string = ' '.join(prm_vals_sort)
                param_string2 = os.environ["INS_ARG_SEP"].join(prm_vals_sort)

                #if cls.__name__=="EventDetector":

                #append on the values for methodID and method to param string
                if params['descriptors']['runtype']!= "no_method":
                #squish together methodID and methodvers to be more consistent with existing R methods etc.
                    param_string = param_string + ' '+ methodparams['methodID'] + "-" + methodparams['methodvers']
                    param_string2 = param_string2 + os.environ["INS_ARG_SEP"]+ methodparams['methodID'] + "-" + methodparams['methodvers']

                    parameters.update(methodparams)

            else:
                param_string = ''
                param_string2 = ''

            #change the n value within parameters
            #if n!= 'default':
            #import code
            #code.interact(local=locals())
            keys = list(parameters.keys())
            any_list = False
            for i in range(len(keys)):
                if type(parameters[keys[i]])==list:
                    any_list=True
                    parameters_n = parameters.copy()
                    #print(n)
                    #import code
                    #code.interact(local=dict(globals(), **locals()))
                    if n == 'default':
                        n_ = 0
                    else:
                        n_ = n
                    parameters_n[keys[i]]=sorted(parameters[keys[i]])[n_]
            if any_list==False:
                parameters_n = parameters
            #else:
                #parameters_n = parameters

        else:
            parameters_n = None
            param_string =""
            param_string2 =""

        #print(parameters_n)

        if 'arguments' in params:
            arguments = params['arguments'].copy()
            if 'splits' not in arguments:
                arguments.update({'splits':'1'})#

            #test for presence of global parameters and populate.
            arg_vlist= list(arguments.values())
            arg_klist= list(arguments.keys())
            for i in range(len(arg_vlist)):
                val = arg_vlist[i]
                if val.startswith("[") and val.endswith("]"):
                    #import code
                    #code.interact(local=dict(globals(), **locals()))
                    arguments[arg_klist[i]] = PARAMSET_GLOBALS['parameters'][val[1:len(val)-1]]
        else:
            arguments = {'splits':'1'}
        if 'descriptors' in params:
            descriptors = params['descriptors'].copy()
        else:
            descriptors = None
        #get outpath and hash from ports

        #print(param_string)

        portsout = cls.getports(upstream)

        #if has an argument to rerun process, test for completion, and delete output if complete.
        #this is useful for tasks where the state of the precursor is intentionally not address (a db for example)
        #and use cases arise where remaining consistent is more important than using the most up to date info

        rerun_key = 0
        if 'rerun_key' in arguments:
            rerun_key = int(arguments['rerun_key'])

        process = cls(parameters=parameters_n,param_string=param_string,param_string2=param_string2,arguments=arguments,descriptors=descriptors,\
                   ports=portsout[0],ports_hash=portsout[1],processID=processID,pipeID=pipeID,rerun_key=rerun_key)

        #downstream will be set to empty to when just populating hashes for combextloop.
        if 'hard_rerun' in arguments:
            if arguments['hard_rerun']=='y' and dag!=None:
                #delete all of the downstream outputs
                dag._clean(process,'downstream')

        if 'trim_tree' in arguments:
            if arguments['trim_tree']=='y' and dag!=None:
                #delete all of the downstream outputs
                dag._clean(process,'upstream')

        return(process)

class INSTINCT_userprocess(INSTINCT_process):

    #instinct userprocess has the same run, but calls a specific instructions
    def get_modules_Art(self):

        from getglobals import PROJECT
        from art import getArt
        import random

        art = getArt(PROJECT,True,random.randint(1,10))

        return art

    def input_w_art(self,message):

        mlen = len(message)
        spaceslen = 40
        messagebuffer = " "* (spaceslen-round(mlen/2))

        out = input("\n " + r"""                  '.'.,'; '. .'.,',; ,' :','..'.,', '. ; ,'""" + "\n" +\
                    messagebuffer + message + "\n" + self.get_modules_Art() + "\n(y/n):")
        return out

    def get_userfile(self,only_file=False):

        file = self.outfile
        outf1 = file[0:file.find(".")]
        outf2 = file[self.outfile.find("."):]
        outmod = outf1 + self.userfile_append + outf2

        if only_file ==False:
            out =self.outpath() + "/" +outmod
        elif only_file==True:
            out = outmod
        return out

    def user_cancelation(self):
        os.environ["USERREVIEW"] = "False"
        os.environ["CANCELREVIEW"] = "True"
    def print_manifest(self):

        os.system("start " + self.outpath().replace('/','\\')) #windows friendly for copy paste (matters on NAS drive)

        text = 'INPUT: file named: "' + self.outfile + '" in opened directory: "' + self.outpath().replace('/','\\') + '"\n\n' +\
               'INFO:' + self.get_info()+ '"\n\n' +\
                "INSTRUCTIONS: " + self.parameters['instructions'] + '\n\nOUTPUT: save a new copy of the file in the opened directory as: "' + self.get_userfile(True) + '" \n'+\
                "When task is completed, enter y to Proceed, or n to abort." + "\n\n" +\
                "Proceed?"

        return text

    def file_modify(self,file): #if you want to modify the file prior to user working on it, override this in specific implementation
        print("")
        #for example, this could take a RAVENx file and add a column for analyst (based on user ID)
    def run(self):

        if os.environ.get('USERREVIEW')!="False":
            #this is the default value

            userreview = self.input_w_art("A userprocess task awaits! Perfom now?") #A userprocess task awaits! Perfom now?

            if userreview == 'y':
                os.environ["USERREVIEW"] = "True"

                #make a change: before calling this, move files into this task output folder- will be much more clear if running different versions, otherwise have already populated
                #files and folders which will be confusing to the user.

                shutil.copy(self.ports[0].outfilegen(), self.outfilegen())

                self.file_modify(self.outfilegen()) #in the specific usage of this class, you can change file_modify to prepare file for analysis.

                taskcomplete = self.input_w_art(self.print_manifest())

                if taskcomplete=='y':
                    #in this case, rename and move the file to correct output format.
                    #import code
                    #code.interact(local=dict(globals(), **locals()))

                    shutil.copy(self.get_userfile(), self.outfilegen())

                elif taskcomplete=='n':
                    self.user_cancelation()
                    os.remove(self.outfilegen())
                else:
                    raise ValueError("Invalid input to (y/n) prompt")

            elif userreview == 'n':
                self.user_cancelation() #this signifies when printing job output that a review was not performed
            else:
                raise ValueError("Invalid input to (y/n) prompt")
        #otherwise, do nothing and let the task fail
        #import code
        #code.interact(local=dict(globals(), **locals()))

        #idea: prompt user whether they want to perform tasks now, save preference as a temporary env variable if not.

class Split_process:

    #reason these are loaded in this way instead of appended to arguments is because arguments is a 'frozendict'
    splits = luigi.IntParameter()
    split_ID = luigi.IntParameter()
    process_ID = luigi.Parameter(significant=False)

    outpath_str = luigi.Parameter(significant=False)

    def outpath(self):
        return self.outpath_str
    def outfilegen(self,only_file=False):

        #import code
        #code.interact(local=locals())
        file = self.outfile
        outf1 = file[0:file.find(".")]
        outf2 = file[self.outfile.find("."):]
        outmod = outf1 + str(self.split_ID) + '_' + str(self.splits) + outf2
        #adding in splitnum here keeps names unique and does not cause partial outputs
        #to be interpreted as different size chunk between runs.
        if only_file ==False:
            out =self.outpath() + "/" +outmod #testing
        elif only_file==True:
            out = outmod
        return out

class SplitRun_process:
    def requires(self):
        return self.SplitInitial(parameters=self.parameters,param_string=self.param_string,param_string2=self.param_string2,arguments=self.arguments,\
                                 descriptors=self.descriptors,ports=self.ports,ports_hash=self.ports_hash,splits=self.splits,\
                                 split_ID=self.split_ID,process_ID=self.process_ID,outpath_str = self.outpath(),processID=self.processID,\
                                 pipeID=self.pipeID,rerun_key=self.rerun_key)

class Unify_process:

    def requires(self):
        for k in range(int(self.arguments['splits'])):
            yield self.SplitRun(parameters=self.parameters,param_string=self.param_string,param_string2=self.param_string2,arguments=self.arguments,\
                                descriptors=self.descriptors,ports=self.ports,ports_hash=self.ports_hash,splits=int(self.arguments['splits']),\
                                split_ID=k+1,process_ID=self.__class__.__name__,outpath_str = self.outpath(),processID=self.processID,\
                                pipeID=self.pipeID,rerun_key=self.rerun_key)
#pipelines:

#note: the looping function currently relies on some AFSC submodule file formats, and supporting functions, to correctly combine outputs.
class CombineExtLoop(INSTINCT_task):

    outfile = luigi.Parameter(significant=False)

    pipeID= luigi.Parameter(significant=False) #reference the pipeline that called the process

    def output(self):
        return luigi.LocalTarget(self.outfilegen())
    def run(self):

        if '.txt' in self.outfile: #this assumes it's going to be a raven format- can get more specific here
            #if you need other .txt formats.

            dataframes = [None] * len(self.ports)
            for L in range(len(self.ports)):
                dataframes[L] = pd.read_csv(self.ports[L].outfilegen(), delimiter = "\t") #
            Dat = pd.concat(dataframes,ignore_index=True)

            Dat["Selection"] = Dat.index + 1

            #import code
            #code.interact(local=locals())
            Dat.to_csv(self.outfilegen(),index=False, sep = "\t")
        elif self.outfile=="no_comb" and ('.tgz' in self.ports[0].outfilegen() or '.tar.gz' in self.ports[0].outfilegen()):

            os.mkdir(self.outfilegen())

            for L in range(len(self.ports)):
                outpath_name = self.outfilegen() +"/" + str(L+1) + "_" + os.path.basename(self.ports[L].outfilegen()[:self.ports[L].outfilegen().index(".")])
                os.mkdir(outpath_name)
                with tarfile.open(self.ports[L].outfilegen(), "r:gz") as tar:

                    #outdir =tarfile.open(self.outfile)
                    for member in tar:
                        if member.isdir():
                            continue
                        fname = member.name.rsplit('/',1)[1]
                        tar.makefile(member, outpath_name + '/' + fname)

                tar.close()

            #import code #unpack before combining into folder! Use commented out code at bottom of elifs
            #code.interact(local=dict(globals(), **locals()))

                    #tar.extract(outpath_name)

            #create a directory called no_comb with all of the files
            #os.mkdir(self.outfilegen())
            #for L in range(len(self.ports)):
            #    shutil.copy(self.ports[L].outfilegen(), self.outfilegen() +"/" + str(L+1) + "_" + self.ports[L].outfile)

        elif self.outfile=="no_comb":
            #create a directory called no_comb with all of the files
            os.mkdir(self.outfilegen())
            for L in range(len(self.ports)):
                shutil.copy(self.ports[L].outfilegen(), self.outfilegen() +"/" + str(L+1) + "_" + self.ports[L].outfile)

        elif 'FileGroupFormat.csv.gz' in self.outfile:

            #FG_dict = file_peek(self.ports[0].outfilegen(),fn_type = object,fp_type = object,st_type = object,dur_type = 'float64')

            dataframes = [None] * len(self.ports)
            for L in range(len(self.ports)):
                dataframes[L] = pd.read_csv(self.ports[L].outfilegen())#, dtype=FG_dict)
            Dat = pd.concat(dataframes,ignore_index=True)
            Dat['StartTime'] = pd.to_datetime(Dat['StartTime'])
            #import code
            #code.interact(local=locals())

            #Note: since I don't have access to the parameter to do this correctly, I should probably not do this at all.
            #however, do I know the implications if i don't do this here?
            Dat.drop(columns=['DiffTime'])

            Dat=get_difftime(Dat) #this is currently broken- difftime doesn't work between original format and format FG

            Dat.to_csv(self.outfilegen(),index=False)
        elif '.csv.gz' or '.csv.' in self.outfile:

            dataframes = [None] * len(self.ports)
            for L in range(len(self.ports)):
                dataframes[L] = pd.read_csv(self.ports[L].outfilegen()) #
            Dat = pd.concat(dataframes,ignore_index=True)

            Dat.to_csv(self.outfilegen(),index=False)
        #elif ".tgz" in self.outfile or ".tar.gz" in self.outfile:

            #pseudo: unpack all files into single directory, then tar that directory.

        #    import code
        #    code.interact(local=dict(globals(), **locals()))

        #    for L in range(len(self.ports)):
        #        outnamepath = self.outfile[:self.ports[L].outfilegen().index(".")]
        #        outdir =tarfile.open(self.outfile)

        #        outdir.extractall(outnamepath)

    @classmethod
    def invoke(cls,params,component,pipe_args={},upstream=[None],process_peek=None,novr='default',pipeID=None,namespace = None,compdef=None,dag=None,
               n_ref=None,n='default',reset_loop=False):
        #params in this case, just means a level above processes/subroutines

        #save these variables as copies so I don't modify self.n_ref

        #n = n.copy()
        #print(n)
        #print(isinstance(n, int))
        #print(n_ref)
        #print(process_peek)
        if n_ref!=None:
            n_ref = n_ref.copy()
        #import code
        #code.interact(local=locals())

        #if this is true, blank out n_ref and n
        if reset_loop:
            n_ref = None
            n = 'default'

        paramssave=params

        n_lens = []

        #make everything into lists
        if n_ref==None or isinstance(n_ref, str):
            n_ref= [n_ref]
            #n=[n]

        if isinstance(n, int):
            n = [n]

        if isinstance(process_peek, str):
            process_peek= [process_peek]

        for f in range(len(process_peek)):

            process_lookup = process_peek[f]

            parameters = params[process_lookup]["parameters"]

            n_len = None
            for i in parameters.keys():
                if type(parameters[i])==list:
                    n_len = len(parameters[i])

            if novr!= 'default':
                n_len = 1
                nval = novr
            else:
                nval = 0

            n_lens.append(range(n_len))

            #print(pipe_args)

        #print(process_peek)
        #print(n_lens)
        #print(n_ref)

        #might work a little better as a dict, but using the itertools below which seems to work better on lists.
        override_matches = set(process_peek) & set(n_ref)

        if len(override_matches)>0:
            #didthis=True
            override_matches_list = list(override_matches)

            for p in range(len(override_matches_list)):
                #new (process peek) will override old (n_ref).

                #find index of match in old
                del_ind = n_ref.index(override_matches_list[p])

                #print("DIDIT")

                n_ref = [x for i,x in enumerate(n_ref) if i!=del_ind]
                n = [x for i,x in enumerate(n) if i!=del_ind]

                #del n_ref[del_ind]
                #del n[del_ind]

                #print(n_ref)
                #print(n_ref != [None] and n_ref != [])
                #print(list(range(len(n))))

                override_matches = []
        #else:
        #    didthis = False

        #if didthis:
        #    print(n_lens)
        #    print(process_peek)

        if n_ref != [None] and n_ref != []:
            #if n_ref is not none need to append
            #append lists

            #if len(n_ref) !=len(n):
            #        import code
             #       code.interact(local=locals())

            for p in range(len(n)):
                n_lens.append(range(n[p],n[p]+1))
                process_peek.append(n_ref[p])

        #if didthis:
        #    print(n_lens)
        #    print(process_peek)

        #if is_list:
        if(len(process_peek)>1):

            import itertools
            perms = list(itertools.product(*n_lens))
            #convert tuples to list
            perms = [list(tup) for tup in perms]

            #assert len(perms[0]) == len(process_peek )
        else:

            perms =  n_lens[0]
            #import code
            #code.interact(local=locals())
        #else
        #    perms = n_lens[0]

        #print(perms)
        #print(process_peek)

        #print(n_ref)

        #[assert len(L)==len(process_peek) for L in perms]

        #here, determine if n_ref should be ignored (if absent), appended to (if unique) or overwritten (if the same).
        #also, see if I can access other parameters on the same level ('loop_reset') for instance.
        #if n_ref != None:

        #for p in n_ref:
        #print(perms)
        #print(process_peek)

        #print(perms)

        upstreamNew=[component.invoke(paramssave,n=L,n_ref=process_peek,pipe_args=pipe_args.copy(),upstream=upstream,pipeID=pipeID,namespace = namespace,\
                                      compdef=compdef,dag=dag) for L in perms]

        #this makes sure only unique inputs are retained. If not unique, why run it?
        upstreamNew = list(set(upstreamNew))
        n_len = len(upstreamNew)

        if n_len!=1:

            portsout = cls.getports(upstreamNew)

            noncombinable_formats = [".png","test",".tgz"]

            if any([(i in upstreamNew[0].outfile) for i in noncombinable_formats]):
                outmod = "no_comb"
            else:
                outmod = upstreamNew[0].outfile

            comb_process = cls(ports=portsout[0],ports_hash=portsout[1],outfile=outmod,pipeID=pipeID)

            return(comb_process)
        else:
            #if it isn't a loop, just return the component
            #import code
            #code.interact(local=locals())
            #print(perms)
            return component.invoke(paramssave,n=nval,pipe_args=pipe_args,upstream=upstream,pipeID=pipeID,namespace = namespace,compdef=compdef,dag=dag)

        #pass values into luigi task as parameters: outpath, hash, file type, etc.

class INSTINCT_job(INSTINCT_task):

    job_out=luigi.Parameter(significant=False) #means if output folder is changed, will keep hash but rerun
    #in new folder
    param_file_name=luigi.Parameter(significant=False) #for saving params. duplication helps with tracking.

    paramset_original=luigi.DictParameter(significant=False)

    pipenames=luigi.ListParameter(significant=False)

    def outpath(self):

        path = PARAMSET_GLOBALS['project_root'] +'Outputs/' + self.job_out + '/' + str(self._Task__hash)

        if not os.path.exists(path): #might be redundant, make sure not slowing anything down
            os.makedirs(path)

        return path

    def output(self):
        for n in range(len(self.ports)):
            filedest =self.outpath() + "/" + self.pipenames[n] + "_" + self.ports[n].outfile
            if '.gz' in filedest:
                filedest = filedest[0:-3]
            elif '.tgz' in filedest or '.tar.gz' in filedest:

                filedest = filedest[:filedest.index(".")]
            yield luigi.LocalTarget(filedest)

    def run(self):

        #find the outputs and dump them in common folder.

        nt.dump(self.paramset_original, self.outpath() +"/" +self.param_file_name +"_archive.nt")

        for n in range(len(self.ports)):

            if self.ports[n].outfile!="no_comb":

                #here, look for .gz and unzip before transfering! Make output not outfile but a non .gz modification of outfile.
                #import code
                #code.interact(local=locals())

                filepath = self.ports[n].outfilegen()

                filedest =self.outpath() + "/" + self.pipenames[n] + "_" + self.ports[n].outfile

                if '.gz' in filepath:
                    outnamepath = filedest[0:-3]
                    with gzip.open(filepath, 'rb') as f_in:
                        with open(outnamepath, 'wb') as f_out:
                            shutil.copyfileobj(f_in, f_out)
                elif '.tgz' in filepath or '.tar.gz' in filepath:

                    outnamepath = filedest[:filedest.index(".")]
                    outdir =tarfile.open(filepath)

                    outdir.extractall(outnamepath)

                    #import code
                    #code.interact(local=dict(globals(), **locals()))
                else:
                    shutil.copy(filepath, filedest)

            else:

                shutil.copytree(self.ports[n].outfilegen(), self.outpath() + "/" + self.pipenames[n] + "_" +  self.ports[n].outfile)

                #import code
                #code.interact(local=locals())

            #add in functionality here to unzip files in outputs!

    @classmethod
    def invoke(cls,job,paramset_original,param_file_name,pipenames,pipeID=None): #testing out change- params was 2nd arg here

        #import code
        #code.interact(local=locals())

        job_out=list(job.keys())[0]
        upstream=list(job.values())[0]

        #print(upstream[0].outpath())

        portsout = cls.getports(upstream)

        return(cls(ports=portsout[0],ports_hash=portsout[1],job_out=job_out,pipenames=pipenames,\
                   paramset_original=paramset_original,param_file_name=param_file_name))

class INSTINCT_job_wrapper(luigi.WrapperTask,INSTINCT_job): #may not work, as both inherit from luigi.task...
    pass

##############################################


from django.urls import path,include
from .views import (User,Profile,PostAPIView,
                    UserPosts,Search,FollowerAPIView,
                    FollowingAPIView,FeedAPIView,StartFollowAPIView,
                    StopFollowAPIView,UniqueEmailCheck,UniqueUsernameCheck,
                    Register,
                    UpdatePicAPIView,Profile_Detail)
from django.views.decorators.csrf import csrf_exempt

urlpatterns = [
    path('users/', User.as_view(),name="users"),
    path('rest-auth/',include("rest_auth.urls")),
    path('api-auth/',include("rest_framework.urls")),
    path("profile/<str:usrname>",Profile.as_view(),name="profile-detail"),
    path("create-post",PostAPIView.as_view(),name="postapi"),
    path("post/<int:pk>",PostAPIView.as_view(),name="single-post"),
    path("post/<int:pk>/edit",PostAPIView.as_view(),name="post-edit"),
    path("profile/<str:username>/posts",UserPosts.as_view(),name="user-posts"),
    path("search",Search.as_view(),name="search"),
    path("profile/<str:username>/followers",FollowerAPIView.as_view(),name="followers"),
    path("profile/<str:username>/following",FollowingAPIView.as_view(),name="following"),
    path("gethomefeed",FeedAPIView.as_view(),name="home-feed"),
    path("addfollow/<str:profile_username>",StartFollowAPIView.as_view(),name="start-follow"),
    path("removefollow/<str:profile_username>",StopFollowAPIView.as_view(),name="stop-follow"),
    path("doesemailexist",UniqueEmailCheck,name="check-email"),
    path("doesusernameexist",UniqueUsernameCheck,name="username-check"),
    path("register",Register,name="my-register"),
    # path("testing",csrf_exempt(Image.as_view()),name="dhgtd"),
    path("update-pic",UpdatePicAPIView.as_view(),name="update-pic"),
    path("profile-detail",Profile_Detail,name="profile-detail"),
]


from .abstract import ShortAssembler, ShortAssemblerWrapper, AsmStats, FilterGF
import os
import itertools
import subprocess
import functools

@functools.lru_cache(maxsize=8, typed=True)
def get_class_location(load):
    cmd = "{load} which class_run.py".format(load=load)
    loc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE).stdout.read().decode()
    if not loc:
        raise OSError("CLASS2 not found!")
    return loc.rstrip()

class Class2Wrapper(ShortAssemblerWrapper):

    __toolname__ = "class2"
    __tag__ = "Abundance"

    def __init__(self, aln_wrapper):

        super().__init__(aln_wrapper)
        if len(self.runs) > 0 and len(self.bams) > 0:
            for bam, run in itertools.product(self.bams, range(len(self.runs))):
                class2 = Class2(bam, run, create_link=False)
                class_filter = FilterGF(class2,
                                        tag=self.tag,
                                        monoexonic_threshold=self.mono_threshold,
                                        multiexonic_threshold=self.multi_threshold)
                self.add_edge(class2, class_filter)
                stat = AsmStats(class_filter)
                self.add_edge(class_filter, stat)
                self.add_to_gf(stat)
                continue

class Class2(ShortAssembler):

    def __init__(self, bam, run, create_link=True):

        super().__init__(bam, run, create_link=create_link)

    __toolname__ = "class2"

    @property
    def loader(self):
        return ["class2", "mikado"]

    @property
    def strand(self):
        return ""

    @property
    def program(self):
        return get_class_location(self.load)

    @property
    def cmd(self):
        load = self.load
        cmd = "{load} "
        extra = self.extra
        if self.extra:
            extra = "-c \"{}\"".format(self.extra)
        else:
            extra = ""
        threads = self.threads
        outdir = self.gfdir
        program = self.program
        link_src = self.link_src
        input = self.input
        output = self.output
        log = self.log
        gtf = os.path.join(self.gfdir, "transcripts.gtf")
        cmd += "{program} --clean --force {extra} -p {threads} {input[bam]} > {output[gf]} 2> {log} "
        if self._create_link:
            cmd += " ln -sf {link_src} {output[link]} && touch -h {output[link]}"
        cmd = cmd.format(**locals())
        return cmd

    @property
    def suffix(self):
        return "gtf"

    @property
    def input_reads(self):
        return ""


def r1():
    from PIL import Image, ImageFilter
    from pathlib import Path

    current_dir = ''
    filenames = Path(current_dir).glob('*')
    Path('new_dir').mkdir(parents=True, exist_ok=True)

    for file in filenames:
        if file.suffix in ['.jpg', '.png']:
            with Image.open(file) as img:
                img.load()
                new_img = img.filter(ImageFilter.EMBOSS)
                new_img.save(Path("new_dir", file))

def r3():
    import csv
    file = open("l9.csv", 'r')
    l = list(csv.reader(file, delimiter=','))
    summ = 0
    for i in l:
        summ = summ + (int(i[1]) * int(i[2]))
    for i in l:
        i.insert(1, ' - ')
        i.insert(3, ' шт. за ')
        i.insert(5, ' руб. ')
        print(*i)

    print('Итоговая сумма:', summ, 'руб.')

r1()
r3()


import json
from collections import OrderedDict

from djblets.testing.testcases import TestCase
from djblets.webapi.encoders import (JSONEncoderAdapter, WebAPIEncoder,
                                     XMLEncoderAdapter)

class EncoderAdapterTests(TestCase):
    """Tests encoding correctness of WebAPIEncoder adapters"""

    @classmethod
    def setUpClass(cls):
        super(EncoderAdapterTests, cls).setUpClass()

        data = OrderedDict()
        data['string_val'] = 'foobar'
        data['none_val'] = None
        data['dict_val'] = {'foo': 'bar'}
        data['bool_val'] = True
        data['scientific_val'] = 2.75e-15
        data['int_val'] = 42
        data['float_val'] = 3.14159
        data['list_val'] = [10, 'baz']

        cls.data = data

    def test_json_encoder_adapter(self):
        """Testing JSONEncoderAdapter.encode"""
        encoder = WebAPIEncoder()
        adapter = JSONEncoderAdapter(encoder)

        content = adapter.encode(self.data)
        self.assertEqual(content, json.dumps(self.data, sort_keys=True))

    def test_xml_encoder_adapter(self):
        """Testing XMLEncoderAdapter.encode"""
        encoder = WebAPIEncoder()
        adapter = XMLEncoderAdapter(encoder)

        expected = (
            '<?xml version="1.0" encoding="utf-8"?>\n'
            '<rsp>\n'
            ' <string_val>foobar</string_val>\n'
            ' <none_val>\n'
            ' </none_val>\n'
            ' <dict_val>\n'
            '  <foo>bar</foo>\n'
            ' </dict_val>\n'
            ' <bool_val>1</bool_val>\n'
            ' <scientific_val>2.75e-15</scientific_val>\n'
            ' <int_val>42</int_val>\n'
            ' <float_val>3.14159</float_val>\n'
            ' <list_val>\n'
            '  <array>\n'
            '   <item>10</item>\n'
            '   <item>baz</item>\n'
            '  </array>\n'
            ' </list_val>\n'
            '</rsp>'
        )

        content = adapter.encode(self.data)
        self.assertEqual(content, expected)


#!/usr/bin/env python

import os
import sys
import time
import usb.core
import argparse
import matplotlib.pyplot as plt

VID             = 0x24aa
PID             = 0x4000
BUF             = [0] * 8
HOST_TO_DEVICE  = 0x40
DEVICE_TO_HOST  = 0xC0
TIMEOUT_MS      = 1000

def get_spectrum():
    print("sending ACQUIRE")
    dev.ctrl_transfer(HOST_TO_DEVICE, 0xad, 0, 0, BUF, TIMEOUT_MS)

    bytes_to_read = args.pixels * 2
    print(f"reading {bytes_to_read} bytes from bulk endpoint")
    data = dev.read(0x82, bytes_to_read)

    if len(data) != bytes_to_read:
        print("ERROR: read %d bytes" % len(data))
        sys.exit(1)

    spectrum = []
    for i in range(args.pixels):
        spectrum.append(data[i*2] | (data[i*2 + 1] << 8))

    return spectrum

################################################################################
# parse command-line arguments
################################################################################

parser = argparse.ArgumentParser()
parser.add_argument("--integration-time-ms", type=int, default=400, help="default 400")
parser.add_argument("--gain-db",             type=int, default=8, help="default 8")
parser.add_argument("--count",               type=int, default=1, help="how many spectra to take")
parser.add_argument("--delay-ms",            type=int, default=10, help="how long to delay between spectra (default 10)")
parser.add_argument("--throwaways",          type=int, default=2, help="how many throwaways to take (default 2)")
parser.add_argument("--pixels",              type=int, default=1920, help="default 1920")
parser.add_argument("--pixel-mode",          type=int, default=None, choices=[0,1,2,3], help="(optional, no default)")
parser.add_argument("--plot",                action="store_true", help="display graph")
parser.add_argument("--outfile",             type=str, help="save spectra")
args = parser.parse_args()

################################################################################
# connect
################################################################################

dev = usb.core.find(idVendor=VID, idProduct=PID)
if dev is None:
    print(f"No matching spectrometer found (VID 0x{VID:04x}, PID 0x{PID:04x})")
    sys.exit(1)

if os.name == "posix":
    dev.set_configuration()
    usb.util.claim_interface(dev, 0)

# print(dev)

################################################################################
# set acquisition parameters
################################################################################

print("sending SET_INTEGRATION_TIME_MS -> %d ms" % args.integration_time_ms)
dev.ctrl_transfer(HOST_TO_DEVICE, 0xb2, args.integration_time_ms, 0, BUF, TIMEOUT_MS)

gain_ff = args.gain_db << 8
print(f"sending GAIN_DB -> {args.gain_db} (FunkyFloat 0x{gain_ff:04x})")
dev.ctrl_transfer(HOST_TO_DEVICE, 0xb7, gain_ff, 0, BUF, TIMEOUT_MS)

if args.pixel_mode is not None:
    print(f"setting pixel mode {args.pixel_mode}")
    dev.ctrl_transfer(HOST_TO_DEVICE, 0xfd, args.pixel_mode, 0, BUF, TIMEOUT_MS)
    print("sleeping 1 sec for pixel mode to 'take'")
    time.sleep(1)

################################################################################
# collect spectra
################################################################################

spectra = []
for i in range(args.count + args.throwaways):
    spectrum = get_spectrum()
    if i > 0:
        print(f"sleeping {args.delay_ms}ms")
        time.sleep(args.delay_ms / 1000.0)

    if i < args.throwaways:
        print("dumping throwaway")
    else:
        spectra.append(spectrum)
        print("storing")

################################################################################
# process spectra
################################################################################

print("max = %d" % max(max(spectra)))
print("min = %d" % min(min(spectra)))
print("sum = %e" % sum([sum(spectrum) for spectrum in spectra]))

if args.outfile:
    print(f"writing {args.outfile}")
    with open(args.outfile, 'w') as f:
        for i in range(args.pixels):
            f.write("%s\n" % ', '.join([str(a[i]) for a in spectra]))

if args.plot:
    [plt.plot(a) for a in spectra]
    plt.title(f"integration time {args.integration_time_ms}ms, gain {args.gain_db}dB, pixel mode {args.pixel_mode}, count {args.count}")
    plt.show()


import unittest
import uuid

import sys
import os

sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
import kafka_helper
from queue_client import QueueConfig
from queue_factory import QueueProducerCreator, QueueConsumerCreator

sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "log"))
from custom_log import LoggerFactory

class TestQueueClient(unittest.TestCase):
    def setUp(self) -> None:
        self.__config = QueueConfig(
            host=os.environ.get("QUEUE_HOST"),
            port=int(os.environ.get("QUEUE_PORT")),
            optional_param={
                "topic_name": "test_queue_helper",
                "timeout": 5000,
            },
        )

        # Topicが残っているケースがあるので、事前にチェックして、削除する
        # TODO : アドホックな対応なので、なぜTearDownでTopicが削除されてないのか？を調査して対処する
        exist_topic = kafka_helper.exist_kafka_topic(
            self.__config.optional_param["topic_name"],
            client_id="test",
            config=self.__config,
        )
        if exist_topic:
            kafka_helper.delete_kafka_topics(
                topics=[self.__config.optional_param["topic_name"]],
                client_id="test",
                config=self.__config,
            )

        # テスト用のtopicの作成
        kafka_helper.create_kafka_topics_if_needed(
            topics=[self.__config.optional_param["topic_name"]],
            client_id="test",
            config=self.__config,
        )

    def tearDown(self) -> None:
        # テスト用のtopicの削除
        kafka_helper.delete_kafka_topics(
            topics=[self.__config.optional_param["topic_name"]],
            client_id="test",
            config=self.__config,
        )

    def test_send_and_receive_body(self):
        # データの作成
        body_text = "これはテストです"

        # 送信
        TestUtil.send_body_into_queue(
            body_text=body_text, config=self.__config
        )

        # 受信
        messages = TestUtil.receive_body_from_queue(config=self.__config)
        self.assertEqual(1, len(messages))
        message = messages[0]
        self.assertEqual(body_text, message["body"])
        try:
            _ = uuid.UUID(message["id"])
        except Exception:
            self.fail("id type don't equal UUID type")

        # 再度受信処理をして、データないことを確認
        messages = TestUtil.receive_body_from_queue(config=self.__config)
        self.assertEqual(0, len(messages))

class TestUtil:
    @staticmethod
    def send_body_into_queue(body_text: str, config: QueueConfig):
        # QueueProducerのインスタンスを取得
        logger = LoggerFactory.get_logger(
            logger_type="print", logger_name="test"
        )
        producer = QueueProducerCreator.create_producer(
            producer_type="kafka", config=config, logger=logger
        )

        # uuidを作成し、本文情報と一緒にjsonに格納
        id = str(uuid.uuid4())
        message = {"id": id, "body": body_text}

        # uuidと本文情報をjsonに格納し、Queueに送付
        producer.produce(messages=[message])

    @staticmethod
    def receive_body_from_queue(config: QueueConfig) -> list:
        # QueueConsumerのインスタンスを取得
        logger = LoggerFactory.get_logger(
            logger_type="print", logger_name="test"
        )
        consumer = QueueConsumerCreator.create_consumer(
            consumer_type="kafka", config=config, logger=logger
        )

        # jsonデータを取得
        messages = consumer.consume()
        return messages

if __name__ == "__main__":
    unittest.main()


import math
from flask.ext.sqlalchemy import SQLAlchemy
from sqlalchemy.ext.hybrid import hybrid_property, hybrid_method
from sqlalchemy import func

db = SQLAlchemy()

class FoodTruck(db.Model):
    id = db.Column(db.Integer, primary_key=True)
    name = db.Column(db.String(100))
    facility_type = db.Column(db.String(20))
    address = db.Column(db.String(200))
    food_items = db.Column(db.Text)
    latitude = db.Column(db.Float)
    longitude = db.Column(db.Float)
    schedule = db.Column(db.String(200))

    def __init__(self, name, facility_type, address, food_items, latitude, longitude, schedule):
        self.name = name
        self.facility_type = facility_type
        self.address = address
        self.food_items = food_items
        self.latitude = latitude
        self.longitude = longitude
        self.schedule = schedule

    # from http://stackoverflow.com/a/11884806
    def as_dict(self):
      return {c.name: getattr(self, c.name) for c in self.__table__.columns}

    # calculate distance (in km) between food truck and geolocation
    # adapted from http://www.movable-type.co.uk/scripts/latlong.html
    @hybrid_method
    def distance(self, latitude, longitude):
        earth_radius = 6371
        dlat = math.radians(latitude - self.latitude)
        dlon = math.radians(longitude - self.longitude)
        lat1 = math.radians(self.latitude)
        lat2 = math.radians(latitude)
        haversine = math.sin(dlat/2) * math.sin(dlat/2) + math.sin(dlon/2) * math.sin(dlon/2) * math.cos(lat1) * math.cos(lat2)
        return earth_radius * 2 * math.atan2(math.sqrt(haversine), math.sqrt(1-haversine))

    # converted distance function for use in SQL query
    @distance.expression
    def distance(cls, latitude, longitude):
        earth_radius = 6371
        pi_converter = func.pi() / 180
        dlat = (latitude - cls.latitude) * pi_converter
        dlon = (longitude - cls.longitude) * pi_converter
        lat1 = (cls.latitude) * pi_converter
        lat2 = (latitude) * pi_converter
        haversine = func.sin(dlat/2) * func.sin(dlat/2) + func.sin(dlon/2) * func.sin(dlon/2) * func.cos(lat1) * func.cos(lat2)
        return earth_radius * 2 * func.atan2(func.sqrt(haversine), func.sqrt(1-haversine))


"""Tests for Gaussian process models."""
from math import isclose

import numpy as np
import pytest
from mlapp_models.gaussian_process import (
    GaussianProcessClassifier,
    GaussianProcessRegressor,
)
from numpy import testing

from .utils import AbstractTestCase

class TestGaussianProcessRegressor(AbstractTestCase):
    """Test gaussian process regressor."""

    def test_no_noise(self):
        """Test that the model behaves as expected when there is no noise."""
        X = np.array([[0], [1], [2]])
        y = np.array([1, 2, 3])
        model = GaussianProcessRegressor(var_eps=0, var0=1, ell=0.5)
        model.fit(X, y)
        preds_mean, preds_covar = model.predict(X)
        preds_var = np.diag(preds_covar)
        testing.assert_allclose(preds_mean, y)
        testing.assert_allclose(preds_var, np.zeros_like(preds_var), atol=1e-8)

        preds_mean, preds_var = model.predict(np.array([[0.5]]))
        assert 1 < preds_mean[0] < 2
        assert preds_var[0, 0] > 1e-2

        # fitting different y with same x should give error
        X = np.array([[0, 1], [0, 1], [2, 0]])
        y = np.array([1, 2, 3])
        with pytest.raises(np.linalg.LinAlgError) as excinfo:
            model.fit(X, y)
        assert "singular" in excinfo.value.args[0].lower()

    def test_some_noise(self):
        """Test that adding some noise to the model makes it behave as expected."""
        X = np.array([[0], [1], [2]])
        y = np.array([1, 2, 3])
        var_eps = 0.001
        var0 = 1
        model = GaussianProcessRegressor(var_eps=var_eps, var0=var0, ell=0.5)
        model.fit(X, y)
        preds_mean, preds_covar = model.predict(X)
        preds_var = np.diag(preds_covar)
        testing.assert_allclose(preds_mean, y, atol=1e-2)
        testing.assert_allclose(preds_var, np.ones_like(preds_var) * 0.002, atol=1e-2)

        # Stack points
        X = np.array([[0], [0]])
        y = np.array([1, 2])
        model.fit(X, y)
        preds_mean, preds_var = model.predict(np.array([[0]]))
        assert isclose(preds_mean[0], 1.5, abs_tol=1e-2)
        assert 0 < preds_var[0, 0] < 0.002

        # predict some far off point
        preds_mean, preds_var = model.predict(np.array([[9999999]]))
        assert isclose(preds_mean[0], 0)
        assert isclose(preds_var[0, 0], var0, abs_tol=1e-2)

class TestGaussianProcessClassifier(AbstractTestCase):
    """Test for GP classifier."""

    def test_fit(self):
        """Test that the model can fit to some data."""
        model = GaussianProcessClassifier(var0=0.001, ell=4)
        self.assert_classifier_fit_blobs(
            model,
            blob_kwargs={"centers": np.array([[5, 0], [0, -5]])},
            zero_one_labels=False,
            min_accuracy=1,
        )
        model = GaussianProcessClassifier(var0=0.001, ell=0.1)
        self.assert_classifier_fit_moons(model, zero_one_labels=False, min_accuracy=1)
        model = GaussianProcessClassifier(var0=0.001, ell=0.1)
        self.assert_classifier_fit_circles(model, zero_one_labels=False, min_accuracy=1)

    def test_easy_data(self):
        """The model should perform as expected on easy data."""
        model = GaussianProcessClassifier(var0=0.01, ell=0.5)

        n = 10
        X = np.array([[1, 1], [-1, -1]])
        y = np.array([1, -1])
        model.fit(X, y)

        X = np.ones((n, 2)) * np.linspace(-1, 1, n).reshape(-1, 1)
        means = model.predict_mean(X)
        assert list(means) == sorted(means)
        assert (means[: n // 2] < 0).all()
        assert (means[n // 2 :] > 0).all()
        variances = np.diag(model.predict_var(X))
        assert list(variances[: n // 2]) == sorted(variances[: n // 2])
        assert list(variances[n // 2 :]) == sorted(variances[n // 2 :], reverse=True)


#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
# @Author: Brian Cherinka, José Sánchez-Gallego, and Brett Andrews
# @Date: 2017-07-02
# @Filename: test_map.py
# @License: BSD 3-clause (http://www.opensource.org/licenses/BSD-3-Clause)
#
# @Last modified by:   andrews
# @Last modified time: 2019-11-22 12:11:29

import operator
from copy import deepcopy

import matplotlib
import numpy as np
import pytest
from astropy import units as u

from marvin.core.exceptions import MarvinError
from tests import marvin_test_if
from marvin.tools.maps import Maps
from marvin.tools.quantities import EnhancedMap, Map
from marvin.utils.datamodel.dap import datamodel
from marvin.utils.general.maskbit import Maskbit

value1 = np.array([[16.35, 0.8],
                   [0, -10.]])
value2 = np.array([[591., 1e-8],
                   [4., 10]])

value_prod12 = np.array([[9.66285000e+03, 8e-9],
                         [0, -100]])

value_log2 = np.array([[2.77158748, -8.],
                       [0.60205999, 1.]])

ivar1 = np.array([[4, 1],
                  [6.97789734e+36, 1e8]])
ivar2 = np.array([[10, 1e-8],
                  [5.76744385e+36, 0]])

ivar_sum12 = np.array([[2.85714286e+00, 9.99999990e-09],
                       [3.15759543e+36, 0]])

ivar_prod12 = np.array([[1.10616234e-05, 1.56250000e-08],
                        [0, 0.]])

ivar_pow_2 = np.array([[5.23472002e-08, 9.53674316e-01],
                      [0, 25]])
ivar_pow_05 = np.array([[3.66072168e-03, 7.81250000e+00],
                       [0, 0]])
ivar_pow_0 = np.array([[0, 0],
                      [0, 0]])
ivar_pow_m1 = np.array([[4, 1.],
                        [0, 1e+08]])
ivar_pow_m2 = np.array([[2.67322500e+02, 1.6e-01],
                        [0, 2.5e+09]])
ivar_pow_m05 = np.array([[0.97859327, 5],
                         [0, 0]])

ivar_log1 = np.array([[3.67423420e-04, 4.34294482e+07],
                      [4.11019127e-20, 4.34294482e-06]])

u_flux = u.erg / u.cm**2 / u.s / u.def_unit('spaxel')
u_flux2 = u_flux * u_flux

ufuncs = [it for it in dir(np) if isinstance(getattr(np, it), np.ufunc)]

def _get_maps_kwargs(galaxy, data_origin):

    if data_origin == 'file':
        maps_kwargs = dict(filename=galaxy.mapspath)
    else:
        maps_kwargs = dict(plateifu=galaxy.plateifu, release=galaxy.release,
                           bintype=galaxy.bintype, template_kin=galaxy.template,
                           mode='local' if data_origin == 'db' else 'remote')

    return maps_kwargs

@pytest.fixture(scope='function', params=[('emline_gflux', 'ha_6564'),
                                          ('emline_gvel', 'oiii_5008'),
                                          ('stellar_vel', None),
                                          ('stellar_sigma', None)])
def map_(request, galaxy, data_origin):
    maps = Maps(**_get_maps_kwargs(galaxy, data_origin))
    map_ = maps.getMap(property_name=request.param[0], channel=request.param[1])
    map_.data_origin = data_origin
    return map_

class TestMap(object):

    def test_map(self, map_, galaxy):

        assert map_.getMaps().release == galaxy.release

        assert tuple(map_.shape) == tuple(galaxy.shape)
        assert map_.value.shape == tuple(galaxy.shape)
        assert map_.ivar.shape == tuple(galaxy.shape)
        assert map_.mask.shape == tuple(galaxy.shape)

        assert (map_.masked.data == map_.value).all()
        assert (map_.masked.mask == map_.mask.astype(bool)).all()

        assert map_.snr == pytest.approx(np.abs(map_.value * np.sqrt(map_.ivar)))

        assert datamodel[map_.getMaps()._dapver][map_.datamodel.full()].unit == map_.unit

    def test_plot(self, map_):
        fig, ax = map_.plot()
        assert isinstance(fig, matplotlib.figure.Figure)
        assert isinstance(ax, matplotlib.axes._subplots.Subplot)
        assert 'Make single panel map or one panel of multi-panel map plot.' in map_.plot.__doc__

    @marvin_test_if(mark='skip', map_={'data_origin': ['db']})
    def test_save_and_restore(self, temp_scratch, map_):

        fout = temp_scratch / 'test_map.mpf'
        map_.save(str(fout))
        assert fout.exists() is True

        map_restored = Map.restore(str(fout), delete=True)
        assert tuple(map_.shape) == tuple(map_restored.shape)

    @pytest.mark.parametrize('property_name, channel',
                             [('emline_gflux', 'ha_6564'),
                              ('stellar_vel', None)])
    def test_deepcopy(self, galaxy, property_name, channel):
        maps = Maps(plateifu=galaxy.plateifu)
        map1 = maps.getMap(property_name=property_name, channel=channel)
        map2 = deepcopy(map1)

        for attr in vars(map1):
            if not attr.startswith('_'):
                value = getattr(map1, attr)
                value2 = getattr(map2, attr)

                if isinstance(value, np.ndarray):
                    assert np.isclose(value, value2).all()

                elif isinstance(value, np.ma.core.MaskedArray):
                    assert (np.isclose(value.data, value2.data).all() and
                            (value.mask == value2.mask).all())

                elif isinstance(value, Maskbit) or isinstance(value[0], Maskbit):

                    if isinstance(value, Maskbit):
                        value = [value]
                        value2 = [value2]

                    for mb, mb2 in zip(value, value2):
                        for it in ['bits', 'description', 'labels', 'mask', 'name']:
                            assert getattr(mb, it) == getattr(mb2, it)

                        assert (mb.schema == mb2.schema).all().all()

                elif isinstance(value, Maps):
                    pass

                else:
                    assert value == value2, attr

    def test_getMap_invalid_property(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        with pytest.raises(ValueError) as ee:
            maps.getMap(property_name='mythical_property')

        assert 'Your input value is too ambiguous.' in str(ee.value)

    def test_getMap_invalid_channel(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        with pytest.raises(ValueError) as ee:
            maps.getMap(property_name='emline_gflux', channel='mythical_channel')

        assert 'Your input value is too ambiguous.' in str(ee.value)

    @marvin_test_if(mark='include', maps={'plateifu': '8485-1901',
                                          'release': 'DR17',
                                          'mode': 'local',
                                          'data_origin': 'file'})
    def test_quatities_reorder(self, maps):
        """Asserts the unit survives a quantity reorder (issue #374)."""

        ha = maps['emline_gflux_ha']

        assert ha is not None
        assert ha.unit is not None

        reordered_ha = np.moveaxis(ha, 0, -1)
        assert reordered_ha.unit is not None

    @marvin_test_if(mark='include', maps={'plateifu': '8485-1901',
                                          'release': 'DR17',
                                          'bintype': ['HYB10']})
    def test_get_spaxel(self, maps):
        """Tests `.Map.getSpaxel`."""

        ha = maps['emline_gflux_ha']

        spaxel = ha.getSpaxel(x=10, y=10, xyorig='lower')

        assert spaxel is not None
        assert spaxel.x == 10 and spaxel.y == 10

    @marvin_test_if(mark='skip', galaxy=dict(release=['DR17']))
    def test_stellar_sigma_values(self, maps, galaxy):
        ''' Assert values for stellar_sigma and stellar_sigmacorr are different (issue #411) '''

        ss = maps.stellar_sigma
        sc = maps.stellar_sigmacorr
        compare = sum(ss.value == sc.value)
        assert len(np.unique(compare)) > 1
        x = galaxy.dap['x']
        y = galaxy.dap['y']
        ssvalue = galaxy.dap['stellar_sigma'][galaxy.bintype.name]
        scvalue = galaxy.dap['stellar_sigmacorr'][galaxy.bintype.name]
        assert ssvalue == pytest.approx(ss[x, y].value, 1e-4)
        assert scvalue == pytest.approx(sc[x, y].value, 1e-4)

    def test_datamodel(self, maps):

        gew_ha = maps.emline_gew_ha_6564
        assert gew_ha.datamodel.description == ('Gaussian-fitted equivalent widths measurements '
                                                '(based on EMLINE_GFLUX). Channel = H-alpha 6564.')
    @marvin_test_if(mark='include', galaxy=dict(release=['DR15']))
    def test_stellar_sigma_mpl6(self, maps, galaxy):
        with pytest.raises(MarvinError) as cm:
            __ = maps.stellar_sigmacorr
        assert 'stellar_sigmacorr is unreliable in DR15. Please use DR17.' in str(cm.value)

class TestMapArith(object):

    def test_add_constant(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']
        ha10 = ha + 10.

        assert ha10.value == pytest.approx(ha.value + 10.)
        assert ha10.ivar == pytest.approx(ha.ivar)
        assert ha10.mask == pytest.approx(ha.mask)

    def test_reflexive_add_constant(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']
        ha10 = 10. + ha

        assert ha10.value == pytest.approx(ha.value + 10.)
        assert ha10.ivar == pytest.approx(ha.ivar)
        assert ha10.mask == pytest.approx(ha.mask)

    def test_subtract_constant(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']
        ha10 = ha - 10.

        assert ha10.value == pytest.approx(ha.value - 10.)
        assert ha10.ivar == pytest.approx(ha.ivar)
        assert ha10.mask == pytest.approx(ha.mask)

    def test_reflexive_subtract_constant(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']
        ha10 = 10. - ha

        assert ha10.value == pytest.approx(10. - ha.value)
        assert ha10.ivar == pytest.approx(ha.ivar)
        assert ha10.mask == pytest.approx(ha.mask)

    def test_multiply_constant(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']
        ha10 = ha * 10.

        assert ha10.value == pytest.approx(ha.value * 10.)
        assert ha10.ivar == pytest.approx(ha.ivar / 10.**2)
        assert ha10.mask == pytest.approx(ha.mask)

    def test_reflexive_multiply_constant(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']
        ha10 = 10. * ha

        assert ha10.value == pytest.approx(ha.value * 10.)
        assert ha10.ivar == pytest.approx(ha.ivar / 10.**2)
        assert ha10.mask == pytest.approx(ha.mask)

    def test_divide_constant(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']
        ha10 = ha / 10.

        assert ha10.value == pytest.approx(ha.value / 10.)
        assert ha10.ivar == pytest.approx(ha.ivar * 10.**2)
        assert ha10.mask == pytest.approx(ha.mask)

    def test_reflexive_divide_constant(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']
        ha10 = 10. / ha

        assert ha10.value == pytest.approx(10. / ha.value)
        assert ha10.ivar == pytest.approx(ha.ivar)
        assert ha10.mask == pytest.approx(ha.mask)

    @pytest.mark.parametrize('ivar1, ivar2, expected',
                             [(ivar1, ivar2, ivar_sum12)])
    def test_add_ivar(self, ivar1, ivar2, expected):
        assert Map._add_ivar(ivar1, ivar2) == pytest.approx(expected)

    @pytest.mark.parametrize('ivar1, ivar2, value1, value2, value_prod12, expected',
                             [(ivar1, ivar2, value1, value2, value_prod12, ivar_prod12)])
    def test_mul_ivar(self, ivar1, ivar2, value1, value2, value_prod12, expected):
        ivar = Map._mul_ivar(ivar1, ivar2, value1, value2, value_prod12)
        ivar[np.isnan(ivar)] = 0
        ivar[np.isinf(ivar)] = 0
        assert ivar == pytest.approx(expected)

    @pytest.mark.parametrize('power, expected',
                             [(2, ivar_pow_2),
                              (0.5, ivar_pow_05),
                              (0, ivar_pow_0),
                              (-1, ivar_pow_m1),
                              (-2, ivar_pow_m2),
                              (-0.5, ivar_pow_m05)])
    @pytest.mark.parametrize('ivar, value,',
                             [(ivar1, value1)])
    def test_pow_ivar(self, ivar, value, power, expected):
        ivar = Map._pow_ivar(ivar, value, power)
        ivar[np.isnan(ivar)] = 0
        ivar[np.isinf(ivar)] = 0
        assert ivar == pytest.approx(expected)

    @pytest.mark.parametrize('power', [2, 0.5, 0, -1, -2, -0.5])
    def test_pow_ivar_none(self, power):
        assert Map._pow_ivar(None, np.arange(4), power) == pytest.approx(np.zeros(4))

    @pytest.mark.parametrize('ivar, value, expected',
                             [(ivar1, value2, ivar_log1)])
    def test_log10_ivar(self, ivar, value, expected):
        actual = Map._log10_ivar(ivar, value)
        assert actual == pytest.approx(expected)

    def test_log10(self, maps_release_only):
        niiha = maps_release_only.emline_gflux_nii_6585 / maps_release_only.emline_gflux_nii_6585
        log_niiha = np.log10(niiha)
        ivar = np.log10(np.e) * niiha.ivar**-0.5 / niiha.value

        assert log_niiha.value == pytest.approx(np.log10(niiha.value), nan_ok=True)
        assert log_niiha.ivar == pytest.approx(ivar, nan_ok=True)
        assert (log_niiha.mask == niiha.mask).all()
        assert log_niiha.unit == u.dimensionless_unscaled

    @pytest.mark.runslow
    @marvin_test_if(mark='skip', ufunc=['log10'])
    @pytest.mark.parametrize('ufunc', ufuncs)
    def test_np_ufunc_notimplemented(self, maps_release_only, ufunc):
        ha = maps_release_only.emline_gflux_ha_6564
        nii = maps_release_only.emline_gflux_nii_6585

        with pytest.raises(NotImplementedError) as ee:
            if getattr(getattr(np, ufunc), 'nargs') <= 2:
                getattr(np, ufunc)(ha)

            else:
                getattr(np, ufunc)(nii, ha)

        expected = 'np.{0} is not implemented for Map.'.format(getattr(np, ufunc).__name__)
        assert str(ee.value) == expected

    @pytest.mark.parametrize('unit1, unit2, op, expected',
                             [(u_flux, u_flux, '+', u_flux),
                              (u_flux, u_flux, '-', u_flux),
                              (u_flux, u_flux, '*', u_flux2),
                              (u_flux, u_flux, '/', u.dimensionless_unscaled),
                              (u.km, u.s, '*', u.km * u.s),
                              (u.km, u.s, '/', u.km / u.s)])
    def test_unit_propagation(self, unit1, unit2, op, expected):
        assert Map._unit_propagation(unit1, unit2, op) == expected

    @pytest.mark.parametrize('unit1, unit2, op',
                             [(u_flux, u.km, '+'),
                              (u_flux, u.km, '-')])
    def test_unit_propagation_mismatch(self, unit1, unit2, op):
        with pytest.warns(UserWarning):
            assert Map._unit_propagation(unit1, unit2, op) is None

    @pytest.mark.parametrize('property1, channel1, property2, channel2',
                             [('emline_gflux', 'ha_6564', 'emline_gflux', 'nii_6585'),
                              ('emline_gvel', 'ha_6564', 'stellar_vel', None)])
    def test_add_maps(self, galaxy, property1, channel1, property2, channel2):
        maps = Maps(plateifu=galaxy.plateifu)
        map1 = maps.getMap(property_name=property1, channel=channel1)
        map2 = maps.getMap(property_name=property2, channel=channel2)
        map12 = map1 + map2

        assert map12.value == pytest.approx(map1.value + map2.value)
        assert map12.ivar == pytest.approx(map1._add_ivar(map1.ivar, map2.ivar))
        assert map12.mask == pytest.approx(map1.mask | map2.mask)

    @pytest.mark.parametrize('property1, channel1, property2, channel2',
                             [('emline_gflux', 'ha_6564', 'emline_gflux', 'nii_6585'),
                              ('emline_gvel', 'ha_6564', 'stellar_vel', None)])
    def test_subtract_maps(self, galaxy, property1, channel1, property2, channel2):
        maps = Maps(plateifu=galaxy.plateifu)
        map1 = maps.getMap(property_name=property1, channel=channel1)
        map2 = maps.getMap(property_name=property2, channel=channel2)
        map12 = map1 - map2

        assert map12.value == pytest.approx(map1.value - map2.value)
        assert map12.ivar == pytest.approx(map1._add_ivar(map1.ivar, map2.ivar))
        assert map12.mask == pytest.approx(map1.mask | map2.mask)

    @pytest.mark.parametrize('property1, channel1, property2, channel2',
                             [('emline_gflux', 'ha_6564', 'emline_gflux', 'nii_6585'),
                              ('emline_gvel', 'ha_6564', 'stellar_vel', None)])
    def test_multiply_maps(self, galaxy, property1, channel1, property2, channel2):
        maps = Maps(plateifu=galaxy.plateifu)
        map1 = maps.getMap(property_name=property1, channel=channel1)
        map2 = maps.getMap(property_name=property2, channel=channel2)
        map12 = map1 * map2

        ivar = map1._mul_ivar(map1.ivar, map2.ivar, map1.value, map2.value, map12.value)
        ivar[np.isnan(ivar)] = 0
        ivar[np.isinf(ivar)] = 0

        assert map12.value == pytest.approx(map1.value * map2.value)
        assert map12.ivar == pytest.approx(ivar)
        assert map12.mask == pytest.approx(map1.mask | map2.mask)

    @pytest.mark.parametrize('property1, channel1, property2, channel2',
                             [('emline_gflux', 'ha_6564', 'emline_gflux', 'nii_6585'),
                              ('emline_gvel', 'ha_6564', 'stellar_vel', None)])
    def test_divide_maps(self, galaxy, property1, channel1, property2, channel2):
        maps = Maps(plateifu=galaxy.plateifu)
        map1 = maps.getMap(property_name=property1, channel=channel1)
        map2 = maps.getMap(property_name=property2, channel=channel2)
        map12 = map1 / map2

        ivar = map1._mul_ivar(map1.ivar, map2.ivar, map1.value, map2.value, map12.value)
        ivar[np.isnan(ivar)] = 0
        ivar[np.isinf(ivar)] = 0

        mask = map1.mask | map2.mask
        bad = np.isnan(map12.value) | np.isinf(map12.value)
        mask[bad] = mask[bad] | map12.pixmask.labels_to_value('DONOTUSE')

        with np.errstate(divide='ignore', invalid='ignore'):
            assert map12.value == pytest.approx(map1.value / map2.value, nan_ok=True)

        assert map12.ivar == pytest.approx(ivar)
        assert map12.mask == pytest.approx(mask)

    @pytest.mark.runslow
    @pytest.mark.parametrize('power', [2, 0.5, 0, -1, -2, -0.5])
    @pytest.mark.parametrize('property_name, channel',
                             [('emline_gflux', 'ha_6564'),
                              ('stellar_vel', None)])
    def test_pow(self, galaxy, property_name, channel, power):
        maps = Maps(plateifu=galaxy.plateifu)
        map_orig = maps.getMap(property_name=property_name, channel=channel)
        map_new = map_orig**power

        sig_orig = np.sqrt(1. / map_orig.ivar)
        sig_new = map_new.value * power * sig_orig * map_orig.value
        ivar_new = 1 / sig_new**2.
        ivar_new[np.isnan(ivar_new)] = 0
        ivar_new[np.isinf(ivar_new)] = 0

        assert map_new.value == pytest.approx(map_orig.value**power, nan_ok=True)
        assert map_new.ivar == pytest.approx(ivar_new)
        assert (map_new.mask == map_orig.mask).all()

    @marvin_test_if(mark='skip', galaxy=dict(release=['MPL-4', 'MPL-6']))
    def test_stellar_sigma_correction(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        stsig = maps['stellar_sigma']
        stsigcorr = maps['stellar_sigmacorr']

        expected = (stsig**2 - stsigcorr**2)**0.5
        expected.ivar = (expected.value / stsig.value) * stsig.ivar
        expected.ivar[stsig.ivar == 0] = 0
        expected.ivar[stsigcorr.value >= stsig.value] = 0
        expected.value[stsigcorr.value >= stsig.value] = 0

        actual = stsig.inst_sigma_correction()

        assert actual.value == pytest.approx(expected.value, nan_ok=True)
        assert actual.ivar == pytest.approx(expected.ivar)
        assert (actual.mask == expected.mask).all()
        assert actual.datamodel == stsig.datamodel

    @marvin_test_if(mark='include', galaxy=dict(release=['MPL-4', 'MPL-6']))
    def test_stellar_sigma_correction_MPL4(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        stsig = maps['stellar_sigma']

        if galaxy.release == 'MPL-4':
            errmsg = 'Instrumental broadening correction not implemented for MPL-4.'
        elif galaxy.release == 'MPL-6':
            errmsg = 'The stellar sigma corrections in MPL-6 are unreliable. Please use MPL-7.'

        with pytest.raises(MarvinError) as ee:
            stsig.inst_sigma_correction()

        assert errmsg in str(ee.value)

    def test_stellar_sigma_correction_invalid_property(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']

        with pytest.raises(MarvinError) as ee:
            ha.inst_sigma_correction()

        assert ('Cannot correct {0}_{1} '.format(ha.datamodel.name, ha.datamodel.channel) +
                'for instrumental broadening.') in str(ee.value)

    def test_emline_sigma_correction(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        hasig = maps['emline_gsigma_ha_6564']
        emsigcorr = maps['emline_instsigma_ha_6564']

        expected = (hasig**2 - emsigcorr**2)**0.5
        expected.ivar = (expected.value / hasig.value) * hasig.ivar
        expected.ivar[hasig.ivar == 0] = 0
        expected.ivar[emsigcorr.value >= hasig.value] = 0
        expected.value[emsigcorr.value >= hasig.value] = 0

        actual = hasig.inst_sigma_correction()

        assert actual.value == pytest.approx(expected.value, nan_ok=True)
        assert actual.ivar == pytest.approx(expected.ivar)
        assert (actual.mask == expected.mask).all()
        assert actual.datamodel == hasig.datamodel

    @marvin_test_if(mark='skip', galaxy=dict(release=['MPL-4', 'MPL-5']))
    @pytest.mark.parametrize('channel, op',
                             [('hb', '*'),
                              ('d4000', '*'),
                              ('cn1', '+'),
                              ])
    def test_specindex_sigma_correction(self, galaxy, channel, op):
        maps = Maps(plateifu=galaxy.plateifu)
        si = maps['specindex_' + channel]
        sicorr = maps['specindex_corr' + channel]

        ops = {'+': operator.add, '-': operator.sub, '*': operator.mul, '/': operator.truediv}
        expected = ops[op](si, sicorr)

        actual = si.specindex_correction()

        assert actual.value == pytest.approx(expected.value, nan_ok=True)
        assert actual.ivar == pytest.approx(expected.ivar)
        assert (actual.mask == expected.mask).all()
        assert actual.datamodel == si.datamodel

class TestMaskbit(object):

    def test_masked(self, maps_release_only):
        params = maps_release_only.datamodel.parent.get_default_plot_params()
        ha = maps_release_only['emline_gflux_ha_6564']
        expected = ha.pixmask.get_mask(params['default']['bitmasks'], dtype=bool)

        assert ha.masked.data == pytest.approx(ha.value)
        assert (ha.masked.mask == expected).all()

    @marvin_test_if(mark='include', maps_release_only=dict(release=['MPL-4']))
    def test_values_to_bits_mpl4(self, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.pixmask.values_to_bits(1) == [0]

    @marvin_test_if(mark='skip', maps_release_only=dict(release=['MPL-4']))
    def test_values_to_bits(self, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.pixmask.values_to_bits(3) == [0, 1]

    @marvin_test_if(mark='include', maps_release_only=dict(release=['MPL-4']))
    def test_values_to_labels_mpl4(self, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.pixmask.values_to_labels(1) == ['DONOTUSE']

    @marvin_test_if(mark='skip', maps_release_only=dict(release=['MPL-4']))
    def test_values_to_labels(self, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.pixmask.values_to_labels(3) == ['NOCOV', 'LOWCOV']

    @marvin_test_if(mark='include', maps_release_only=dict(release=['MPL-4']))
    def test_labels_to_value_mpl4(self, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.pixmask.labels_to_value('DONOTUSE') == 1

    @marvin_test_if(mark='skip', maps_release_only=dict(release=['MPL-4']))
    @pytest.mark.parametrize('names, expected',
                             [(['NOCOV', 'LOWCOV'], 3),
                              ('DONOTUSE', 1073741824)])
    def test_labels_to_value(self, maps_release_only, names, expected):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert ha.pixmask.labels_to_value(names) == expected

    @pytest.mark.parametrize('flag',
                             ['manga_target1',
                              'manga_target2',
                              'manga_target3',
                              'target_flags',
                              'pixmask'])
    def test_flag(self, flag, maps_release_only):
        ha = maps_release_only['emline_gflux_ha_6564']
        assert getattr(ha, flag, None) is not None

class TestEnhancedMap(object):

    def test_overridden_methods(self, galaxy):
        maps = Maps(plateifu=galaxy.plateifu)
        ha = maps['emline_gflux_ha_6564']
        nii = maps['emline_gflux_nii_6585']
        n2ha = nii / ha

        assert isinstance(n2ha, EnhancedMap)

        methods = ['_init_map_from_maps', '_get_from_file', '_get_from_db', '_get_from_api',
                   'inst_sigma_correction']

        for method in methods:
            with pytest.raises(AttributeError) as ee:
                meth = getattr(n2ha, method)
                meth()

            assert "'EnhancedMap' has no attribute '{}'.".format(method) in str(ee.value)


from django import template

register = template.Library()

@register.filter
def has_lock(user, obj):
    if getattr(obj, "is_locked", None):
        return obj.is_locked() and obj.locked_by == user
    elif getattr(obj, "related_lock_object", None):
        # things that are not lockable themselves like original texts
        # but who can be edited if a related object is locked e.g. fragment
        return has_lock(user, obj.related_lock_object())

@register.filter
def lock_request(from_user, obj):
    try:
        return (
            obj.get_object_lock()
            .objectlockrequest_set.filter(from_user=from_user)
            .order_by("created")
            .last()
        )
    except AttributeError:
        return None


#!/usr/bin/env python

import pika
import queue
import random
import time
from Request import *

""" constants """
BROADCAST_EXCHANGE = 'BROADCAST'
REQUEST_MSG = 'REQUEST'
RESPONSE_MSG = 'RESPONSE'
RELEASE_MSG = 'RELEASE'
NETWORK_SIZE_REQ_MSG = 'NETWORK_SIZE_REQ'
NETWORK_SIZE_ACK_MSG = 'NETWORK_SIZE_ACK'

""" global varaibles """
req_list = queue.PriorityQueue()
clock = 0
network_size = 1
nbr_acknowledgement = 1
queue_name = ''

def increment_clock():
	global clock
	clock += 1
	print(f'[increment clock] new clock value: {clock}')

def update_clock(sender_clock):
	global clock
	clock = max(clock, sender_clock)
	print(f'[update clock] new clock value: {clock}')

def send_msg(msg_type, routing_key, is_broadcast = False):
	props = pika.BasicProperties(reply_to=queue_name, timestamp=clock)
	exchange_name = BROADCAST_EXCHANGE if is_broadcast else ''
	routing_name = routing_key if exchange_name == '' else ''
	if msg_type == REQUEST_MSG:
		increment_clock()
		request = Request(queue_name, clock)
		req_list.put_nowait(request)
	channel.basic_publish(exchange=exchange_name, routing_key=routing_name, body=msg_type, properties=props)

def process_request_msg(properties):
	req_list.put_nowait(Request(properties.reply_to, properties.timestamp))
	first_req_in_queue = req_list.get()
	if first_req_in_queue.timestamp == properties.timestamp:
		send_msg(RESPONSE_MSG, properties.reply_to, False)

	req_list.put_nowait(first_req_in_queue)

def process_permission_msg():
	global nbr_acknowledgement
	nbr_acknowledgement += 1
	if nbr_acknowledgement == network_size:
		first_req_in_queue = req_list.get()
		if queue_name == first_req_in_queue.queue_name:
			enter_critical_section(first_req_in_queue)
		else:
			req_list.put(first_req_in_queue)

def simulate_critcal_section():
	print('[critical_Secton]: made it to the critical section')
	time.sleep(random.randint(5,10))

def enter_critical_section(request):
	global nbr_acknowledgement
	nbr_acknowledgement = 0
	simulate_critcal_section()
	send_msg(RELEASE_MSG, queue_name, True)
	process_next_requests_in_queue()

def process_next_requests_in_queue():
	if req_list.empty():
		return
	req = req_list.get()
	if req.queue_name == queue_name and nbr_acknowledgement == network_size:
		enter_critical_section(req)
		return
	if req.queue_name != queue_name:
		send_msg(RESPONSE_MSG, queue_name, False)
	req_list.put(req)

# callback function, called whenever a message is received
def callback(ch, method, properties, body):
	global network_size
	global nbr_acknowledgement
	sender_name = properties.reply_to
	sender_timestamp = properties.timestamp
	msg_type = body.decode('UTF-8')

	update_clock(sender_timestamp)
	increment_clock()
	if sender_name == queue_name:
		return

	print(f'[MSG] msg type: {msg_type}')

	if msg_type == NETWORK_SIZE_REQ_MSG:
		network_size += 1
		send_msg(NETWORK_SIZE_ACK_MSG, sender_name, False)
		print("[NETWORK_SIZE] New client joined the network:", network_size, "clients")
	elif msg_type == NETWORK_SIZE_ACK_MSG:
		network_size += 1
		print("[NETWORK_SIZE] client already in the network:", network_size, "clients")
	elif msg_type == REQUEST_MSG:
		process_request_msg(properties)
	elif msg_type == RESPONSE_MSG:
		process_permission_msg()
	elif msg_type == RELEASE_MSG:
		process_next_requests_in_queue()

	else:
		print(f"[error] msg type is unknown {msg_type}")

if __name__ == '__main__':
	# start connection
	connection = pika.BlockingConnection(pika.ConnectionParameters(host='localhost'))
	channel = connection.channel()

	# connect to the broadcast exchange
	channel.exchange_declare(exchange=BROADCAST_EXCHANGE, exchange_type='fanout')
	# The exclusive flag is used so once the client connection is closed, the queue is deleted
	result = channel.queue_declare('', exclusive=True)
	queue_name = result.method.queue

	# Bind the queue with the exchange to get messages from all clients
	channel.queue_bind(exchange=BROADCAST_EXCHANGE, queue=queue_name)
	print("[CLIENT] client name: %r" %queue_name)

	time.sleep(random.randint(5,10))

	# send message to check for other clients in the network
	send_msg(NETWORK_SIZE_REQ_MSG, queue_name, True)
	# send request to enter the critical section
	send_msg(REQUEST_MSG, 'queue_name', True)

	channel.basic_consume(queue=queue_name, on_message_callback=callback, auto_ack=True)
	channel.start_consuming()