# -*- coding: utf-8 -*-
import click
import json
import os
from pathlib import Path
from dotenv import find_dotenv, load_dotenv

from ..logging import logger
from ..utils import save_json
from ..paths import model_path, trained_model_path
from .train import train_model, save_model
from ..data.datasets import available_datasets
from .algorithms import available_algorithms

@click.command()
@click.argument('model_list')
@click.option('--output_file', '-o', nargs=1, type=str,
              default='trained_models.json')
@click.option('--hash-type', '-H', type=click.Choice(['md5', 'sha1']),
              default='sha1')
def main(model_list, *, output_file, hash_type):
    """Trains models speficied in the supplied `model_list` file

    output is a dictionary of trained model metadata keyed by
    model_key := {algorithm}_{dataset}_{run_number}, where:

    dataset:
        name of dataset to use
    algorithm:
        name of algorithm (estimator) to run on the dataset
    run_number:
        Arbitrary integer.

    The combination of these 3 things must be unique.

    trained models are written to `trained_model_path`.

    For every model, we write:

    {model_key}.model:
        the trained model
    {model_key}.metadata:
        Metadata for this model

    Parameters
    ----------
    model_list: filename
        json file specifying list of options dictionaries to be passed to
        `train_model`
    output_file: str
        name of json file to write metadata to
    hash_name: {'sha1', 'md5'}
        type of hash to use for caching of python objects

    """
    logger.info(f'Building models from {model_list}')

    os.makedirs(trained_model_path, exist_ok=True)

    with open(model_path / model_list) as f:
        training_dicts = json.load(f)

    dataset_list = available_datasets()
    algorithm_list = available_algorithms()

    metadata_dict = {}  # Used to ensure uniqueness of keys
    for td in training_dicts:
        ds_name = td.get('dataset', None)
        assert ds_name in dataset_list, f'Unknown Dataset: {ds_name}'

        alg_name = td.get('algorithm', None)
        assert alg_name in algorithm_list, f'Unknown Algorithm: {alg_name}'

        run_number = td.get('run_number', 0)
        model_key = f"{td['algorithm']}_{td['dataset']}_{run_number}"
        if model_key in metadata_dict:
            raise Exception("{id_base} already exists. Give a unique " +
                            "`run_number` to avoid collisions.")
        else:
            td['run_number'] = run_number
            metadata_dict[model_key] = td

        saved_meta = {}
        for model_key, td in metadata_dict.items():
            logger.debug(f'Creating model for {model_key}')
            trained_model, added_metadata = train_model(hash_type=hash_type,
                                                        **td)
            # replace specified params with full set of params used
            # td['algorithm_params'] = dict(trained_model.get_params())
            new_metdata = {**td, **added_metadata}
            saved_meta[model_key] = save_model(model_name=model_key,
                                               model=trained_model,
                                               metadata=new_metdata)

    logger.debug(f"output dir: {model_path}")
    logger.debug(f"output filename: {output_file}")
    save_json(model_path / output_file, saved_meta)
    logger.info("Training complete! Saved metdata to " +
                f"{model_path / output_file}")

if __name__ == '__main__':

    # not used in this stub but often useful for finding various files
    project_dir = Path(__file__).resolve().parents[2]

    # find .env automagically by walking up directories until it's found, then
    # load up the .env entries as environment variables
    load_dotenv(find_dotenv())

    main()


from pyblish import api
import pyblish.plugin
import json
import io
import logging

# does register_config imply we can have multiple filters?
# let's name it register because the filter only applies after discover has run.

# options
# - multiple pipeline filters. apply settings to plugins in order (could error on clash)
# - multiple pipeline filters, but only 1 active pipeline filter, able to switch pipeline
# - single pipeline only

# we need to support discover since the pyblish GUI uses it
# skip_attr = []#'repair', 'id', 'log', 'process', 'version', 'requires']

# plugin config: the config/settings of a single plugin

# pipeline config: the config/settings of all plugins in your pipeline, a dict of dicts of plugin configs
# since we use names you cant have 2 plugins  with the same name

# when it just says config this likely is pipeline config

class PipelineConfig(dict):
    # TODO doesnt yet support methods like deepcopy, returns dict instead of Config.

    def filter_default_attributes(config):  # filter_default_attrs
        """ filter out default pyblish plugin attributes. example: families, id, ... """
        default_attrs = [x for x in iter_default_plugin_attrs()]
        for plugin_name, plugin_config in config.items():
            for attr_name in list(plugin_config.keys()):
                if attr_name in default_attrs:
                    plugin_config.pop(attr_name)

    def filter_attributes(config, attributes):  # filter_attrs
        """ filter out custom attributes. """
        for plugin_name, plugin_config in config.items():
            for attr_name in list(plugin_config.keys()):
                if attr_name in attributes:
                    plugin_config.pop(attr_name)

    def filter_private_attributes(config):
        """ filter out custom attributes. """
        for plugin_name, plugin_config in config.items():
            for attr_name in list(plugin_config.keys()):
                if attr_name.startswith('_'):
                    plugin_config.pop(attr_name)

    def filter_empty_plugins(config):  # filter_empty_plugins
        for plugin_name in list(config.keys()):
            plugin_config = config[plugin_name]
            if not plugin_config:
                config.pop(plugin_name)

    def dump(config, path):  # save_config_as_json
        """save a config to a json. can be used on both pipeline and plugin configs"""
        with io.open(path, 'w', encoding='utf-8') as outfile:
            try:
                # python 2
                my_json_str = json.dumps(config, ensure_ascii=False, indent=4)
                if isinstance(my_json_str, str):
                    my_json_str = my_json_str.decode("utf-8")
                outfile.write(my_json_str)

            except:
                # python 3
                json.dump(config, outfile, indent=4)

    def register(config):

        def config_filter_callback(plugins):

            # raise error when an expected plugin is not found
            plugin_names = [plugin.__name__ for plugin in plugins]
            for plugin_name in config:
                if plugin_name not in plugin_names:
                    raise Exception("plugin not found in pipeline: " + plugin_name)

            # get settings per plugin
            for plugin in plugins[:]:

                logging.debug("applying config to plugin: " + plugin.__name__)
                plugin_config = config.get(plugin.__name__, None)  # plugins with same name might clash

                # skip Pyblish plugins not in the config, this might be unwanted in same cases
                if plugin_config is None:
                    logging.debug("disabling plugin not in config: " + plugin.__name__)
                    plugin.active = False
                    continue

                for attr_name, value in plugin_config.items():
                    # TODO  differentiate between input classes (action, plugin...) and raw input int,str...
                    # custom load if value starts with #action| or #plugin|. ex: #action|modulename.action_name

                    # apply settings to plugins
                    setattr(plugin, attr_name, value)

        api.register_discovery_filter(config_filter_callback)

def iter_default_plugin_attrs(private=False):
    """ filter default pyblish attributes """
    variables = vars(pyblish.plugin.Plugin)
    for x in variables:
        if not private and x.startswith('_'):
            continue
        yield(x)

def iter_non_default_plugin_attrs(plugin):
    """ get non default pyblish attributes """
    default_attrs = list(iter_default_plugin_attrs())
    for x in vars(plugin):
        if x.startswith('_'):
            continue
        if x in default_attrs:
            continue
        yield(x)

# TODO remove this convenience function
def register_pipeline_config_filter(config_path=None, config_dict=None):
    """
    register a plugin settings filter from a settings file,
    which overwrites the default plugin settings,
    and is applied when running pyblish discover

    :param config_path: path to config dict to load plugin settings from (json file)
    :param config_dict: pass a dict directly
    :return: None
    """

    # we expect either the config path or the config data
    assert config_path or config_dict, "register_config_filter requires either a config path or a config dict, not both"

    if config_path:
        f = open(config_path)
        config_dict = json.load(f)
    elif not config_dict:
        raise Exception

    # default_settings (ex all enabled/disabled ....)
    # any plugins not in pipeline, should they be included or exluded

    PipelineConfig(config_dict).register()

# def apply_config(config_path=None, config_dict=None):
#     register_config_filter(config_path=config_path)
#     plugins = api.discover()
#     return plugins

def load_config_from_json(config_path):
    """load a config from a json. can be used on both pipeline and plugin configs"""
    f = open(config_path)
    config_dict = json.load(f)
    return PipelineConfig(config_dict)

def diff_pipeline_configs(config_new, config_original):
    """get differences between config and plugin settings"""
    config_data = {}
    for k, v in config_new.items():
        config_data[k] = {}
        for k2, v2 in v.items():
            original_value = config_original[k][k2]
            if original_value != v2:

                if not original_value and not v2:  # example empty string and None
                    continue

                config_data[k][k2] = v2
    return PipelineConfig(config_data)

def get_pipeline_config_from_plugins(plugins):
    """
    from a list of plugins, get all plugin configs.
    save these in a single dict, the pipeline config
    """

    # store all plugin configs in the pipeline config
    pipeline_config = {}
    for plugin in plugins:
        pipeline_config[plugin.__name__] = get_plugin_config(plugin)

    return PipelineConfig(pipeline_config)

def type_is_supported(value):
    if hasattr(value, '__call__'):  # skip functions
        return
    if isinstance(value, type):  # skip classes
        return

    if isinstance(value, (list, tuple)) and len(value) > 0:
        value2 = value[0]
        return type_is_supported(value2)

    return True

def get_plugin_config(plugin):
    """
    get all attributes from the plugin class, and save them in a dict
    :param plugin: Pyblish plugin to get settings from
    :return: a dict containing plugin settings
    """
    # for every plugin, store all settings in the plugin config
    plugin_config = {}
    for attr in dir(plugin):
        if not attr.startswith('_') and attr not in ('id', 'log'):  # skip private, skip id & log, you should never change these
            value = getattr(plugin, attr)

            if not type_is_supported(value):
                continue

            plugin_config[attr] = value
    plugin_config['__doc__'] = plugin.__doc__
    return plugin_config


# -*- coding: utf-8 -*-
"""Input text file of terms, output text file of term pair and frequency"""
def terms_to_term_pair_freq(if_name, of_name, min_freq=3, max_term_len=10):
    """
        description='generate term pairs from terms')
    Returns:
        (input_file_name, output_file_name, min_freq, max_term_len)

        min_freq: the minimum frequency of a term pair to be stored
    """
    # parser.add_argument('-m', '--min_freq',
    #                     help='minimum frequency (default: %(default)s)',
    #                     type=int, default=3)
    # parser.add_argument('-l', '--max_term_len',
    #                     help='maximum length of terms (default: %(default)s)',
    #                     type=int, default=10)

    import itertools
    import collections
    import logging

    def get_term_list(line, term_len):
        """Return a term list with the specific term length

        Args:
            line: input line of terms, should be '/term1/term2/...'
            term_len: desire length of the return terms

        Returns:
            list of terms of term_len
        """
        black_list = ('',)
        # don't know why terms str such as
        # /sylvester/speech/issue/letter/s/sweetie-pie//s//name
        # will be generated, so filter '' here

        return [term for term in line[1:-1].split('/')
                if term not in black_list and len(term) == term_len]
        # remove line[0]: '/' and line[last]: '\n'
        # python2 cht version used to use pat = u'[\u4e00-\u9fa5]'  一 to 龥
        # and remove term that (re.search(pat, y.decode('utf-8')) == None)
        # not sure if it's still needed in python3

    def get_term_lists(line, term1_len, term2_len):
        """Return a tuple of 2 term list with the specific term lengths

        Args:
            line: input line of terms, should be '/term1/term2/...'
            term1_len: desire length of the 1st return terms
            term2_len: desire length of the 2nd return terms

        Returns:
            tuple of 2 list of terms of term1_len, term2_len respectively
        """
        black_list = ('',)

        terms = [term for term in line[1:-1].split('/') if term not in black_list]

        term1s = [term for term in terms if len(term) == term1_len]
        term2s = [term for term in terms if len(term) == term2_len]

        return (term1s, term2s)

    def yield_term_pairs_from_term_lists(term1s, term2s):
        """Return generator of all combinations of (term1, term2)

        This function generates all the combinations of (term1 from term1s, term2..)

        Args:
            term1s: list of term1s
            term2s: list of term2s

        Returns:
            generator of term pairs (term1, term2)
        """
        for term1 in term1s:
            for term2 in term2s:
                yield (term1, term2)

    def yield_term_pairs_from_term_list(term_list):
        """Return generator of all combinations of tuple(term_pair)

        ['an', 'apple', 'a'] generates ['a', 'an'], ['a', 'apple'], ...
        Pairs will always be ['apple', 'doctor'], not ['doctor', 'apple'].

        Args:
            term_list: list of terms

        Returns:
            generator of term pairs
        """
        for term_pair in itertools.combinations(term_list, 2):
            if term_pair[0] == term_pair[1]:  # two terms are identical
                continue
            yield sorted(list(term_pair))

    def yield_term_pair_freq_str(if_name, min_freq, term1_len, term2_len):
        """Generate term pair frequency str

        Args:
            if_name: name of input file of '/term1/term2\n/t1/t2/t3\n....'
            min_freq: minimum frequency of a term pair to be stored
            term1_len: length of term1
            term2_len: length of term2
        """
        tp_f_d = collections.defaultdict(int)
        with open(if_name, 'r') as input_file:
            for line in input_file:
                if term1_len == term2_len:
                    term_len = term1_len
                    terms = get_term_list(line, term_len)
                    for t1, t2 in yield_term_pairs_from_term_list(terms):
                        tp_f_d['{t1} {t2}'.format(**locals())] += 1

                else:  # term1 len != term2 len
                    (term1s, term2s) = get_term_lists(line, term1_len, term2_len)
                    for t1, t2 in yield_term_pairs_from_term_lists(term1s, term2s):
                        tp_f_d['{t1} {t2}'.format(**locals())] += 1

        for tp_str, f in tp_f_d.items():
            if not f < min_freq:
                yield '{tp_str} {f}\n'.format(**locals())
    """Main function"""
    logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)

    logging.info('Begin of script')
    with open(of_name, 'w') as output_file:
        for term1_len in range(1, max_term_len + 1):
            for term2_len in range(term1_len, max_term_len + 1):
                # generate term_pair_freq_(term1_len)-(term2_len)?
                logging.info('begin term1 length: {term1_len}, '
                             'term2 length: {term2_len}'
                             '\n\n'.format(**locals()))

                for tp_f_str in yield_term_pair_freq_str(if_name, min_freq,
                                                         term1_len, term2_len):
                    output_file.write(tp_f_str)

                logging.info('term1 length: {term1_len}, '
                             'term2 length: {term2_len} finished\n\n'
                             .format(**locals()))

if __name__ == "__main__":
    terms_to_term_pair_freq(if_name, of_name)


#!/usr/bin/python

import numpy as np
import json
import collections,random,sys
from keras.models import model_from_json
from keras.optimizers import Adam
import rospy
from environment import Environment

class Agent(object):
    def __init__(self, env, model, weight):
        self.alpha = 0.001
        self.env = env
        self.ntrials = 6
        self.nstates = env.observation_space.get_size()
        self.model = self.load_model(model)
        self.weight = weight
        self.configure = {}

    def load_model(self,filename):
        json_file = open(filename,'r')
        model = model_from_json(json_file.read())
        json_file.close()
        return model

    def epsilon_greedy(self,state):
        if np.random.rand() < self.epsilon:
            return self.env.action_space.sample()
        else:
            Q = self.model.predict(state)
            return np.argmax(Q[0])

    def train(self):
        self.model.load_weights(self.weight)
        self.model.compile(loss='mse', optimizer=Adam(lr=self.alpha))
        self.epsilon = 0.1
        for trial in range(self.ntrials):
            s = self.env.reset(test=trial+1)
            s = np.reshape(s,[1,self.nstates])
            treward = 0
            while True:
                a = self.epsilon_greedy(s)
                s2, r, done = self.env.step(a)
                s = np.reshape(s2, [1,self.nstates])
                treward += r
                if done:
                    rospy.loginfo('test: ' + str(trial+1) + ' reward: ' + str(treward))
                    break

if __name__ == '__main__':
    if not len(sys.argv) > 2:
        assert False, 'missing model and/or weight'
    rospy.init_node('ddqn_test')
    rospy.loginfo('start testing')
    env = Environment()
    agent = Agent(env, str(sys.argv[1]),str(sys.argv[2]))
    agent.train()
    env.reset()
    rospy.loginfo('COMPLETE TESTING')
    rospy.spin()


# -*- coding:utf-8 -*-
from http.client import HTTPConnection
from xml.etree.ElementTree import fromstring

#http://openapi.seoul.go.kr:8088/4f6a6547456b6368333355736a714f/xml/MosquitoStatus/1/10/2020-03-02
f = open("C:\\Users\\sdedu\\Desktop\\test/exam_mosquito.txt", "a", encoding = "utf-8")
con = HTTPConnection("openapi.seoul.go.kr:8088")

for y in range(2020,2022):
    for m in range(1,13):
        for d in range(1,31):
            when = "%d-%02d-%02d" % (y, m, d)
            con.request("GET", "/4f6a6547456b6368333355736a714f/xml/MosquitoStatus/1/5/" + when)
            resBody = con.getresponse().read()
            mqData = fromstring(resBody)
            mqData = mqData.find("row")
            for i in mqData:
                if mqData != None:
                    print(mqData.find("MOSQUITO_DATE"))
                    print(mqData.find("MOSQUITO_VALUE_HOUSE"))
                    print(mqData.find("MOSQUITO_VALUE_PARK"))
                # f.write("%s," % mqData.find("MOSQUITO_DATE"))
                # f.write("%s," % mqData.find("MOSQUITO_VALUE_HOUSE"))
                # f.write("%s," % mqData.find("MOSQUITO_VALUE_PARK"))

            # print(mqData)

con.close()
f.close()


import zmq
import time

owner=[0,0,0]
list=[]
def circ_buffer(x1):
	if not (x1 in list):
		list.append(x1)
	return list

def create_publisher(x, y):
	context = zmq.Context()
	socket= context.socket(zmq.PUB)
	socket.bind("tcp://127.0.0.1:%s" %x)
	time.sleep(20)
	t=0
	for x1 in range (0,20):
		t=circ_buffer(x1)
		socket.send_string("%s %s %s" %(y, str(x1),t))
		time.sleep(5)

	socket.send_string('dieing')
	print('Will die in 100 seconds')
	time.sleep(100)
	socket.close()

def register_pub(x, y):
	print('Registering the publisher information to broker\n')
	context = zmq.Context()
	socket= context.socket(zmq.REQ)
	socket.connect("tcp://127.0.0.1:5621")
	socket.send('hi....are you there....................')
	msg1=socket.recv()
	#time.sleep(5)
	print("answering the broker with the information about publisher and topic")
	socket.send_string("%s %s" %(x, y))
	msg123=socket.recv()
	print(msg123)
	socket.close()

#function to notify the subscriber info about available publishers and ownership strength
def notify():
	print("Asking the broker for publishers and topics available")
	context = zmq.Context()
	socket= context.socket(zmq.REQ)
	socket.connect("tcp://localhost:5618")
	print("trying to get topics and publisher info from the broker\n")
	socket.send("Please send me the topic and publisher info\n")
	msg2=socket.recv()
	print("The available publishers and topics are: %s" %msg2)
	socket.send('send me the ownership strength')
	msg13=socket.recv_string()
	print(msg13)
	owner[0],owner[1],owner[2]=msg13.split()
	time.sleep(10)
	return(owner)
	msg13=None
	socket.close()

#function to bind the subscriber to the available publisher
def publisher_connect(pub_num, topic,owner):
	poller=zmq.Poller()

	context = zmq.Context()
	socket_sub= context.socket(zmq.SUB)
	socket_sub.connect("tcp://localhost:%s" %pub_num)
	poller.register(socket_sub, zmq.POLLIN)
	socket_sub.setsockopt(zmq.SUBSCRIBE, '')
	print("connected to publisher with port number %s and topic %s" %(pub_num,topic))
	while True:
		event=dict(poller.poll(10))
		if socket_sub in event:
			message=socket_sub.recv_string()
			print(message)
			if(message=='dieing'):
				if(pub_num==owner[0]):
					print("hi")
					socket_sub1= context.socket(zmq.SUB)
					socket_sub1.connect("tcp://localhost:%s" %owner[1])
					poller.register(socket_sub1, zmq.POLLIN)
					socket_sub1.setsockopt(zmq.SUBSCRIBE, '')
					print("Trying to connect to publisher with port number %s" %(owner[1]))
					while True:
						event=dict(poller.poll(10))
						#time.sleep(20)
						if socket_sub1 in event:
							message2=socket_sub1.recv_string()
							print(message2)
							if(message2=='dieing'):
								print('connect to the next in the order.....')
								socket_sub2= context.socket(zmq.SUB)
								socket_sub2.connect("tcp://localhost:%s" %owner[2])
								poller.register(socket_sub2, zmq.POLLIN)
								socket_sub2.setsockopt(zmq.SUBSCRIBE, '')
								print("Trying to connect to publisher with port number %s" %(owner[2]))
								while True:
									event=dict(poller.poll(10))
									print('waiting..............................')
									#time.sleep(20)
									if socket_sub2 in event:
										message3=socket_sub2.recv_string()
										print(message3)
										if(message3=='dieing'):
											print('nobody is avaialble........I will listen until anybody comes')
											time.sleep(5)
											#socket_sub2.close()

									'''
									else:
										print('no service.............................')
										time.sleep(5)
										#socket_sub2.close()

						else:
							print('no service.............................')
							time.sleep(5)
							#socket_sub1.close()

						'''

				elif(pub_num==owner[1]):
					print("looking for next available ")
					socket_sub3= context.socket(zmq.SUB)
					socket_sub3.connect("tcp://localhost:%s" %owner[0])
					poller.register(socket_sub3, zmq.POLLIN)
					socket_sub3.setsockopt(zmq.SUBSCRIBE, '')
					print(" Trying to connect to the publisher with port number %s and topic %s" %(owner[0],topic))
					while True:
						event=dict(poller.poll(10))
						print('waiting..............................')
						#time.sleep(20)
						if socket_sub3 in event:
							message4=socket_sub3.recv_string()
							print(message4)
							if(message4=='dieing'):
								print('connect to the next in the order.....')
								socket_sub4= context.socket(zmq.SUB)
								socket_sub4.connect("tcp://localhost:%s" %owner[2])
								poller.register(socket_sub4, zmq.POLLIN)
								socket_sub4.setsockopt(zmq.SUBSCRIBE, '')
								print("Trying to connect to publisher with port number %s and topic %s" %(owner[2],topic))
								while True:
									event=dict(poller.poll(10))
									print('waiting..............................')
									#time.sleep(20)
									if socket_sub4 in event:
										message5=socket_sub4.recv_string()
										print(message5)
										if(message5=='dieing'):
											print('nobody is avaialble........I will listen until anybody comes')
											time.sleep(5)
											#socket_sub4.close()

									else:
										print('no service.............................')
										time.sleep(5)
										#socket_sub4.close()
						'''

						else:
							print('no service.............................')
							time.sleep(5)
							#socket_sub3.close()

						'''

				elif(pub_num==owner[2]):
					print("hi")
					socket_sub5= context.socket(zmq.SUB)
					socket_sub.connect("tcp://localhost:%s" %owner[0])
					poller.register(socket_sub5, zmq.POLLIN)
					socket_sub.setsockopt(zmq.SUBSCRIBE, '')
					print("Trying to connect to publisher with port number %s and topic %s" %(owner[0],topic))
					while True:
						event=dict(poller.poll(10))
						#time.sleep(20)
						if socket_sub in event:
							message6=socket_sub5.recv_string()
							print(message6)
							if(message6=='dieing'):
								print('connect to the next in the order.....')
								socket_sub6= context.socket(zmq.SUB)
								socket_sub6.connect("tcp://localhost:%s" %owner[2])
								poller.register(socket_sub6, zmq.POLLIN)
								socket_sub6.setsockopt(zmq.SUBSCRIBE, '')
								print("Trying to connect to publisher with port number %s and topic %s" %(owner[2],topic))
								while True:
									event=dict(poller.poll(10))
									#time.sleep(20)
									if socket_sub6 in event:
										message7=socket_sub6.recv_string()
										print(message7)
										if(message7=='dieing'):
											print('nobody is avaialble........I will listen until anybody comes')
											time.sleep(5)
											#socket_sub6.close()

									else:
										print('no service.............................')
										time.sleep(5)
										#socket_sub6.close()

						'''
						else:
							print('no service.............................')
							time.sleep(5)
							#socket_sub5.close()
						'''

	socket_sub.close()

def main():

	print('Hi from publisher middleware')

if __name__=="__main__":
	main()


# -*- coding: utf-8 -*-
import re
import logging
from typing import Any, Dict, List, Text, Union, Optional
# from itertools import islice

from rasa_sdk import Action, Tracker
# from rasa_sdk.executor import CollectingDispatcher
from rasa_sdk.events import (
    SlotSet,
    UserUtteranceReverted,
    ConversationPaused,
    EventType,
    FollowupAction,
    ActionExecuted,
    UserUttered
)

# from actions import config
# from actions.api.algolia import AlgoliaAPI
# from actions.utils import text_to_float

logger = logging.getLogger(__name__)

class ActionInitializeAKissStory(Action):

    def name(self):
        return 'action_initialize_a_kiss_story'

    def run(self, dispatcher, tracker, domain):

        return [SlotSet("lesson_topic", 'a_kiss_story'),
                SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive'),
                SlotSet('will_return', None),
                SlotSet("lesson_history", []),
                SlotSet("a_kiss_progress", 0)]

class ActionStoreLessonHistory__a_kiss(Action):

    def name(self):
        return 'action_store_lesson_history__a_kiss'

    def _extract_question_number(self, action_name):
        p = re.compile(r"_(\d{1,2})_")
        result = p.findall(action_name)

        return int(result[0]) if len(result) > 0 else None

    def run(self, dispatcher, tracker, domain):
        current_state = tracker.current_state()
        logger.debug(f"Current state: {current_state}")

        latest_action_name = tracker.latest_action_name

        question_num = self._extract_question_number(latest_action_name)
        if not question_num:
            return []

        question_num = int(self._extract_question_number(latest_action_name))

        logger.debug(f"latest utterance action: {latest_action_name}")
        logger.debug(f"latest question number: {question_num}")

        data = tracker.get_slot("lesson_history")
        if not data:
            data = []
        data.append(question_num)

        return [SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive'),
                SlotSet('will_return', None),
                SlotSet("lesson_history", data),
                SlotSet("a_kiss_progress", question_num)]

class ActionResetNLUConfusedSlot(Action):

    def name(self):
        return 'action_reset_nlu_confused_slot'

    def run(self, dispatcher, tracker, domain):

        return [SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive')]

class ActionNotUnderstandFallback(Action):
    def name(self):
        return 'action_not_understand_fallback'

    def run(self, dispatcher, tracker, domain):
        current_state = tracker.current_state()
        logger.debug(f"Current state: {current_state}")

        return [SlotSet("nlu_confused", "positive"),
                SlotSet('nlu_confident', None),
                FollowupAction('utter_can_not_understand')]

class ActionNotSureWhatToDoFallback(Action):
    def name(self):
        return 'action_not_sure_what_to_do_fallback'

    def _latest_user_uttered(self, events) -> UserUttered:
        utters = list(filter(lambda e: e["event"] == "user", events))
        length = len(utters)

        if length <= 0:
            return None

        latest_user_utter_data = self._latest_user_utter(events)

        user_uttered = UserUttered(
                text=latest_user_utter_data['text'],
                parse_data=latest_user_utter_data['parse_data'],
                input_channel=latest_user_utter_data['input_channel']
            )

        return user_uttered

    def run(self, dispatcher, tracker, domain):
        # events = tracker.current_state()['events']

        return [SlotSet('will_return', "positive"),
                FollowupAction('utter_return_to_previous_question')]

class ActionInitializeChangedStory(Action):

    def name(self):
        return 'action_initialize_changed_story'

    def run(self, dispatcher, tracker, domain):

        return [SlotSet("lesson_topic", 'changed_story'),
                SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive'),
                SlotSet('will_return', None),
                SlotSet("lesson_history", []),
                SlotSet("changed_progress", 0)]

class ActionStoreLessonHistory__changed(Action):

    def name(self):
        return 'action_store_lesson_history__changed'

    def _extract_question_number(self, action_name):
        p = re.compile("_(\d{2})_")
        result = p.findall(action_name)

        return int(result[0]) if len(result) > 0 else None

    def run(self, dispatcher, tracker, domain):
        latest_action_name = tracker.latest_action_name

        question_num = self._extract_question_number(latest_action_name)
        if not question_num:
            return []

        question_num = int(self._extract_question_number(latest_action_name))

        logger.debug(f"latest utterance action: {latest_action_name}")
        logger.debug(f"latest question number: {question_num}")

        data = tracker.get_slot("lesson_history")
        if not data:
            data = []
        data.append(question_num)

        return [SlotSet('nlu_confused', None),
                SlotSet('nlu_confident', 'positive'),
                SlotSet('will_return', None),
                SlotSet("lesson_history", data),
                SlotSet("changed_progress", question_num)]


from typing import Optional, Any, Union, List, Tuple, Callable

from src.items.items import Item
import dearpygui.dearpygui as dpg

class Tooltip(Item):
    def __init__(self,
                 parent: Union[int, str],
                 *,
                 label: str = None,
                 user_data: Any = None,
                 use_internal_label: bool = True,
                 tag: Union[int, str] = 0,
                 show: bool = True,
                 **kwargs
                 ):
        self.identifier = dpg.add_tooltip(
            parent=parent,
            label=label,
            user_data=user_data,
            use_internal_label=use_internal_label,
            tag=tag,
            show=show,
            **kwargs
        )


import pandas as pd

df = pd.read_csv("Book\Sukkiri_ML\ProjectA\datafiles\cinema.csv")
df2 = df.fillna(df.mean())

no = df2[(df2['SNS2']>1000) & (df2["sales"]<8500)].index
df3 = df2.drop(no,axis=0)

x = df3[['SNS1','SNS2','actor','original']]
t = df3["sales"]

from sklearn.model_selection import train_test_split
x_train,x_test,y_train,y_test = train_test_split(x,t,test_size=0.2,random_state=0)

from sklearn.linear_model import LinearRegression
model = LinearRegression()

print(model.fit(x_train,y_train))
print(model.score(x_test,y_test))

new = [[150,700,300,0]]
print(model.predict(new))

from sklearn.metrics import median_absolute_error
pred = model.predict(x_test)

print(median_absolute_error(y_pred=pred,y_true=y_test))

import pickle

with open("Book\Sukkiri_ML\ProjectA\cinema.pkl","wb") as f:
    pickle.dump(model,f)


# 1 cross-stitch
import torch
import torch.nn as nn
import torch.nn.functional as F
from model_part import *
class HNet(nn.Module):
    def __init__(self):
        super(HNet, self).__init__()
        # 1 input image channel, 2 output channels, 5x5 square convolution
        # kernel
        self.conv1 = double_conv(3,64)
        self.conv2 = double_conv(64,192)
        self.conv3 = double_conv(192,384)
        ##################
        self.conv4_a = double_conv1(384, 256)
        self.fc_a = outfully(4096,4,1024)
        self.conv4_b = double_conv1(384, 256)
        self.fc_b = outfully(4096,2,1024)

        self.cross_stich1 = cross_stich(8192, 8192);  #####
        ########################################
    def forward(self, x):
        # Max pooling over a (2, 2) window
        x = F.max_pool2d(self.conv1(x), 2);
        x = F.max_pool2d(self.conv2(x), 2);
        x = F.max_pool2d(self.conv3(x), 2);

        x1 = self.conv4_a(x)
        x2 = self.conv4_b(x)
        x1, x2 = self.cross_stich1(x1, x2);
        x1 = self.fc_a(x1)
        x2 = self.fc_b(x2)

        output1 = F.log_softmax(x1, dim=1)
        output2 = F.log_softmax(x2, dim=1)

        return output1, output2


import time
from framework.problems.singleobjective.ackley import Ackley
from evolutionary_memetic.memetic_cognitive import MemeticCognitiveAlgorithm, Species
import matplotlib.pyplot as plt
from jmetal.algorithm.singleobjective import GeneticAlgorithm
from jmetal.operator import PolynomialMutation, SBXCrossover, BinaryTournamentSelection
from jmetal.util.termination_criterion import StoppingByEvaluations
from evolutionary_memetic.memetic import MemeticAlgorithm, MemeticLocalSearch

if __name__ == '__main__':
    problem = Ackley(number_of_variables=150)
    max_evaluations = 500000
    mutation = PolynomialMutation(probability=1.0 / problem.number_of_variables, distribution_index=20)
    local_search = MemeticLocalSearch(problem, mutation, StoppingByEvaluations(500))

    memetic_algo = MemeticCognitiveAlgorithm(
        problem=problem,
        population_size=5000,
        offspring_population_size=1000,
        mutation=mutation,
        crossover=SBXCrossover(probability=1.0, distribution_index=20),
        selection=BinaryTournamentSelection(),
        species1=Species(
            mutation=mutation,
            crossover=SBXCrossover(probability=1.0, distribution_index=20),
            selection=BinaryTournamentSelection(),
            local_search=local_search,
            termination_criterion=StoppingByEvaluations(max_evaluations=1000)
        ),
        species2=Species(
            mutation=mutation,
            crossover=SBXCrossover(probability=1.0, distribution_index=20),
            selection=BinaryTournamentSelection(),
            local_search=local_search,
            termination_criterion=StoppingByEvaluations(max_evaluations=1000)
        ),
        local_search=local_search,
        termination_criterion=StoppingByEvaluations(max_evaluations=max_evaluations)
    )

    genetic_algo = GeneticAlgorithm(
        problem=problem,
        population_size=5000,
        offspring_population_size=1000,
        mutation=mutation,
        crossover=SBXCrossover(probability=1.0, distribution_index=20),
        selection=BinaryTournamentSelection(),
        termination_criterion=StoppingByEvaluations(max_evaluations=max_evaluations)
    )

    memetic_algo.run()
    genetic_algo.run()

    memetic_result = memetic_algo.get_result()
    genetic_result = genetic_algo.get_result()

    print('Algorithm: ' + memetic_algo.get_name())
    print('Problem: ' + problem.get_name())
    print('Solution: ' + str([var for var in memetic_result.variables]))
    print('Fitness:  ' + str(memetic_result.objectives[0]))
    print('Computing time: ' + str(memetic_algo.total_computing_time))

    print('Algorithm: ' + genetic_algo.get_name())
    print('Problem: ' + problem.get_name())
    print('Solution: ' + str([var for var in genetic_result.variables]))
    print('Fitness:  ' + str(genetic_result.objectives[0]))
    print('Computing time: ' + str(genetic_algo.total_computing_time))

    for o in range(problem.number_of_objectives):
        plt.plot(range(len(genetic_algo.history)), [s.objectives[o] for s in genetic_algo.history])
    for o in range(problem.number_of_objectives):
        plt.plot(range(len(memetic_algo.history)), [s.objectives[o] for s in memetic_algo.history])

    legend = ["GENETIC", "COGNITIVE-MEMETIC"]

    plt.legend(legend)
    plt.title(f"{problem.get_name()} with {problem.number_of_variables} variables")
    plt.savefig(f"{problem.get_name()}_{problem.number_of_variables}_comparison_history_{time.time()}.jpg")
    plt.show()


import numpy as np
import random
import matplotlib.pyplot as plt
import math

simlen = 10000
x = [2,3,4,5,6,7,8,9,10]
count = [0,0,0,0,0,0,0,0,0]
for j in range(2,11):
  for i in range(simlen):
    random_matrix_array = np.random.randint(0,2,size=(j,j))
    det = np.linalg.det(random_matrix_array)
    if det > 0:
      count[j-2] = count[j-2] + 1

theoretical = [0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]
theoreticalValue = 0.5;
for k in range(2,11):
  theoreticalValue = theoreticalValue * (1 - math.pow(2,-k));
  theoretical[k-2] = theoretical[k-2] * theoreticalValue

simulated = [0,0,0,0,0,0,0,0,0]
for i in range(0,9):
  simulated[i] = count[i] / simlen

#x = np.array([1, 1])
#y = np.array([P_X1_Y0, analysis_ans])
y1 = np.array(simulated)
y2 = np.array(theoretical)

print(theoretical)
print(simulated)
#plt.stem(unique,psim, markerfmt='o', use_line_collection=True, label='Simulation')
plt.ylim(0, 1)
#plt.xlim(2,10)
plt.stem(x,y1, markerfmt='o', use_line_collection=True, label = 'Simulation')
plt.stem(x,y2, markerfmt='o', use_line_collection=True, label = 'Theoretical')
plt.ylabel('$Pr($det $M > 0)$')
plt.xlabel('$n$')
plt.legend()
plt.grid()


#!/usr/bin/env python3

import re
import sys
import pysam
import tabix
import cyvcf2
import natsort
import argparse
import multiprocessing as mp
from collections import defaultdict
from typing import Dict, List, Tuple, Set

def parse_args(args):
    parser = argparse.ArgumentParser(
        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument(
        "-i",
        "--bam",
        type=str,
        required=True,
        help="BAM file to read",
    )
    parser.add_argument(
        "--sbs",
        type=str,
        required=True,
        help="himut VCF file to read",
    )
    parser.add_argument(
        "--vcf",
        type=str,
        required=True,
        help="deepvariant VCF file to read",
    )
    parser.add_argument(
        "--min_bq",
        type=int,
        default=93,
        required=False,
        help="minimum base quality score",
    )
    parser.add_argument(
        "--min_mapq",
        type=int,
        default=60,
        required=False,
        help="minimum mapping quality score",
    )
    parser.add_argument(
        "-t",
        "--threads",
        type=int,
        default=1,
        required=False,
        help="maximum number of threads",
    )
    parser.add_argument(
        "-o",
        "--out",
        type=str,
        required=True,
        help="file to write",
    )
    args = args[1:]
    return parser.parse_args(args)

class BAM:
    def __init__(self, line):
        # target
        self.tname = line.reference_name
        self.tstart = line.reference_start
        self.tend = line.reference_end
        self.target_alignment_length = self.tend - self.tstart
        # query
        self.qname = line.query_name.replace("/ccs", "")
        self.qstart = line.query_alignment_start
        self.qend = line.query_alignment_end
        self.qseq = line.query_sequence
        self.qlen = len(self.qseq)
        self.mapq = line.mapping_quality
        self.bq_int_lst = line.query_qualities
        self.qv = sum(self.bq_int_lst)/self.qlen
        self.query_alignment_length = self.qend - self.qstart
        self.query_alignment_proportion = self.query_alignment_length/float(self.qlen)
        self.cs_tag = line.get_tag("cs") if line.has_tag("cs") else "."
        if line.has_tag("tp"):
            if line.get_tag("tp") == "P":
                self.is_primary = True
            else:
                self.is_primary = False
        else:
            self.is_primary = False

    def cs2lst(self):
        cslst = [cs for cs in re.split("(:[0-9]+|\*[a-z][a-z]|[=\+\-][A-Za-z]+)", self.cs_tag)]
        cslst = [cs.upper() for cs in cslst if cs != ""]
        return cslst

    def cs2tuple(self) -> List[Tuple[int, str, str, int, int]]:
        qpos = self.qstart
        self.cstuple_lst = []
        cs_lst = self.cs2lst()
        for cs in cs_lst:
            m = cs[1:]
            mlen = len(m)
            qstart = qpos
            if cs.startswith("="):  # match # --cs=long
                cs = ":{}".format(mlen)
                t = (1, m, m, mlen, mlen)
            elif cs.startswith(":"):  # match # --cs=short
                mlen = int(m)
                qend = qpos + mlen
                m = self.qseq[qstart:qend]
                t = (1, m, m, mlen, mlen)
            elif cs.startswith("*"):  # snp # target and query
                mlen = 1
                ref, alt = list(m)
                t = (2, ref, alt, 1, 1)
            elif cs.startswith("+"):  # insertion # query
                ref = self.qseq[qpos - 1]
                alt = ref + m
                t = (3, ref, alt, 0, mlen)
            elif cs.startswith("-"):  # deletion # target
                alt = self.qseq[qpos - 1]
                ref = alt + m
                t = (4, ref, alt, mlen, 0)
                mlen = 0
            self.cstuple_lst.append(t)
            qpos += mlen

    def cs2subindel(self):
        self.cs2tuple()
        tpos = self.tstart
        qpos = self.qstart
        self.tsbs_lst = []
        self.qsbs_lst = []
        self.qsbs_bq_lst = []
        self.tindel_lst = []
        for cstuple in self.cstuple_lst:
            state, ref, alt, ref_len, alt_len, = cstuple
            if state == 2:  # snp
                if ref == "N":
                    continue
                self.qsbs_bq_lst.append(self.bq_int_lst[qpos])
                self.tsbs_lst.append((tpos + 1, ref, alt))
                self.qsbs_lst.append((qpos + 1, alt, ref))
            elif state == 3 or state == 4:  # insertion
                self.tindel_lst.append((tpos, ref, alt))
            tpos += ref_len
            qpos += alt_len

class VCF:
    def __init__(self, line):
        arr = line.strip().split()
        self.chrom = arr[0]
        self.pos = int(arr[1])
        self.id = arr[2]
        self.ref = arr[3]
        self.alt_lst = arr[4].split(",")
        self.qual = arr[5]
        self.qual = float(self.qual) if self.qual != "." else self.qual
        self.is_pass = True if arr[6] == "PASS" else False
        self.info = arr[7]
        self.format_lst = arr[8].split(":")
        self.sample_format_lst = arr[9].split(":")
        hsh = {i:j for i,j in zip(self.format_lst, self.sample_format_lst)}
        if "GT" in hsh: self.sample_gt = hsh["GT"]
        if "PS" in hsh: self.sample_phase_set = hsh["PS"]
        if "AD" in hsh:
            arr = hsh["AD"].split(",")
            self.ref_count = arr[0]
            self.alt_count_arr = arr[1:]

        self.is_snp = False
        self.is_dbs = False
        self.is_indel = False
        if len(self.alt_lst) == 1:  # bi-allelic
            self.is_biallelic = True
            self.alt = self.alt_lst[0]
            if len(self.ref) == 1 and len(self.alt) == 1:  # snp
                self.is_snp = True
            elif len(self.ref) == len(self.alt) == 2:
                self.is_dbs = True
            elif len(self.ref) > len(self.alt): # del
                self.is_indel = True
            elif len(self.ref) < len(self.alt): # ins
                self.is_indel = True
        else:
            self.is_biallelic = False

def get_tname2tsize(bam_file: str) -> Tuple[List[str], Dict[str, int]]:

    tname2tsize = {}
    alignments = pysam.AlignmentFile(bam_file, "rb")
    bam_header_lst = str(alignments.header).strip().split("\n")
    for h in bam_header_lst:
        if h.startswith("@SQ"):
            _tag, tname, tsize = h.split("\t")
            tname = tname.replace("SN:", "")
            tsize = tsize.replace("LN:", "")
            tname2tsize[tname] = int(tsize)
    alignments.close()
    return tname2tsize

def load_vcf_file(
    chrom: str,
    chrom_len: int,
    vcf_file: str
) -> Set[Tuple[int, str, str]]:

    snp_set = set()
    if vcf_file.endswith(".vcf"):
        for line in open(vcf_file):
            if line.startswith("#"):
                continue
            v = VCF(line)
            if v.chrom != chrom:
                continue
            if v.is_pass:
                if v.is_biallelic:
                    if v.is_snp:
                        snp_set.add((v.pos, v.ref, v.alt))
                else:
                    for alt in v.alt_lst:
                        if len(v.ref) == 1 and len(alt) == 1:
                            snp_set.add((v.pos, v.ref, alt))
    elif vcf_file.endswith(".bgz"):
        tb = tabix.open(vcf_file)
        records = tb.query(chrom, 0, chrom_len)
        for record in records:
            v = VCF("\t".join(record))
            if v.is_snp and v.is_pass:
                if v.is_biallelic:
                    if v.is_snp:
                        snp_set.add((v.pos, v.ref, v.alt))
                else:
                    for alt in v.alt_lst:
                        if len(v.ref) == 1 and len(alt) == 1:
                            snp_set.add((v.pos, v.ref, alt))
    return snp_set

def load_sbs_file(vcf_file: str):

    chrom2sbs_lst = defaultdict(list)
    if vcf_file.endswith(".vcf"):
        for line in open(vcf_file).readlines():
            if line.startswith("#"):
                continue
            v = VCF(line)
            if v.is_pass:
                chrom2sbs_lst[v.chrom].append((v.pos, v.ref, v.alt))
    elif vcf_file.endswith(".bgz"):
        for i in cyvcf2.VCF(vcf_file):
            v = VCF(str(i))
            if v.is_pass:
                chrom2sbs_lst[v.chrom].append((v.pos, v.ref, v.alt))
    return chrom2sbs_lst

def get_ccs(
    chrom: str,
    chrom_len: int,
    bam_file: str,
    vcf_file: str,
    sbs_lst: List[Tuple[int, str, str]],
    min_bq: int,
    min_mapq: int,
    chrom2ccs_lst: Dict[str, str]
):

    ccs_lst = []
    himut_set = set(sbs_lst)
    alignments = pysam.AlignmentFile(bam_file, "rb")
    deepvariant_set = load_vcf_file(chrom, chrom_len, vcf_file)
    for sbs in sbs_lst:
        pos, ref, alt = sbs
        for line in alignments.fetch(chrom, pos, pos + 1):
            ccs = BAM(line)
            if not ccs.is_primary:
                continue
            if ccs.mapq >= min_mapq:
                ccs.cs2subindel()
                tsbs_set = set(ccs.tsbs_lst)
                if sbs in tsbs_set:

                    himut_sbs_lst = []
                    deepvariant_snp_lst = []
                    himut_candidate_lst = []
                    himut_q93_candidate_lst = []
                    for tsbs, qbq in zip(ccs.tsbs_lst, ccs.qsbs_bq_lst):
                        if tsbs in deepvariant_set:
                            deepvariant_snp_lst.append(tsbs)
                        else:
                            himut_candidate_lst.append(tsbs)
                            if qbq >= min_bq:
                                himut_q93_candidate_lst.append(tsbs)

                        if tsbs in himut_set and qbq >= min_bq:
                            himut_sbs_lst.append(tsbs)

                    ccs_lst.append(
                        "\t".join(
                            [
                            "{}:{}_{}/{}".format(chrom, pos, ref, alt),
                            ccs.qname,
                            str(ccs.qv),
                            str(ccs.qlen),
                            str(ccs.mapq),
                            str(len(himut_sbs_lst)),
                            str(len(himut_q93_candidate_lst)),
                            str(len(himut_candidate_lst)),
                            str(len(deepvariant_snp_lst)),
                            str(len(ccs.tsbs_lst)),
                            ]
                        )
                    )
    chrom2ccs_lst[chrom] = ccs_lst

def sbs2ccs(
    bam_file: str,
    sbs_file: str,
    vcf_file: str,
    min_bq: int,
    min_mapq: int,
    threads: int,
    out_file: str
):

    tname2tsize = get_tname2tsize(bam_file)
    chrom2sbs_lst = load_sbs_file(sbs_file)
    chrom_lst = natsort.natsorted(chrom2sbs_lst.keys())

    p = mp.Pool(threads)
    manager = mp.Manager()
    chrom2ccs_lst = manager.dict()
    get_ccs_arg_lst = [
        (
            chrom,
            tname2tsize[chrom],
            bam_file,
            vcf_file,
            chrom2sbs_lst[chrom],
            min_bq,
            min_mapq,
            chrom2ccs_lst
        )
        for chrom in chrom_lst
    ]
    p.starmap(
        get_ccs, get_ccs_arg_lst,
    )
    p.close()
    p.join()

    o = open(out_file, "w")
    o.write("{}\n".format("\t".join(["sbs", "qname", "qv", "qlen", "mapq", "himut_count", "himut_q93_candidate_count", "himut_candidate_count", "snp_count", "sbs_count"])))
    for chrom in chrom_lst:
        for p in chrom2ccs_lst[chrom]:
            o.write("{}\n".format(p))
    o.close()

def main():
    options = parse_args(sys.argv)
    sbs2ccs(
        options.bam,
        options.sbs,
        options.vcf,
        options.min_bq,
        options.min_mapq,
        options.threads,
        options.out
    )
    sys.exit(0)

if __name__ == "__main__":
    main()


from turtle import right
import matplotlib.pyplot as plt
import math
# how to

def sd(l):
    N = len(l)

    mu = mean(l)
    sum = 0
    for xi in l:
        sum += (xi-mu)**2
    var = sum/N
    return math.sqrt(var)

def mode(l):
    s = set(l)
    if len(s) == len(l):
        return False
    ss = list(s)
    print(ss)
    freq = [data.count(i) for i in ss]
    print(freq)
    maxIndx = freq.index(max(freq))
    return ss[maxIndx]

def median(l):
    l.sort()
    n = len(l)
    if n % 2 != 0:
        return l[int(n/2)]
    else:
        return (l[int((n/2)-1)]+l[int(n/2)])/2

def mean(l):
    sum = 0
    for i in l:
        sum = sum + i
    return sum/len(l)

data = [10, 13, 14, 14, 14, 14, 15, 15, 15, 15, 16, 17, 20, 20]
y = [0 for i in data]
m = mean(data)
print(m)
sigma = sd(data)
print(sigma)
plt.xlim(left=0, right=0)
plt.scatter(data, y, marker='x', color='g')
plt.scatter([m], [0], marker='o', color='b', s=100)
plt.errorbar(m, 0.1, xerr=sigma, fmt='o')

k = 1
for i in data:
    plt.errorbar(i+(m-i)/2, k, xerr=(m-i)/2, fmt='o')
    k += .1
plt.xlim(0,20)
plt.boxplot(data,vert=False)
plt.boxplot(data,horizt=False)

y2018=[1,2,3,6,6,9,9,9,9,9,9,9,7,8,9,7,8,7,4,1,2,3,6,5,8,9,4,2,8]
y2019=[9,6,3,2,5,8,7,4,1,2,3,6,5,4,7,8,9,9,6,3,1,5,7,8,2,7,8,1,2]
plt.xlim(0,20)
plt.boxplot([y2018,y2019],vert=False)

# Histogram=(hist) =>draw diagrams for data by using arrays by repeat values in list or arrays

data=[10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,9,7,8,1,5,7,48,15,6]
# to give you range or limits for number on horizental line 0->20
plt.xlim(0,20)
# xticks => to make every plot above number to generate list from 0 to 20
plt.xticks([i for i in range(21)])
plt.yticks([i for i in range(31)])
plt.grid()
# align=> it take 3 values left or right or mid
# rwidth =relative width=> default value it equal to 1
# bins =>to generate list from0 to 21
# cumulative => it take boolean value it use to repeat data and organize it on diagram
plt.hist([y2018,y2019],color='skyblue',rwidth=0.5,align='left',bins=[i for i in range(22)],cumulative=True)


import torch
import torch.nn as nn
import torch.optim as optim
import torchvision
import torchvision.datasets as datasets
import torchvision.transforms as transforms
from torch.utils.data import DataLoader
from torch.utils.tensorboard import SummaryWriter
from tqdm import tqdm

class Discriminator(nn.Module):
    def __init__(self, num_channels, features_d, num_classes, image_size):
        super().__init__()
        self.image_size = image_size
        self.discriminator = nn.Sequential(
            nn.Conv2d(num_channels + 1, features_d, kernel_size = 4, stride = 2, padding = 1),
            nn.LeakyReLU(0.2),
            self._block(features_d, features_d * 2, 4, 2, 1),
            self._block(features_d * 2, features_d * 4, 4, 2, 1),
            self._block(features_d * 4, features_d * 8, 4, 2, 1),
            nn.Conv2d(features_d * 8, 1, kernel_size = 4, stride = 2, padding = 0)
        )
        self.embed = nn.Embedding(num_classes, image_size * image_size)
    def _block(self, in_channels, out_channels, kernel_size, stride, padding):
        return nn.Sequential(
            nn.Conv2d(in_channels, out_channels, kernel_size, stride, padding, bias = False),
            nn.InstanceNorm2d(out_channels, affine = True),
            nn.LeakyReLU(0.2)
        )
    def forward(self, x, labels):
        embedding = self.embed(labels).view(labels.shape[0], 1, self.image_size, self.image_size)
        x = torch.cat([x, embedding], dim = 1)
        return self.discriminator(x)

class Generator(nn.Module):
    def __init__(self, latent_dim, num_channels, features_g, num_classes, image_size, embed_size):
        super().__init__()
        self.image_size = image_size
        self.generator = nn.Sequential(
            self._block(latent_dim + embed_size, features_g * 16, 4, 1, 0),
            self._block(features_g * 16, features_g * 8, 4, 2, 1),
            self._block(features_g * 8, features_g * 4, 4, 2, 1),
            self._block(features_g * 4, features_g * 2, 4, 2, 1),
            nn.ConvTranspose2d(features_g * 2, num_channels, kernel_size = 4, stride = 2, padding = 1),
            nn.Tanh()
        )
        self.embed = nn.Embedding(num_classes, embed_size)
    def _block(self, in_channels, out_channels, kernel_size, stride, padding):
        return nn.Sequential(
            nn.ConvTranspose2d(in_channels, out_channels, kernel_size, stride, padding, bias = False),
            nn.BatchNorm2d(out_channels),
            nn.ReLU()
        )
    def forward(self, x, labels):
        embedding = self.embed(labels).unsqueeze(2).unsqueeze(3)
        x = torch.cat([x, embedding], dim = 1)
        return self.generator(x)

def initialize_weights(model):
    for m in model.modules():
        if isinstance(m, (nn.Conv2d, nn.ConvTranspose2d, nn.BatchNorm2d)):
            nn.init.normal_(m.weight.data, 0.0, 0.02)

def gradient_penalty(discriminator, labels, real, fake, device = "cpu"):

    BATCH_SIZE, Channels, Height, Width = real.shape
    epsilon = torch.rand((BATCH_SIZE, 1, 1, 1)).repeat(1, Channels, Height, Width).to(device)
    interpolated_images = real * epsilon + fake * (1 - epsilon) # 10 percent of the real image and 90 percent of the fake images
    mixed_scores = discriminator(interpolated_images, labels)
    gradient = torch.autograd.grad(inputs = interpolated_images, outputs = mixed_scores, grad_outputs = torch.ones_like(mixed_scores), create_graph = True, retain_graph = True)[0]
    gradient = gradient.view(gradient.shape[0], -1)
    gradient_norm = gradient.norm(2, dim = 1)
    gradient_penalty = torch.mean((gradient_norm - 1) ** 2)
    return gradient_penalty

device = "cuda" if torch.cuda.is_available() else "cpu"

LR = 1e-4
BATCH_SIZE = 64
IMAGE_SIZE = 64
NUM_CHANNELS = 1
NUM_CLASSES = 10
GEN_EMBED = 100
LATENT_DIM = 100
EPOCHS = 100
FEATURES_D = 16
FEATURES_G = 16
DISC_ITERATIONS = 5
LAMBDA_GRADIENT_PEN = 10
BETAS = (0.0, 0.9)

transforms = transforms.Compose([
    transforms.ToTensor(),
    transforms.Resize(IMAGE_SIZE),
transforms.Normalize([0.5 for _ in range(NUM_CHANNELS)], [0.5 for _ in range(NUM_CHANNELS)])
])

dataset = datasets.MNIST(root = "dataset/", transform = transforms, download = True)
data_loader = DataLoader(dataset, batch_size = BATCH_SIZE, shuffle = True)
generator = Generator(LATENT_DIM, NUM_CHANNELS, FEATURES_G, NUM_CLASSES, IMAGE_SIZE, GEN_EMBED).to(device)
discriminator = Discriminator(NUM_CHANNELS, FEATURES_D, NUM_CLASSES, IMAGE_SIZE).to(device)
initialize_weights(generator)
initialize_weights(discriminator)

gen_optimizer = optim.Adam(generator.parameters(), lr = LR, betas = BETAS)
disc_optimizer = optim.Adam(discriminator.parameters(), lr = LR, betas = BETAS)

fixed_noise = torch.randn(32, LATENT_DIM, 1, 1).to(device)
def set_writers(first, second, step = 0):
    writer_real = SummaryWriter(f"logs/{first}")
    writer_fake = SummaryWriter(f"logs/{second}")
    step = step
    return writer_real, writer_fake, step

writer_real, writer_fake, step = set_writers("real", "fake", step = 0)

generator.train()
discriminator.train()

for epoch in range(EPOCHS):
    for batch_idx, (real, labels) in enumerate(data_loader):
        real = real.to(device)
        labels = labels.to(device)
        for _ in range(DISC_ITERATIONS):
            noise = torch.randn(BATCH_SIZE, LATENT_DIM, 1, 1).to(device)
            fake = generator(noise, labels)
            critic_real = discriminator(real, labels).reshape(-1)
            critic_fake = discriminator(fake, labels).reshape(-1)
            gp = gradient_penalty(discriminator, labels, real, fake, device = device)
            # loss_disc = -(torch.mean(critic_real) - torch.mean(critic_fake))
            loss_disc = (-(torch.mean(critic_real) - torch.mean(critic_fake)) + LAMBDA_GRADIENT_PEN * gp)
            loss_disc.backward(retain_graph = True)
            disc_optimizer.step()

        output = discriminator(fake, labels).reshape(-1)
        loss_gen = -torch.mean(output)
        generator.zero_grad()
        loss_gen.backward()
        gen_optimizer.step()

        if batch_idx % 100 == 0:
            print(f"Epoch [{epoch}/{EPOCHS}] Batch {batch_idx}/{len(data_loader)} Loss Discriminator: {loss_disc:.4f}, Loss Generator: {loss_gen:.4f}")

            with torch.no_grad():
                fake = generator(noise, labels)
                img_grid_real = torchvision.utils.make_grid(real[:64], normalize = True)
                img_grid_fake = torchvision.utils.make_grid(fake[:64], normalize = True)

                writer_real.add_image("Real", img_grid_real, global_step = step)
                writer_fake.add_image("Fake", img_grid_fake, global_step = step)

            step += 1
            generator.train()
            discriminator.train()


from pynput.keyboard import Key, Listener, Controller
from PIL import Image
import pyautogui, os, time

print('Hello, I am Python bot and I will try to play Tetris game on NES emulator :)')
print('Press q to start, set focus on the emulator window, and start the game :)')

scriptPath = os.path.dirname(__file__)

#keys:
keyRotate='z'
keyLeft='n'
keyRight='m'
keyPutDown='j'

keyboard = Controller()

widthOfWell = 80
heightOfWell = 160

screenshotImg = 'screenshot.png'

def determineCoordinatesOfBricksWell():
	coordinatesOfBottomFrame = pyautogui.locateOnScreen(scriptPath + '/' + 'bottom-of-well.png', region=(0, 0, 250, 270))
	if coordinatesOfBottomFrame != None:
		print('Bottom of well detected')
		wellLeft = coordinatesOfBottomFrame[0] + 3
		wellTop = coordinatesOfBottomFrame[1] - 3 - heightOfWell
		return (wellLeft, wellTop, widthOfWell, heightOfWell)
	return None

def takeScreenshot(screenshotRegion):
	image = pyautogui.screenshot(screenshotImg, region=screenshotRegion)

def getPixel(x, y):
	imageObject = Image.open(scriptPath + '/' + screenshotImg).convert('RGB')
	r, g, b = imageObject.getpixel((x, y))
	if r != 0 or g != 0 or b != 0:
		return 1
	else:
		return 0

def scanScreenshot():
	posX = 4
	posY = 4
	stepX = 8
	stepY = 8
	wellMatrix = [[0]*10]*20
	matrixX = 0
	matrixY = 0
	while posY < heightOfWell and matrixY <= 19 :
		while posX < widthOfWell:
			result = getPixel(posX, posY)
			posX+=stepX
			wellMatrix[matrixY][matrixX]=result
			matrixX+=1
		posY+=stepY
		posX=4
		matrixX=0
		matrixY+=1
	return wellMatrix

def playGame():
	print('initialized....')
	try:
		coordinatesOfBricksWell = determineCoordinatesOfBricksWell()
		takeScreenshot(coordinatesOfBricksWell)
		wellMatrix = scanScreenshot()
		print(wellMatrix)
		while 1==1 :
			return None
	except Exception as e:
		print(e)

def on_press(key):
    if str(key)=="'q'":
        playGame()
    if key == Key.esc:
        return False

with Listener(on_press=on_press) as listener:
    listener.join()


import time

import torch
import torch.nn as nn
import torch.optim as optim
import torchvision.models as models
import torchvision.transforms as transforms

import model
import utility

if __name__ == '__main__':
    BATCH_SIZE = 1
    INITIAL_LEARNING_RATE = 0.0001
    EPOCHS = 500
    adobe_train_path = "/Users/posoo/GitRepos/adobeset/train"

    '''
    define our device as the first visible cuda device if we have CUDA available
    '''
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    print(device)

    """
    using pre-trained VGG-16 to output conv_4_3 feature
    """
    vgg16 = models.vgg16(pretrained=True)
    vgg16_conv_4_3 = nn.Sequential(*list(vgg16.children())[0][:23])

    """
    prepare training set
    """
    transform_to_Tensor = transforms.Compose([transforms.ToTensor()])
    train_dataset = utility.AdobeDataset(adobe_train_path, transform=transform_to_Tensor)
    print("DATASET SIZE: {}".format(len(train_dataset)))
    train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)

    """
    initialize net models and optimizer
    """
    l1_loss = nn.L1Loss()
    l2_loss = nn.MSELoss()

    flow_computation = model.UNet(6, 4)
    flow_computation.to(device)
    arbitrary_time_flow_interpolation = model.UNet(20, 5)
    arbitrary_time_flow_interpolation.to(device)

    backward_warping = model.BackwardWarping(352, 352, device)

    learning_rate = INITIAL_LEARNING_RATE
    optimizer = optim.Adam(list(flow_computation.parameters()) +
                           list(arbitrary_time_flow_interpolation.parameters()),
                           lr=learning_rate)

    """
    start training
    """
    for epoch in range(EPOCHS):
        for i, (I_0, I_t, I_1, t) in enumerate(train_loader, 0):
            I_0 = I_0.to(device)
            I_t = I_t.to(device)
            I_1 = I_1.to(device)

            """
            zero the parameter gradients
            """
            optimizer.zero_grad()

            """
            calculate loss
            """
            flow_outputs = flow_computation(torch.cat((I_0, I_1), dim=1))
            F_0_1 = flow_outputs[:, :2, :, :]
            F_1_0 = flow_outputs[:, 2:, :, :]

            slomo_cp = model.SuperSlomo(arbitrary_time_flow_interpolation, backward_warping, I_0, I_1, device, F_0_1,
                                        F_1_0, True)
            g_t_0_hat, g_t_1_hat, I_t_hat, g_0_hat, g_1_hat = slomo_cp.forward(2, t)
            loss_reconstruction = l1_loss(I_t_hat, I_t)
            loss_perceptual = l2_loss(vgg16_conv_4_3(I_t_hat), vgg16_conv_4_3(I_t_hat))
            loss_warping = l1_loss(I_1, g_0_hat) + l1_loss(I_0, g_1_hat) + \
                           l1_loss(I_t, g_t_0_hat) + l1_loss(I_t, g_t_1_hat)

            loss_smooth = torch.mean(torch.abs(F_1_0[:, :, :, :-1] - F_1_0[:, :, :, 1:])) + \
                          torch.mean(torch.abs(F_1_0[:, :, :-1, :] - F_1_0[:, :, 1:, :])) + \
                          torch.mean(torch.abs(F_0_1[:, :, :, :-1] - F_0_1[:, :, :, 1:])) + \
                          torch.mean(torch.abs(F_0_1[:, :, :-1, :] - F_0_1[:, :, 1:, :]))
            loss_total = 0.8 * loss_reconstruction + 0.005 * loss_perceptual + 0.4 * loss_warping + 1 * loss_smooth

            """
            BP update
            """
            loss_total.backward()
            optimizer.step()

            """
            print every 200 iterations
            """
            if i % 200 == 0:
                print("EPOCHS: {}/{} - {}, lr: {}, lp: {}, lw: {}, ls: {}, ltotal:{}".format(epoch, EPOCHS, i,
                                                                                loss_reconstruction,
                                                                                loss_perceptual,
                                                                                loss_warping,
                                                                                loss_smooth,
                                                                                loss_total))

            """
            Learning Rate decreased by a factor of 10 every 200 epochs
            """
            if epoch % 200 == 199:
                learning_rate = learning_rate / 10
                optimizer = optim.Adam(list(flow_computation.parameters()) +
                                       list(arbitrary_time_flow_interpolation.parameters()),
                                       lr=learning_rate)

            """
            Save checkpoint per 50 epochs
            """
            if epoch % 50 == 49:
                checkpoint_dict = {
                    "epoch": epoch,
                    "learning_rate": learning_rate,
                    "loss_total": loss_total,
                    "loss_reconstruction": loss_reconstruction,
                    "loss_perceptual": loss_perceptual,
                    "loss_warping": loss_warping,
                    "loss_smooth": loss_smooth,
                    "flow_computation_state": flow_computation.state_dict(),
                    "arbitrary_time_interpolation_state": arbitrary_time_flow_interpolation.state_dict()
                }
                torch.save(checkpoint_dict, "superslomo-{}.ckpt".format(int(time.time())))


# from requests import request
from flask import Flask, render_template, request, send_file
from apriori import main as mainApriori
from metricas import main as mainMetricas
from cluster import main as mainCluster
# from pronostico import main as mainPronostico
from arboles import main as mainArboles
from bosques import main as mainBosques
from glob import glob
from io import BytesIO
from zipfile import ZipFile
import os

app = Flask(__name__)

@app.route('/')
def index():
  return render_template('index.html')

@app.route('/presentacion')
def presentacion():
  return render_template('algoritmos/presentacion.html')

# @app.route('/apriori', methods=['GET','POST'])
# def apriori():
#   if request.method == "GET":
#     return render_template('algoritmos/apriori_settings.html')
#   if request.method == "POST":
#     f = request.files['dataset']
#     f.save(secure_filename(f.filename))
#     soporte = float(request.form['soporte'])
#     confi = float(request.form['confi'])
#     lifto = float(request.form['lifto'])
#     Resultados = mainApriori(soporte,confi,lifto)
#     return render_template('algoritmos/apriori.html', resultados=Resultados)

@app.route('/apriori', methods=['GET','POST'])
def apriori():
  if request.method == "GET":
    return render_template('algoritmos/apriori_settings.html')
  if request.method == "POST":
    # f = request.files['dataset']
    # f.save(secure_filename(f.filename))
    soporte = float(request.form['soporte'])
    confi = float(request.form['confi'])
    lifto = float(request.form['lifto'])
    Resultados = mainApriori(soporte,confi,lifto)
    return render_template('algoritmos/apriori.html', resultados=Resultados)

@app.route('/aprioriDownload')
def downloadApriori():
  target = './static/images/apriori/'

  stream = BytesIO()
  with ZipFile(stream, 'w') as zf:
      for file in glob(os.path.join(target, '*.pdf')):
          zf.write(file, os.path.basename(file))
  stream.seek(0)

  return send_file(
      stream,
      as_attachment=True,
      download_name='apriori.zip'
  )

@app.route('/metricas', methods=['GET','POST'])
def metricas():
  if request.method == "GET":
    return render_template('algoritmos/metricas.html')
  if request.method == "POST":
    obj1 = request.form['objeto1']
    if obj1 == '':
      obj1 = 0
    else:
      obj1 = int(obj1)
    obj2 = request.form['objeto2']
    if obj2 == '':
      obj2 = 1
    else:
      obj2 = int(obj2)
    dM = mainMetricas(obj1,obj2)
    distanceM = [{'distanceM': dM}]
    return render_template('algoritmos/metricasShow.html', distanceM=distanceM)

@app.route('/metricasDownload')
def downloadMetricas():
  target = './static/images/metricas/'

  stream = BytesIO()
  with ZipFile(stream, 'w') as zf:
      for file in glob(os.path.join(target, '*.pdf')):
          zf.write(file, os.path.basename(file))
  stream.seek(0)

  return send_file(
      stream,
      as_attachment=True,
      download_name='metricas.zip'
  )

@app.route('/cluster')
def cluster():
  top10, CentroidesH, CentroidesP, elbow = mainCluster()
  return render_template('algoritmos/cluster.html', top10=top10, CentroidesH=CentroidesH, CentroidesP=CentroidesP, elbow=elbow)

@app.route('/clusterDownload')
def downloadCluster():
  target = './static/images/cluster/'

  stream = BytesIO()
  with ZipFile(stream, 'w') as zf:
      for file in glob(os.path.join(target, '*.pdf')):
          zf.write(file, os.path.basename(file))
  stream.seek(0)

  return send_file(
      stream,
      as_attachment=True,
      download_name='cluster.zip'
  )

@app.route('/arboles')
def arboles():
  acc_score, Matriz_Clasificacion1, criterio, reporte_clasif, ImportanciaMod1 = mainArboles(False)
  return render_template('algoritmos/arboles.html', acc_score=acc_score, Matriz_Clasificacion1=Matriz_Clasificacion1, criterio=criterio, reporte_clasif=reporte_clasif, ImportanciaMod1=ImportanciaMod1)

@app.route('/arbolesDownload')
def downloadArboles():
  target = './static/images/arboles/'

  stream = BytesIO()
  with ZipFile(stream, 'w') as zf:
      for file in glob(os.path.join(target, '*.pdf')):
          zf.write(file, os.path.basename(file))
  stream.seek(0)

  return send_file(
      stream,
      as_attachment=True,
      download_name='arboles.zip'
  )

@app.route('/bosques')
def bosques():
  acc_score, Matriz_Clasificacion2, criterio, reporte_clasif, ImportanciaMod = mainBosques()
  return render_template('algoritmos/bosques.html', acc_score=acc_score, Matriz_Clasificacion2=Matriz_Clasificacion2, criterio=criterio, reporte_clasif=reporte_clasif, ImportanciaMod2=ImportanciaMod)

@app.route('/bosquesDownload')
def downloadBosques():
  target = './static/images/bosques/'

  stream = BytesIO()
  with ZipFile(stream, 'w') as zf:
      for file in glob(os.path.join(target, '*.pdf')):
          zf.write(file, os.path.basename(file))
  stream.seek(0)

  return send_file(
      stream,
      as_attachment=True,
      download_name='bosques.zip'
  )


import matplotlib.pyplot as plt
import os
from scipy import optimize
from utils import stats as ss
from utils import fio
import numpy as np

plot = True
debug = False
ttfname = fio.tmpfile('distr-{}.pdf')
p2 = (.025,.975)
p3 = (.025,.5,.975)
p5 = (.025,.25,.5,.75,.975)

# FUNCTIONS ----------------------------------------------------------------------------------------

def pdfmerge():
  os.system(' && '.join([
    'pdftk {} cat output {}'.format(ttfname.format('*'),ttfname.format('all')),
    'mv {} pyplots.pdf'.format(ttfname.format('all')),
    'rm {}'.format(ttfname.format('*')),
  ]))

def section(name):
  print('-'*50+'\n'+name+'\n'+'-'*50)

def wfun(x,w):
  x,w = np.asarray(x),np.asarray(w)
  return np.sum(x*w)/np.sum(w)

def p2r(p,t=1):
  return -np.log(1-p)/t

def q_err(q,distrx,p=p2,pwr=2):
  dq = distrx.ppf(p)
  return np.sum(np.abs(q - dq)**pwr)

def print_distr(distr,x,name,m=None,q=None,p=p2,ok=True):
  xstr = {
    'betabin':     'p = {:.4f}, n = {:5.0f}',
    'gamma':       'm = {:.5f}, sd = {:.5f}',
    'lnorm':       'm = {:.5f}, sd = {:.5f}',
    'skewnorm':    'm = {:.5f}, sd = {:.5f}, a = {:.4f}',
    'invgauss':    'm = {:.5f}, sd = {:.5f}, z = {:.5f}',
    'uniform':     'l = {:.3f}, h = {:.3f}',
    'ratio_binom': 'p1 = {:.4f}, n1 = {:5.0f}, p2 = {:.4f}, n2 = {:5.0f}',
  }[distr.__name__].format(*x)
  qstr = '('+',~'.join(['{:.4f}']*len(p)).format(*distr(*x).ppf(p))+')'
  s = ' --- '.join((distr.__name__,xstr,qstr,name))
  print(s if ok else '(!) FAILED: '+s,flush=True)
  if plot: plot_distr(distr(*x),m,q,p,name)

def fit_distr(distr,m,q,w=1,p=p2,name='',n0=100,sd0=1,a0=0,z0=0,pwr=2):
  m  = wfun(m,w)
  q = [wfun(qi,w) for qi in q]
  dname = distr.__name__
  x0 = {
    'betabin': [m,n0],
    'gamma': [m,sd0],
    'lnorm': [m,sd0],
    'invgauss': [m,sd0,z0],
    'skewnorm': [m,sd0,a0],
  }[distr.__name__]
  xb = {
    'betabin': [(.001,.999),(5,1e5)],
    'gamma': None,
    'lnorm': None,
    'invgauss': [(-1e5,+1e5),(1e-9,1e5),(1e-9,100)],
    'skewnorm': [(-1e5,+1e5),(1e-9,1e5),(-100,100)],
  }[distr.__name__]

  jfun = lambda x: q_err(q,distr(*x),p=p,pwr=pwr)
  opt = optimize.minimize(jfun,x0,method='L-BFGS-B',bounds=xb,
    options=dict(ftol=1e-15,gtol=1e-12,eps=1e-9))
  ok = opt.success and np.sqrt(opt.fun) < (1e-2*m)
  print_distr(distr,opt.x,name+('' if ok else ' (!)'),m,q,p,ok)
  if debug and not ok: print(opt)
  return opt.x

def plot_distr(distrx,m=None,q=None,p=p2,name=''):
  dq = distrx.ppf(p); q = dq if q is None else q
  dm = distrx.mean(); m = dm if m is None else m
  xq = (min(*q,*dq),max(*q,*dq))
  x = np.linspace(1.5*xq[0]-.5*xq[1],1.5*xq[1]-.5*xq[0],256)
  d = distrx.pdf(x)
  y = np.quantile(d,.99)*1.01
  plt.figure(figsize=(5,3))
  plt.plot(x,d,color='#CC0033')
  for dk,k,pk in zip((dm,*dq),(m,*q),(False,*p)):
    plt.plot([ k, k],[0,y],'-',color='#0099CC')
    plt.plot([dk,dk],[0,y],':',color='#CC0033')
    if pk: plt.text(k,y*pk,' {:.4f} '.format(distrx.cdf(k)),ha='left',va='center')
  plt.ylim((0,y))
  plt.title(name)
  plt.savefig(ttfname.format(name))
  plt.close()

# PARAMETERS ---------------------------------------------------------------------------------------

def beta():
  section('beta')
  fit_distr(ss.gamma,m=.00072,q=(.0005,.00097),sd0=.0001,name='beta_0') # Boily2009
  fit_distr(ss.gamma,m=.0013,q=(.0005,.0025),sd0=.0005,name='beta_0_adj') # Boily2009
  fit_distr(ss.gamma,m=5.3,q=(1,5.3,15),p=p3,pwr=1,name='Rbeta_acute') # Bellan2015 (CI adj: 15 <- 56)
  fit_distr(ss.gamma,m=.1417,q=(.046,.1417,.5),p=p3,pwr=1,sd0=.1,name='dur_acute') # Bellan2015 - years (CI adj: )
  fit_distr(ss.gamma,m=1.6,q=(1.3,1.9),name='Rbeta_350') # Wawer2005, Boily2009, Donnell2010
  fit_distr(ss.gamma,m=8.3,q=(4.5,13),name='Rbeta_200') # Wawer2005, Boily2009, Donnell2010
  fit_distr(ss.gamma,m=1.45,q=(1,2),name='Rbeta_mtf') # assume
  fit_distr(ss.gamma,m=5.29,q=(1.43,5.29,15),p=p3,name='Rbeta_gud_sus') # Boily2009
  fit_distr(ss.gamma,m=2.9, q=(1.03,2.9,5.69),p=p3,name='Rbeta_gud_inf') # Gray2001
  fit_distr(ss.gamma,m=2.15,q=(0.2,6.0),name='aRbeta_gud_sus_adj') # Boily2009
  fit_distr(ss.gamma,m=0.95,q=(0.2,2.4),name='aRbeta_gud_inf_adj') # Gray2001
  fit_distr(ss.betabin,m=.3,q=(.2,.4),name='P_gud_fsw_l') # (JK) & 18/124 EswIBBS2022
  fit_distr(ss.gamma,m=1.3,q=(1,1.6),name='RP_gud_fsw_h:l') # (JK)
  # @FYI SDHS2006 Table 13.14: ~.07 GUD among wider pop P12M
  # @FYI SDHS2006 Table 13.14: ~.61 GUD among STI/symp/GUD (7% of 11.4%)
  # fit_distr(ss.betabin,m=.25,q=(.015,.65),name='Rbeta_uvls') # Donnell2010 (CI adj)
  return

def circumcision():
  section('circumcision')
  print_distr(ss.betabin,x=[.082,4156],name='circum_2006') # SDHS2006
  print_distr(ss.betabin,x=[.171,8329],name='circum_2011') # SHIMS1 Bicego2013
  print_distr(ss.betabin,x=[.300,3988],name='circum_2016') # SHIMS2
  fit_distr(ss.betabin,m=.7,q=(.5,.9),name='circum_2050') # assume
  return

def condoms():
  section('condoms')
  fit_distr(ss.betabin,m=1-.26,q=(1-.43,1-.13),name='Rbeta_condom') # Giannou2016
  # ai vs vi
  fit_distr(ss.betabin,m=.540,q=(.387,.692),name='PF_condom_ai)') # Owen2020a
  fit_distr(ss.betabin,m=.684,q=(.555,.813),name='PF_condom_vi)') # Owen2020a
  ru = .789; rlse = np.sqrt(1/22 - 1/40 + 1/33 - 1/48); ci = np.exp(np.log(ru)+[-rlse,+rlse])
  fit_distr(ss.betabin,m=.789,q=ci,name='RPF_condom_av') # Owen2020a
  fit_distr(ss.betabin,m=.80,q=(.50,.95),name='RPF_condom_av') # Owen2020a (adj) - TODO
  fit_distr(ss.betabin,m=.80,q=(.7,.9),name='RPF_condom_av (FSW,2011)') # FSW 2011
  fit_distr(ss.betabin,m=.55,q=(.4,.7),name='RPF_condom_av (FSW,2014)') # FSW 2014
  # sex work
  fit_distr(ss.betabin,m=.744*.5,q=(.744*.2,.744),name='PF_condom_swo_2002 (vi)') # SBSS2002
  fit_distr(ss.betabin,m=.600*.5,q=(.600*.2,.600),name='PF_condom_swr_2002 (vi)') # SBSS2002
  fit_distr(ss.betabin,m=.848,q=(.772*.75,.924),name='PF_condom_swo_2011 (vi)') # Baral2014
  fit_distr(ss.betabin,m=.829,q=(.758*.75,.900),name='PF_condom_swr_2011 (vi)') # Baral2014
  # fit_distr(ss.betabin,m=.511,q=(.418,.604), name='PF_condom_npp_2011 (vi)') # Baral2014 [omit]
  print_distr(ss.betabin,x=[.856,100],name='PF_condom_swo_2014') # EswKP2014 (n adj, was 620)
  print_distr(ss.betabin,x=[.885,100],name='PF_condom_swr_2014') # EswKP2014 (n adj, was 595)
  # print_distr(ss.betabin,x=[.806,100],name='PF_condom_npp_2014') # EswKP2014 (n adj, was 395) [omit]
  fit_distr(ss.betabin,m=.884,q=(.549,.95),name='PF_condom_swo_2014 (vi)') # EswKP2014
  fit_distr(ss.betabin,m=.853,q=(.479,.95),name='PF_condom_swr_2014 (vi)') # EswKP2014
  # fit_distr(ss.betabin,m=.801,q=(.647,.90),name='PF_condom_npp_2014 (vi)') # EswKP2014 [omit]
  print_distr(ss.betabin,x=[.500,100],name='PF_condom_swx_2021 (vi)') # EswIBBS2022 always
  print_distr(ss.betabin,x=[.455,11],name='PF_condom_swx_2021 (ai)') # EswIBBS2022 always
  # wider pop
  # fit_distr(ss.betabin,m=.006,q=(.004,.013),name='PF_condom_nsw_wom_1998') # SFHS1988 [omit]
  # fit_distr(ss.betabin,m=.073,q=(.059,.121),name='PF_condom_nsw_men_1998') # SFHS1988 [omit]
  fit_distr(ss.betabin,m=.040,q=(.004,.121),name='PF_condom_cas_1998') # SFHS1988 (JK)
  fit_distr(ss.betabin,m=.600,q=(.535,.660),name='PF_condom_cas_2006') # SDHS2006 Tables 14.7.1, 14.7.2
  fit_distr(ss.betabin,m=.693,q=(.649,.737),name='PF_condom_cas_2016') # SHIMS2 Tables Table 15.4.A, Table 15.4.B
  fit_distr(ss.betabin,m=.222,q=(.194,.269),name='PF_condom_msp_2006') # SDHS2006 Tables 14.7.1, 14.7.2
  fit_distr(ss.betabin,m=.414,q=(.308,.529),name='PF_condom_msp_2016') # SHIMS2 Tables Table 15.4.A, Table 15.4.B
  return

def PX():
  section('pop sizes')
  # FSW
  fit_distr(ss.betabin,m=.029, q=(.007,.065),name='PX_w_fsw_2011') # EswKP2014 Table 4
  fit_distr(ss.betabin,m=.0243,q=(.0117,.0502),name='PX_w_fsw_2021') # EswKP2014 Table 4
  fit_distr(ss.betabin,m=.164,q=(.096,.278),name='PX_w_h') # (JK)
  # fit_distr(ss.betabin,m=.252,q=(.150,.440),name='PX_m_h') # (JK) [omit]
  fit_distr(ss.betabin,m=.130,q=(.100,.170),name='PX_m_m') # (JK)
  # fit_distr(ss.betabin,m=.130,q=(.021,.385),name='PX_m_cli') # (JK) [omit]
  return

def CF():
  section('partnership numbers')
  # fit_distr(ss.betabin,m=.38,q=(.21,.57),name='C_msp_wl') # (JK) [omit]
  # fit_distr(ss.betabin,m=.32,q=(.09,.55),name='C_msp_wh') # (JK) [omit]
  # fit_distr(ss.betabin,m=.35,q=(.23,.50),name='C_msp_ml') # (JK) [omit]
  fit_distr(ss.betabin,m=.37,q=(.25,.50),name='C_msp_xl') # (JK) synthesis
  fit_distr(ss.betabin,m=.35,q=(.20,.55),name='C_cas_xl') # (JK) synthesis
  fit_distr(ss.gamma,m=1.5,q=(1.2,2),name='C_cas_wm') # assume
  print_distr(ss.uniform,x=[.25,1],name='RC_cas_cli:wm') # assume
  fit_distr(ss.gamma,m=8.4,q=(6,11),name='CF_swr_l') # (JK)
  print_distr(ss.gamma,x=[3.5,.70],name='C_swo_fsw_l') # (JK) Baral2014,EswKP2014
  print_distr(ss.gamma,x=[6.0,1.2],name='C_swr_fsw_l') # (JK) Baral2014,EswKP2014
  print_distr(ss.gamma,x=[14,2.8], name='C_swo_fsw_h') # (JK) Baral2014,EswKP2014
  print_distr(ss.gamma,x=[21,4.2], name='C_swr_fsw_h') # (JK) Baral2014,EswKP2014
  # fit_distr(ss.gamma,m=4.1,q=(2.5,6.0),name='C_swo_fsw_l') # (JK) Baral2014,EswKP2014 [omit]
  # fit_distr(ss.gamma,m=2.0,q=(1.6,2.5),name='RC_swo_fsw_h:l') # (JK) Baral2014,EswKP2014 [omit]
  # fit_distr(ss.gamma,m=8.4,q=(6.0,11.0),name='C_swr_fsw_l') # (JK) Baral2014,EswKP2014 [omit]
  # fit_distr(ss.gamma,m=8.4*5/6,q=(6.0*5/6,11.0*5/6),name='C_swr_fsw_l_adj') # (JK) Baral2014,EswKP2014 [omit]
  # fit_distr(ss.gamma,m=1.5,q=(1.3,1.7),name='RC_swr_fsw_h:l') # (JK) Baral2014,EswKP2014 [omit]
  fit_distr(ss.gamma,m=36,q=(18,72),name='KF_swx_cli') # assume
  fit_distr(ss.gamma,m=2.0,q=(1.6,2.5),name='RKF_swx_cli_h:l') # assume
  section('sex frequency')
  fit_distr(ss.gamma,m=52*1.5,q=(.5*52,3*52),name='F_msp') # Shisana2005,Delva2013
  print_distr(ss.uniform,x=[12,48],name='F_swr') # assume
  fit_distr(ss.gamma,m=.05,q=(.006,.165),sd0=.01,name='PF_ai_mcx') # Owen2017
  print_distr(ss.uniform,x=[1,2],name='RPF_ai_swx:mcx') # Owen2017
  # fit_distr(ss.gamma,m=.08,q=(.006,.292),sd0=.01,name='PF_ai_swx_old') # Owen2017 [omit]
  # fit_distr(ss.gamma,m=.08,q=(.024,.159),sd0=.01,name='PF_ai_swx') # Owen2020 [omit]
  return

def mixing():
  section('mixing')
  # fit_distr(ss.gamma,m=9,q=(2,19),name='pref_mcx_swx') # assume [omit]
  # fit_distr(ss.gamma,m=2,q=(1.5,3),name='pref_msp_xl') # (JK) [omit]
  print_distr(ss.uniform,x=[-2,+2],name='lpref_mxc_swx') # assume
  print_distr(ss.uniform,x=[-2,+2],name='lpref_msp_xl') # assume
  return

def dur():
  section('durations: risk groups')
  # fit_distr(ss.betabin,m=.383,q=(.275,.383,.491),p=p3,name='dur_fsw_0-2')  # Baral2014 [prelim]
  # fit_distr(ss.betabin,m=.321,q=(.236,.321,.407),p=p3,name='dur_fsw_3-5')  # Baral2014 [prelim]
  # fit_distr(ss.betabin,m=.202,q=(.132,.202,.271),p=p3,name='dur_fsw_6-10') # Baral2014 [prelim]
  # fit_distr(ss.betabin,m=.094,q=(.044,.094,.144),p=p3,name='dur_fsw_11+')  # Baral2014 [prelim]
  fit_distr(ss.gamma,m=4.06,q=(2.29,6.34),name='dur_fsw') # (JK) - Knight2024* bias
  fit_distr(ss.gamma,m=0.5,q=(1/6,1.00),name='dur_sw_h') # assume
  # fit_distr(ss.invgauss,m=2,q=(1.54,2,3.25),p=p3,name='Rdur_fsw_hl') # (JK)
  # fit_distr(ss.gamma,m=4.07,q=(2.96, 5.48),name='dur_fsw_lr') # (JK)
  # fit_distr(ss.gamma,m=9.33,q=(6.30,13.13),name='dur_fsw_hr') # (JK)
  # fit_distr(ss.betabin,m=.5,q=(.4,.6),name='Pturn_e_fsw') # assume
  # fit_distr(ss.betabin,.075,q=(.056,.075,.100),p=p3,name='ybs_ever_15-24') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.088,q=(.065,.088,.117),p=p3,name='ybs_ever_25-34') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.077,q=(.057,.077,.103),p=p3,name='ybs_ever_35-54') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.051,q=(.036,.051,.071),p=p3,name='ybs_p12m_15-24') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.039,q=(.027,.039,.056),p=p3,name='ybs_p12m_25-34') # Hodgins2022 [prelim]
  # fit_distr(ss.betabin,.022,q=(.015,.022,.032),p=p3,name='ybs_p12m_35-54') # Hodgins2022 [prelim]
  fit_distr(ss.gamma,m=8,q=(4,15),name='dur_cli') # (JK)
  fit_distr(ss.gamma,m=0.5,q=(1/6,1.00),name='dur_cli_h') # assume
  fit_distr(ss.gamma,m=.2,q=(.05,.50),sd0=.1,name='turn_xm_xl') # assume
  fit_distr(ss.betabin,m=.75,q=(.50,.90),name='Pturn_fsw_m:l') # assume
  fit_distr(ss.betabin,m=.60,q=(.25,.90),name='Pturn_cli_m:l') # assume
  section('durations: partnership types')
  print_distr(ss.uniform,x=[14.5,18.5],name='dur_msp') # (JK)
  # print_distr(ss.uniform,x=[.01,.02],name='pcr_msp') # (JK) [omit]
  fit_distr(ss.gamma,m=.75,q=(.25,1.5),name='dur_cas') # (JK)
  fit_distr(ss.gamma,m=.5,q=(1/6,1),name='dur_swr') # (JK)
  return

def dx():
  section('HIV test p12m')
  # FSW
  fit_distr(ss.gamma,m=p2r(.617),q=(p2r(.556),p2r(.675)),name='dx_fsw_2011') # Baral2014
  print_distr(ss.gamma,[p2r(.617)/p2r(.468)-1,.2],name='aRdx_fsw:w_2011_adj') # (JK)
  print_distr(ss.betabin,[.746,100],name='test_fsw_2014') # EswKP2014 (adj n was 781)
  fit_distr(ss.gamma,m=p2r(.75),q=(p2r(.657),p2r(.826)),name='dx_fsw_2014_adj') # EswKP2014 (adj)
  print_distr(ss.gamma,[p2r(.746)/p2r(.571)-1,.2],name='aRdx_fsw:w_2016_adj') # (JK)
  # 2000 / 2002
  print_distr(ss.betabin,[.133,5642],name='test_ever_women_2000') # EswMICS2000
  fit_distr(ss.betabin,m=.1,q=(.05,.15),name='dx_w_2002') # assume
  # SDHS2006
  fit_distr(ss.betabin,m=p2r(.219),q=(p2r(.206),p2r(.233)),name='dx_w_2006') # SDHS2006
  fit_distr(ss.betabin,m=p2r(.089),q=(p2r(.078),p2r(.100)),name='dx_m_2006') # SDHS2006
  print_distr(ss.betabin,[p2r(.219),100],name='dx_w_2006_adj') # SDHS2006 (adj n)
  print_distr(ss.gamma,[p2r(.089)/p2r(.219),.1],name='Rdx_m:w_2006_adj') # SDHS2006 (adj n)
  # SHIMS
  print_distr(ss.betabin,[.5051,100],name='test_w_2011') # SHIMS1T (adj n was 9838)
  print_distr(ss.betabin,[.3167,100],name='test_m_2011') # SHIMS1T (adj n was 8318)
  print_distr(ss.betabin,[.468,100],name='test_w_2011_adj') # SHIMS1T (adj 15-17 & n was 9838)
  print_distr(ss.betabin,[.284,100],name='test_m_2011_adj') # SHIMS1T (adj 15-17 & n was 8318)
  fit_distr(ss.gamma,m=p2r(.468),q=(p2r(.372),p2r(.566)),sd0=.01,name='dx_w_2011_adj') # (JK)
  fit_distr(ss.gamma,m=p2r(.284),q=(p2r(.200),p2r(.376)),sd0=.01,name='dx_m_2011_adj') # (JK)
  print_distr(ss.gamma,[p2r(.284)/p2r(.468),.1],name='Rdx_m:w_2011_adj') # (JK)
  # 2016
  print_distr(ss.betabin,[.529,100],name='test_all_2016') # SHIMS2 (adj n was 9075) [omit]
  print_distr(ss.betabin,[p2r(.571),100],name='dx_w_2016') # SHIMS2 (adj n was 5127)
  print_distr(ss.betabin,[p2r(.478),100],name='dx_m_2016') # SHIMS2 (adj n was 3948)
  print_distr(ss.gamma,[p2r(.571)/p2r(.505)-1,.05],name='Rdx_w_16:11') # (JK)
  print_distr(ss.gamma,[p2r(.478)/p2r(.317)-1,.05],name='Rdx_m_16:11') # (JK) [omit]
  print_distr(ss.gamma,[p2r(.479)/p2r(.571),.1],name='Rdx_m:w_2016_adj') # (JK)
  return

def tx():
  section('ART initiation rate amond diagnosed')
  fit_distr(ss.gamma,m=1.5,q=(.5,3),name='tx_2010') # assume
  fit_distr(ss.gamma,m=9,q=(6,12),name='tx_2012') # assume
  fit_distr(ss.gamma,m=.75,q=(.5,1),name='revx_2010') # assume
  fit_distr(ss.gamma,m=.6,q=(.33,1),sd0=.1,name='ivx') # assume
  return

# TARGETS ------------------------------------------------------------------------------------------

def prevalence():
  section('prevalence')
  # prevalence ratios
  print_distr(ss.ratio_binom,x=[.605,127,.280,7747],p=p3,name='prev_fsw_w_2011') # Baral2014, Bicego2013
  fit_distr(ss.gamma,m=1.46,q=(1.30,1.63),name='prev_fsw_hl_2011') # Baral2014 (JK)
  fit_distr(ss.gamma,m=2.3, q=(1.92,2.75),name='prev_fsw_hl_2014') # EswKP2014 (JK) [omit] SR
  args = dict(p=p5,pwr=2,sd0=.1)
  fit_distr(ss.invgauss,m=1.979,q=(1.843,1.918,1.977,2.063,2.335),**args,name='prev_women_hl_2006_adj') # SDHS2006 (adj wp)
  fit_distr(ss.invgauss,m=2.571,q=(2.155,2.361,2.572,2.927,5.280),**args,name='prev_men_hl_2006_adj')   # SDHS2006 (adj wp)
  fit_distr(ss.invgauss,m=1.526,q=(1.467,1.501,1.526,1.558,1.660),**args,name='prev_women_hl_2011_adj') # Bicego2013 (adj wp & 15-17)
  fit_distr(ss.invgauss,m=1.236,q=(1.200,1.220,1.236,1.261,1.344),**args,name='prev_men_hl_2011_adj')   # Bicego2013 (adj wp & 15-17)
  fit_distr(ss.invgauss,m=1.410,q=(1.368,1.391,1.409,1.434,1.506),**args,name='prev_women_hl_2016_adj') # SHIMS2 (adj wp)
  fit_distr(ss.invgauss,m=1.305,q=(1.258,1.283,1.305,1.337,1.454),**args,name='prev_men_hl_2016_adj')   # SHIMS2 (adj wp)
  # FSW
  fit_distr(ss.betabin,m=.605,q=(.521,.690),name='prev_fsw_2011') # Baral2014 (RDS-adj)
  fit_distr(ss.betabin,m=.588,q=(.539,.636),name='prev_fsw_2021') # EswIBBS2022 (RDS-adj)
  # lr: 0-1 pp6m; nlr: 2+ p6m
  # SDHS2006 (Tables B.2, 14.3, 14.6, 14.7)
  fit_distr(ss.betabin,m=.259,q=(.244,.273),name='prev_all_2006')
  fit_distr(ss.betabin,m=.311,q=(.294,.329),name='prev_women_2006')
  fit_distr(ss.betabin,m=.197,q=(.179,.214),name='prev_men_2006')
  print_distr(ss.betabin,x=[wfun([.033,.298,.330],[1971,922,4696]),1971+922+4696],name='prev_all_lr_2006') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.052,.388,.358],[ 780,577,2989]), 780+577+2989],name='prev_women_lr_2006') # (unadj wp
  print_distr(ss.betabin,x=[wfun([.021,.149,.280],[1191,345,1707]),1191+345+1707],name='prev_men_lr_2006') # (unadj wp)
  fit_distr(ss.betabin,m=0.268,q=(.227,.287),name='prev_women_lr_2006_adj') # (adj wp) [omit->pr]
  fit_distr(ss.betabin,m=0.141,q=(.065,.167),name='prev_men_lr_2006_adj')   # (adj wp) [omit->pr]
  print_distr(ss.betabin,x=[wfun([.380,.414],[524,55]),524+55],name='prev_all_nlr_2006')   # [omit->pr]
  print_distr(ss.betabin,x=[wfun([.523,.645],[ 68, 4]), 68+ 4],name='prev_women_nlr_2006') # [omit->pr]
  print_distr(ss.betabin,x=[wfun([.359,.380],[456,50]),456+50],name='prev_men_nlr_2006')   # [omit->pr]
  # SHIMS1 Bicego2013 (Table 3) - adj n by fratio '06 & '16
  print_distr(ss.betabin,x=[.321,18172],name='prev_all_2011')       # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.388, 9843],name='prev_women_2011')     # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.241, 8329],name='prev_men_2011')       # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.280, 7747],name='prev_all_2011_adj')   # (adj n & 15-17)
  print_distr(ss.betabin,x=[.342, 6015],name='prev_women_2011_adj') # (adj n & 15-17)
  print_distr(ss.betabin,x=[.207, 4977],name='prev_men_2011_adj')   # (adj n & 15-17)
  print_distr(ss.betabin,x=[wfun([.198,.360],[4062,12083]),4062+12083],name='prev_all_lr_2011') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.338,.392],[1794,7618]),1794+7618],name='prev_women_lr_2011') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.087,.306],[2267,4466]),2267+4466],name='prev_men_lr_2011')   # (unadj wp)
  fit_distr(ss.betabin,m=.315,q=(.289,.328),name='prev_women_lr_2011_adj') # (adj wp & 15-17) [omit->pr]
  fit_distr(ss.betabin,m=.195,q=(.180,.201),name='prev_men_lr_2011_adj')   # (adj wp & 15-17) [omit->pr]
  print_distr(ss.betabin,x=[.333,1887],name='prev_all_nlr_2011')       # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.545, 373],name='prev_women_nlr_2011')     # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.281,1515],name='prev_men_nlr_2011')       # (unadj n & 15-17)
  print_distr(ss.betabin,x=[.290, 792],name='prev_all_nlr_2011_adj')   # (adj n & 15-17) [omit->pr]
  print_distr(ss.betabin,x=[.481, 216],name='prev_women_nlr_2011_adj') # (adj n & 15-17) [omit->pr]
  print_distr(ss.betabin,x=[.241, 955],name='prev_men_nlr_2011_adj')   # (adj n & 15-17) [omit->pr]
  # SHIMS 2 (Table C.2, [6.3.B], 15.3.A)
  fit_distr(ss.betabin,m=.272,q=(.258,.287),name='prev_all_2016')
  fit_distr(ss.betabin,m=.343,q=(.326,.360),name='prev_women_2016')
  fit_distr(ss.betabin,m=.189,q=(.173,.204),name='prev_men_2016')
  print_distr(ss.betabin,x=[wfun([.324,.321],[1723,6164]),1723+6164],name='prev_all_lr_2016') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.361,.367],[1355,3848]),1355+3848],name='prev_women_lr_2016') # (unadj wp)
  print_distr(ss.betabin,x=[wfun([.226,.255],[ 368,2316]), 368+2316],name='prev_men_lr_2016') # (unadj wp)
  fit_distr(ss.betabin,m=.321,q=(.300,.331),name='prev_women_lr_2016_adj') # (adj wp) [omit->pr]
  fit_distr(ss.betabin,m=.175,q=(.157,.181),name='prev_men_lr_2016_adj')   # (adj wp) [omit->pr]
  print_distr(ss.betabin,x=[.287,914],name='prev_all_nlr_2016')   # [omit->pr]
  print_distr(ss.betabin,x=[.453,263],name='prev_women_nlr_2016') # [omit->pr]
  print_distr(ss.betabin,x=[.228,651],name='prev_men_nlr_2016')   # [omit->pr]
  # SHIMS 3 (Summary Sheet)
  fit_distr(ss.betabin,m=.237,q=(.226,.249),name='prev_all_2021')
  fit_distr(ss.betabin,m=.316,q=(.298,.334),name='prev_women_2021')
  fit_distr(ss.betabin,m=.156,q=(.143,.169),name='prev_men_2021')
  return

def incidence():
  section('incidence')
  # EswIBBS2022 (Table 13)
  q = ss.betabin(p=-np.log(1-30/676)/141*365,n=676).ppf(p3) * [141/160,1,141/119]
  fit_distr(ss.invgauss,m=q[1],q=q,p=p3,sd0=.1,name='inc_fsw_2021')
  # incidence ratios
  args = dict(p=(.05,.25,.50,.75,.95)) # p != p5
  fit_distr(ss.invgauss,m=5.891,q=(4.398,5.088,5.891,7.310,13.394),**args,name='inc_women_hl_2011_adj') # Bicego2013 (adj wp & 15-17)
  fit_distr(ss.invgauss,m=3.939,q=(2.908,3.395,3.939,5.055,10.864),**args,name='inc_men_hl_2011_adj')   # Bicego2013 (adj wp & 15-17)
  # SHIMS1 Justman2016 (Tables 2,1)
  fit_distr(ss.skewnorm,m=.031,q=(.026,.037),a0=2,name='inc_women_2011')        # (unadj 15-17)
  fit_distr(ss.skewnorm,m=.017,q=(.013,.021),name='inc_men_2011')               # (unadj 15-17)
  fit_distr(ss.skewnorm,m=.0294,q=(.0252,.0347),a0=2,name='inc_women_2011_adj') # (adj 15-17)
  fit_distr(ss.skewnorm,m=.0150,q=(.0116,.0184),name='inc_men_2011_adj')        # (adj 15-17)
  fit_distr(ss.skewnorm,m=[.011,.036],q=([.006,.030],[.022,.044]),w=[789,4135],a0= 3,name='inc_women_lr_2011') # (unadj wp & 15-17)
  fit_distr(ss.skewnorm,m=[.004,.020],q=([.001,.015],[.018,.027]),w=[909,2946],a0=10,name='inc_men_lr_2011')  # (unadj wp & 15-17)
  fit_distr(ss.skewnorm,m=.0163,q=(.0040,.0224),a0=-10,name='inc_women_lr_2011_adj')  # (adj wp & 15-17) [omit->pr]
  fit_distr(ss.skewnorm,m=.0084,q=(.0001,.0117),a0=-10,name='inc_men_lr_2011_adj')    # (adj wp & 15-17) [omit->pr]
  fit_distr(ss.skewnorm,m=.100,q=(.050,.192),a0=8,name='inc_women_nlr_2011')          # (unadj 15-17)
  fit_distr(ss.skewnorm,m=.038,q=(.025,.056),a0=2,name='inc_men_nlr_2011')            # (unadj 15-17)
  fit_distr(ss.skewnorm,m=.0948,q=(.0476,.1829),a0=8,name='inc_women_nlr_2011_adj')   # (adj 15-17) [omit->ir]
  fit_distr(ss.skewnorm,m=.0339,q=(.0221,.0494),a0=2,name='inc_men_nlr_2011_adj')     # (adj 15-17) [omit->ir]
  # SHIMS2 (Table C.1)
  fit_distr(ss.skewnorm,m=.0148,q=(.0096,.0199),name='inc_all_2016')
  fit_distr(ss.skewnorm,m=.0199,q=(.0116,.0280),name='inc_women_2016')
  fit_distr(ss.skewnorm,m=.0099,q=(.0039,.0159),name='inc_men_2016')
  # SHIMS 3 (Summary Sheet)
  fit_distr(ss.skewnorm,m=.0077,q=(.0039,.0115),name='inc_all_2021')
  fit_distr(ss.skewnorm,m=.0145,q=(.0069,.0220),name='inc_women_2021')
  fit_distr(ss.skewnorm,m=.0020,q=(.0005,.0048),a0=10,name='inc_men_2021')
  return

def diagnosed():
  section('diagnosed')
  # EswR2P2013 (Table 10)
  fit_distr(ss.betabin,m=.450,q=(.395,.506),name='diag_u_fsw_2011') # n.b. among FSW not FSW-LHIV + prev (simulate) -> below
  fit_distr(ss.skewnorm,m=.741,q=(.617,.741,.898),p=p3,pwr=1,a0=2,name='diag_fsw_2011')
  # EswIBBS2022 (Table 13)
  print_distr(ss.betabin,x=[363/411,411],name='diag_fsw_2021')
  # SHIMS1T (Table 6)
  print_distr(ss.betabin,x=[.626,5807],name='diag_all_2011')
  print_distr(ss.betabin,x=[.691,3810],name='diag_women_2011')
  print_distr(ss.betabin,x=[.501,1997],name='diag_men_2011')
  # SHIMS2 (Table C.7)
  fit_distr(ss.betabin,m=.861,q=(.847,.876),name='diag_all_2016')
  fit_distr(ss.betabin,m=.902,q=(.886,.918),name='diag_women_2016')
  fit_distr(ss.betabin,m=.773,q=(.740,.806),name='diag_men_2016')
  return

def treated():
  section('treated')
  # EswKP2014 (Table 10)
  fit_distr(ss.betabin,m=.369,q=(.301,.442),name='treat_c_fsw_2011')
  fit_distr(ss.skewnorm,m=.274,q=(.209,.274,.357),p=p3,pwr=1,a0=2,name='treat_u_fsw_2011')
  print_distr(ss.betabin,x=[354/363,363],name='treat_c_fsw_2021')
  print_distr(ss.betabin,x=[354/411,411],name='treat_u_fsw_2021')
  # SHIMS1T (Table 6)
  print_distr(ss.betabin,x=[.521,3635],name='treat_c_all_2011')
  print_distr(ss.betabin,x=[.480,2633],name='treat_c_women_2011')
  print_distr(ss.betabin,x=[.627,1002],name='treat_c_men_2011')
  print_distr(ss.betabin,x=[.319,5807],name='treat_u_all_2011')
  print_distr(ss.betabin,x=[.332,3810],name='treat_u_women_2011')
  print_distr(ss.betabin,x=[.314,1997],name='treat_u_men_2011')
  # SHIMS2 (Tables C.6 C.7)
  fit_distr(ss.betabin,m=.878,q=(.860,.896),name='treat_c_all_2016')
  fit_distr(ss.betabin,m=.875,q=(.854,.896),name='treat_c_women_2016')
  fit_distr(ss.betabin,m=.884,q=(.852,.916),name='treat_c_men_2016')
  fit_distr(ss.betabin,m=.756,q=(.736,.775),name='treat_u_all_2016')
  fit_distr(ss.betabin,m=.789,q=(.768,.811),name='treat_u_women_2016')
  fit_distr(ss.betabin,m=.683,q=(.647,.720),name='treat_u_men_2016')
  return

def vls():
  section('vls')
  # SHIMS2 (Tables C.6 C.7)
  fit_distr(ss.betabin,m=.903,q=(.890,.916),name='vls_c_all_2016')
  fit_distr(ss.betabin,m=.914,q=(.899,.928),name='vls_c_women_2016')
  fit_distr(ss.betabin,m=.876,q=(.844,.909),name='vls_c_men_2016')
  fit_distr(ss.betabin,m=.682,q=(.661,.704),name='vls_u_all_2016')
  fit_distr(ss.betabin,m=.721,q=(.697,.745),name='vls_u_women_2016')
  fit_distr(ss.betabin,m=.599,q=(.561,.637),name='vls_u_men_2016')
  # SHIMS 3 (Summary Sheet)
  fit_distr(ss.betabin,m=.866,q=(.850,.881),name='vls_u_all_2021')
  fit_distr(ss.betabin,m=.886,q=(.870,.902),name='vls_u_women_2021')
  fit_distr(ss.betabin,m=.824,q=(.793,.855),name='vls_u_men_2021')
  return

def cascade_ideal():
  section('cascade: 90-90-90 / 95-95-95')
  # assume
  fit_distr(ss.betabin,m=.90**1,q=(.99*.90**1,1.01*.90**1),name='90^1')
  fit_distr(ss.betabin,m=.90**2,q=(.99*.90**2,1.01*.90**2),name='90^2')
  fit_distr(ss.betabin,m=.90**3,q=(.99*.90**3,1.01*.90**3),name='90^3')
  fit_distr(ss.betabin,m=.95**1,q=(.99*.95**1,1.01*.95**1),name='95^1')
  fit_distr(ss.betabin,m=.95**2,q=(.99*.95**2,1.01*.95**2),name='95^2')
  fit_distr(ss.betabin,m=.95**3,q=(.99*.95**3,1.01*.95**3),name='95^3')
  return

# MAIN ---------------------------------------------------------------------------------------------

# beta()
# circumcision()
# condoms()
# PX()
# CF()
# mixing()
# dur()
# dx()
# tx()
# prevalence()
# incidence()
# diagnosed()
# treated()
# vls()
# cascade_ideal()

if plot: pdfmerge()