from urllib.parse import unquote

import dash  # type: ignore
from dash import dcc, html  # type: ignore
from dash_extensions.EventListener import EventListener  # type: ignore

import lecture_search.app.pages.asset_readers.reader_slides as reader_slides
import lecture_search.app.pages.asset_readers.reader_video as reader_video

dash.register_page(__name__, path_template="/<course_name>/<lecture_name>/<asset_name>")
event = {"event": "scroll", "props": ["target.scrollTop", "target.scrollLeft"]}

def layout(course_name, lecture_name, asset_name, page=None, startTime=None):
    if page is None:
        page = 0
    if startTime is None:
        startTime = 0

    app = dash.get_app()
    # decode url and special characters
    course_name = unquote(course_name)
    lecture_name = unquote(lecture_name)
    asset_name = unquote(asset_name)
    path = f"{course_name}/{lecture_name}/{asset_name}"
    print(path)
    if asset_name.endswith(".pdf"):
        content = reader_slides.bp.embed(app)
    elif asset_name.endswith(".mp4"):
        content = reader_video.bp.embed(app)
    else:
        content = html.Div("Not implemented yet")
    return html.Div(
        [
            dcc.Store("file_path", data=path),
            dcc.Store("page", data=page),
            dcc.Store("startTime", data=startTime),
            html.H2(lecture_name),
            EventListener(
                content,
                events=[event],
                logging=True,
                id="el",
            ),
        ]
    )


from __future__ import print_function
from __future__ import division
from __future__ import absolute_import

import sys
import os
sys.path.insert(0, os.path.abspath(__file__ + "/../../"))

import json
import argparse
import pickle
import numpy as np

from data_processing.utils import *
from functools import partial

def main():
    parser = argparse.ArgumentParser(
        description="Maps the tokenized reviews onto the provided vocabulary.")
    parser.add_argument(
        'review_path',
        help="The location of json file of tokenized reviews to map over",
        type=str)
    parser.add_argument(
        'vocab_pickle',
        help="The location of the vocabulary pickle file",
        type=str)
    parser.add_argument(
        'mapped_vocab_path',
        help="The location of the json file to save the vocab mapped json to",
        type=str)
    parser.add_argument(
        '--progress_interval',
        metavar='secs',
        help="The progress printing interval in seconds",
        type=restricted_int,
        default=5)
    parser.add_argument(
        '--map_to_index',
        help='Whether the text gets mapped to its index in the vocabulary or not',
        default=False,
        type=bool)
    args = parser.parse_args()

    directory = os.path.dirname(args.mapped_vocab_path)
    if not os.path.exists(directory):
        os.makedirs(directory)

    with open(args.review_path, 'r') as review_file,\
            open(args.vocab_pickle, 'rb') as vocab_pickle,\
            open(args.mapped_vocab_path, 'x+') as mapped_vocab_file:
        print("Beginning vocabulary mapping!")
        num_reviews = sum(1 for _ in review_file)
        print("There are %d reviews." % num_reviews)

        vocab_dict = pickle.load(vocab_pickle)
        num_words = len(vocab_dict)
        vocab_list = list(key for key, _ in vocab_dict.items())
        print("There are %d words in the vocabulary." % num_words)

        review_file.seek(0)  # Reset stream position to start
        reviews = (json.loads(line) for line in review_file)

        unknown_idx = vocab_list.index(UNKNOWN_TOKEN)
        numerical_idx = vocab_list.index(NUMERICAL_TOKEN)

        def map_word_to_vocab(word):
            if word.isnumeric():
                if args.map_to_index:
                    return numerical_idx
                else:
                    return NUMERICAL_TOKEN
            try:
                if args.map_to_index:
                    return vocab_list.index(word)
                else:
                    return word
            except ValueError:
                if args.map_to_index:
                    return unknown_idx
                else:
                    return UNKNOWN_TOKEN

        def map_to_vocab(review, acc):
            text = review['text']
            review['text'] = [[map_word_to_vocab(word) for word in sentence] for sentence in text]
            json.dump(review, mapped_vocab_file)
            mapped_vocab_file.write('\n')
            return acc + 1

        apply_count = 0
        apply_count = apply_over_generator(
            reviews, map_to_vocab, acc=apply_count, num_elements=num_reviews, progress_interval=args.progress_interval)

        if apply_count != num_reviews:
            print(("ERROR! The vocab mapper ran on %d reviews but the number of reviews in the file is %d. " +
                  "They should match! Something went wrong.") % (apply_count, num_reviews))
            return -1
        print("Vocabulary mapping complete!")

if __name__ == '__main__':
    main()


from setuptools import setup, find_packages
import os

def read(fname):
    return open(os.path.join(os.path.dirname(__file__), fname)).read()

setup(
    name='quix.pay',
    version='0.1.4',
    description='Abstract credit card processing with online payment gateways such as Authorize.Net.',
    long_description=read('README'),
    author='Micah Carrick',
    url='http://www.quixotix.com',
    author_email='software@quixotix.com',
    packages=find_packages(),
    namespace_packages=['quix'],
    install_requires=['setuptools'],
    license='BSD',
    classifiers=[
        "Development Status :: 4 - Beta",
        "Environment :: Web Environment",
        "Intended Audience :: Developers",
        "Operating System :: OS Independent",
        "Programming Language :: Python",
        "Topic :: Office/Business :: Financial",
        "Topic :: Software Development :: Libraries :: Python Modules",
        "License :: OSI Approved :: BSD License",
    ],
    keywords = 'payment gateway processor credit card authorize.net AIM paypal'
)


import argparse

from spark_functions import *

def _country_name_process(name: str) -> str:
    """Function made to add space in country names that contains two words splited by upper case
    Example: UnitedKingdom -> United Kingdom

    Args:
        name (str): string to proces

    Returns:
        str: proccesed string
    """
    prevletter = None
    output = ""
    for letter in name:
        if letter.isupper():
            if prevletter == "lower":
                output += " "
            output += letter
        else:
            prevletter = "lower"
            output += letter
    return output

def main(dataset1_path: str, dataset2_path: str, filter_countries):
    logging.basicConfig(level=logging.INFO)
    schema1, schema2 = create_scheams()
    spark = setSparkSession("spark_app")
    df1 = read_file(dataset1_path, spark, schema1)
    df2 = read_file(dataset2_path, spark, schema2)
    df1 = drop_columns(df1, ["first_name", "last_name"])
    df2 = drop_columns(df2, ["cc_n"])
    df1 = filter_df_equal(df1, {"country": filter_countries})
    df = inner_join(df1, df2, "id")
    rename_dict = {
        "id": "client_identifier",
        "btc_a": "bitcoin_address",
        "cc_t": "credit_card_type",
    }
    df = rename_columns(df, rename_dict)
    write_df_to_file(df)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "--d1",
        "--dataset1",
        type=str,
        help="Provide path to datset 1",
        default="dataset_one.csv",
    )
    parser.add_argument(
        "--d2",
        "--dataset2",
        type=str,
        help="Provide path to dataset 2",
        default="dataset_two.csv",
    )
    parser.add_argument("--countries", help="Countries to fillter", nargs="*")
    args = vars(parser.parse_args())
    countries = [_country_name_process(name) for name in args["countries"]]
    main(args["d1"], args["d2"], countries)


from functools import reduce as reduceIns

import te.lang.cce
from te import tvm
from te.platform.fusion_manager import fusion_manager
from topi import generic
from topi.cce import util

SHAPE_SIZE_LIMIT = 2147483648

# pylint: disable=unidiomatic-typecheck,too-many-branches,too-many-locals
# pylint: disable=no-member,dangerous-default-value,invalid-name
@fusion_manager.register("eltwise")
def eltwise_compute(x, y, mode=1, coeff=[], kernel_name="eltwise"):
    '''
    Compute elementwise operation
    '''
    tensor_num = len(x)
    inp_dtype = x[0].dtype
    data0_tmp = x[0]

    if mode == 1:
        if len(coeff) != 0 and len(coeff) != tensor_num:
            raise RuntimeError("lenth of coeff must equal to tensor_num or 0.")
        if len(coeff) == tensor_num:
            if type(coeff[0]) != int and type(coeff[0]) != float:
                raise RuntimeError("ele of coeff must be a number.")
            if coeff[0] != 1:
                coeff1 = tvm.const(coeff[0], dtype=inp_dtype)
                data0_tmp = te.lang.cce.vmuls(data0_tmp, coeff1)

    if tensor_num == 1:
        const_val_0 = tvm.const(0, dtype=inp_dtype)
        data0_tmp = te.lang.cce.vadds(data0_tmp, const_val_0)
        res = data0_tmp
    elif tensor_num > 1:
        for i in range(1, tensor_num):
            datan_tmp = x[i]
            if mode == 0:
                data0_tmp = te.lang.cce.vmul(data0_tmp, datan_tmp)
            elif mode == 2:
                data0_tmp = te.lang.cce.vmax(data0_tmp, datan_tmp)
            else:
                if len(coeff) == 0:
                    data0_tmp = te.lang.cce.vadd(data0_tmp, datan_tmp)
                elif coeff[i] == 1:
                    data0_tmp = te.lang.cce.vadd(data0_tmp, datan_tmp)
                else:
                    coeff2 = tvm.const(coeff[i], dtype=inp_dtype)
                    datan_tmp = te.lang.cce.vmuls(datan_tmp, coeff2)
                    data0_tmp = te.lang.cce.vadd(data0_tmp, datan_tmp)
        res = data0_tmp
    return res

def _eltwise_check_para(x, y, mode=1, coeff=[],
                        kernel_name="eltwise"):

    util.check_kernel_name(kernel_name)

    shape = x[0].get("shape")
    dtype = x[0].get("dtype").lower()

    util.check_shape_rule(shape)
    util.check_shape_size(shape, SHAPE_SIZE_LIMIT)

    dtype_check_list = ["float16", "float32"]
    if not dtype in dtype_check_list:
        raise RuntimeError("dtype is not support,"
                           "only support fp16 and fp32.")

    tensor_num = len(x)
    if tensor_num < 1 or tensor_num > 32:
        raise RuntimeError("tensor_num need in range [1,32].")

    # all input data should be same shape and dtype
    if tensor_num > 1:
        for i in range(1, tensor_num):

            shape_tmp = x[i].get("shape")
            dtype_tmp = x[i].get("dtype").lower()

            if shape_tmp != shape:
                raise RuntimeError("input shape is not same.")

            if dtype_tmp != dtype:
                raise RuntimeError("input dtype is not same.")

    shape_output = y.get("shape")
    util.check_shape_rule(shape_output)
    util.check_shape_size(shape_output, SHAPE_SIZE_LIMIT)
    if shape_output != shape:
        raise RuntimeError("output shape is not same with input.")

    dtype_output = y.get("dtype").lower()
    if dtype_output != dtype:
        raise RuntimeError("output dtype is not same with input.")

    #mode type must be 0, 1 or 2
    op_list = (0, 1, 2)
    if mode not in op_list:
        raise RuntimeError("mode is wrong, only suppor 0, 1, 2.")

@util.check_input_type((list, tuple), dict, int, (list, tuple), str)
def eltwise(x, y, mode=1, coeff=[], kernel_name="eltwise"):
    """
    Compute elementwise modes, such as 0:PRODUCT, 1:SUM and 2:MAX

    Parameters
    ----------
    x : the list of input data, it's element is dict:{"shape":[], "dtype":""}

    y : the dict of output

    mode : 0:product,1:sum,2:max;default is 1:sum.

    coeff : input_num should be equal with coeff size.

    kernel_name : cce kernel name, default value is "eltwise"

    Returns
    -------
    None

    """

    _eltwise_check_para(x, y, mode=mode,
                        coeff=coeff, kernel_name=kernel_name)
    shape = x[0].get("shape")
    dtype = x[0].get("dtype").lower()
    tensor_num = len(x)

    shape = util.shape_refine(shape)
    fuseshape = [1]
    fuseshape[0] = reduceIns(lambda x, y: x*y, shape)

    tlist = []
    with tvm.target.cce():
        for i in range(0, tensor_num):
            datan_name = 'data%d' % i
            datan_tmp = tvm.placeholder(fuseshape, name=datan_name, dtype=dtype)
            tlist.append(datan_tmp)

        y_datan_name = 'y_data'
        data_y_tmp = tvm.placeholder(fuseshape, name=y_datan_name, dtype=dtype)

        res = eltwise_compute(tlist, data_y_tmp,
                              mode, coeff, kernel_name)
        sch = generic.auto_schedule(res)
    tlist.append(res)

    config = {"print_ir": False,
              "need_build": False,
              "name": kernel_name,
              "tensor_list": tlist}
    te.lang.cce.cce_build_code(sch, config)


#!/usr/bin/env python3
import pathlib
import re
import sys

ROOT_DIR = pathlib.Path(__file__).parent.parent
README = ROOT_DIR / "README.md"
INDEX = ROOT_DIR / "docs" / "index.md"
SETUP = ROOT_DIR / "setup.py"
QUICKSTART = ROOT_DIR / "examples" / "quickstart.py"

USED_FILES = {str(path.relative_to(ROOT_DIR)) for path in (INDEX, SETUP, QUICKSTART)}

def main():
    version_match = re.search(r"version=\"(\d+\.\d+)", SETUP.read_text())
    assert version_match is not None
    version = version_match.group(1)
    content = INDEX.read_text()
    # Set title
    content = re.sub(r"# Overview\s*## apischema", "# apischema", content)
    # Remove FAQ
    content = content[: content.index("## FAQ")]
    # Remove admonitions
    content = re.sub(r"!!! note\n\s*(.*)\n", lambda m: f"> {m.group(1)}\n", content)
    # Add chart
    content = content.replace(
        r"<!--insert chart-->",
        "\n".join(
            f"![benchmark chart](https://wyfo.github.io/apischema/{version}/"
            f"benchmark_chart_{theme}#gh-{theme}-mode-only)"
            for theme in ("light", "dark")
        ),
    )
    # Uncomment
    content = re.sub(r"<!--\n(\s*(.|\n)*?\s*)\n-->", lambda m: m.group(1), content)
    content = re.sub(
        r"(\d+\.\d+)/benchmark_chart\.svg", f"{version}/benchmark_chart.svg", content
    )
    # Rewrite links
    content = re.sub(
        r"\(([\w/]+)\.(md|svg)(#[\w-]+)?\)",
        lambda m: f"(https://wyfo.github.io/apischema/{version}/{m.group(1)}"
        + (".svg" if m.group(2) == "svg" else "")
        + (m.group(3) or "")
        + ")",
        content,
    )
    # Add quickstart
    content = re.sub(
        "```python(.|\n)*?```", f"```python\n{QUICKSTART.read_text()}```", content
    )
    README.write_text(content)

if __name__ == "__main__":
    if not set(sys.argv).isdisjoint(USED_FILES):
        main()


import numpy as np
import cv2
import copy
import argparse
import logging
import os

''' config '''
num_lmks = 5
target_shape = (64, 64)

def process_txt(txt):
    with open(txt, 'r') as f:
        lines = f.readlines()

    # each line: img_path, rects(4), lmks(10)
    info_dict_list = []
    for line in lines:
        import pdb
        #pdb.set_trace()
        info_dict = dict()
        data = line.strip().split(' ')
        img_path = data[0]
        rects = np.array([float(x) for x in data[1:5]])
        lmks = np.array([float(x) for x in data[(5 + num_lmks * 0):(5 + num_lmks * 2)]])
        img_name = img_path.split('/')[-1]
        label = data[-1]
        #img_name = '_'.join(img_path.split('AntiSpoofingData/')[-1].split('/'))
        info_dict = {'img_path': img_path,
                    'img_name': img_name,
                    'rects': rects,
                    'lmks': lmks,
                    'label':label}
        info_dict_list.append(info_dict)

    return info_dict_list

def plot(info_dict_list, output_file):
    last_name = ''
    for i, info_dict in enumerate(info_dict_list):
        img_name = info_dict['img_name']
        img_path = info_dict['img_path']
        rects = info_dict['rects']
        lmks = info_dict['lmks']
        label = info_dict['label']
        if img_name != last_name:
            last_name = img_name
            # img = cv2.imread(img_path)
            img = cv2.imdecode(np.fromfile(img_path, dtype=np.uint8), -1)
        else:
            img = img
        ref = (rects[2] - rects[0] + rects[3] - rects[1]) / 2
        cv2.rectangle(img, (int(rects[0]), int(rects[1])), (int(rects[2]), int(rects[3])), (0, 255, 192), int(ref*0.02))

        ''' vis / invis metric '''
        for j in range(num_lmks):
            if lmks[j] > 0.5:
                # green
                cv2.circle(img, (int(lmks[2 * j]), int(lmks[2 * j + 1])), int(ref*0.02), (0, 255, 0), -1)
            else:
                # yellow
                cv2.circle(img, (int(lmks[2 * j]), int(lmks[2 * j + 1])), int(ref*0.02), (0, 255, 255), -1)
            # blue
            cv2.circle(img, (int(lmks[2 * j]), int(lmks[2 * j + 1])), int(ref*0.02), (255, 128, 0), -1)
            cv2.putText(img, 'label:{}'.format(label),(int(rects[0]), int(rects[1]*0.98)),color=(255, 255, 192), fontFace=cv2.FONT_HERSHEY_COMPLEX, fontScale=0.5)

        if i+1 == len(info_dict_list):
            cv2.imwrite('./{}/{}'.format(output_file, img_name), img)
            continue
        if img_name != info_dict_list[i+1]['img_name']:
            cv2.imwrite('./{}/{}'.format(output_file, img_name), img)

def main(args):
    output_file = './Vis_rects_lmks_results'
    if os.path.exists('./{}'.format(output_file)):
        os.system('rm -r {}'.format(output_file))
    os.system('mkdir {}'.format(output_file))

    # get lmks info
    info_dict_list = process_txt(args.file)
    # plot lmks on image
    plot(info_dict_list, output_file)

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Plot Lmks rects')
    parser.add_argument('--file', help='lmks rects txt file path', required=True, type=str)

    args = parser.parse_args()

    main(args)


platform = "linux"

import dbus
from dbus.mainloop.glib import DBusGMainLoop
from twisted.internet import defer

from ubuntuone.platform.linux.os_helper import (
    access,
    allow_writes,
    can_write,
    get_path_list,
    is_link,
    is_root,
    listdir,
    make_dir,
    make_link,
    move_to_trash,
    normpath,
    open_file,
    path_exists,
    read_link,
    recursive_move,
    remove_dir,
    remove_file,
    remove_link,
    remove_tree,
    rename,
    set_application_name,
    set_dir_readonly,
    set_dir_readwrite,
    set_file_readonly,
    set_file_readwrite,
    set_no_rights,
    stat_path,
    walk,
)
from ubuntuone.platform.linux.logger import setup_filesystem_logging, get_filesystem_logger
from ubuntuone.platform.linux.filesystem_notifications import FilesystemMonitor
from ubuntuone.platform.linux.notification import Notification

class ExternalInterface(object):
    """An ExternalInterface implemented with a DBus interface."""

    def __init__(self, main, glib_loop=False, broadcast_events=False,
                 dbus_iface=None):
        # avoid circular dependencies

        if dbus_iface is None:
            from ubuntuone.platform.linux import dbus_interface
            if not glib_loop:
                self.bus = dbus.SessionBus()
            else:
                loop = DBusGMainLoop(set_as_default=True)
                self.bus = dbus.SessionBus(loop)

            self.dbus_iface = dbus_interface.DBusInterface(
                self.bus, main, send_events=broadcast_events)

        else:
            self.dbus_iface = dbus_iface
            self.bus = None

    def _get_credentials(self):
        return self.dbus_iface.oauth_credentials

    def _set_credentials(self, credentials):
        self.dbus_iface.oauth_credentials = credentials

    oauth_credentials = property(fget=_get_credentials, fset=_set_credentials)

    def shutdown(self, with_restart):
        self.dbus_iface.shutdown(with_restart)

    def connect(self, *args, **kwargs):
        self.dbus_iface.connect(*args, **kwargs)

def is_already_running():
    """Check if there is another instance registered in DBus."""
    from ubuntuone.platform.linux import dbus_interface
    bus = dbus.SessionBus()
    request = bus.request_name(dbus_interface.DBUS_IFACE_NAME,
                               dbus.bus.NAME_FLAG_DO_NOT_QUEUE)
    if request == dbus.bus.REQUEST_NAME_REPLY_EXISTS:
        return defer.succeed(True)
    else:
        return defer.succeed(False)


# Take a gzipped pileup file and a .tsv files from oncotator reformat (/data/ipadioleau/code/oncotator_extract_information_from_MAF_tumor_Vs_normal.py)
# Add strand coverage of variant and reference allele to the tsv file
import sys
from collections import Counter
import gzip
import io

# MPILEUP = gzip.open(sys.argv[1], 'rb')
ONCOTATOR_FILE = open(sys.argv[2], 'r')
COMPLETED_ONCOTATOR_FILES = open(sys.argv[3], 'w')

# A function that return a dictionnary with all alternatives count on forward and reverse strand
def return_alternativ_count(seq):
	nucleotides_list = {}
	acceptable = ['A', 'a', 'C', 'c', 'G', 'g', 'T', 't', '+', '-', '1', '2', '3', '4', '5', '6', '7', '8', '9', '0']
	getsize = 0
	indel = []
	tmp_size = []
	size = 0
	for nuc in seq:
		# skip all non informative value
		if nuc in acceptable:
			# If there is an indel set getsize to 1 so we get it's size next turn
			if (nuc=='+' or nuc=='-'):
				getsize = 1
				# if previous variant was a indel, save information
				if indel != []:
					indel_seq = ''.join(indel)
					indel = []
					if not indel_seq.upper() in nucleotides_list:
						nucleotides_list[indel_seq.upper()] = [0,0]
					if indel_seq.isupper():
						nucleotides_list[indel_seq.upper()][0] += 1
					else:
						nucleotides_list[indel_seq.upper()][1] += 1
				signe = nuc
			# Get indel size
			elif getsize == 1:
				if nuc.isdigit() :
					tmp_size.append(nuc)
				else:
					if tmp_size == []:
						indel.append(signe + nuc)
					else:
						size = int(''.join(tmp_size)) - 1
						tmp_size = []
						getsize = 0
						indel.append(signe + nuc)
			# If we are processing an indel, get next nucleotide
			elif size > 0 :
				indel.append(nuc)
				size -= 1
			# Save count for each alternative in a dictionary
			else:
				if indel != []:
					indel_seq = ''.join(indel)
					indel = []
					signe=''
					if not indel_seq.upper() in nucleotides_list:
						nucleotides_list[indel_seq.upper()] = [0,0]
					if indel_seq.isupper():
						nucleotides_list[indel_seq.upper()][0] += 1
					else:
						nucleotides_list[indel_seq.upper()][1] += 1
				if  not nuc.isdigit():
					if not nuc.upper() in nucleotides_list:
						nucleotides_list[nuc.upper()] = [0,0]
					if nuc.isupper() :
						nucleotides_list[nuc.upper()][0] += 1
					else:
						nucleotides_list[nuc.upper()][1] += 1
	if indel != []:
		indel_seq = ''.join(indel)
		indel = []
		if not indel_seq.upper() in nucleotides_list:
			nucleotides_list[indel_seq.upper()] = [0,0]
		if indel_seq.isupper():
			nucleotides_list[indel_seq.upper()][0] += 1
		else:
			nucleotides_list[indel_seq.upper()][1] += 1
	return(nucleotides_list)

variant_dict = {}
with io.TextIOWrapper(io.BufferedReader(gzip.open(sys.argv[1]))) as MPILEUP:
	for line in MPILEUP:
		info = line.split('\t')
		nucleotides = Counter(info[4])
		ref_forward = '0'
		ref_reverse = '0'
		if '.' in nucleotides:
			ref_forward = str(nucleotides['.'])
		if ',' in nucleotides:
			ref_reverse = str(nucleotides[','])
		alternativ = return_alternativ_count(info[4])
		for alt in alternativ:
			if alt[0] == '-' :
				info[1] = str(int(info[1]) + 1)
			chr_pos = info[0] + '_' + info[1]
			if chr_pos in variant_dict:
				variant_dict[chr_pos][alt] = [alternativ[alt][0],alternativ[alt][1], ref_forward, ref_reverse]
			else:
				variant_dict[chr_pos] = {}
				variant_dict[chr_pos][alt] = [alternativ[alt][0],alternativ[alt][1], ref_forward, ref_reverse]
			if alt[0] == '-' :
				info[1] = str(int(info[1]) - 1)
			#OUTFILE.write(info[0] + '\t' + info[1] + '\t' + info[2] + '\t' + alt + '\t' + ref_forward + '\t' + ref_reverse + '\t' + str(alternativ[alt][0]) + '\t' + str(alternativ[alt][1]) + '\n')
			#print(info[0] + '\t' + info[1] + '\t' + info[2] + '\t' + alt + '\t' + ref_forward + '\t' + ref_reverse + '\t' + str(alternativ[alt][0]) + '\t' + str(alternativ[alt][1]) + '\n')

#####
header = ONCOTATOR_FILE.readline().strip('\n')
COMPLETED_ONCOTATOR_FILES.write(header + '\tt_alt_forward_strand\tt_alt_reverse_strand\tt_ref_forward_strand\tt_ref_reverse_strand\n')
header_info = header.split('\t')
Variant_Type_index = header_info.index('Variant_Type')
Tumor_Seq_Allele2_index = header_info.index('Tumor_Seq_Allele2')

Start_position_index = header_info.index('Start_position')
Chromosome_index = header_info.index('Chromosome')
Tumor_Seq_Allele1_index = header_info.index('Tumor_Seq_Allele1')

for line in ONCOTATOR_FILE:
	info = line.strip().split('\t')
	chr_pos = info[Chromosome_index] + '_' + info[Start_position_index]
	variant_counts = [9999999999,9999999999, 9999999999, 9999999999]
	# My variant checker script do not detect DNP or TNP
	# To report DNP/TNP counts we report the minimum counts per strand for all alternatives nucleotides composing the variant
	if info[Variant_Type_index] == 'DNP' or info[Variant_Type_index] == 'TNP' :
		nucleotides = info[Tumor_Seq_Allele2_index]
		#print(nucleotides)
		tmp_pos = int(info[Start_position_index])
		for tmp_nuc in nucleotides:
			tmp_chr_pos = info[Chromosome_index] + '_' + str(tmp_pos)
			if (tmp_chr_pos in variant_dict) and (tmp_nuc in variant_dict[tmp_chr_pos]):
				if (int(variant_dict[tmp_chr_pos][tmp_nuc][0]) < variant_counts[0]):
					variant_counts[0] = int(variant_dict[tmp_chr_pos][tmp_nuc][0])
				if (int(variant_dict[tmp_chr_pos][tmp_nuc][1]) < variant_counts[1]):
					variant_counts[1] = int(variant_dict[tmp_chr_pos][tmp_nuc][1])
				if (int(variant_dict[tmp_chr_pos][tmp_nuc][2]) < variant_counts[2]):
					variant_counts[2] = int(variant_dict[tmp_chr_pos][tmp_nuc][2])
				if (int(variant_dict[tmp_chr_pos][tmp_nuc][3]) < variant_counts[3]):
					variant_counts[3] = int(variant_dict[tmp_chr_pos][tmp_nuc][3])
			tmp_pos += 1
	elif info[Variant_Type_index] == 'DEL' or info[Variant_Type_index] == 'ONP' :
		seq = '-' + info[Tumor_Seq_Allele1_index]
		if (chr_pos in variant_dict) and (seq in variant_dict[chr_pos]):
			if (int(variant_dict[chr_pos][seq][0]) < variant_counts[0]):
				variant_counts[0] = variant_dict[chr_pos][seq][0]
			if (int(variant_dict[chr_pos][seq][1]) < variant_counts[1]):
				variant_counts[1] = variant_dict[chr_pos][seq][1]
			if (int(variant_dict[chr_pos][seq][2]) < variant_counts[2]):
				variant_counts[2] = variant_dict[chr_pos][seq][2]
			if (int(variant_dict[chr_pos][seq][3]) < variant_counts[3]):
				variant_counts[3] = variant_dict[chr_pos][seq][3]
	elif info[Variant_Type_index] == 'INS' :
		seq = '+' + info[Tumor_Seq_Allele2_index]
		if (chr_pos in variant_dict) and (seq in variant_dict[chr_pos]):
			if (int(variant_dict[chr_pos][seq][0]) < variant_counts[0]):
				variant_counts[0] = variant_dict[chr_pos][seq][0]
			if (int(variant_dict[chr_pos][seq][1]) < variant_counts[1]):
				variant_counts[1] = variant_dict[chr_pos][seq][1]
			if (int(variant_dict[chr_pos][seq][2]) < variant_counts[2]):
				variant_counts[2] = variant_dict[chr_pos][seq][2]
			if (int(variant_dict[chr_pos][seq][3]) < variant_counts[3]):
				variant_counts[3] = variant_dict[chr_pos][seq][3]
	else:
		seq = info[Tumor_Seq_Allele2_index]
		if (chr_pos in variant_dict) and (seq in variant_dict[chr_pos]):
			if (int(variant_dict[chr_pos][seq][0]) < variant_counts[0]):
				variant_counts[0] = variant_dict[chr_pos][seq][0]
			if (int(variant_dict[chr_pos][seq][1]) < variant_counts[1]):
				variant_counts[1] = variant_dict[chr_pos][seq][1]
			if (int(variant_dict[chr_pos][seq][2]) < variant_counts[2]):
				variant_counts[2] = variant_dict[chr_pos][seq][2]
			if (int(variant_dict[chr_pos][seq][3]) < variant_counts[3]):
				variant_counts[3] = variant_dict[chr_pos][seq][3]
	if variant_counts[0] == 9999999999 and variant_counts[1] == 9999999999:
		if variant_counts[2] == 9999999999 :
			variant_counts[2] = 0
		if variant_counts[3] == 9999999999 :
			variant_counts[3] = 0
		variant_counts = ['0','0', str(variant_counts[2]), str(variant_counts[3])]

	COMPLETED_ONCOTATOR_FILES.write(line.strip('\n') + '\t' + str(variant_counts[0]) + '\t' + str(variant_counts[1]) + '\t' + str(variant_counts[2]) + '\t' + str(variant_counts[3]) + '\n')

COMPLETED_ONCOTATOR_FILES.close()
ONCOTATOR_FILE.close()
MPILEUP.close()


from pages.base_page import Page
from selenium.webdriver.common.by import By
from time import sleep

class CartPage(Page):

    MINUS_BUTTON = (By.CSS_SELECTOR, "button[name='minus']")
    EMPTY_CART_TEXT = (By.CSS_SELECTOR, "h2.cart__empty-text")

    def click_on_minus_button(self):
        self.click(*self.MINUS_BUTTON)
        sleep(5)

    def verify_cart_is_empty(self):
        self.verify_text('Your cart is currently empty', *self.EMPTY_CART_TEXT)


#!/usr/bin/env python
# -*- coding: utf-8 -*

# Licensed under a 3-clause BSD style license - see LICENSE.rst
from __future__ import print_function

from abc import abstractmethod, ABC
from enum import Enum

class PropertyConstraint(object):
    """
    PropertyConstraint's class definition

    The user can specify a constraint acting only
    on properties too. The MOCServer asks for an
    algebraic expression dealing with properties.
    Example : a user can ask the MOCServer all datasets
    having the word CDS(Center of astronomical Data of
    Strasbourg) in its ID property or/and a moc_sky_fraction
    < 1 %

    """

    def __init__(self, expression):
        """
        PropertyConstraint's constructor

        :param expression:
            can be a PropertiesExpr or a string
        """

        if not isinstance(expression, PropertiesExpr) and not isinstance(expression, str):
            raise TypeError

        self.expr = expression
        self.request_payload = dict()
        self.compute_payload()

    def compute_payload(self):
        """Update the property constraints payload"""
        if isinstance(self.expr, str):
            self.request_payload = {'expr': self.expr}
        else:
            self.request_payload = {'expr': self.expr.eval()}

    def __repr__(self):
        result = "Properties constraints' request payload :\n{0}".format(self.request_payload)
        return result

class OperandExpr(Enum):
    """Operand Enum which allow the user to define relationship between expr"""
    Inter = 1,
    Union = 2,
    Subtr = 3

class PropertiesExpr(ABC):
    """
    General expression interface

    Expressions are built like a binary tree. A parent expression can
    have one or two children. The parent defines an operand between
    its two children (if it is a ParentNode) or nothing in
    the other case (only one child).

    """

    @abstractmethod
    def eval(self):
        """
        Evaluate recursively the whole expression

        Returns a str understandable by the MOCServer
        """

        pass

class ChildNode(PropertiesExpr):
    """Leaf expression node of the binary tree expression"""

    def __init__(self, condition):
        assert condition is not None
        self.condition = condition

    def eval(self):
        return str(self.condition)

class ParentNode(PropertiesExpr):
    """Parent expression node of the binary tree expression"""

    def __init__(self, operand, left_expr, right_expr):
        if not isinstance(right_expr, PropertiesExpr) or not isinstance(left_expr, PropertiesExpr):
            raise TypeError

        if operand not in (OperandExpr.Inter, OperandExpr.Union, OperandExpr.Subtr):
            raise TypeError

        self.left_expr = left_expr
        self.right_expr = right_expr
        self.operand = operand

    def eval(self):
        left_expr_str = str(self.left_expr.eval())
        right_expr_str = str(self.right_expr.eval())

        operand_str = " &! "
        if self.operand is OperandExpr.Inter:
            operand_str = " && "
        elif self.operand is OperandExpr.Union:
            operand_str = " || "

        if isinstance(self.left_expr, ParentNode):
            left_expr_str = '(' + left_expr_str + ')'
        if isinstance(self.right_expr, ParentNode):
            right_expr_str = '(' + right_expr_str + ')'

        return left_expr_str + operand_str + right_expr_str


# IMPORTS
import argparse
import logging
import os
import time
import warnings

import numpy as np
import wandb
from sklearn.preprocessing import StandardScaler
from tensorflow.keras.utils import set_random_seed
from tqdm import tqdm

import datasets
from predictive_models import *
from self_supervised_tasks import *
from utils import (
    compute_interval_metrics,
    compute_deficet,
    compute_excess,
    process_results,
    write_to_file,
)

logging.getLogger().setLevel(logging.INFO)

warnings.filterwarnings("ignore")

def seed_everything(seed: int):
    import random, os
    import numpy as np
    import torch

    random.seed(seed)
    os.environ["PYTHONHASHSEED"] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = True
    set_random_seed(seed)

logging.getLogger().setLevel(logging.INFO)

import yaml
# Load the WANDB YAML file
with open('../wandb.yaml') as file:
    wandb_data = yaml.load(file, Loader=yaml.FullLoader)

os.environ["WANDB_API_KEY"] = wandb_data['wandb_key']
wandb_entity = wandb_data['wandb_entity']

###########################################################################################################
#
# SEEDING AND PROCESSING THE SAME AS THE LABELLED EXPERIMENTS
#
###########################################################################################################

def init_arg():
    parser = argparse.ArgumentParser()
    parser.add_argument("--dataset", default="bike", type=str)
    parser.add_argument("--test-size", default=0.33, type=float)
    parser.add_argument("--labeled-prop", default=0.4, type=float)
    parser.add_argument("--runs", default=3, type=int)
    parser.add_argument("--epochs", default=10, type=int)
    return parser.parse_args()

# If you want to write the different intervals to a file for post-hoc analysis
save_intervals = False

if __name__ == "__main__":
    project_name = "crf_labeled_sanity"
    logging.info("WANDB init...")
    run = wandb.init(
        project=project_name,
        entity=wandb_entity,
    )

    args = init_arg()

    dataset_name = args.dataset
    epochs = args.epochs

    labeled_prop = args.labeled_prop

    assert dataset_name in [
        "star",
        "bike",
        "bio",
        "concrete",
        "community",
        "facebook_2",
        "blog_data",
    ]

    start = time.time()

    logging.info(f"Getting dataset {dataset_name}...")

    X, y, seed = datasets.GetDataset(dataset_name, "../datasets/", seeded=True)

    cfr_results_list = []
    batch_size = 128
    wandb.log(
        {
            "Dataset": args.dataset,
            "Test Size": args.test_size,
            "Runs": args.runs,
            "Samples": len(y),
            "Epochs": epochs,
            "Labeled Prop": labeled_prop,
        }
    )

    start_seed = seed

    log_res = False

    for i in tqdm(range(args.runs)):
        seed += 1 #increment per run so we test different random seeds
        seed_everything(seed)

        alpha = 0.1

        # Preprocess data
        (
            X_train_sc,
            X_residual_sc,
            X_cal_sc,
            X_test_sc,
            y_train_sc,
            y_residual_sc,
            y_cal_sc,
            y_test_sc,
        ) = datasets.process_data(X=X, y=y, seed=seed)

        # Fit the predictive model on the proper training set
        pred_model = predictive_model(
            input_shape=X_train_sc.shape[1],
            epochs=epochs,
            batch_size=batch_size,
            seed=seed,
        )
        pred_model.fit(X_train_sc, y_train_sc)

        #######################################################
        #
        # Self-supervised learning
        #
        ######################################################

        # Fit the autoencoder
        ssl_task = AutoEncoder(
            input_shape=pred_model.extract_encoding(X_train_sc).shape[1],
            reconstruct_shape=X_train_sc.shape[1],
            epochs=epochs,
            batch_size=batch_size,
            seed=seed,
        )
        ssl_task.fit(x_train=X_train_sc, pred_model=pred_model)

        # Fit VIME
        vime = Vime_Task(epochs=epochs, seed=seed)
        vime.fit(pred_model.extract_encoding(X_train_sc))

        results = {}

        ###########################################################################################################
        # Make predictions with the main model
        # Predict on the calibration set
        ypred_cal = pred_model.predict(X_cal_sc).reshape(-1, 1)

        # Predict on the test set
        ypred_test = pred_model.predict(X_test_sc).reshape(-1, 1)

        # Compute the main model errors
        main_model_errors = np.abs(ypred_test - y_test_sc)

        N = len(y_cal_sc)

        ###########################################################################################################

        ###########################################################################################################
        #
        # CRF
        #
        ###########################################################################################################

        # RUN CONFORMALIZED RESIDUAL FITTING  - CRF

        logging.info("Running Conformalized Residual Fitting")

        # calculate residuals
        res_y = np.abs(y_residual_sc - pred_model.predict(X_residual_sc).reshape(-1, 1))

        if log_res:
            res_y += 0.00001  # prevent log(0)
            res_y = np.log(res_y)

        # fit residual (normalization) model
        model_r = predictive_model(
            input_shape=X_residual_sc.shape[1],
            epochs=epochs,
            batch_size=batch_size,
            seed=seed,
        )
        model_r.fit(X_residual_sc, res_y)

        # calculate q_yhat for the calibration set
        res_ypred_cal = model_r.predict(X_cal_sc).reshape(-1, 1)

        if log_res:
            res_ypred_cal = np.exp(res_ypred_cal)

        # compute critical nonconformity value
        q_yhat = np.quantile(
            np.abs(y_cal_sc - ypred_cal) / res_ypred_cal,
            np.ceil((N + 1) * (1 - alpha)) / N,
        )

        res_ypred_test = model_r.predict(X_test_sc).reshape(-1, 1)

        if log_res:
            res_ypred_test = np.exp(res_ypred_test)

        # predict with 1-alpha confidence - scaled by residual predictions
        lower_bound = ypred_test - (q_yhat * res_ypred_test.reshape(-1, 1))
        upper_bound = ypred_test + (q_yhat * res_ypred_test.reshape(-1, 1))

        # COMPUTE METRICS
        intervals = np.array(abs(upper_bound - lower_bound))

        if save_intervals:
             write_to_file(intervals, f"../dists/{dataset_name}_{i}_crf.p")

        residual_model_errors = np.abs(ypred_test - y_test_sc)

        _, avg_length = compute_interval_metrics(lower_bound, upper_bound, y_test_sc)
        avg_excess, proportion_excess = compute_excess(
            lower_bound, upper_bound, y_test_sc
        )
        avg_deficet, proportion_deficet = compute_deficet(
            lower_bound, upper_bound, y_test_sc
        )

        # Log results
        results["CFR"] = {
            "avg_length": avg_length,
            "avg_excess": avg_excess,
            "avg_deficet": avg_deficet,
        }

        ###########################################################################################################
        #
        # SANITY CHECK 1: CFR+ONLYSSL
        #
        ###########################################################################################################

        logging.info('SANITY CHECK 1: CFR+ONLYSSL')

        # predict using VIME
        ssl_train_vime = vime.predict(pred_model.extract_encoding(X_train_sc)).reshape(-1,1)
        ssl_residual_vime = vime.predict(pred_model.extract_encoding(X_residual_sc)).reshape(-1,1)
        ssl_cal_vime = vime.predict(pred_model.extract_encoding(X_cal_sc)).reshape(-1,1)
        ssl_test_vime = vime.predict(pred_model.extract_encoding(X_test_sc)).reshape(-1,1)

        # use the SSL
        res_ypred_cal= ssl_cal_vime.reshape(-1,1)
        q_yhat=np.quantile(np.abs(y_cal_sc-ypred_cal)/res_ypred_cal,np.ceil((N+1)*(1-alpha))/N)

        # use the SSL loss as the residual, rather than fitting a model
        res_ypred_test= ssl_test_vime.reshape(-1,1)
        lower_bound=ypred_test-(q_yhat*res_ypred_test.reshape(-1,1))
        upper_bound=ypred_test+(q_yhat*res_ypred_test.reshape(-1,1))

        onlyssl_residual_model_errors =res_ypred_test
        onlyssl_upper = upper_bound
        onlyssl_lower = lower_bound

        _, avg_length = compute_interval_metrics(lower_bound,upper_bound,y_test_sc)
        avg_excess, proportion_excess = compute_excess(lower_bound,upper_bound,y_test_sc)
        avg_deficet, proportion_deficet = compute_deficet(lower_bound,upper_bound,y_test_sc)

        results['CFR+ONLYSSL'] = {'avg_length': avg_length,
                                  'avg_excess': avg_excess,
                                  'avg_deficet': avg_deficet}

        ###########################################################################################################
        #
        # SANITY CHECK 2: CFR+ISOLATION FOREST
        #
        ###########################################################################################################
        logging.info('SANITY CHECK 2: CFR+ISOLATION FOREST')

        # use isolation forest instead of SSL loss
        from sklearn.ensemble import IsolationForest
        isoforest = IsolationForest().fit(X_train_sc)

        # predict using Isolation Forest
        ssl_train_vime = isoforest.score_samples(X_train_sc).reshape(-1,1)
        ssl_residual_vime = isoforest.score_samples(X_residual_sc).reshape(-1,1)
        ssl_cal_vime = isoforest.score_samples(X_cal_sc).reshape(-1,1)
        ssl_test_vime = isoforest.score_samples(X_test_sc).reshape(-1,1)

        # scale the isolation forest
        scaler_ssl = StandardScaler()
        scaler_ssl.fit(ssl_train_vime)
        ssl_train_sc_vime = scaler_ssl.transform(ssl_train_vime)
        ssl_residual_sc_vime = scaler_ssl.transform(ssl_residual_vime)
        ssl_cal_sc_vime = scaler_ssl.transform(ssl_cal_vime)
        ssl_test_sc_vime = scaler_ssl.transform(ssl_test_vime)

        res_y=np.abs(y_residual_sc-pred_model.predict(X_residual_sc).reshape(-1,1))

        # augment the residual with the IsoForest output
        X_train_r  = np.hstack([X_residual_sc, ssl_residual_sc_vime])
        # fit residual model
        model_r = predictive_model(
            input_shape= X_train_r.shape[1],
            epochs=epochs,
            batch_size=128,
        )
        model_r.fit(X_train_r,res_y)

        #calculate q_yhat for the calibration set
        X_cal_r  = np.hstack([X_cal_sc, ssl_cal_sc_vime])
        res_ypred_cal=model_r.predict(X_cal_r).reshape(-1,1)
        q_yhat=np.quantile(np.abs(y_cal_sc-ypred_cal)/res_ypred_cal,np.ceil((N+1)*(1-alpha))/N)

        vime_qyhat = q_yhat

        #predict with 1-alpha confidence - scaled by residual predictions
        X_test_r  = np.hstack([X_test_sc, ssl_test_sc_vime])
        res_ypred_test=model_r.predict(X_test_r).reshape(-1,1)
        lower_bound=ypred_test-(q_yhat*res_ypred_test.reshape(-1,1))
        upper_bound=ypred_test+(q_yhat*res_ypred_test.reshape(-1,1))

        if_residual_model_errors =res_ypred_test

        if_upper = upper_bound
        if_lower = lower_bound

        _, avg_length = compute_interval_metrics(lower_bound,upper_bound,y_test_sc)
        avg_excess, proportion_excess = compute_excess(lower_bound,upper_bound,y_test_sc)
        avg_deficet, proportion_deficet = compute_deficet(lower_bound,upper_bound,y_test_sc)

        results['CFR+IF'] = {'avg_length': avg_length,
                             'avg_excess': avg_excess,
                             'avg_deficet': avg_deficet,
                             }

        cfr_results_list.append(results)

        seed += 1

    # Process results
    final_cfr = process_results(cfr_results_list)
    print(f"CFR: {final_cfr}")

    # Log to wandb
    wandb.log({"CFR": final_cfr})

    end = time.time()
    print(f"TIME TAKEN = {end - start}")


import rethinkdb as r
import semantic_version
from flask_restful import Resource, Api, request, fields, marshal_with
from lone_jupiter import app, appconfig, api
from helpers.rethinkdb_helper import RethinkDbHelper
from helpers.server_selector import ServerSelector
import requests
import json

rethinkdb_host = appconfig['rethinkdb_host']
rethinkdb_port = appconfig['rethinkdb_port']
rethinkdb_db = appconfig['rethinkdb_db']
serverpool_port_start = appconfig['serverpool_port_start']

class ApiDeploymentSelector(Resource):
    def get(self, appmap_name):

        appmaps_table = "applicationmaps"

        r.connect(rethinkdb_host, rethinkdb_port).repl()
        RethinkDbHelper.ensure_database_and_table(rethinkdb_host, rethinkdb_port, rethinkdb_db, appmaps_table)

        keys = r.db(rethinkdb_db).table(appmaps_table).run()
        db_appmaps = list(keys)

        if db_appmaps.__len__() == 0:
            #not found
            pass
        if db_appmaps.__len__() < 1:
            #too many
            pass

        appmap = db_appmaps[0]
        desired_instancecount = appmap['instancecount']

        #figure out the existing number of deployments
        url = request.url_root + "api/deployments/appmap/" + appmap_name
        result = requests.get(url)
        result_data = json.loads(result.content)

        current_deployments = [x for x in result_data if x['package_version'] == appmap['gitfilter']['version'] and x['package_branch'] == appmap['gitfilter']['branch']]

        current_deployments_count = current_deployments.__len__()

        resultobj = {}
        resultobj['current_deployments'] = current_deployments_count
        resultobj['desired_deployments'] = desired_instancecount

        deployment_candidates = []
        if current_deployments_count < desired_instancecount:
            needed_deployments = desired_instancecount - current_deployments_count
            deployment_candidates = ServerSelector.get_deployment_candidates(rethinkdb_host, rethinkdb_port,
                                                                             rethinkdb_db, appmap['targettags'],
                                                                             needed_deployments, current_deployments)

            for candidate in deployment_candidates:
                port = ServerSelector.get_free_port(candidate['name'], request.url_root, serverpool_port_start)
                candidate['port'] = port
                result = ServerSelector.create_reservation(candidate, appmap, request.url_root)

        return deployment_candidates

api.add_resource(ApiDeploymentSelector, '/api/deploymentselector/<string:appmap_name>')


#!/usr/bin/env python3

"""
Setup script for the HyperCanny Climate module.
"""

from pathlib import Path
from setuptools import setup

# Get the long description from the README file
here = Path(__file__).parent.absolute()
with (here / 'README.rst').open(encoding='utf-8') as f:
    long_description = f.read()

setup(
    name='hypercc',
    version='0.1.0',
    description='Runs the Canny edge detector on climate data.',
    long_description=long_description,
    author='Johan Hidding',
    url='https://github.com/abrupt-climate/hypercc',
    packages=['hypercc', 'hypercc.data'],
    classifiers=[
        'License :: OSI Approved :: Apache Software License',
        'Intended Audience :: Science/Research',
        'Environment :: Console',
        'Development Status :: 4 - Beta',
        'Programming Language :: Python :: 3.6'],
    install_requires=[
        'HyperCanny', 'noodles[numpy]', 'matplotlib', 'scipy', 'numpy',
        'netCDF4', 'jupyter', 'cartopy', 'pint', 'pyparsing'],
    extras_require={
        'develop': [
            'pytest', 'coverage', 'pep8', 'tox', 'flake8',
            'sphinx', 'sphinx_rtd_theme', 'nbsphinx', 'sphinx-argparse',
            'nbsphinx'
        ],
    },
)


import datetime
import os
import re
import string
import subprocess
import webbrowser as wb
from os import path

from unidecode import unidecode

import Microfone as m

import pyttsx3

import comandos
from pln.calssificador import classificar
import urllib.request, json

### ----------- TEXTO EM VOZ ----------- ###
engine = pyttsx3.init()  # object creation

voices = engine.getProperty('voices')
engine.setProperty('voice', voices[0].id)
engine.setProperty('rate', 135)

def falar(texto):
    engine.say(texto)
    engine.runAndWait()

### ----------- TEXTO EM VOZ ----------- ###

def boas_vindas():
    Horario = int(datetime.datetime.now().hour)

    if Horario >= 0 and Horario < 12:
        falar('Bom dia, em que posso ajudar ?')

    elif Horario >= 12 and Horario < 18:
        falar('Boa tarde, em que posso ajudar ?')

    elif Horario >= 18 and Horario != 0:
        falar('Boa noite, em que posso ajudar ?')

class SystemInfo:
    def __init__(self):
        pass

    @staticmethod
    def obter_horario():
        now = datetime.datetime.now()
        resposta = 'São {} horas e {} minutos.'.format(now.hour, now.minute)
        return resposta

    @staticmethod
    def obter_data():
        now = datetime.datetime.now()
        resposta = 'Hoje é dia {} de {} de {}'.format(now.day, now.strftime("%B"), now.year)
        return resposta

    @staticmethod
    def abrir_chrome():
        chrome_path = 'open -a /Applications/Google\ Chrome.app %s'
        return wb.get(chrome_path)

    @staticmethod
    def abrir_navegador():
        return wb.open_new_tab('http://www.google.com')

    @staticmethod
    def abrir_google():
        return wb.open_new_tab('http://www.google.com')

    @staticmethod
    def abrir_youtube():
        return wb.open_new_tab('https://www.youtube.com')

    @staticmethod
    def abrir_facebook():
        return wb.open_new_tab('https://www.facebook.com')

    @staticmethod
    def abrir_instagram():
        return wb.open_new_tab('https://www.instagram.com')

    @staticmethod
    def obter_dolar():
        with urllib.request.urlopen("https://economia.awesomeapi.com.br/json/last/USD-BRL") as url:
            data = json.loads(url.read().decode())

        now = datetime.datetime.now()
        valor = round(float(data['USDBRL']['ask']), 2)

        resposta = 'O valor do Dólar é R$ {}, valor atualizado às {} horas e {} minutos'.format(valor, now.hour,
                                                                                now.minute)
        return resposta

    @staticmethod
    def obter_euro():
        with urllib.request.urlopen("https://economia.awesomeapi.com.br/json/last/EUR-BRL") as url:
            data = json.loads(url.read().decode())

        now = datetime.datetime.now()
        valor = round(float(data['EURBRL']['ask']), 2)

        resposta = 'O valor do Euro é R$ {}, valor atualizado às {} horas e {} minutos'.format(valor, now.hour,
                                                                                now.minute)
        return resposta

    @staticmethod
    def buscar_endereco(cep):

        temp_string = cep
        out = temp_string.translate(str.maketrans('', '', string.punctuation))
        numbers = [int(temp) for temp in out.split() if temp.isdigit()]
        cep_limpo = ''.join(char for char in str(numbers) if char.isalnum())

        with urllib.request.urlopen(f"https://viacep.com.br/ws/{cep_limpo}/json/") as url:
            data = json.loads(url.read().decode())

        resposta = 'O CEP informado está localizado no estado {}, na cidade de {}, no bairro {}, {}'.format(data['uf'],
                                                                                data[
                                                                                'localidade'],
                                                                                data[
                                                                                'bairro'],
                                                                                data[
                                                                                'logradouro'])

        return resposta

    @staticmethod
    def cumprimentar():

        Horario = int(datetime.datetime.now().hour)

        if Horario >= 0 and Horario < 12:
            resposta = 'Olá, Bom dia, tudo bem com você ? espero que sim. Em que posso ajudar ?'

        elif Horario >= 12 and Horario < 18:
            resposta = 'Olá, Boa tarde, tudo bem com você ? espero que sim. Em que posso ajudar ?'

        elif Horario >= 18 and Horario != 0:
            resposta = 'Olá, Boa noite, tudo bem com você ? espero que sim. Em que posso ajudar ?'

        return resposta

    @staticmethod
    def pesquisar_internet(pesquisa):

        remover_palavras = ['pesquisar', 'pesquise', 'busque', 'buscar', 'procure', 'procurar']
        lista_frase = pesquisa.split()

        result = [palavra for palavra in lista_frase if palavra.lower() not in remover_palavras]

        retorno = ' '.join(result)

        return wb.open_new_tab('http://google.com/search?q='+ retorno)

    @staticmethod
    def lower_lista(lista):
        a = (map(lambda x: x.lower(), lista))
        b = list(a)
        return b

    @staticmethod
    def navegacao():
        mic = m.cMicrofone()

        alfabeto = list(string.ascii_lowercase)

        diretorio = ""
        diretorio_anterior = ""
        listadir = []

        while (True):
            comandos.falar("Qual disco inicial ?")
            print("❖ - Assistente: Qual disco inicial ?")
            palavra = mic.Ouvir()
            print('🎤 - Audio captado: '+palavra)

            # if ((palavra.lower() in alfabeto) or ("de" in palavra.lower())):

            if "de" in palavra.lower():

                diretorio = "D:\\"

            elif((" c" in palavra.lower()) or ("se" in palavra.lower())):

                diretorio = "C:\\"
            else:
                print('🎤 - Audio captado: '+ palavra)
                print("❖ - Assistente: Desculpe, Não consegui encontrar o disco... \n ❖ - Assistente: Saindo do modo navegação por diretórios")
                print('\n________________________________\n')
                falar("Desculpe, Não consegui encontrar o disco...")
                falar("Saindo do modo navegação por diretórios")
                break

            diretorio_anterior = diretorio

            listadir.append(diretorio_anterior)

            while (True):

                diretorio_anterior = diretorio

                try:

                    lista = os.listdir(diretorio)
                    comandos.falar("Qual diretório?")
                    print('________________________________')
                    print("❖ - Assistente: Qual diretorio ?\n")
                    print(str(lista))
                    #for i in lista:
                        #print(f"-- {i}")

                    print('________________________________')
                    palavra = unidecode((mic.Ouvir()).lower())
                    print('🎤 - Audio captado: '+palavra)

                    if palavra == "voltar" and len(listadir) > 1:
                        diretorio = listadir.__getitem__(len(listadir) - 2)
                        listadir.remove(listadir.__getitem__(len(listadir) - 1))
                    else:
                        str_match = [x for x in comandos.SystemInfo.lower_lista(lista) if re.search(palavra, x)]

                        palavra = str(str_match)
                        simbolos = '[]\''

                        if (str_match.__len__() > 1):
                            dirduplicado = (f"Foram encontrados os seguintes diretórios: {str(str_match)}. Diga qual deles deseja acessar.")
                            print('❖ - Assistente: ' +dirduplicado)
                            comandos.falar(dirduplicado)

                            while True:
                                diroficial = mic.Ouvir()
                                palavra = [x for x in comandos.SystemInfo.lower_lista(lista) if diroficial == x]
                                if (palavra != ''):
                                    palavra = palavra[0]
                                    break

                        for i in simbolos:
                            palavra = palavra.replace(i, '')

                        if palavra != "":
                            diretorio += "\\" + palavra

                    # if (('.txt' or '.lnk' or '.exe' or '.docx' or '.xls') in diretorio):
                    if ("." in diretorio):
                        os.startfile(diretorio)
                        diretorio = diretorio_anterior
                    else:
                        os.listdir(diretorio)
                        if not listadir.__contains__(diretorio):
                            listadir.append(diretorio)

                    #print(listadir)

                except:
                    print("❖ - Assistente: diretorio não encontrado")
                    if (diretorio == str(palavra).upper() + ":\\"):
                        print("Por favor selecione um disco válido!")
                        break
                    diretorio = diretorio_anterior

            else:
                print('Por favor selecione um disco válido!')

class Executar:

    @staticmethod
    def excutar_comandos(frase):

        grupo = classificar(frase)
        fala = ''

        if grupo == 'horario|retornarHorario':
            fala = SystemInfo.obter_horario()
            falar(fala)
        elif grupo == 'data|retornarData':
            fala = SystemInfo.obter_data()
            falar(fala)
            # Abrir softwares listados
        elif grupo == 'cumprimento|responderCumprimento':
            fala = SystemInfo.cumprimentar()
            falar(fala)
            # Abrir softwares listados
        elif grupo == 'abrir|notepad':
            fala = 'Abrindo o bloco de notas'
            falar(fala)
            subprocess.Popen('notepad.exe')
        elif grupo == 'abrir|abrirCalculadora':
            fala = 'Abrindo calculadora'
            falar(fala)
            subprocess.Popen('calc.exe')
        elif grupo == 'abrir|abrirNavegador':
            fala = 'Abrindo o navegador'
            falar(fala)
            SystemInfo.abrir_navegador()
            # Abrir softwares sites
        elif grupo == 'abrir|acessarGoogle' and 'google' in frase:
            fala = 'Acessando Google'
            falar(fala)
            SystemInfo.abrir_google()
        elif grupo == 'abrir|acessarGoogle' and 'google' not in frase:
            fala = 'Acho que não entendi muito bem oque você disse, estou realizando uma pesquisa na web para ajudar!'
            falar(fala)
            SystemInfo.pesquisar_internet(frase)
        elif grupo == 'abrir|acessarYoutube' and 'youtube' in frase:
            fala = 'Acessando Youtube'
            falar(fala)
            SystemInfo.abrir_youtube()
        elif grupo == 'abrir|acessarYoutube' and 'youtube' not in frase:
            fala = 'Acho que não entendi muito bem oque você disse, estou realizando uma pesquisa na web para ajudar!'
            falar(fala)
            SystemInfo.pesquisar_internet(frase)
        elif grupo == 'abrir|acessarFacebook' and 'facebook' in frase:
            fala = 'Acessando Facebook'
            falar(fala)
            SystemInfo.abrir_facebook()
        elif grupo == 'abrir|acessaracebook' and 'facebook' not in frase:
            fala = 'Acho que não entendi muito bem oque você disse, estou realizando uma pesquisa na web para ajudar!'
            falar(fala)
            SystemInfo.pesquisar_internet(frase)
        elif grupo == 'abrir|acessarInstagram' and 'instagram' in frase:
            fala = 'Acessando Instragram'
            falar(fala)
            SystemInfo.abrir_instagram()
        elif grupo == 'abrir|acessarInstagram' and 'instagram' not in frase:
            fala = 'Acho que não entendi muito bem oque você disse, estou realizando uma pesquisa na web para ajudar!'
            falar(fala)
            SystemInfo.pesquisar_internet(frase)
        elif (grupo == 'cotacao|retornarCotacaoAtual') and 'dolar' in frase:
            falar('Buscando valor atual do dólar ...')
            fala = SystemInfo.obter_dolar()
            falar(fala)
        elif (grupo == 'cotacao|retornarCotacaoAtual') and 'euro' in frase:
            falar('Buscando valor atual do euro ...')
            fala = SystemInfo.obter_euro()
            falar(fala)
        elif grupo == 'pesquisa|buscarCep':
            falar('Buscando o CEP informado ...')
            fala = SystemInfo.buscar_endereco(frase)
            falar(fala)
        elif grupo == 'pesquisa|pesquisaWeb':
            fala = 'Ok, já estou pesquisando o\'que você me disse!'
            falar(fala)
            SystemInfo.pesquisar_internet(frase)
        elif grupo == 'navegacao|navegarDiretorio':
            falar('Ok, abrindo a navegação por diretório!')
            SystemInfo.navegacao()

        # Mostrar informações (resultado e a qual grupo ele pertence)
        print("\n\n❖ - Assistente: Ouvindo...")
        print('\n________________________________\n')
        print('🎤 - Audio captado: {} - Grupo: {}\n'.format(frase, grupo))
        print('\n________________________________\n')
        print('❖ - Assistente: '+fala)


import re
from string import Template
import io
import numpy as np
from .utils import journal_macros, get_month_num, tex2unicode

class Author:
    # author structure which parses an author's
    # first and last name from the bibtex entry
    def __init__(self, bib_entry):
        bib_entry = re.sub(r'[^\\]~', ' ', bib_entry)

        # figure out if it is lastname, firstname
        # or firstname lastname

        if "," in bib_entry:
            nnames = bib_entry.split(', ')
            self.firstname = nnames[1]
            self.lastname = nnames[0]
        else:
            nnames = re.findall('[{]?([\S]+)[}]?', bib_entry)
            if len(nnames) > 1:
                self.lastname = " ".join(nnames[1:])
                self.firstname = nnames[0]
            else:
                self.lastname = nnames[0]
                self.firstname = ""

        if ("{" in self.lastname) or ("}" in self.lastname):
            self.lastname = self.lastname.replace("{", "")
            self.lastname = self.lastname.replace("}", "")

        if ("{" in self.firstname) or ("}" in self.firstname):
            self.firstname = self.firstname.replace("{", "")
            self.firstname = self.firstname.replace("}", "")

    def short_name(self):
        # format as [last name], [first name letters].
        # e.g. Sankar, R. or Sankar, R. G.
        first = self.firstname.split(' ')
        short_first = ""
        if first[0] != '':
            for name in first:
                # account for first letter unicodes
                if name[0] == "{":
                    fname = re.findall(r'(\{.+\})', name)[0]
                else:
                    fname = name[0]
                short_first = short_first + fname + ". "
        return "%s, %s" % (self.lastname, short_first[:-1])

    def long_name(self):
        # format as [last name], [first names].
        # e.g. Sankar, Ramanakumar  or Sankar, Ramanakumar Ganapathy
        first = self.firstname.split(' ')
        long_first = ""
        for name in first:
            long_first = long_first + name + " "
        return "%s %s" % (long_first[:-1], self.lastname)

class Records(object):
    def __init__(self, rec_type, entry_name):
        self.authors = []
        self.rec_type = rec_type
        self.entry_name = entry_name

    def parse_text(self, text):
        self.text = text

        # concatenate the entries together into a single
        # string so that we can parse the entries
        text = re.sub(r"[\s]{2,50}?|[\n?]", r"", text)

        # the entry pattern is [key] = [entry],
        # text_pattern = r"(\w+)\s*=\s*\"?\{*\"?(.*?)\"?\}*\"?,?(,|$)"
        text_pattern = r"(((\w+)\s*=\s*([^=]+))(,|$))"

        matches = re.findall(text_pattern, text)

        for i, (_, _, key, entry, _) in enumerate(matches):
            key = key.lower().strip()
            line = entry.strip()
            line = line.replace('"', '')

            # special case author list
            # break down individual authors
            if (key == 'author') or (key == 'authors'):
                authors = entry.split(' and')
                for author in authors:
                    authi = Author(author.strip())
                    self.authors.append(authi)
            else:
                entry = entry.replace("{", "")
                entry = entry.replace("}", "")

            # if the journaly contains a macro,
            # parse it using the dictionary
            if key == "journal":
                self.journal = entry

                if self.journal[0] == '\\':
                    try:
                        self.journal = journal_macros[self.journal]
                    except KeyError:
                        print(
                            f"Found macro {self.journal} for journal name but"
                            "no matching journal name..."
                        )
                        self.journal = entry

            # year and volume are numbers
            elif key == "year":
                try:
                    self.year = int(entry)
                except ValueError:
                    self.year = entry

            elif key == "month":
                try:
                    self.month = int(entry)
                except ValueError:
                    self.month = entry

            elif key == "volume":
                try:
                    self.volume = int(entry)
                except ValueError:
                    self.volume = entry

            # parse the DOI into a URL
            elif key == "doi":
                self.doi = entry
                self.doiurl = "https://doi.org/%s" % self.doi

            # default case, save the entry as self.[key] = [entry]
            else:
                setattr(self, key, entry)

    def __getattribute__(self, __name):
        try:
            value = object.__getattribute__(self, __name)
        except AttributeError:
            raise AttributeError(f"{self.entry_name} has no keyword {__name}")

        if __name == 'year':
            try:
                return int(value)
            except Exception:
                raise ValueError(
                    f"Year entry '{value}' cannot be converted to integer for"
                    f"{self.entry_name}")
        else:
            return value

class bibtexParser:
    '''
        Parses bibtex entries from a file and creates
        a Record object of each entry

        Attributes
        ----------
        records : list
            array of ``records`` objects

        Methods
        -------
        get_records()
            Gets the records from input filename (run automatically)
        to_html()
            Parse the entries as HTML output with each entry as an
            <li> element
        to_latex_item()
            Parse the entries to a .text file with each entry as an
            \\item
        to_text()
            Parse all entries as a plain text (.txt) file with
            each entry separated by a newline

    '''

    def __init__(self, fname, fileIO=None):
        '''
            Initialize

            Pareters
            ------
            fname : str
                Input filename (.bib)

        '''
        self.fname = fname

        if (fileIO is not None) and isinstance(fileIO, io.StringIO):
            self.lines = fileIO.readlines()
            fileIO.close()
            self.filetype = 'buffer'
        else:
            file = open(self.fname, "r", encoding='utf-8')
            self.lines = file.readlines()
            file.close()
            self.filetype = 'file'

        self.get_records()

    def get_records(self):
        self.records = []

        # start pattern is @[type]{[name],
        pattern = r"@(\w+)\s*\{(.*?),\n?([\s\S\n\t]*?[^\w\.\)\!])\}\n"

        lines = "".join(self.lines) + "\n"

        records = re.findall(pattern, lines)

        for record in records:
            entry_type, entry_name, ref_text = record
            reci = Records(entry_type, entry_name)
            reci.parse_text(ref_text)

            self.records.append(reci)

        print("Found %d records from %s" % (len(self.records), self.fname))

    def to_out(self, outname, templatefile,
               convunicode=True, clean=False, sort=False):
        # output to [outname] using the template given

        # open the output file
        if isinstance(outname, io.StringIO):
            outfile = outname
            outfile.seek(0)
        else:
            outfile = open(outname, "w", encoding='utf-8')

        # no unicode output for .tex files
        if convunicode:
            encodeunicode = True
        else:
            encodeunicode = False

        # check if we want to clean or sort
        if clean:
            recs = self.cleanup(sort=sort)
            self.records = [reci[0] for reci in recs]

        # get the template string -- we will replace this for
        # each entry
        if isinstance(templatefile, io.StringIO):
            templatelines = templatefile.readlines()
        else:
            templatelines = open(templatefile, 'r').readlines()

        templates = []
        for linei in templatelines:
            if linei[0] == '@':
                temptype = re.findall(r'@([A-Za-z]*?):', linei)[0]
                templates.append(
                    [temptype.lower(), linei.replace('@%s:' % temptype, '')])
            else:
                templates.append(['all', linei])
        nalls = 0
        for template in templates:
            if template[0] == 'all':
                generictempstring = template[1]
                nalls += 1

        if nalls > 1:
            print("Error! Can only have one general template")
            return

        # loop through all the records
        for record in self.records:
            rectype = record.rec_type

            templatestring = generictempstring
            for template in templates:
                if rectype.lower() == template[0]:
                    templatestring = template[1]

            # first, find the author template because this is going to
            # be common to all
            authtemplate = re.findall(r'auth([sf])([0-9]+|[a]{1})?', templatestring)
            try:
                if len(authtemplate) > 1:
                    raise ValueError(
                        "Error! Only one entry for author is allowed!")
                authstyle = authtemplate[0][0]
            except IndexError:
                raise IndexError("Please enter an author template")

            if authstyle == 's':
                try:
                    if authtemplate[0][1] != 'a':
                        authnum = int(authtemplate[0][1])
                    else:
                        authnum = int(1e10)
                except Exception:
                    raise ValueError(
                        "Please use $authsa for all authors or enter a number "
                        "after $auths")
            else:
                authnum = 1

            # then find the groups
            groups = re.findall(
                r"(?<!\\)([\{]{1}.*?(?<!\\)[\}]{1})", templatestring)

            # then find all the templates
            kws = re.findall(r'(\$[a-z0-9]+)', templatestring)
            # copy the template string to replace later
            tempstring = templatestring

            # this will hold all the attribute values
            tempdict = {}
            for tempi in kws:

                # get the variable name
                tempstr = tempi.replace('$', '')

                # special case -- authors
                if "auth" in tempi:
                    authtext = ''
                    if authstyle == 's':  # short author (lastname initials)
                        if len(record.authors) > authnum:
                            author = record.authors[0]
                            if encodeunicode:
                                short_name = tex2unicode(author.short_name())
                            else:
                                short_name = author.short_name()

                            authtext = authtext + short_name + " et al."
                        else:
                            nauths = min([len(record.authors), authnum])
                            for authi in range(nauths):
                                author0 = record.authors[authi]
                                if encodeunicode:
                                    short_name = tex2unicode(
                                        author0.short_name())
                                else:
                                    short_name = author0.short_name()

                                authtext = authtext + short_name
                                if authi == (nauths - 2):
                                    authtext = authtext + " and "
                                elif authi == (nauths - 1):
                                    authtext = authtext + " "
                                else:
                                    authtext = authtext + ", "
                    elif authstyle == 'f':
                        for i, author in enumerate(record.authors):
                            if encodeunicode:
                                long_name = tex2unicode(author.long_name())
                            else:
                                long_name = author.long_name()

                            if i != (len(record.authors) - 1):
                                authtext = authtext + long_name + ", "
                            else:
                                authtext = authtext + long_name
                    # save the author string to the output dictionary
                    tempdict[tempstr] = authtext
                else:  # for everything else
                    if hasattr(record, tempstr):
                        if encodeunicode:
                            try:
                                tempdict[tempstr] = tex2unicode(
                                    record.__getattribute__(tempstr))
                            except AttributeError:
                                tempdict[tempstr] = record.__getattribute__(
                                    tempstr)
                        else:
                            tempdict[tempstr] = record.__getattribute__(
                                tempstr)

                    else:
                        # if the record does not have this attribute
                        # find the group that this attribute is in
                        remgroup = None
                        for groupi in groups:
                            if tempstr in groupi:
                                remgroup = groupi

                        if remgroup is None:
                            raise KeyError(
                                f"entry '{tempstr}' is not available "
                                f"in record {record.entry_name}")

                        # remove the group
                        tempstring = tempstring.replace(remgroup, '')
                        tempdict[tempstr] = ''

            # finally, remove all the group {}'s and replace the string
            for groupi in groups:
                repgroup = re.sub(r'{(.*?)}', r'\1', groupi)
                tempstring = tempstring.replace(groupi, repgroup)

            # and also replace all the actual escaped \{ to {
            tempstring = tempstring.replace('\\{', '{')
            tempstring = tempstring.replace('\\}', '}')
            # and then substitute
            newtemplate = Template(tempstring)
            outfile.write(newtemplate.safe_substitute(tempdict))

    def cleanup(self, sort=False, save=False, outfile=None):
        if save:
            if outfile is None:
                if self.filetype == 'file':
                    outname = self.fname.replace(".bib", "_clean.bib")
                else:
                    raise RuntimeError(
                        "Please provide an output writer for non-file inputs")

        # sort the data first
        if sort:
            names = []

            # create a list of author/year/month so that we can sort by this
            # key later
            record_sub = []
            for record in self.records:
                if 'Yoden' in record.entry_name:
                    print(record.entry_name)
                if hasattr(record, "month"):
                    monthnum = get_month_num(record.month)
                else:
                    monthnum = 0
                try:
                    namei = "%s%d%02d_%s" % (
                        record.authors[0].lastname, record.year,
                        monthnum, record.title[:5])
                except IndexError:
                    try:
                        namei = "%d%02d" % (record.year, monthnum)
                    except Exception:
                        continue
                except Exception as e:
                    print(f"failing on {record.entry_name}")
                    print(e)
                    continue
                names.append(namei)
                record_sub.append(record)
            # namesort = sorted(names)
            # records = [record_sub[names.index(name)] for name in namesort]
            records = np.asarray(record_sub)[np.argsort(names)]

        else:
            # if we don't want to sort then just use the same list
            records = self.records

        recs = []
        rec_list = []

        for record in records:
            try:
                # [record.authors[0].short_name(), record.title, record.year]
                reci = f"{record.authors[0].lastname}_{record.title[:5]}"
                f"_{record.year}_{record.entry_name}"
            except IndexError:
                try:
                    reci = f"{record.title[:5]}_{record.year}"
                    f"_{record.entry_name}"
                except Exception as e:
                    print(f"failing on {record.entry_name}")
                    print(e)
                    continue
            # find duplicates
            if reci not in rec_list:
                recs.append([record, record.entry_name])
                rec_list.append(reci)
            # recs.append([record, record.entry_name])
        print(f"Down to {len(recs)} records")

        # check if we want to save this out
        if save:
            # for input from the web UI
            # or if a file handler is passed in
            if outfile is not None:
                for rec in recs:
                    outfile.write("@%s{%s,\n" % (rec[0].rec_type, rec[1]))
                    for text in rec[0].text:
                        outfile.write("%s" % text)
                    outfile.write("\n}\n\n")
            # for a file input
            elif self.filetype == 'file':
                with open(outname, "w", encoding='utf-8') as outfile:
                    for rec in recs:
                        outfile.write("@%s{%s,\n" % (rec[0].rec_type, rec[1]))
                        for text in rec[0].text:
                            outfile.write("%s" % text)
                        outfile.write("\n}\n\n")

        return recs


# -*- coding: utf-8 -*-
"""
Created on Sun Nov 26 09:36:19 2017

@author: majin
"""
from PIL import Image
import matplotlib.pyplot as plt
import numpy as np
import DBScan_segment as dbss

def get_my_img_1x784(img_path, show=True):
    '''
    获取输入的单数字图片
    '''
    #获取输入图片  将输入图片大小调整成28x28
    img_28x28 = resizeImage(img_path,28,28)
    gray_img_28x28 = rgb2gray(img_28x28)
    if show:
        plt.imshow(gray_img_28x28)
        plt.show()

    #将图片转成矩阵，feed需要1x784
    img_1x784 = np.matrix(gray_img_28x28.getdata()).reshape([1,-1])
    return img_1x784

def openImage(fileImg):
    return Image.open(fileImg)

def resizeImage(img,  width, height, imtype="png",fileout=None):
    '''
    filein: 输入图片
    fileout: 输出图片
    width: 输出图片宽度
    height:输出图片高度
    type:输出图片类型（png, gif, jpeg...）
    '''

    #resize image with high-quality
    out = img.resize((width, height),Image.ANTIALIAS)
    if fileout:
        out.save(fileout, imtype)
    return out

def rgb2gray(pic,show=False):
    '''
     Y' = 0.299 R + 0.587 G + 0.114 B

     L模式：灰度模式
     1模式：二值模式
    '''
    #将灰度值
    pic = pic.convert("L")
    #tva = [ (255-x)*1.0/255.0 for x in pic.getdata()]
    for i in range(pic.width):
        for j in range(pic.height):
            h = pic.getpixel((i,j))
            if h>200:
                h=255   #设成白色
            pic.putpixel((i,j), h)
    if show:
        pic.show()
    return pic

def segment_picture(filein,blank_padding = 10):
    '''
    切分图片
    ---------------
    paramters
        @filein：图像的路径
        @blannk_padding: 图像切分时，保留多少的padding
    ---------------
    return
        @[0] : 返回切分的图像矩阵,维度(c,784)
                c:类数，聚了几类切成几类
                784：切下来的图片，resize
    '''

    #使用DBScan聚类来分割图片
    #DBScan后，得到类的边界（使用路径参数的原因，为了序列化）
    bound = dbss.runner(filein)

    pic = openImage(filein)  #打开图片
    pic = rgb2gray(pic)      #转成纯灰度图像

    imgmatric = []           #保存切出来的图片
                             #(c,784),每行是一个数字，784是选框确定后resize的结果

    for i in bound:
        height = pic.height

        #数字上下左右的边界
        max_y = i[0]
        min_y = i[1]
        min_x = i[2]
        max_x = i[3]

        #图像的左上
        x1 = min_x-blank_padding
        y1 = height-max_y-blank_padding

        #图像的右下
        x2 = max_x+blank_padding
        y2 = height-min_y+blank_padding

        #切分图片
        sp_im = pic.crop((x1,y1,x2,y2))

        #保存的图像是28x28的
        resize_sp_im = resizeImage(sp_im,28,28)

        #转成行向量
        img_1x784 = np.matrix(resize_sp_im.getdata()).reshape([1,-1])

        #矩阵添加一行
        imgmatric.append(img_1x784[0].tolist()[0])
    return np.matrix(imgmatric)

if __name__ == "__main__":

    #filein = r'./image/r2.jpg'
    filein = r'./image/realphoto.jpg'

    '''
    #测试去掉真实世界的背景
    a = openImage(filein)
    rgb2gray(a,show=True)
    '''

    #测试分割图像
    res = segment_picture(filein,blank_padding=3)
    clazz_num = res.shape[0]
    for i in range(clazz_num):
        plt.subplot(clazz_num/4+1,4,i+1)
        plt.imshow(res[i].reshape([28,28]))